@pressgang-wp/shakedown 0.1.0 → 0.1.1

package/README.md

@@ -13,7 +13,7 @@ Point it at a running site and in under a minute you'll know: does every page re
 You need Node 20+, WP-CLI, and a PressGang site running locally (any server — Herd, Valet, DDEV, MAMP… it's just a URL). From inside your theme:
 
 ```bash
-npm i -D github:pressgang-wp/pressgang-shakedown
+npm i -D @pressgang-wp/shakedown
 npx playwright install chromium   # once per machine
 npx shakedown                     # ⚓ derive the matrix, run every pass
 ```
@@ -45,6 +45,8 @@ Optional `shakedown.config.json` in the theme, for overrides only:
 
 Authored journeys (form submissions, checkout flows) live in your theme's `tests/e2e/` — when present they run alongside the derived passes as the `journeys` project.
 
+**Seeding is convention-first.** If your theme ships [Muster](https://github.com/pressgang-wp/pressgang-muster) seeders (`src/Muster/*`), the sandbox runs them via `wp capstan seed` as the baseline — your real menus, terms and pages — then layers the derived populated/minimal ACF state fixtures on top. A theme that ships no seeders is unaffected: the derived layer covers it on its own, so it does a good job out of the box.
+
 Sandbox fixture randomness and time are separate deterministic inputs. `seed`
 controls generated values; `epoch` fixes relative dates used by Muster,
 including ACF date fields and the fixture posts themselves. It must be a
@@ -132,7 +134,7 @@ Shakedown is part of the [PressGang](https://pressgang.dev) ecosystem and is des
 | --- | --- |
 | [pressgang](https://github.com/pressgang-wp/pressgang) | The parent theme framework (Timber + Twig, config-driven) |
 | [capstan](https://github.com/pressgang-wp/pressgang-capstan) | WP-CLI scaffolding & introspection — future source of the route matrix and per-URL controller/template oracle |
-| [muster](https://github.com/pressgang-wp/pressgang-muster) | Seeds deterministic populated/minimal ACF states in the disposable sandbox |
+| [muster](https://github.com/pressgang-wp/pressgang-muster) | Runs the theme's own seeders as the sandbox baseline, and seeds deterministic populated/minimal ACF states on top |
 | [bosun](https://github.com/pressgang-wp/pressgang-bosun) | AI-agent guidelines & skills — future distribution channel for Shakedown's QA skills |
 
 ## 🛠️ Roadmap
@@ -142,7 +144,6 @@ Shakedown is part of the [PressGang](https://pressgang.dev) ecosystem and is des
 - More passes: **accessibility** (axe-core), **visual snapshots**
 - **Trial Report** — a client-readable HTML report with screenshots and coverage
 - Engines: self-booting **WordPress Playground**, per-PR **InstaWP** CI sites, **wp-env** fidelity lane
-- Published as `@pressgang-wp/shakedown` on npm
 
 ## 📋 Requirements
 

package/bin/shakedown.mjs

@@ -19,7 +19,7 @@ import { fileURLToPath } from 'node:url';
 import { existsSync } from 'node:fs';
 import { resolveTarget } from '../lib/target.mjs';
 import { capstanDoctor, deriveMatrix, mergeRoutes } from '../lib/derive.mjs';
-import { bootSandbox, DEFAULT_FIXTURE_EPOCH, seedAcfStates } from '../lib/sandbox.mjs';
+import { bootSandbox, DEFAULT_FIXTURE_EPOCH, seedAcfStates, seedThemeMuster } from '../lib/sandbox.mjs';
 
 const pkgRoot = dirname(dirname(fileURLToPath(import.meta.url)));
 const workspace = process.cwd();
@@ -130,13 +130,20 @@ try {
         // ACF state fixtures — populated + minimal per field group. Seeding
         // is sandbox-only by design: the isolation witness has already
         // proven this database is throwaway.
+        const seed = target.sandbox?.seed ?? 42;
+        const epoch = target.sandbox?.epoch ?? DEFAULT_FIXTURE_EPOCH;
+
+        // Convention-first baseline: run the theme's own Muster seeders so its
+        // real menus, terms and pages are present. Falls through silently when
+        // the theme ships none — the derived ACF states below still cover it.
+        if (seedThemeMuster(sandbox, { seed })) {
+          console.log('⚓ seeded theme baseline via `wp capstan seed`');
+        }
+
         let states = [];
         const muster = findMusterAutoload(target);
         if (muster) {
-          states = seedAcfStates(sandbox, muster, {
-            seed: target.sandbox?.seed ?? 42,
-            epoch: target.sandbox?.epoch ?? DEFAULT_FIXTURE_EPOCH,
-          });
+          states = seedAcfStates(sandbox, muster, { seed, epoch });
           console.log(`⚓ seeded ${states.length} ACF state fixtures via Muster`);
         } else {
           console.log('⚓ Muster not found — skipping ACF state fixtures (set sandbox.musterPath)');

package/lib/sandbox.mjs

@@ -306,6 +306,47 @@ export async function bootSandbox(target, options = {}) {
   };
 }
 
+/**
+ * Convention-first baseline seeding: run the theme's OWN Muster seeders
+ * (anything under its `src/Muster/`) via `wp capstan seed`, so a theme that
+ * ships fixtures gets its real content — menus assigned to locations, terms,
+ * pages, relationships — exactly as `wp capstan seed` produces on a dev site.
+ * The ACF state fixtures (see {@link seedAcfStates}) then layer on top.
+ *
+ * This is a deliberate no-op — returning false so the caller falls back to
+ * deriving everything from acf-json — when the theme ships no Muster classes
+ * or Capstan isn't installed to discover them. Out of the box, a theme with no
+ * fixtures still gets a fully derived suite; a theme with fixtures gets its own.
+ *
+ * @param {{root: string, theme: string|null}} sandbox
+ * @param {{seed?: number}} options Deterministic random seed passed through to Muster.
+ * @returns {boolean} True when theme seeders ran; false when none were applied.
+ */
+export function seedThemeMuster(sandbox, { seed = 42 } = {}) {
+  const musterDir = join(sandbox.root, 'wp-content', 'themes', sandbox.theme ?? '', 'src', 'Muster');
+
+  if (!sandbox.theme || !existsSync(musterDir)) {
+    return false;
+  }
+
+  try {
+    // `wp capstan seed` is Capstan's discovery-and-run of the theme's Muster
+    // subclasses; probe for it so a missing Capstan degrades to the fallback
+    // rather than throwing.
+    execFileSync('wp', ['cli', 'has-command', 'capstan seed', `--path=${sandbox.root}`], { stdio: 'ignore' });
+  } catch {
+    return false;
+  }
+
+  execFileSync(
+    'wp',
+    ['capstan', 'seed', `--seed=${seed}`, `--path=${sandbox.root}`],
+    { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }
+  );
+
+  return true;
+}
+
 /**
  * Seed ACF state fixtures (populated + minimal per field group) inside a
  * booted sandbox, via Muster. Only ever called for sandboxes — the isolation

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pressgang-wp/shakedown",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Sea trials for PressGang themes — config-derived e2e smoke testing",
   "type": "module",
   "license": "MIT",
@@ -18,6 +18,7 @@
     "bin",
     "lib",
     "php",
+    "tests",
     "playwright.config.mjs",
     "README.md"
   ],

package/tests/00-availability.spec.mjs

@@ -0,0 +1,66 @@
+import { test, expect } from '@playwright/test';
+import { loadMatrix, controllerHeaderName, ERROR_SIGNATURES } from './matrix.mjs';
+
+/**
+ * Pass 00 — availability.
+ *
+ * Every derived route must return its intended status with a body free of
+ * PHP/Twig error signatures and carrying a <title>. HTTP-only (no browser),
+ * so this pass sweeps the whole site in seconds. Where the observer answers
+ * (sandbox), each route is also checked against the Capstan oracle and for
+ * silent PHP notices.
+ */
+const matrix = loadMatrix();
+
+/** Redirect statuses that count as "handled" for the unknown-URL probe. */
+const REDIRECT_STATUSES = [301, 302, 303, 307, 308];
+
+for (const route of matrix.routes) {
+  test(`00 ${route.kind} ${route.url}`, async ({ request }) => {
+    if (route.expect === 404) {
+      // Unknown URLs must not render a page: a 404, or a redirect away
+      // (e.g. the Redirection plugin's catch-all), both count as handled.
+      const res = await request.get(route.url, { maxRedirects: 0 });
+      expect(
+        [...REDIRECT_STATUSES, 404],
+        `unknown URL should 404 or redirect away, got ${res.status()}`
+      ).toContain(res.status());
+      return;
+    }
+
+    const res = await request.get(route.url, { maxRedirects: 5 });
+    expect(res.status(), `status for ${route.url}`).toBe(route.expect);
+
+    const body = await res.text();
+
+    // Collect-then-assert so a failure names the signatures found instead of
+    // dumping the page body into the report.
+    const found = ERROR_SIGNATURES.filter((sig) => body.includes(sig));
+    expect(found, `PHP/Twig error signatures in ${route.url}`).toEqual([]);
+
+    if (route.expect === 200) {
+      expect(body, `missing <title> on ${route.url}`).toMatch(/<title>[^<]+<\/title>/i);
+    }
+
+    // Oracle assertions — active when the observer answered (sandbox) and
+    // the matrix carries capstan --resolve expectations for this route.
+    const headers = res.headers();
+
+    if (route.template && headers['x-shakedown-template']) {
+      expect(headers['x-shakedown-template'], `template oracle for ${route.url}`).toBe(route.template);
+    }
+
+    if (route.controller && headers['x-shakedown-controller']) {
+      expect(headers['x-shakedown-controller'], `controller oracle for ${route.url}`).toBe(
+        controllerHeaderName(route.controller)
+      );
+    }
+
+    // PHP issues are counted by the observer even when display/log are off —
+    // a page can look perfect and still be raising notices on every request.
+    if (headers['x-shakedown-php-issues'] !== undefined) {
+      const sample = decodeURIComponent(headers['x-shakedown-php-sample'] ?? '');
+      expect(Number(headers['x-shakedown-php-issues']), `PHP notices/warnings on ${route.url}: ${sample}`).toBe(0);
+    }
+  });
+}

package/tests/01-integrity.spec.mjs

@@ -0,0 +1,46 @@
+import { test, expect } from '@playwright/test';
+import { loadMatrix } from './matrix.mjs';
+
+/**
+ * Pass 01 — integrity.
+ *
+ * Renders each 200 route in a real browser and asserts a clean runtime:
+ * no JS exceptions, no console errors, no failed same-origin requests,
+ * and no broken images. Third-party request failures are ignored — the
+ * site under trial is the subject, not its CDNs.
+ */
+const matrix = loadMatrix();
+const origin = new URL(matrix.baseUrl).origin;
+
+for (const route of matrix.routes.filter((r) => r.expect === 200)) {
+  test(`01 ${route.kind} ${route.url}`, async ({ page }) => {
+    const pageErrors = [];
+    const consoleErrors = [];
+    const failedRequests = [];
+
+    page.on('pageerror', (err) => pageErrors.push(err.message));
+    page.on('console', (msg) => {
+      if (msg.type() === 'error') consoleErrors.push(msg.text());
+    });
+    page.on('response', (res) => {
+      if (res.status() >= 400 && res.url().startsWith(origin)) {
+        failedRequests.push(`${res.status()} ${res.url()}`);
+      }
+    });
+
+    await page.goto(route.url, { waitUntil: 'load' });
+
+    // Lazy images below the fold never load in this pass, so only eagerly
+    // loaded, "complete" images with zero natural width count as broken.
+    const brokenImages = await page.evaluate(() =>
+      Array.from(document.querySelectorAll('img'))
+        .filter((img) => img.loading !== 'lazy' && img.complete && img.naturalWidth === 0 && !!img.src)
+        .map((img) => img.currentSrc || img.src)
+    );
+
+    expect(pageErrors, 'JS exceptions').toEqual([]);
+    expect(consoleErrors, 'console errors').toEqual([]);
+    expect(failedRequests, 'failed same-origin requests').toEqual([]);
+    expect(brokenImages, 'broken images').toEqual([]);
+  });
+}

package/tests/02-accessibility.spec.mjs

@@ -0,0 +1,53 @@
+import { test, expect } from '@playwright/test';
+import AxeBuilder from '@axe-core/playwright';
+import { loadMatrix } from './matrix.mjs';
+
+/**
+ * Pass 02 — accessibility.
+ *
+ * Runs axe-core against every 200 route, scoped to WCAG 2.1 A/AA rules.
+ *
+ * Gate policy (per RFC-001: gates start advisory, promoted once stable):
+ * `serious` and `critical` violations fail the route; `moderate` and `minor`
+ * are reported to the console but do not fail — promote them once the
+ * serious/critical set is clean.
+ */
+const matrix = loadMatrix();
+
+const WCAG_TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'];
+const FAILING_IMPACTS = ['serious', 'critical'];
+
+/**
+ * One-line summary of an axe violation for readable failure output.
+ *
+ * @param {{id: string, impact?: string|null, nodes: unknown[], helpUrl: string}} v
+ * @returns {string}
+ */
+function describeViolation(v) {
+  return `[${v.impact}] ${v.id} × ${v.nodes.length} — ${v.helpUrl}`;
+}
+
+for (const route of matrix.routes.filter((r) => r.expect === 200)) {
+  test(`02 ${route.kind} ${route.url}`, async ({ page }) => {
+    await page.goto(route.url, { waitUntil: 'load' });
+
+    // Iframe contents are excluded: violations inside third-party embeds
+    // (YouTube player chrome etc.) are not the site's remediation surface.
+    const results = await new AxeBuilder({ page })
+      .withTags(WCAG_TAGS)
+      .exclude('iframe')
+      .analyze();
+
+    const failing = results.violations.filter((v) => FAILING_IMPACTS.includes(v.impact ?? ''));
+    const advisory = results.violations.filter((v) => !FAILING_IMPACTS.includes(v.impact ?? ''));
+
+    if (advisory.length > 0) {
+      console.log(`advisory a11y (${route.url}):\n  ${advisory.map(describeViolation).join('\n  ')}`);
+    }
+
+    expect(
+      failing.map(describeViolation),
+      `serious/critical WCAG 2.1 A/AA violations on ${route.url}`
+    ).toEqual([]);
+  });
+}

package/tests/03-visual.spec.mjs

@@ -0,0 +1,50 @@
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { test, expect } from '@playwright/test';
+import { loadMatrix } from './matrix.mjs';
+
+/**
+ * Pass 03 — visual regression.
+ *
+ * Full-page screenshots of every 200 route against baselines committed in
+ * the workspace (`tests/__screenshots__/`, per-platform). Deterministic
+ * fixtures (seeded values, pinned dates) are what make these stable —
+ * a diff should mean the THEME changed, not the content.
+ *
+ * First run: no baselines exist yet — the whole pass skips with a pointer
+ * to `shakedown sandbox --update-snapshots`, so CI isn't red for the wrong
+ * reason. `<time>` elements are masked: sample content created by WP core
+ * install carries the install date.
+ */
+const matrix = loadMatrix();
+const workspace = process.env.SHAKEDOWN_WORKSPACE ?? process.cwd();
+const hasBaselines = existsSync(join(workspace, 'tests', '__screenshots__'));
+
+/**
+ * Stable, unique snapshot name: kind plus the URL path.
+ * Mirrored in lib/trial-reporter.mjs — keep in sync.
+ *
+ * @param {{kind: string, url: string}} route
+ * @returns {string}
+ */
+function snapshotName(route) {
+  const path = new URL(route.url).pathname.replace(/\W+/g, '-').replace(/^-|-$/g, '') || 'home';
+  return `${route.kind.replace(/\W+/g, '-')}--${path}.png`;
+}
+
+for (const route of matrix.routes.filter((r) => r.expect === 200)) {
+  test(`03 ${route.kind} ${route.url}`, async ({ page }) => {
+    test.skip(!hasBaselines && test.info().config.updateSnapshots === 'missing',
+      'No visual baselines yet — create them with: shakedown sandbox --update-snapshots');
+
+    await page.goto(route.url, { waitUntil: 'load' });
+    await page.evaluate(() => document.fonts.ready);
+
+    await expect(page).toHaveScreenshot(snapshotName(route), {
+      fullPage: true,
+      animations: 'disabled',
+      mask: [page.locator('time')],
+      maxDiffPixelRatio: 0.001,
+    });
+  });
+}

package/tests/matrix.mjs

@@ -0,0 +1,70 @@
+import { readFileSync } from 'node:fs';
+
+/**
+ * @typedef {object} Route One derived route.
+ * @property {string} url Absolute URL to test.
+ * @property {string} kind `family:detail` label, e.g. `archive:event`.
+ * @property {number} expect Intended HTTP status (200, or 404 for the probe).
+ * @property {string} [template] Oracle (capstan --resolve): basename of the PHP template WP should choose.
+ * @property {string|null} [controller] Oracle: FQCN of the controller that should render (dispatched routes only).
+ */
+
+/**
+ * @typedef {object} Matrix The derived matrix written by `shakedown matrix`.
+ * @property {string} target
+ * @property {string} baseUrl
+ * @property {Route[]} routes
+ */
+
+/**
+ * Load the persisted matrix from the workspace (the directory the CLI was
+ * invoked from), with a friendly nudge when it hasn't been derived.
+ *
+ * @returns {Matrix}
+ */
+export function loadMatrix() {
+  const path = process.env.SHAKEDOWN_WORKSPACE
+    ? `${process.env.SHAKEDOWN_WORKSPACE}/.shakedown/matrix.json`
+    : new URL('../.shakedown/matrix.json', import.meta.url);
+  try {
+    return JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    throw new Error('No matrix found. Run `shakedown matrix` (or `npm run matrix`) first.');
+  }
+}
+
+/**
+ * The observer reports controllers as the snake_case short name used in the
+ * pressgang_render_{key} action; the oracle stores an FQCN. Normalise the
+ * FQCN to the observer's shape for comparison.
+ *
+ * @param {string} fqcn
+ * @returns {string}
+ */
+export function controllerHeaderName(fqcn) {
+  const short = fqcn.split('\\').pop() ?? fqcn;
+  return short.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase();
+}
+
+/**
+ * Signatures of PHP/Twig failure output in a rendered page body.
+ *
+ * Covers both plain-text (CLI/log style) and PHP's HTML display format
+ * (`<b>Warning</b>:`). Trade-off: a page whose *content* legitimately contains
+ * one of these strings will false-positive — accepted, because a marketing
+ * page reading "Fatal error" deserves a human look anyway.
+ */
+export const ERROR_SIGNATURES = [
+  'Fatal error',
+  'Parse error',
+  'Warning: ',
+  'Notice: ',
+  'Deprecated: ',
+  '<b>Fatal error</b>',
+  '<b>Warning</b>:',
+  '<b>Notice</b>:',
+  '<b>Deprecated</b>:',
+  'Uncaught Error',
+  'Twig\\Error',
+  'Stack trace:',
+];

package/tests/unit/sandbox-seeding.test.mjs

@@ -0,0 +1,68 @@
+import assert from 'node:assert/strict';
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import test from 'node:test';
+import { seedAcfStates } from '../../lib/sandbox.mjs';
+
+test('seedAcfStates passes the configured seed and epoch to Muster', () => {
+  const root = mkdtempSync(join(tmpdir(), 'shakedown-seed-test-'));
+  const theme = 'fixture-theme';
+  const acfJson = join(root, 'wp-content', 'themes', theme, 'acf-json');
+  const fakeBin = join(root, 'bin');
+  const capture = join(root, 'wp-args.txt');
+  const fakeWp = join(fakeBin, 'wp');
+  const oldPath = process.env.PATH;
+  const oldCapture = process.env.SHAKEDOWN_WP_CAPTURE;
+
+  mkdirSync(acfJson, { recursive: true });
+  mkdirSync(fakeBin, { recursive: true });
+  writeFileSync(fakeWp, `#!/bin/sh
+printf '%s\n' "$@" > "$SHAKEDOWN_WP_CAPTURE"
+printf '{"routes":[]}'
+`);
+  chmodSync(fakeWp, 0o755);
+  process.env.PATH = `${fakeBin}:${oldPath}`;
+  process.env.SHAKEDOWN_WP_CAPTURE = capture;
+
+  try {
+    const routes = seedAcfStates(
+      { root, theme },
+      '/tmp/muster/vendor/autoload.php',
+      { seed: 1978, epoch: '2030-04-05T06:07:08+00:00' }
+    );
+
+    assert.deepEqual(routes, []);
+    const args = readFileSync(capture, 'utf8').trim().split('\n');
+    assert.equal(args[0], 'eval-file');
+    assert.equal(args[2], '/tmp/muster/vendor/autoload.php');
+    assert.equal(args[3], acfJson);
+    assert.equal(args[4], '1978');
+    assert.equal(args[5], '2030-04-05T06:07:08+00:00');
+    assert.equal(args[6], `--path=${root}`);
+  } finally {
+    process.env.PATH = oldPath;
+    if (oldCapture === undefined) delete process.env.SHAKEDOWN_WP_CAPTURE;
+    else process.env.SHAKEDOWN_WP_CAPTURE = oldCapture;
+    rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test('seedAcfStates rejects ambiguous deterministic inputs', () => {
+  assert.throws(
+    () => seedAcfStates({ root: '/tmp', theme: 'theme' }, '/tmp/autoload.php', { seed: '42' }),
+    /seed must be an integer/
+  );
+  assert.throws(
+    () => seedAcfStates({ root: '/tmp', theme: 'theme' }, '/tmp/autoload.php', { epoch: '' }),
+    /timezone-qualified ISO 8601/
+  );
+  assert.throws(
+    () => seedAcfStates(
+      { root: '/tmp', theme: 'theme' },
+      '/tmp/autoload.php',
+      { epoch: '2030-04-05T06:07:08' }
+    ),
+    /timezone-qualified ISO 8601/
+  );
+});