@equinor/fusion-framework-cli-plugin-copilot 1.0.0

package/src/commands/app/command.ts

@@ -0,0 +1,204 @@
+import { createCommand } from 'commander';
+import { basename } from 'node:path';
+
+import { resetDaemon, cleanup, createRunDir } from '../../utils/index.js';
+import { runLogin } from './login.js';
+import { resolveEvalCommandInput, resolveEvalFiles } from '../../eval-resolve.js';
+import { createSession } from './eval.js';
+import { startAppServer } from './server.js';
+import { attachSessionLogger } from './session-logger.js';
+import type { CopilotEvalOptions, Plan, Verdict } from './types.js';
+import { formatVerdict } from './format.js';
+import { existsSync, readFileSync } from 'node:fs';
+import { createPlanPrompt } from './prompts/plan.prompt.js';
+import { createStepPrompt } from './prompts/step.prompt.js';
+import { createJudgePrompt } from './prompts/judge.prompt.js';
+import chalk from 'chalk';
+
+/**
+ * CLI command: `copilot app eval <path-or-eval>`
+ *
+ * Evaluates a Fusion application using the GitHub Copilot SDK with
+ * `agent-browser` exposed as tools. The Copilot agent autonomously
+ * navigates, interacts, and judges the application against eval criteria.
+ */
+const _appEvalCommand = createCommand('eval')
+  .description('Evaluate a Fusion app using Copilot SDK with agent-browser tools')
+  .argument('<path-or-eval>', 'Path to the Fusion application directory or an eval markdown file')
+  .option('--eval <name-or-path>', 'Specific eval to run (name or file path)')
+  .option('--port <port>', 'Port for the app server', '3333')
+  .option('--host <host>', 'Host for the app server', '0.0.0.0')
+  .option('--url <url>', 'Skip server start, use an already-running URL')
+  .option('--verbose', 'Show detailed output', false)
+  .option('--logon, --login', 'Open a headed browser for interactive MSAL login', false)
+  .option('-m, --model <model>', 'LLM model to use (e.g. claude-sonnet-4)')
+  .option('-r, --reasoning <effort>', 'Reasoning effort: low, medium, high, xhigh')
+  .option('-o, --output <dir>', 'Output directory for run artifacts')
+  .addHelpText(
+    'after',
+    [
+      '',
+      'Uses the GitHub Copilot SDK to create an agentic session where the LLM',
+      'autonomously drives agent-browser to verify eval criteria.',
+      '',
+      'Examples:',
+      '  $ ffc copilot app eval ./cookbooks/app-react',
+      '  $ ffc copilot app eval . --eval smoke',
+      '  $ ffc copilot app eval . --eval 06-people-api-page.md',
+      '  $ ffc copilot app eval ./cookbooks/app-react-router/eval/06-people-api-page.md',
+      '  $ ffc copilot app eval . --model claude-sonnet-4',
+      '  $ ffc copilot app eval . --login',
+      '  $ ffc copilot app eval . --logon',
+      '  $ ffc copilot app eval . --url http://localhost:3333/apps/my-app',
+    ].join('\n'),
+  )
+  .action(async (appOrEvalPath: string, options: CopilotEvalOptions) => {
+    const resolvedInput = resolveEvalCommandInput(appOrEvalPath);
+    const absAppPath = resolvedInput.appPath;
+    const port = parseInt(options.port, 10);
+    const verbose = options.verbose;
+
+    if (resolvedInput.evalFile && options.eval) {
+      console.error('❌ Do not combine a positional eval markdown file with --eval');
+      process.exit(1);
+    }
+
+    // ── Login mode: open headed browser for interactive MSAL login ──
+    if (options.login) {
+      await runLogin(absAppPath, port, options.host);
+      return;
+    }
+
+    // Resolve eval files
+    const evalFiles = resolvedInput.evalFile
+      ? [resolvedInput.evalFile]
+      : resolveEvalFiles(absAppPath, options.eval);
+    console.log(
+      `📋 Found ${evalFiles.length} eval(s): ${evalFiles.map((f) => basename(f)).join(', ')}`,
+    );
+
+    // Start the app dev server (unless --url provides an already-running URL)
+    const { serverProcess, appUrl } = await startAppServer(absAppPath, {
+      port,
+      host: options.host,
+      url: options.url,
+      verbose,
+    });
+
+    // Kill any stale daemon and clean up profile locks before the run
+    resetDaemon();
+
+    // Track active session logger + session for SIGINT cleanup
+    let logger: ReturnType<typeof attachSessionLogger> | null = null;
+    let activeSession: Awaited<ReturnType<typeof createSession>> | null = null;
+
+    process.on('SIGINT', () => {
+      logger?.stop();
+      console.log('\n\n👋 Shutting down...');
+      activeSession?.disconnect();
+      resetDaemon();
+      cleanup(serverProcess);
+      process.exit(130);
+    });
+
+    let failures = 0;
+    try {
+      for (const evalFilePath of evalFiles) {
+        const evalName = basename(evalFilePath, '.md');
+        const query = readFileSync(evalFilePath, 'utf-8').trim();
+        console.log(`\n${'═'.repeat(60)}`);
+        console.log(`📄 Running eval: ${evalName}`);
+        console.log(`${'═'.repeat(60)}`);
+
+        const runDir = createRunDir(evalName, options.output);
+
+        const ctx = { url: appUrl, outDir: runDir };
+
+        try {
+          const sessionConfig = {
+            model: options.model,
+            ...(options.reasoning ? { reasoningEffort: options.reasoning } : {}),
+          };
+
+          // Single session with all tools (browser + file)
+          const session = await createSession({ ctx, config: sessionConfig });
+          activeSession = session;
+          logger = attachSessionLogger(session, { requestedModel: options.model });
+
+          // ── Phase 1: Plan ──────────────────────────────────────────
+          await session.sendAndWait({ prompt: createPlanPrompt(query, ctx) }, 300_000);
+
+          const planPath = `${runDir}/plan.json`;
+          if (!existsSync(planPath)) {
+            console.error(
+              chalk.red(
+                `❌ Agent did not produce ${planPath}. The planning phase may have failed silently.`,
+              ),
+            );
+            session.disconnect();
+            logger.stop();
+            failures++;
+            continue;
+          }
+
+          const plan = JSON.parse(readFileSync(planPath, 'utf-8')) as Plan;
+          console.log(plan.summary);
+
+          // ── Phase 2: Execute + Evaluate each step ──────────────────
+          for (const step of plan.steps) {
+            console.log(chalk.cyan(`\n── Scenario: ${step.scenario}`));
+            for (const c of step.criteria) {
+              console.log(chalk.dim(`   • ${c}`));
+            }
+            await session.sendAndWait({ prompt: createStepPrompt(step, ctx) }, 300_000);
+          }
+
+          // ── Phase 3: Judge ─────────────────────────────────────────
+          console.log(chalk.cyan('\n── Judging results...'));
+          await session.sendAndWait({ prompt: createJudgePrompt(ctx) }, 300_000);
+
+          session.disconnect();
+          logger.stop();
+
+          // Print the verdict
+          const verdictPath = `${runDir}/verdict.json`;
+          if (existsSync(verdictPath)) {
+            const verdict = JSON.parse(readFileSync(verdictPath, 'utf-8')) as Verdict;
+            console.log(formatVerdict(verdict));
+            if (!verdict.pass) failures++;
+          } else {
+            console.error(chalk.red(`❌ Judge did not produce ${verdictPath}.`));
+            failures++;
+          }
+        } catch (err) {
+          logger?.stop();
+          const msg = err instanceof Error ? err.message : String(err);
+          console.error(chalk.red(`\n❌ Eval "${evalName}" failed: ${msg}`));
+          if (err instanceof Error && err.stack) {
+            console.error(chalk.dim(err.stack));
+          }
+          failures++;
+        }
+      }
+    } finally {
+      logger?.stop();
+      resetDaemon();
+      cleanup(serverProcess);
+      process.exit(failures > 0 ? 1 : 0);
+    }
+  });
+
+/** Intermediate `app` command group: `copilot app` */
+const _appCommand = createCommand('app')
+  .description('Copilot-powered application commands')
+  .addCommand(_appEvalCommand);
+
+/**
+ * Top-level `copilot` command group.
+ *
+ * Subcommands:
+ *   app eval — Evaluate a Fusion app with Copilot SDK + agent-browser tools
+ */
+export const copilotCommand = createCommand('copilot')
+  .description('GitHub Copilot SDK powered commands')
+  .addCommand(_appCommand);

package/src/commands/app/eval.ts

@@ -0,0 +1,114 @@
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { ab, abErrorMessage } from '../../utils/index.js';
+import { createAgentBrowserTools } from './tools/index.js';
+import { createWriteFileTool, createAppendFileTool } from './tools/write-file.js';
+import { createReadFileTool } from './tools/read-file.js';
+import type { RuntimeExecutionContext } from './types.js';
+import type { CopilotSession, SessionConfig } from '@github/copilot-sdk';
+import { createSystemPrompt } from './prompts/system.prompt.js';
+import { createDevServerContext } from './prompts/system.dev-server.prompt.js';
+
+/**
+ * Lazily imports the Copilot SDK to avoid module resolution errors at CLI startup.
+ *
+ * The SDK depends on `vscode-jsonrpc/node` which may fail under certain Node versions
+ * when loaded eagerly via static imports.
+ *
+ * @returns The Copilot SDK module namespace
+ */
+async function loadCopilotSdk(): Promise<typeof import('@github/copilot-sdk')> {
+  return await import('@github/copilot-sdk');
+}
+
+type SessionOptions = {
+  ctx: RuntimeExecutionContext;
+  defaultToolTimeoutMs?: number;
+  config?: Omit<
+    SessionConfig,
+    'onPermissionRequest' | 'systemMessage' | 'availableTools' | 'excludedTools'
+  >;
+};
+
+export const createSession = async (options: SessionOptions): Promise<CopilotSession> => {
+  const { ctx, config, defaultToolTimeoutMs } = options;
+
+  const { CopilotClient, defineTool, approveAll } = await loadCopilotSdk();
+  const client = new CopilotClient({ autoStart: true });
+  await client.start();
+
+  const fileTools = [
+    createReadFileTool(ctx.outDir, defineTool),
+    createWriteFileTool(ctx.outDir, defineTool),
+    createAppendFileTool(ctx.outDir, defineTool),
+  ];
+
+  const tools = [
+    ...createBrowserTools(defineTool, {
+      cwd: ctx.outDir,
+      timeoutMs: defaultToolTimeoutMs,
+    }),
+    ...fileTools,
+    ...(config?.tools ?? []),
+  ];
+
+  // Exclude dangerous built-in tools; names vary across model providers
+  // so a blocklist is more robust than an allowlist.
+  const excludedTools = [
+    'bash',
+    'list_bash',
+    'read_bash',
+    'write_bash',
+    'stop_bash',
+    'sql',
+    'web_fetch',
+    'skill',
+    'list_agents',
+    'read_agent',
+    'glob',
+    'rg',
+    'view',
+    'task',
+  ];
+
+  const session = await client.createSession({
+    ...config,
+    tools,
+    excludedTools,
+    systemMessage: {
+      mode: 'replace',
+      content: `${createSystemPrompt(ctx)}\n\n## Dev Server${createDevServerContext(ctx)}`,
+    },
+    streaming: true,
+    onPermissionRequest: approveAll,
+  });
+
+  return session;
+};
+
+/**
+ * Creates the `agent-browser` tool definitions for the Copilot SDK session.
+ *
+ * Each tool wraps an `agent-browser` CLI subcommand so the Copilot agent can
+ * autonomously navigate and inspect a running Fusion application.
+ *
+ * @param defineTool - The `defineTool` helper from the Copilot SDK
+ * @returns Array of Copilot SDK tool definitions for browser interaction
+ */
+function createBrowserTools(
+  defineTool: Awaited<ReturnType<typeof loadCopilotSdk>>['defineTool'],
+  options?: { cwd?: string; timeoutMs?: number },
+): ReturnType<typeof createAgentBrowserTools> {
+  const outDir = join(options?.cwd ?? process.cwd(), 'evidence');
+  mkdirSync(outDir, { recursive: true });
+  const invoke = (args: string[], timeoutMs = options?.timeoutMs): string => {
+    try {
+      return ab(args, timeoutMs);
+    } catch (err) {
+      const msg = abErrorMessage(err);
+      return `Error: ${msg}`;
+    }
+  };
+  return createAgentBrowserTools({ outDir, invoke }, defineTool);
+}

package/src/commands/app/format.ts

@@ -0,0 +1,110 @@
+import chalk from 'chalk';
+
+import type { Plan, StepResult, Verdict } from './types.js';
+
+// ── Plan ────────────────────────────────────────────────────────────────────
+
+/** Renders a structured plan as a human-readable summary. */
+export function formatPlan(plan: Plan): string {
+  const lines = [
+    '',
+    chalk.bold(`📋 ${plan.summary}`),
+    '',
+    ...plan.steps.flatMap((s, si) => [
+      `  ${chalk.dim(`${si + 1}.`)} ${s.scenario}`,
+      ...s.criteria.map((c) => chalk.dim(`     • ${truncate(c, 70)}`)),
+    ]),
+    '',
+  ];
+  return lines.join('\n');
+}
+
+// ── Step result (execution line) ────────────────────────────────────────────
+
+/** Renders a single step execution result. */
+export function formatStepResult(result: StepResult): string {
+  const icon = statusIcon(result.ok);
+  const files = result.evidence.length ? chalk.dim(` [${result.evidence.join(', ')}]`) : '';
+  return `  ${icon}  ${result.criterion}\n${chalk.dim(`     ${result.note}`)}${files}`;
+}
+
+// ── Verdict ─────────────────────────────────────────────────────────────────
+
+/** Renders the final verdict as a full-width coloured report. */
+export function formatVerdict(verdict: Verdict): string {
+  const bar = '═'.repeat(60);
+  const banner = verdict.pass
+    ? chalk.bgGreen.black.bold(' ✅ PASS ')
+    : chalk.bgRed.white.bold(' ❌ FAIL ');
+
+  const lines = [
+    '',
+    bar,
+    `  ${banner}`,
+    bar,
+    '',
+    ...verdict.steps.map(
+      (s) => `  ${statusIcon(s.ok)}  ${s.criterion}\n${chalk.dim(`     ${s.note}`)}`,
+    ),
+    '',
+    chalk.dim(`  ${verdict.reasoning}`),
+    '',
+  ];
+
+  if (verdict.ux && verdict.ux.length > 0) {
+    lines.push(chalk.bold('  💡 UX Feedback'), '');
+    for (const hint of verdict.ux) {
+      lines.push(chalk.yellow(`  • ${hint}`));
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+// ── Dispatcher ──────────────────────────────────────────────────────────────
+
+/**
+ * Attempts to detect the JSON shape of an assistant message and format it.
+ *
+ * Returns `null` when the content is not a recognised structure, so the
+ * caller can fall back to plain-text rendering.
+ */
+export function tryFormatMessage(content: string): string | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content.trim());
+  } catch {
+    return null;
+  }
+  if (!parsed || typeof parsed !== 'object') return null;
+
+  // Verdict: { pass, reasoning, steps[] }
+  if ('pass' in parsed && 'steps' in parsed && Array.isArray((parsed as Verdict).steps)) {
+    return formatVerdict(parsed as Verdict);
+  }
+
+  // Plan: { summary, steps[] }
+  if ('summary' in parsed && 'steps' in parsed && Array.isArray((parsed as Plan).steps)) {
+    return formatPlan(parsed as Plan);
+  }
+
+  // Step result: { criterion, ok, note }
+  if ('criterion' in parsed && 'ok' in parsed && 'note' in parsed) {
+    return formatStepResult(parsed as StepResult);
+  }
+
+  return null;
+}
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+function statusIcon(ok: boolean | string): string {
+  if (ok === true) return chalk.green('✔');
+  if (ok === false) return chalk.red('✖');
+  return chalk.yellow('⚠'); // "blocked", "flaky", etc.
+}
+
+function truncate(str: string, max: number): string {
+  return str.length > max ? `${str.slice(0, max - 1)}…` : str;
+}

package/src/commands/app/login.ts

@@ -0,0 +1,92 @@
+import { spawn, type ChildProcess } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import {
+  ab,
+  resolveAppKey,
+  cleanup,
+  resetDaemon,
+  sleep,
+  stripAnsi,
+  waitForServer,
+  pollConsole,
+} from '../../utils/index.js';
+
+/**
+ * Interactive login mode for the `copilot app` command.
+ *
+ * Starts the application dev server, opens a headed browser window for
+ * manual MSAL authentication, and polls the browser console for the Fusion
+ * `"Done"` message that signals successful initialisation. The browser is
+ * left open after login so the user can finish any manual steps before
+ * pressing Ctrl+C.
+ *
+ * @param absAppPath - Absolute path to the Fusion application directory
+ * @param port - Port number for the dev server
+ * @param host - Host address for the dev server
+ */
+export async function runLogin(absAppPath: string, port: number, host: string): Promise<void> {
+  const pkgJsonPath = join(absAppPath, 'package.json');
+  if (!existsSync(pkgJsonPath)) {
+    console.error(`❌ No package.json found at ${absAppPath}`);
+    process.exit(1);
+  }
+  const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')) as { name?: string };
+  if (!pkgJson.name) {
+    console.error(`❌ package.json at ${absAppPath} has no "name" field`);
+    process.exit(1);
+  }
+
+  let serverProcess: ChildProcess | undefined;
+
+  console.log('📦 Resolving app manifest...');
+  const appKey = resolveAppKey(absAppPath);
+  console.log(`🚀 Starting server for ${appKey}...`);
+  serverProcess = spawn('ffc', ['app', 'dev', '--host', host, '--port', String(port)], {
+    cwd: absAppPath,
+    stdio: 'inherit',
+    detached: false,
+  });
+
+  const serverUrl = `http://localhost:${port}`;
+  const ready = await waitForServer(serverUrl, 60);
+  if (!ready) {
+    console.error('❌ Server failed to start within 60s');
+    cleanup(serverProcess);
+    process.exit(1);
+  }
+
+  const appUrl = `${serverUrl}/apps/${appKey}`;
+  console.log(`✅ Server ready at ${appUrl}`);
+  console.log('🌐 Opening headed browser — log in with your Equinor account...');
+
+  // Kill any stale daemon/Chrome so the new --headed --profile flags take effect
+  resetDaemon();
+  ab(['open', appUrl, '--headed'], 60_000);
+
+  // Ctrl+C handler: stop the dev server but leave the browser open
+  process.on('SIGINT', () => {
+    console.log('\n👋 Stopping server... (browser left open)');
+    cleanup(serverProcess);
+    process.exit(0);
+  });
+
+  // Wait for the Fusion app to initialise (browser console emits "✓ Done" after auth)
+  console.log('⏳ Waiting for app to initialize after login... (Ctrl+C to finish early)');
+  const initialized = await pollConsole(
+    (logs: string) => stripAnsi(logs).includes('Done'),
+    300_000,
+  );
+
+  if (initialized) {
+    await sleep(2000);
+    console.log('✅ Login complete — session saved.');
+  } else {
+    console.log('⚠ Timed out waiting for app init. Session may still be saved.');
+  }
+
+  // Always wait for manual Ctrl+C so the user can finish what they're doing
+  console.log('🌐 Browser still open. Press Ctrl+C to exit.');
+  await new Promise(() => {}); // block until SIGINT
+}

package/src/commands/app/prompts/execute-step.prompt.ts

@@ -0,0 +1,48 @@
+import type { RuntimeExecutionContext } from '../types.js';
+
+/**
+ * Creates the execution prompt for a single test step.
+ *
+ * The agent receives one plan step and executes its actions against the
+ * running application using browser tools, collecting evidence artifacts
+ * into the evidence directory.
+ *
+ * @param actions - The action strings to execute in order
+ * @param ctx - Runtime context with output directory and app URL
+ * @returns The fully interpolated execution prompt string
+ */
+export const createExecuteStepPrompt = (
+  actions: string[],
+  ctx: RuntimeExecutionContext,
+): string => {
+  return `
+You are now EXECUTING one test step using real browser tools.
+
+ACTIONS: ${JSON.stringify(actions, null, 2)}
+
+APPLICATION URL: ${ctx.url}
+EVIDENCE DIRECTORY: ${ctx.outDir}/evidence
+
+Execute the "actions" array in order using the available browser tool calls.
+Collect rich evidence at every stage — especially on failure.
+
+Evidence rules (mandatory):
+- Before important actions: screenshot --annotate evidence/step-{criterion-slug}-before.png
+- After the step (always): screenshot --full --annotate evidence/step-{criterion-slug}-after.png
+- On any failure or suspicion:
+    errors > evidence/step-{criterion-slug}-errors.log
+    console > evidence/step-{criterion-slug}-console.log
+    snapshot -i --json > evidence/step-{criterion-slug}-snapshot.json
+- Use descriptive, unique filenames
+  Examples: step-login-button-missing-screenshot.png, step-dashboard-url.txt
+
+Important patterns:
+- Start most steps with: snapshot -i --annotate → use @eN refs for element stability
+- After navigation: wait --load networkidle or wait --url "**/pattern*"
+- On failure (non-zero exit, timeout, missing element, JS error): capture extra evidence
+- If flaky (timing, network): take 2–3 extra screenshots ~1 s apart
+
+When done, your work will be evaluated against the step's passEvidence and failEvidence.
+Do NOT output markdown or explanations — only use browser tool calls and collect evidence.
+  `.trim();
+};

package/src/commands/app/prompts/judge.prompt.ts

@@ -0,0 +1,121 @@
+import { readdirSync, readFileSync, existsSync } from 'node:fs';
+import { join, extname } from 'node:path';
+import type { RuntimeExecutionContext } from '../types.js';
+
+/** Text file extensions that are safe to embed in the prompt. */
+const TEXT_EXTENSIONS = new Set(['.txt', '.json', '.jsonl', '.log', '.html', '.csv']);
+
+/**
+ * Reads `executions.jsonl` and all text evidence files from the run directory
+ * and returns them as a single string block suitable for prompt embedding.
+ *
+ * Binary files (screenshots) are listed by name but not inlined.
+ *
+ * @param outDir - The run output directory containing executions.jsonl and evidence/
+ * @returns Formatted string with execution records and text evidence
+ */
+function collectEvidence(outDir: string): string {
+  const parts: string[] = [];
+
+  // ── executions.jsonl ──
+  const execPath = join(outDir, 'executions.jsonl');
+  if (existsSync(execPath)) {
+    parts.push('### executions.jsonl', readFileSync(execPath, 'utf-8').trim());
+  } else {
+    parts.push('### executions.jsonl', '(file not found)');
+  }
+
+  // ── evidence/ directory ──
+  const evidenceDir = join(outDir, 'evidence');
+  if (existsSync(evidenceDir)) {
+    const files = readdirSync(evidenceDir).sort();
+    for (const file of files) {
+      const ext = extname(file).toLowerCase();
+      if (TEXT_EXTENSIONS.has(ext)) {
+        const content = readFileSync(join(evidenceDir, file), 'utf-8').trim();
+        parts.push(`### evidence/${file}`, content);
+      } else {
+        parts.push(`### evidence/${file}`, `(binary file — ${ext} screenshot)`);
+      }
+    }
+  }
+
+  return parts.join('\n\n');
+}
+
+/**
+ * Creates the final-verdict prompt for the judge phase.
+ *
+ * The judge reviews ALL step executions and evidence holistically, then writes
+ * a single `verdict.json` with an overall pass/fail decision. This differs
+ * from the per-step evaluate prompt which assesses one criterion at a time.
+ *
+ * Execution records and text evidence files are embedded directly in the prompt
+ * so the model does not need file-reading tools or SDK attachment resolution.
+ *
+ * @param ctx - Runtime context containing the output directory and app URL
+ * @returns The fully interpolated judge prompt string
+ */
+export const createJudgePrompt = (ctx: RuntimeExecutionContext): string => {
+  const evidence = collectEvidence(ctx.outDir);
+
+  return `
+All test steps have been executed. You are now the JUDGE.
+
+Review the evidence below and any screenshots using read_file, then produce a final verdict.
+
+<evidence>
+${evidence}
+</evidence>
+
+To inspect screenshots, use read_file with paths like "evidence/step-hello-fusion-header.jpg".
+Images are returned as inline vision content you can see.
+
+Save your verdict using write_file with path "verdict.json".
+You MUST call write_file. Do NOT output the JSON as a message.
+
+Evaluation rules — apply exactly:
+1. Auto-fail a criterion if:
+   - Any "Uncaught", TypeError, or ReferenceError appears in console/errors during that step
+   - The page shows an error, HTTP 500, white screen, or spinner lasting >15 s
+   - The expected element, text, or URL never appeared
+   - An explicit assertion (is visible, get text, get count) returned a failing result
+2. A criterion passes only when:
+   - All passEvidence conditions are clearly confirmed in screenshots (use read_file to view them), console, or URL
+   - No critical functional deviation is present
+3. Be lenient on:
+   - Console warnings, deprecations, analytics 404s
+   - Cosmetic shifts <5 px or layout differences that do not hide content
+   - Minor visual differences without functional impact
+
+Overall "pass" = true ONLY if every criterion is "ok": true.
+
+Output ONLY this JSON — no markdown fences, no prose:
+
+{
+  "pass": boolean,
+  "reasoning": "One-paragraph summary explaining the overall pass/fail outcome",
+  "steps": [
+    {
+      "criterion": "...",
+      "ok": true | false | "blocked" | "flaky",
+      "note": "Evidence-based verdict for this criterion"
+    }
+  ],
+  "ux": [
+    "Short UX observation or improvement suggestion based on what you saw in the screenshots and evidence"
+  ]
+}
+
+The "ux" array is for observations about the user experience that go beyond
+pass/fail — things a product owner or designer would want to know:
+- Accessibility issues (contrast, missing labels, keyboard traps)
+- Loading performance (slow renders, layout shifts, flash of unstyled content)
+- Visual polish (alignment, spacing, truncation, responsive issues)
+- Interaction hints (unclear affordances, missing hover states, confusing flow)
+- Content quality (typos, unclear copy, missing help text)
+Only include genuine observations from the evidence. Skip "ux" if nothing stands out.
+
+If evidence for a criterion is missing or ambiguous, fail it.
+  `.trim();
+};