@equinor/fusion-framework-cli-plugin-copilot 1.0.0

package/src/commands/app/tools/type-text.ts

@@ -0,0 +1,30 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the text typing tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for typing text into the focused element or a target
+ */
+export function createTypeTextTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_type', {
+    description: 'Type text character by character (useful for inputs that respond to keystrokes).',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        target: {
+          type: 'string',
+          description: 'Element ref or CSS selector (omit to type into focused element)',
+        },
+        text: { type: 'string', description: 'Text to type' },
+      },
+      required: ['text'],
+    },
+    handler: async (args) => {
+      const { target, text } = args as { target?: string; text: string };
+      const command = target ? ['type', target, text] : ['keyboard', 'type', text];
+      return context.invoke(command);
+    },
+  });
+}

package/src/commands/app/tools/types.ts

@@ -0,0 +1,15 @@
+import type { SessionConfig, defineTool } from '@github/copilot-sdk';
+
+/** Shared runtime context passed to every agent-browser tool factory. */
+export interface AgentBrowserToolContext {
+  /** Directory where evidence artifacts for the current eval run are stored. */
+  outDir: string;
+  /** Executes an `agent-browser` command and returns its stdout. */
+  invoke: (args: string[], timeoutMs?: number) => string;
+}
+
+/** Type-only alias for the Copilot SDK `defineTool` helper. */
+export type DefineTool = typeof defineTool;
+
+/** Copilot SDK tool list type used by the registry. */
+export type AgentBrowserToolList = NonNullable<SessionConfig['tools']>;

package/src/commands/app/tools/wait.ts

@@ -0,0 +1,43 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the wait tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for waiting on load state, text, elements, or time
+ */
+export function createWaitTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_wait', {
+    description:
+      'Wait for a condition: page load (networkidle), text to appear, an element, or a timeout in ms.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        text: { type: 'string', description: 'Wait for this text to appear on the page' },
+        selector: { type: 'string', description: 'Wait for this element ref or CSS selector' },
+        load: {
+          type: 'string',
+          description: 'Wait for page load state: "networkidle", "domcontentloaded", or "load"',
+        },
+        timeout: { type: 'number', description: 'Wait for this many milliseconds' },
+      },
+    },
+    handler: async (args) => {
+      const { text, selector, load, timeout } = args as {
+        text?: string;
+        selector?: string;
+        load?: string;
+        timeout?: number;
+      };
+      if (text) return context.invoke(['wait', '--text', text], 60_000);
+      if (load) return context.invoke(['wait', '--load', load], 60_000);
+      if (selector) return context.invoke(['wait', selector], 60_000);
+      if (timeout) {
+        await new Promise((resolve) => setTimeout(resolve, timeout));
+        return `Waited ${timeout}ms`;
+      }
+      return context.invoke(['wait', '--load', 'networkidle'], 60_000);
+    },
+  });
+}

package/src/commands/app/tools/write-file.ts

@@ -0,0 +1,104 @@
+import { appendFileSync, mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, isAbsolute, join } from 'node:path';
+
+import type { DefineTool } from './types.js';
+
+/**
+ * Resolves a path within a base directory, rejecting escapes.
+ *
+ * Accepts both relative paths and absolute paths that fall within
+ * the base directory. Strips the base prefix from absolute paths
+ * before resolving to handle models that echo full paths from prompts.
+ *
+ * @param baseDir - The allowed root directory
+ * @param input - The user-provided path (relative or absolute within baseDir)
+ * @returns The resolved absolute path
+ * @throws {Error} When the path escapes the base directory
+ */
+function resolveSafe(baseDir: string, input: string): string {
+  // If the model echoed the full absolute path from the prompt, strip the base prefix.
+  const relative = isAbsolute(input)
+    ? input.startsWith(baseDir + '/') || input.startsWith(baseDir + '\\')
+      ? input.slice(baseDir.length + 1)
+      : (() => {
+          throw new Error(`Absolute path outside output directory: ${input}`);
+        })()
+    : input;
+  const resolved = join(baseDir, relative);
+  if (!resolved.startsWith(baseDir)) {
+    throw new Error(`Path escapes the output directory: ${input}`);
+  }
+  return resolved;
+}
+
+/**
+ * Creates the write-file tool so the agent can save artifacts
+ * (plan.json, verdict.json, etc.) to the run output directory.
+ *
+ * @param baseDir - The run output directory that scopes all writes
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for writing a file
+ */
+export function createWriteFileTool(baseDir: string, defineTool: DefineTool) {
+  return defineTool('write_file', {
+    description:
+      'Write content to a file in the output directory. Creates parent directories as needed.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        path: {
+          type: 'string',
+          description: 'File path relative to the output directory (e.g. "plan.json")',
+        },
+        content: {
+          type: 'string',
+          description: 'The full file content to write',
+        },
+      },
+      required: ['path', 'content'],
+    },
+    handler: async (args) => {
+      const { path, content } = args as { path: string; content: string };
+      const resolved = resolveSafe(baseDir, path);
+      mkdirSync(dirname(resolved), { recursive: true });
+      writeFileSync(resolved, content);
+      return `Wrote ${content.length} bytes to ${path}`;
+    },
+  });
+}
+
+/**
+ * Creates the append-file tool so the agent can append lines
+ * to artifacts like executions.jsonl.
+ *
+ * @param baseDir - The run output directory that scopes all writes
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for appending to a file
+ */
+export function createAppendFileTool(baseDir: string, defineTool: DefineTool) {
+  return defineTool('append_file', {
+    description:
+      'Append content to a file in the output directory. Creates the file if it does not exist.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        path: {
+          type: 'string',
+          description: 'File path relative to the output directory (e.g. "executions.jsonl")',
+        },
+        content: {
+          type: 'string',
+          description: 'Content to append (a trailing newline is added automatically)',
+        },
+      },
+      required: ['path', 'content'],
+    },
+    handler: async (args) => {
+      const { path, content } = args as { path: string; content: string };
+      const resolved = resolveSafe(baseDir, path);
+      mkdirSync(dirname(resolved), { recursive: true });
+      appendFileSync(resolved, content.endsWith('\n') ? content : content + '\n');
+      return `Appended to ${path}`;
+    },
+  });
+}

package/src/commands/app/types.ts

@@ -0,0 +1,76 @@
+import type { Observable } from 'rxjs';
+
+export interface RuntimeExecutionContext {
+  outDir: string;
+  url: string;
+}
+
+/** CLI option set for the `copilot app eval` command. */
+export interface CopilotEvalOptions {
+  /** Port for the local app dev server. */
+  port: string;
+  /** Host address for the local app dev server. */
+  host: string;
+  /** Skip server start and use an already-running URL. */
+  url?: string;
+  /** Show detailed output from the dev server and agent-browser. */
+  verbose: boolean;
+  /** LLM model override (e.g. "claude-sonnet-4"). */
+  model?: string;
+  /** Reasoning effort: low, medium, high, xhigh. */
+  reasoning?: 'low' | 'medium' | 'high' | 'xhigh';
+  /** Output directory override for run artifacts. */
+  output?: string;
+  /** Open a headed browser for interactive MSAL login. */
+  login: boolean;
+  /** Specific eval to run (name or file path). */
+  eval?: string;
+  /** Observable that emits on SIGINT for graceful shutdown. */
+  shutdown$?: Observable<string>;
+}
+
+// ── Eval plan types ─────────────────────────────────────────────────────────
+
+/** A logical test step that groups related criteria under one scenario. */
+export interface PlanStep {
+  /** Short scenario description (e.g. "Open the landing page"). */
+  scenario: string;
+  /** Acceptance criteria to verify (e.g. ["Header shows Hello Fusion", ...]). */
+  criteria: string[];
+  /** What constitutes a pass for each criterion (same order as criteria). */
+  pass: string[];
+  /** What constitutes a fail for each criterion (same order as criteria). */
+  fail: string[];
+}
+
+/** Structured plan produced by Phase 1. */
+export interface Plan {
+  /** Brief summary of the feature under test. */
+  summary: string;
+  /** Ordered acceptance criteria to execute. */
+  steps: PlanStep[];
+}
+
+/** Result of executing a single {@link PlanStep}. */
+export interface StepResult {
+  /** The criterion that was tested. */
+  criterion: string;
+  /** Whether the criterion passed. */
+  ok: boolean;
+  /** Agent's observation or explanation. */
+  note: string;
+  /** Evidence artifact filenames collected during this step. */
+  evidence: string[];
+}
+
+/** Structured verdict produced by the Copilot agent at the end of an eval session. */
+export interface Verdict {
+  /** Whether all acceptance criteria passed. */
+  pass: boolean;
+  /** Overall summary explaining the pass/fail outcome. */
+  reasoning: string;
+  /** Per-criterion results with evidence references. */
+  steps: StepResult[];
+  /** Optional UX observations and improvement suggestions based on what was seen. */
+  ux?: string[];
+}

package/src/eval-resolve.ts

@@ -0,0 +1,113 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path';
+
+/**
+ * Normalized positional input for the `copilot app eval` command.
+ *
+ * When the user passes an application directory, `evalFile` is undefined.
+ * When the user passes a direct `eval/*.md` file path, `appPath` is derived
+ * from the file location and `evalFile` points to the selected markdown file.
+ */
+export interface EvalCommandInput {
+  /** Absolute path to the Fusion application directory. */
+  appPath: string;
+  /** Optional absolute path to the directly selected eval markdown file. */
+  evalFile?: string;
+}
+
+/**
+ * Prints a CLI resolution error and exits the current process.
+ *
+ * @param message - Primary error message.
+ * @param detail - Optional follow-up detail shown on the next line.
+ * @returns This function never returns because the process exits.
+ */
+function exitWithResolutionError(message: string, detail?: string): never {
+  console.error(`❌ ${message}`);
+  if (detail) {
+    console.error(`   ${detail}`);
+  }
+  process.exit(1);
+}
+
+/**
+ * Resolves the positional `copilot app eval` argument.
+ *
+ * Supports both application directory paths and direct paths to markdown files
+ * inside an application's `eval/` directory.
+ *
+ * @param inputPath - Positional CLI argument passed to the eval command.
+ * @returns Normalized application path and optional direct eval file path.
+ */
+export function resolveEvalCommandInput(inputPath: string): EvalCommandInput {
+  const absInput = resolve(inputPath);
+
+  if (extname(absInput) !== '.md') {
+    return { appPath: absInput };
+  }
+
+  if (!existsSync(absInput)) {
+    exitWithResolutionError(`Eval file not found: ${absInput}`);
+  }
+
+  if (!statSync(absInput).isFile()) {
+    exitWithResolutionError(`Eval path is not a file: ${absInput}`);
+  }
+
+  const evalDir = dirname(absInput);
+  if (basename(evalDir) !== 'eval') {
+    exitWithResolutionError(
+      `Direct eval file path must point to a markdown file inside an eval directory: ${absInput}`,
+    );
+  }
+
+  return {
+    appPath: dirname(evalDir),
+    evalFile: absInput,
+  };
+}
+
+/**
+ * Resolves eval markdown files for a Fusion application.
+ *
+ * @param appPath - Path to the Fusion application directory
+ * @param evalOpt - Value of the `--eval` CLI option (name, path, or `undefined`)
+ * @returns Array of absolute paths to eval markdown files
+ */
+export function resolveEvalFiles(appPath: string, evalOpt?: string): string[] {
+  const absApp = resolve(appPath);
+
+  if (!evalOpt) {
+    const evalDir = join(absApp, 'eval');
+    if (!existsSync(evalDir)) {
+      exitWithResolutionError(
+        `No eval directory found at ${evalDir}`,
+        'Create eval/*.md files or use --eval <file>',
+      );
+    }
+    const files = readdirSync(evalDir)
+      .filter((f) => f.endsWith('.md'))
+      .sort()
+      .map((f) => join(evalDir, f));
+
+    if (files.length === 0) {
+      exitWithResolutionError(`No .md files found in ${evalDir}`);
+    }
+    return files;
+  }
+
+  if (isAbsolute(evalOpt) || evalOpt.includes('/') || evalOpt.includes('\\')) {
+    const absPath = isAbsolute(evalOpt) ? evalOpt : resolve(evalOpt);
+    if (!existsSync(absPath)) {
+      exitWithResolutionError(`Eval file not found: ${absPath}`);
+    }
+    return [absPath];
+  }
+
+  const evalFileName = extname(evalOpt) === '.md' ? evalOpt : `${evalOpt}.md`;
+  const evalFile = join(absApp, 'eval', evalFileName);
+  if (!existsSync(evalFile)) {
+    exitWithResolutionError(`Eval file not found: ${evalFile}`);
+  }
+  return [evalFile];
+}

package/src/index.ts

@@ -0,0 +1,20 @@
+import type { Command } from 'commander';
+import { copilotCommand } from './commands/app/command.js';
+
+/**
+ * Registers the `copilot` CLI plugin with the Fusion Framework CLI.
+ *
+ * Attaches the top-level `copilot` command group, providing Copilot SDK
+ * powered evaluation tools with agent-browser.
+ *
+ * Usage:
+ *   ffc copilot app eval ./cookbooks/app-react
+ *   ffc copilot app eval . --eval smoke --model claude-sonnet-4
+ *
+ * @param program - The root Commander program instance
+ */
+export function registerCopilotPlugin(program: Command): void {
+  program.addCommand(copilotCommand);
+}
+
+export default registerCopilotPlugin;

package/src/utils/agent-browser.ts

@@ -0,0 +1,66 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, unlinkSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+/** Persistent Chrome profile directory used across agent-browser sessions. */
+export const PROFILE_DIR = `${homedir()}/.fusion-smoke-profile`;
+
+/**
+ * Runs an agent-browser command with the persistent Chrome profile.
+ *
+ * Calls the `agent-browser` binary resolved from `PATH` (homebrew or npm global).
+ * Every invocation automatically injects `--profile <PROFILE_DIR>` so browser
+ * state (cookies, session storage, MSAL tokens) persists across calls.
+ *
+ * @param args - Subcommand and arguments passed directly to `agent-browser`
+ * @param timeoutMs - Maximum execution time in milliseconds (default: 30 000)
+ * @returns Trimmed stdout from the command
+ * @throws {Error} When `agent-browser` exits with a non-zero code or times out
+ */
+export function ab(args: string[], timeoutMs = 30_000): string {
+  // Remove stale SingletonLock left by a previous Chrome crash to prevent
+  // "Failed to create SingletonLock: File exists" errors.
+  const lockPath = join(PROFILE_DIR, 'SingletonLock');
+  if (existsSync(lockPath)) {
+    try {
+      unlinkSync(lockPath);
+    } catch {
+      // Best-effort — if we can't remove it Chrome will fail with a clear message anyway.
+    }
+  }
+  const fullArgs = ['--profile', PROFILE_DIR, ...args];
+  return execFileSync('agent-browser', fullArgs, {
+    encoding: 'utf-8',
+    timeout: timeoutMs,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    cwd: process.cwd(),
+  }).trim();
+}
+
+/**
+ * Extracts the user-facing error message from an agent-browser failure.
+ *
+ * Strips ANSI escape codes from stderr and returns only the diagnostic
+ * lines prefixed with `✗` or `⚠`. Falls back to the first 200 characters
+ * of the error message when no diagnostic lines are found.
+ *
+ * @param err - The caught error (typically from {@link ab})
+ * @returns A single-line summary suitable for console output
+ */
+export function abErrorMessage(err: unknown): string {
+  if (!(err instanceof Error)) return String(err);
+  const e = err as Error & { stderr?: string };
+  if (e.stderr) {
+    // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI escape sequences
+    const ansiPattern = /\x1B\[[0-9;]*m/g;
+    const lines = e.stderr
+      .replace(ansiPattern, '')
+      .split('\n')
+      .map((l) => l.trim())
+      .filter((l) => l.startsWith('✗') || l.startsWith('⚠'));
+    if (lines.length) return lines.join(' | ');
+  }
+  const match = e.message.match(/[✗⚠].+/);
+  return match ? match[0] : e.message.slice(0, 200);
+}

package/src/utils/browser-poll.ts

@@ -0,0 +1,27 @@
+import { ab } from './agent-browser.js';
+import { sleep } from './process.js';
+
+/**
+ * Polls the browser's console log output until a predicate returns `true`
+ * or the timeout expires.
+ *
+ * @param predicate - Callback receiving the raw console log text; return `true` to stop
+ * @param timeoutMs - Maximum polling duration in milliseconds
+ * @returns `true` if the predicate matched before timeout, `false` otherwise
+ */
+export async function pollConsole(
+  predicate: (logs: string) => boolean,
+  timeoutMs: number,
+): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const logs = ab(['console']);
+      if (predicate(logs)) return true;
+    } catch {
+      // Browser not ready — retry
+    }
+    await sleep(2000);
+  }
+  return false;
+}

package/src/utils/daemon.ts

@@ -0,0 +1,84 @@
+import { execFileSync, execSync } from 'node:child_process';
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+
+import { ab, PROFILE_DIR } from './agent-browser.js';
+
+/** PID file written by the agent-browser daemon to track its own process. */
+const DAEMON_PID_FILE = join(homedir(), '.agent-browser', 'default.pid');
+
+/**
+ * Kills any running agent-browser daemon and its Chrome children,
+ * then removes stale state files so the next run starts clean.
+ *
+ * Cleanup sequence:
+ * 1. Graceful `agent-browser close` (5 s timeout)
+ * 2. Force-kill the daemon PID read from {@link DAEMON_PID_FILE}
+ * 3. `pkill` leftover headless Chrome processes
+ * 4. Delete Session Storage to clear stuck MSAL `interaction.status` flags
+ */
+export function resetDaemon(): void {
+  // 1. Try graceful close first
+  try {
+    execFileSync('agent-browser', ['close'], {
+      encoding: 'utf-8',
+      timeout: 5_000,
+      stdio: 'ignore',
+    });
+    console.log('🔄 Closed agent-browser daemon');
+  } catch {
+    // Daemon not responding — fall through to force-kill
+  }
+
+  // 2. Kill the daemon process tracked by agent-browser's own pid file
+  if (existsSync(DAEMON_PID_FILE)) {
+    try {
+      const pid = parseInt(readFileSync(DAEMON_PID_FILE, 'utf-8').trim(), 10);
+      if (pid > 0) process.kill(pid);
+      console.log('🔄 Killed agent-browser daemon (pid %d)', pid);
+    } catch {
+      // Process already dead — acceptable
+    }
+    rmSync(DAEMON_PID_FILE, { force: true });
+  }
+
+  // 3. Force-kill any leftover headless Chrome processes
+  try {
+    execSync("pkill -f 'Google Chrome for Testing.*--headless'", {
+      stdio: 'ignore',
+      timeout: 3_000,
+    });
+  } catch {
+    // No matching processes — acceptable
+  }
+
+  // 4. Remove stale SingletonLock left by crashed Chrome
+  const lockPath = join(PROFILE_DIR, 'SingletonLock');
+  if (existsSync(lockPath)) {
+    try {
+      rmSync(lockPath, { force: true });
+    } catch {
+      // Best-effort
+    }
+  }
+
+  // 5. Clear Session Storage to remove stuck MSAL interaction.status flags
+  const sessionStorage = join(PROFILE_DIR, 'Default', 'Session Storage');
+  if (existsSync(sessionStorage)) {
+    rmSync(sessionStorage, { recursive: true, force: true });
+    console.log('🔄 Cleared Session Storage (MSAL interaction flags)');
+  }
+}
+
+/**
+ * Clears the MSAL `interaction.status` flag from `sessionStorage` in the
+ * running browser via `agent-browser eval`.
+ */
+export function clearMsalInteraction(): void {
+  try {
+    ab(['eval', 'sessionStorage.removeItem("msal.interaction.status")'], 5_000);
+  } catch {
+    // Browser may not be ready yet — non-fatal
+  }
+}

package/src/utils/index.ts

@@ -0,0 +1,5 @@
+export { ab, abErrorMessage, PROFILE_DIR } from './agent-browser.js';
+export { resetDaemon, clearMsalInteraction } from './daemon.js';
+export { cleanup, sleep, stripAnsi } from './process.js';
+export { createRunDir, resolveAppKey, waitForServer } from './server.js';
+export { pollConsole } from './browser-poll.js';

package/src/utils/process.ts

@@ -0,0 +1,33 @@
+import type { ChildProcess } from 'node:child_process';
+
+/**
+ * Sends `SIGTERM` to a child process if it is still running.
+ *
+ * @param proc - The child process to terminate (no-op when `undefined` or already killed)
+ */
+export function cleanup(proc?: ChildProcess): void {
+  if (proc && !proc.killed) {
+    proc.kill('SIGTERM');
+  }
+}
+
+/**
+ * Returns a promise that resolves after the given delay.
+ *
+ * @param ms - Delay in milliseconds
+ * @returns A promise that resolves after `ms` milliseconds
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+/**
+ * Strips ANSI escape codes (color / formatting) from a string.
+ *
+ * @param s - The string potentially containing ANSI sequences
+ * @returns The input string with all ANSI escape sequences removed
+ */
+export function stripAnsi(s: string): string {
+  // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI escape sequences
+  return s.replace(/\x1b\[[0-9;]*m/g, '');
+}