@equinor/fusion-framework-cli-plugin-copilot 1.0.0

package/src/commands/app/prompts/plan.prompt.ts

@@ -0,0 +1,48 @@
+import type { RuntimeExecutionContext } from '../types.js';
+
+/**
+ * Creates the planning prompt that turns a feature specification into
+ * a structured test plan.
+ *
+ * The agent produces a `plan.json` with one step per atomic acceptance
+ * criterion, each containing browser actions, pass evidence, and fail evidence.
+ *
+ * @param query - The feature specification / eval content to plan against
+ * @param ctx - Runtime context with output directory and app URL
+ * @returns The fully interpolated planning prompt string
+ */
+export const createPlanPrompt = (query: string, ctx: RuntimeExecutionContext): string => {
+  return `
+Create a test plan for this feature specification:
+
+${query}
+
+Application URL: ${ctx.url}
+
+Save the plan using write_file with path "plan.json".
+
+JSON structure:
+
+{
+  "summary": "One-sentence description of what is being tested",
+  "steps": [
+    {
+      "scenario": "A logical user scenario (e.g. 'Open the landing page')",
+      "criteria": ["First thing to verify", "Second thing to verify"],
+      "pass": ["What pass looks like for first", "What pass looks like for second"],
+      "fail": ["What fail looks like for first", "What fail looks like for second"]
+    }
+  ]
+}
+
+Rules:
+- Group related criteria under one step when they share the same scenario
+  (e.g. same page, same state). Do NOT create separate steps for things
+  that can be verified in a single browser visit.
+- Only create multiple steps when there is a distinct user action between them
+  (navigation, click, form submission, etc.).
+- Do NOT include HOW to test — no actions, selectors, or tool calls.
+- passEvidence and failEvidence describe what a human would see.
+- Keep criteria directly traceable to the spec. Do NOT invent criteria.
+  `.trim();
+};

package/src/commands/app/prompts/step.prompt.ts

@@ -0,0 +1,45 @@
+import type { PlanStep, RuntimeExecutionContext } from '../types.js';
+
+/**
+ * Creates the execution prompt for a single test step.
+ *
+ * The agent autonomously verifies all criteria for the scenario,
+ * takes evidence, and records results.
+ *
+ * @param step - The plan step with scenario and criteria
+ * @param ctx - Runtime context with output directory and app URL
+ * @returns The fully interpolated prompt string
+ */
+export const createStepPrompt = (step: PlanStep, ctx: RuntimeExecutionContext): string => {
+  const slug = slugify(step.scenario);
+  const criteriaBlock = step.criteria
+    .map((c, i) => `${i + 1}. ${c}\n   Pass: ${step.pass[i] ?? ''}\n   Fail: ${step.fail[i] ?? ''}`)
+    .join('\n');
+
+  return `
+Scenario: ${step.scenario}
+
+Verify ALL of these criteria:
+${criteriaBlock}
+
+Use browser tools to verify. Take a screenshot when done:
+  browser_screenshot path="evidence/step-${slug}.jpg" annotate=true
+
+Then for EACH criterion, append ONE JSON line using append_file with path "executions.jsonl":
+{"criterion": "...", "ok": true|false, "note": "what you observed", "evidence": ["evidence/step-${slug}.jpg"]}
+  `.trim();
+};
+
+/**
+ * Converts text to a URL-safe slug for filenames.
+ *
+ * @param text - The text to slugify
+ * @returns Lowercase hyphenated string safe for filenames
+ */
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-|-$/g, '')
+    .slice(0, 60);
+}

package/src/commands/app/prompts/system.dev-server.prompt.ts

@@ -0,0 +1,22 @@
+import type { RuntimeExecutionContext } from '../types.js';
+
+/**
+ * Creates the Fusion dev-server–specific system prompt supplement.
+ *
+ * Describes the DOM structure of a Fusion Framework application served
+ * by the local dev server / dev-portal so the agent knows where to
+ * find app content in the page.
+ *
+ * @param ctx - Runtime context with the application URL
+ * @returns The dev-server DOM context block to append to the system prompt
+ */
+export const createDevServerContext = (ctx: RuntimeExecutionContext): string => {
+  return `
+Fusion dev-server DOM:
+- App content lives inside <section id="app-section" style="display:contents">.
+- While loading, #app-section contains an EquinorLoader spinner — wait for it to disappear.
+- Once loaded, the app renders in an anonymous <div> child of #app-section.
+- Target #app-section for assertions, not <body>.
+- URL pattern: /apps/{appKey} → ${ctx.url}
+  `.trim();
+};

package/src/commands/app/prompts/system.prompt.ts

@@ -0,0 +1,78 @@
+import type { RuntimeExecutionContext } from '../types.js';
+
+/**
+ * Creates the system prompt that sets the agent's identity and ground rules
+ * for the entire eval session.
+ *
+ * Injected once as the session-level system message. Individual phase prompts
+ * (plan, execute, evaluate, judge) are sent as user messages within this session.
+ *
+ * @param ctx - Runtime context with the application URL and output directory
+ * @returns The fully interpolated system prompt string
+ */
+export const createSystemPrompt = (ctx: RuntimeExecutionContext): string => {
+  return `
+You are an expert software test agent with perfect knowledge of the feature specification,
+the application under test, and the testing tools at your disposal.
+
+Application under test: ${ctx.url}
+
+## File tools use RELATIVE paths
+
+The write_file, append_file, and read_file tools are scoped to an output directory.
+Always use **relative paths** like "plan.json", "executions.jsonl", "evidence/screenshot.jpg".
+Never use absolute paths with these tools.
+
+You will be guided through phases in order:
+1. PLAN    — produce a structured test plan (plan.json)
+2. EXECUTE — run each step's browser actions and collect evidence
+3. JUDGE   — review all evidence and deliver a final pass/fail verdict (verdict.json)
+
+All phases run in this single session. You have access to both browser tools
+and file tools throughout.
+
+## Tool inventory
+
+### File tools
+- **write_file** — write a file to the artifacts directory (plan.json, verdict.json)
+- **append_file** — append a line to a file (executions.jsonl)
+- **read_file** — read a file or list a directory from the artifacts directory.
+  Image files (png, jpg) are returned as inline vision content you can inspect.
+  Use this in the JUDGE phase to view screenshots taken during execution.
+
+### Browser tools
+- **browser_navigate** — open a URL
+- **browser_snapshot** — capture an accessibility snapshot (DOM tree). Use \`-i --json\` for interactive-only, JSON format.
+- **browser_screenshot** — capture a screenshot. Returns the image inline so you can see it.
+  Use \`--annotate --full\` for labeled full-page screenshots.
+- **browser_get_styles** — get computed styles of an element
+- **browser_eval** — evaluate JavaScript in the page context
+- **browser_click** — click an element (use @eN refs from snapshot, or CSS/role selectors)
+- **browser_fill** — fill an input field
+- **browser_type** — type text character by character
+- **browser_press_key** — press a keyboard key
+- **browser_hover** — hover over an element
+- **browser_select** — select an option from a dropdown
+- **browser_scroll** — scroll the page or an element
+- **browser_wait** — wait for a condition: \`networkidle\`, text, selector, or time (ms)
+- **browser_find** — find elements by role, text, label, placeholder, testid, or CSS
+- **browser_errors** — get console errors and uncaught exceptions
+- **browser_get_url** — get the current URL
+- **browser_go_back** — navigate back
+- **browser_reload** — reload the current page
+
+## Tool use rules
+- Act immediately. Do not deliberate or plan between tool calls — execute then evaluate.
+- Keep tool calls minimal. Prefer fewer, targeted calls over exhaustive probing.
+- After navigation, always \`browser_wait\` for \`networkidle\` before interacting.
+- Use \`@eN\` element refs from \`browser_snapshot\` for clicks, fills, and assertions.
+- Take ONE screenshot per step (after the action). Skip redundant before-screenshots.
+- Capture \`browser_errors\` only when something looks wrong, not on every step.
+- Do NOT use browser tools during the PLAN phase — only describe actions as strings.
+- Do NOT use browser tools during the JUDGE phase — use \`read_file\` to view evidence.
+
+## General rules
+- Save artifacts using write_file / append_file with RELATIVE paths (e.g. "plan.json").
+- Never fabricate evidence. If a tool call fails, report the failure honestly.
+  `.trim();
+};

package/src/commands/app/server.ts

@@ -0,0 +1,105 @@
+import { spawn, execSync, type ChildProcess } from 'node:child_process';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { resolveAppKey, cleanup, waitForServer } from '../../utils/index.js';
+
+/** Result of starting (or skipping) the app dev server. */
+export interface AppServerResult {
+  /** The child process running the dev server, if one was spawned. */
+  serverProcess: ChildProcess | undefined;
+  /** The fully-qualified app URL (e.g. `http://localhost:3000/apps/my-app`). */
+  appUrl: string;
+}
+
+/**
+ * Kills any process listening on the given port.
+ *
+ * Uses `lsof` to find PIDs bound to the port and sends SIGTERM.
+ * Failures are silently ignored (port may already be free).
+ *
+ * @param port - The TCP port to free
+ */
+function freePort(port: number): void {
+  try {
+    const pids = execSync(`lsof -ti tcp:${port}`, { encoding: 'utf-8' })
+      .trim()
+      .split('\n')
+      .filter(Boolean);
+    for (const pid of pids) {
+      try {
+        process.kill(parseInt(pid, 10), 'SIGTERM');
+      } catch {
+        // already dead
+      }
+    }
+    if (pids.length) {
+      console.log(`🔄 Killed stale process(es) on port ${port}: ${pids.join(', ')}`);
+    }
+  } catch {
+    // lsof exits non-zero when no matches — port is free
+  }
+}
+
+/**
+ * Starts the Fusion app dev server for the given application path,
+ * or returns the caller-provided URL when the server is already running.
+ *
+ * @param absAppPath - Absolute path to the Fusion application directory
+ * @param options - Server configuration options
+ * @param options.port - Port number for the dev server
+ * @param options.host - Host address for the dev server
+ * @param options.url - When provided, skips server startup and uses this URL directly
+ * @param options.verbose - When true, inherits stdio from the spawned server process
+ * @returns The spawned server process (if any) and the resolved app URL
+ */
+export async function startAppServer(
+  absAppPath: string,
+  options: { port: number; host: string; url?: string; verbose: boolean },
+): Promise<AppServerResult> {
+  // Caller already has a running server — skip startup entirely
+  if (options.url) {
+    return { serverProcess: undefined, appUrl: options.url };
+  }
+
+  const pkgJsonPath = join(absAppPath, 'package.json');
+  if (!existsSync(pkgJsonPath)) {
+    console.error(`❌ No package.json found at ${absAppPath}`);
+    process.exit(1);
+  }
+  const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf-8')) as { name?: string };
+  if (!pkgJson.name) {
+    console.error(`❌ package.json at ${absAppPath} has no "name" field`);
+    process.exit(1);
+  }
+
+  console.log('📦 Resolving app manifest...');
+  const appKey = resolveAppKey(absAppPath);
+  console.log(`🚀 Starting app server (${appKey})...`);
+
+  // Kill stale dev servers on the target port from previous crashed runs
+  freePort(options.port);
+
+  const serverProcess = spawn(
+    'ffc',
+    ['app', 'serve', '--host', options.host, '--port', String(options.port)],
+    {
+      cwd: absAppPath,
+      stdio: options.verbose ? 'inherit' : 'ignore',
+      detached: false,
+    },
+  );
+
+  const serverUrl = `http://localhost:${options.port}`;
+  const ready = await waitForServer(serverUrl, 60);
+  if (!ready) {
+    console.error('❌ Server failed to start within 60s');
+    cleanup(serverProcess);
+    process.exit(1);
+  }
+
+  const appUrl = `${serverUrl}/apps/${appKey}`;
+  console.log(`✅ Server ready at ${appUrl}`);
+
+  return { serverProcess, appUrl };
+}

package/src/commands/app/session-logger.ts

@@ -0,0 +1,187 @@
+import chalk from 'chalk';
+import ora from 'ora';
+
+import type { CopilotSession } from '@github/copilot-sdk';
+
+import { tryFormatMessage } from './format.js';
+
+const TOOL_ICONS: Record<string, string> = {
+  browser_screenshot: '📷',
+  browser_navigate: '🌐',
+  browser_click: '👆',
+  browser_wait: '⏳',
+  browser_type: '⌨️',
+  browser_scroll: '📜',
+  browser_snapshot: '🔍',
+};
+
+/**
+ * Attaches a console logger to a Copilot session that renders tool progress
+ * with a single shared ora spinner (avoiding concurrent-spinner warnings).
+ *
+ * @param session - The active Copilot SDK session
+ * @param options - Optional config (e.g. requested model name for mismatch detection)
+ * @returns A handle with `stop()` to tear down the spinner (e.g. on SIGINT)
+ */
+export function attachSessionLogger(
+  session: CopilotSession,
+  options?: { requestedModel?: string },
+): { stop: () => void } {
+  const pendingTools = new Map<string, string>();
+  let spinner: ReturnType<typeof ora> | null = null;
+
+  function syncSpinner(): void {
+    if (pendingTools.size === 0) {
+      if (spinner) {
+        spinner.stop();
+        spinner = null;
+      }
+      return;
+    }
+    const text = [...pendingTools.values()].join(chalk.dim(' | '));
+    if (spinner) {
+      spinner.text = text;
+    } else {
+      spinner = ora({ text, color: 'cyan' }).start();
+    }
+  }
+
+  function pauseSpinner(): void {
+    if (spinner) {
+      spinner.stop();
+      spinner = null;
+    }
+  }
+
+  function logCompletion(label: string, ok: boolean): void {
+    pauseSpinner();
+    const prefix = ok ? chalk.green('✔') : chalk.red('✖');
+    console.log(`${prefix} ${label}`);
+    syncSpinner();
+  }
+
+  session.on((event) => {
+    switch (event.type) {
+      case 'tool.execution_start': {
+        const { toolCallId, toolName, arguments: args } = event.data;
+        const detail = args?.url ?? args?.path ?? args?.load ?? args?.selector;
+        const icon = TOOL_ICONS[toolName] ?? '🔧';
+        const label =
+          typeof detail === 'string' ? `${icon} ${toolName} (${detail})` : `${icon} ${toolName}`;
+        pendingTools.set(toolCallId, label);
+        syncSpinner();
+        break;
+      }
+      case 'tool.execution_complete': {
+        const label = pendingTools.get(event.data.toolCallId);
+        if (label) {
+          pendingTools.delete(event.data.toolCallId);
+          logCompletion(label, event.data.success);
+        }
+        break;
+      }
+      case 'external_tool.completed': {
+        const label = pendingTools.get(event.data.requestId);
+        if (label) {
+          pendingTools.delete(event.data.requestId);
+          logCompletion(label, true);
+        }
+        break;
+      }
+      case 'assistant.reasoning': {
+        pauseSpinner();
+        console.log(chalk.dim(`🧠 ${event.data.content}`));
+        syncSpinner();
+        break;
+      }
+      case 'assistant.message': {
+        if (event.data.content) {
+          pauseSpinner();
+          const formatted = tryFormatMessage(event.data.content);
+          if (formatted) {
+            console.log(formatted);
+          } else {
+            console.log(chalk.green(`🤖 ${event.data.content}`));
+          }
+          syncSpinner();
+        }
+        break;
+      }
+      case 'permission.requested': {
+        const { requestId, permissionRequest } = event.data;
+        const toolName =
+          permissionRequest.kind === 'custom-tool'
+            ? permissionRequest.toolName
+            : permissionRequest.kind;
+        pendingTools.set(requestId, `🔐 ${toolName}`);
+        syncSpinner();
+        break;
+      }
+      case 'permission.completed': {
+        const label = pendingTools.get(event.data.requestId);
+        if (label) {
+          const approved = event.data.result?.kind === 'approved';
+          pendingTools.delete(event.data.requestId);
+          logCompletion(label, approved);
+        }
+        break;
+      }
+      // @ts-expect-error - missing case types in SDK typings
+      case 'session.tools_updated': {
+        // @ts-expect-error - model property missing in SDK typings
+        const negotiatedModel = event.data.model as string | undefined;
+        if (negotiatedModel) {
+          console.log(chalk.dim(`💾  ${negotiatedModel}`));
+          if (options?.requestedModel && negotiatedModel !== options.requestedModel) {
+            console.log(
+              chalk.yellow(
+                `⚠️  Requested model "${options.requestedModel}" but server negotiated "${negotiatedModel}"`,
+              ),
+            );
+          }
+        }
+        break;
+      }
+      case 'external_tool.requested':
+      case 'user.message': {
+        break;
+      }
+      case 'assistant.message_delta':
+      case 'assistant.reasoning_delta':
+      case 'assistant.streaming_delta':
+      case 'assistant.turn_start':
+      case 'assistant.turn_end':
+      case 'assistant.usage':
+      case 'pending_messages.modified':
+      case 'session.idle':
+      case 'session.usage_info': {
+        break;
+      }
+      case 'session.info': {
+        const { message } = event.data as { message?: string };
+        if (message) {
+          pauseSpinner();
+          console.log(chalk.dim(`ℹ️  ${message}`));
+          syncSpinner();
+        }
+        break;
+      }
+      case 'assistant.intent': {
+        const { intent } = event.data as { intent?: string };
+        if (intent) {
+          pauseSpinner();
+          console.log(chalk.yellow(`🎯 ${intent}`));
+          syncSpinner();
+        }
+        break;
+      }
+      default:
+        console.log(event);
+        break;
+    }
+  });
+
+  return {
+    stop: () => pauseSpinner(),
+  };
+}

package/src/commands/app/tools/click.ts

@@ -0,0 +1,28 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the click interaction tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for clicking a page element
+ */
+export function createClickTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_click', {
+    description: 'Click an element identified by a ref (@e1), CSS selector, or text selector.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        target: {
+          type: 'string',
+          description: 'Element ref (e.g. @e3), CSS selector, or text selector',
+        },
+      },
+      required: ['target'],
+    },
+    handler: async (args) => {
+      const { target } = args as { target: string };
+      return context.invoke(['click', target]);
+    },
+  });
+}

package/src/commands/app/tools/errors.ts

@@ -0,0 +1,26 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the browser error collection tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for retrieving JavaScript errors
+ */
+export function createErrorsTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_errors', {
+    description: 'Get all JavaScript console errors from the browser.',
+    parameters: {
+      type: 'object' as const,
+      properties: {},
+    },
+    handler: async () => {
+      const output = context.invoke(['errors']);
+      writeFileSync(join(context.outDir, 'errors.txt'), output, 'utf-8');
+      return output || 'No errors';
+    },
+  });
+}

package/src/commands/app/tools/eval-js.ts

@@ -0,0 +1,40 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the JavaScript evaluation tool.
+ *
+ * Runs a JS expression in the browser's page context and returns the result.
+ * Ideal for extracting computed values such as `getComputedStyle(document.body).backgroundColor`
+ * that cannot be reliably inferred from screenshots.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for evaluating arbitrary JavaScript in the browser
+ */
+export function createEvalJsTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_eval', {
+    description:
+      'Evaluate a JavaScript expression in the browser page context and return the result. Use for reading DOM values, computed styles, or element properties that need exact values (e.g. getComputedStyle(document.body).backgroundColor).',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        script: {
+          type: 'string',
+          description:
+            'A single JavaScript expression to evaluate (e.g. "getComputedStyle(document.body).backgroundColor")',
+        },
+      },
+      required: ['script'],
+    },
+    handler: async (args) => {
+      const { script } = args as { script: string };
+      const output = context.invoke(['eval', script, '--json']);
+      const timestamp = Date.now();
+      writeFileSync(join(context.outDir, `eval-${timestamp}.json`), output, 'utf-8');
+      return output;
+    },
+  });
+}

package/src/commands/app/tools/fill.ts

@@ -0,0 +1,29 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the form fill tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for filling input fields
+ */
+export function createFillTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_fill', {
+    description: 'Fill a form field with text. Clears existing content first.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        target: {
+          type: 'string',
+          description: 'Element ref or CSS selector for the input field',
+        },
+        value: { type: 'string', description: 'Text to fill into the field' },
+      },
+      required: ['target', 'value'],
+    },
+    handler: async (args) => {
+      const { target, value } = args as { target: string; value: string };
+      return context.invoke(['fill', target, value]);
+    },
+  });
+}

package/src/commands/app/tools/find.ts

@@ -0,0 +1,49 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the semantic locator tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for semantic element lookup and optional action execution
+ */
+export function createFindTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_find', {
+    description:
+      'Find an element using semantic locators (role, text, label) and optionally perform an action on it.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        by: {
+          type: 'string',
+          description: 'Locator type: "role", "text", or "label"',
+        },
+        value: {
+          type: 'string',
+          description: 'Value to search for (role name, text content, or label)',
+        },
+        action: {
+          type: 'string',
+          description: 'Action to perform: "click" or "fill"',
+        },
+        name: { type: 'string', description: 'Accessible name filter (for role locators)' },
+        fillValue: { type: 'string', description: 'Text to fill (when action is "fill")' },
+      },
+      required: ['by', 'value'],
+    },
+    handler: async (args) => {
+      const { by, value, action, name, fillValue } = args as {
+        by: string;
+        value: string;
+        action?: string;
+        name?: string;
+        fillValue?: string;
+      };
+      const command = ['find', by, value];
+      if (action) command.push(action);
+      if (name) command.push('--name', name);
+      if (fillValue) command.push(fillValue);
+      return context.invoke(command);
+    },
+  });
+}