@equinor/fusion-framework-cli-plugin-copilot 1.0.0

package/src/commands/app/tools/get-styles.ts

@@ -0,0 +1,40 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the computed CSS styles inspection tool.
+ *
+ * Use this tool instead of (or alongside) screenshots to verify color and layout criteria.
+ * It returns the actual browser-computed style values for a given selector, which cannot
+ * be misread or hallucinated the way image data can.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for retrieving computed CSS style properties
+ */
+export function createGetStylesTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_get_styles', {
+    description:
+      'Get computed CSS styles of an element. Use this for verifying visual properties such as background-color, color, or font-size WITHOUT relying on screenshot interpretation. Returns the computed style object as plain text.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        selector: {
+          type: 'string',
+          description:
+            'Element ref (e.g. @e1), CSS selector, or tag name to inspect (e.g. "body", "main", "h1")',
+        },
+      },
+      required: ['selector'],
+    },
+    handler: async (args) => {
+      const { selector } = args as { selector: string };
+      const output = context.invoke(['get', 'styles', selector]);
+      const safeSelector = selector.replace(/[^a-z0-9]/gi, '_').slice(0, 40);
+      writeFileSync(join(context.outDir, `styles-${safeSelector}.txt`), output, 'utf-8');
+      return output;
+    },
+  });
+}

package/src/commands/app/tools/get-url.ts

@@ -0,0 +1,26 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the current URL inspection tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for retrieving the current page URL
+ */
+export function createGetUrlTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_get_url', {
+    description: 'Get the current page URL.',
+    parameters: {
+      type: 'object' as const,
+      properties: {},
+    },
+    handler: async () => {
+      const output = context.invoke(['get', 'url']);
+      writeFileSync(join(context.outDir, 'url.txt'), output, 'utf-8');
+      return output;
+    },
+  });
+}

package/src/commands/app/tools/go-back.ts

@@ -0,0 +1,16 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the back navigation tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for going back in browser history
+ */
+export function createGoBackTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_go_back', {
+    description: 'Navigate back in browser history.',
+    parameters: { type: 'object' as const, properties: {} },
+    handler: async () => context.invoke(['back']),
+  });
+}

package/src/commands/app/tools/hover.ts

@@ -0,0 +1,25 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the hover interaction tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for hovering a page element
+ */
+export function createHoverTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_hover', {
+    description: 'Hover over an element.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        target: { type: 'string', description: 'Element ref or CSS selector' },
+      },
+      required: ['target'],
+    },
+    handler: async (args) => {
+      const { target } = args as { target: string };
+      return context.invoke(['hover', target]);
+    },
+  });
+}

package/src/commands/app/tools/index.ts

@@ -0,0 +1,57 @@
+import { createClickTool } from './click.js';
+import { createErrorsTool } from './errors.js';
+import { createFillTool } from './fill.js';
+import { createFindTool } from './find.js';
+import { createGetUrlTool } from './get-url.js';
+import { createGoBackTool } from './go-back.js';
+import { createHoverTool } from './hover.js';
+import { createNavigateTool } from './navigate.js';
+import { createPressKeyTool } from './press-key.js';
+import { createReloadTool } from './reload.js';
+import { createEvalJsTool } from './eval-js.js';
+import { createGetStylesTool } from './get-styles.js';
+import { AgentBrowserToolRegistry } from './registry.js';
+import { createScreenshotTool } from './screenshot.js';
+import { createScrollTool } from './scroll.js';
+import { createSelectTool } from './select.js';
+import { createSnapshotTool } from './snapshot.js';
+import { createTypeTextTool } from './type-text.js';
+import type { AgentBrowserToolContext, AgentBrowserToolList, DefineTool } from './types.js';
+import { createWaitTool } from './wait.js';
+
+/**
+ * Creates the full set of Copilot browser tools for an eval session.
+ *
+ * @param context - Shared execution context for all browser tool wrappers
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Registered browser tools in the order they should be exposed to the model
+ */
+export function createAgentBrowserTools(
+  context: AgentBrowserToolContext,
+  defineTool: DefineTool,
+): AgentBrowserToolList {
+  const agentBrowser = new AgentBrowserToolRegistry();
+
+  agentBrowser.addTools(
+    createNavigateTool(context, defineTool),
+    createSnapshotTool(context, defineTool),
+    createScreenshotTool(context, defineTool),
+    createGetStylesTool(context, defineTool),
+    createEvalJsTool(context, defineTool),
+    createClickTool(context, defineTool),
+    createFillTool(context, defineTool),
+    createTypeTextTool(context, defineTool),
+    createPressKeyTool(context, defineTool),
+    createHoverTool(context, defineTool),
+    createSelectTool(context, defineTool),
+    createScrollTool(context, defineTool),
+    createWaitTool(context, defineTool),
+    createFindTool(context, defineTool),
+    createErrorsTool(context, defineTool),
+    createGetUrlTool(context, defineTool),
+    createGoBackTool(context, defineTool),
+    createReloadTool(context, defineTool),
+  );
+
+  return agentBrowser.getTools();
+}

package/src/commands/app/tools/navigate.ts

@@ -0,0 +1,32 @@
+import { clearMsalInteraction } from '../../../utils/index.js';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the browser navigation tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for opening a URL
+ */
+export function createNavigateTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_navigate', {
+    description: 'Navigate the browser to a URL. Use this to open the application or follow links.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        url: { type: 'string', description: 'The URL to navigate to' },
+      },
+      required: ['url'],
+    },
+    handler: async (args) => {
+      const { url } = args as { url: string };
+      try {
+        const result = context.invoke(['open', url], 120_000);
+        return result || 'Navigation complete';
+      } finally {
+        clearMsalInteraction();
+      }
+    },
+  });
+}

package/src/commands/app/tools/press-key.ts

@@ -0,0 +1,25 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the keyboard press tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for pressing a single key
+ */
+export function createPressKeyTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_press_key', {
+    description: 'Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.).',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        key: { type: 'string', description: 'Key name (e.g. Enter, Tab, Escape)' },
+      },
+      required: ['key'],
+    },
+    handler: async (args) => {
+      const { key } = args as { key: string };
+      return context.invoke(['press', key]);
+    },
+  });
+}

package/src/commands/app/tools/read-file.ts

@@ -0,0 +1,113 @@
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { basename, extname, isAbsolute, join } from 'node:path';
+
+import type { ToolResultObject } from '@github/copilot-sdk';
+
+import type { DefineTool } from './types.js';
+
+/** Image extensions that should be returned as vision-capable binary results. */
+const IMAGE_EXTENSIONS = new Set(['.png', '.jpg', '.jpeg', '.gif', '.webp']);
+
+/** MIME types for supported image formats. */
+const IMAGE_MIME: Record<string, string> = {
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+};
+
+/**
+ * Resolves a path within a base directory, rejecting escapes.
+ *
+ * @param baseDir - The allowed root directory
+ * @param input - The user-provided path (relative or absolute within baseDir)
+ * @returns The resolved absolute path
+ * @throws {Error} When the path escapes the base directory
+ */
+function resolveSafe(baseDir: string, input: string): string {
+  const relative = isAbsolute(input)
+    ? input.startsWith(baseDir + '/') || input.startsWith(baseDir + '\\')
+      ? input.slice(baseDir.length + 1)
+      : (() => {
+          throw new Error(`Absolute path outside output directory: ${input}`);
+        })()
+    : input;
+  const resolved = join(baseDir, relative);
+  if (!resolved.startsWith(baseDir)) {
+    throw new Error(`Path escapes the output directory: ${input}`);
+  }
+  return resolved;
+}
+
+/**
+ * Creates the read-file tool so the judge agent can inspect evidence artifacts.
+ *
+ * For image files the tool returns a `ToolBinaryResult` with base64-encoded
+ * data that multimodal models can view directly. Text files are returned
+ * as plain text. Directories return a listing of their contents.
+ *
+ * @param baseDir - The run output directory that scopes all reads
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for reading evidence files
+ */
+export function createReadFileTool(baseDir: string, defineTool: DefineTool) {
+  return defineTool('read_file', {
+    description:
+      'Read a file or list a directory from the output directory. ' +
+      'Image files (png, jpg, jpeg, gif, webp) are returned as inline images the model can see. ' +
+      'Text files are returned as plain text. Directories return a file listing.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        path: {
+          type: 'string',
+          description:
+            'File or directory path relative to the output directory (e.g. "evidence/step-hello.png" or "executions.jsonl")',
+        },
+      },
+      required: ['path'],
+    },
+    handler: async (args): Promise<string | ToolResultObject> => {
+      const { path } = args as { path: string };
+      const resolved = resolveSafe(baseDir, path);
+
+      if (!existsSync(resolved)) {
+        return `File not found: ${path}`;
+      }
+
+      const stat = statSync(resolved);
+
+      // ── Directory listing ──
+      if (stat.isDirectory()) {
+        const entries = readdirSync(resolved).sort();
+        return entries.length > 0
+          ? `Directory ${path}/:\n${entries.join('\n')}`
+          : `Directory ${path}/ is empty`;
+      }
+
+      const ext = extname(resolved).toLowerCase();
+
+      // ── Image: return as binary vision result ──
+      if (IMAGE_EXTENSIONS.has(ext)) {
+        const mime = IMAGE_MIME[ext] ?? 'application/octet-stream';
+        return {
+          textResultForLlm: `Image file: ${path} (${mime}, ${stat.size} bytes). Inspect the attached image.`,
+          binaryResultsForLlm: [
+            {
+              data: readFileSync(resolved, 'base64'),
+              mimeType: mime,
+              type: 'image',
+              description: `Evidence artifact: ${basename(resolved)}`,
+            },
+          ],
+          resultType: 'success',
+          toolTelemetry: {},
+        };
+      }
+
+      // ── Text file ──
+      return readFileSync(resolved, 'utf-8');
+    },
+  });
+}

package/src/commands/app/tools/registry.ts

@@ -0,0 +1,38 @@
+import type { AgentBrowserToolList } from './types.js';
+
+/**
+ * Ordered collection of agent-browser tool registrations for a Copilot session.
+ *
+ * Tools are exposed to the model in insertion order. Use {@link addTools} to
+ * register multiple tools at once during session setup.
+ */
+export class AgentBrowserToolRegistry {
+  private readonly tools: AgentBrowserToolList = [];
+
+  /**
+   * Adds a tool to the registry.
+   *
+   * @param tool - Tool definition created with the Copilot SDK `defineTool` helper
+   */
+  addTool(tool: AgentBrowserToolList[number]): void {
+    this.tools.push(tool);
+  }
+
+  /**
+   * Adds multiple tools to the registry in order.
+   *
+   * @param tools - Tool definitions created with the Copilot SDK `defineTool` helper
+   */
+  addTools(...tools: AgentBrowserToolList): void {
+    this.tools.push(...tools);
+  }
+
+  /**
+   * Returns a shallow copy of the registered tools in insertion order.
+   *
+   * @returns The registered Copilot SDK tool list
+   */
+  getTools(): AgentBrowserToolList {
+    return [...this.tools];
+  }
+}

package/src/commands/app/tools/reload.ts

@@ -0,0 +1,16 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the page reload tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for reloading the current page
+ */
+export function createReloadTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_reload', {
+    description: 'Reload the current page.',
+    parameters: { type: 'object' as const, properties: {} },
+    handler: async () => context.invoke(['reload']),
+  });
+}

package/src/commands/app/tools/screenshot.ts

@@ -0,0 +1,168 @@
+import { mkdirSync, readFileSync } from 'node:fs';
+import { basename, extname, join, relative, resolve, sep } from 'node:path';
+
+import type { ToolResultObject } from '@github/copilot-sdk';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Resolves the requested screenshot format to a supported CLI value.
+ *
+ * @param format - Optional model-provided image format
+ * @returns Validated format string, defaulting to `jpeg`
+ */
+function resolveScreenshotFormat(format: string | undefined): 'jpeg' | 'png' {
+  return format === 'png' ? 'png' : 'jpeg';
+}
+
+/**
+ * Ensures the screenshot artifact filename extension matches the selected image format.
+ *
+ * @param requestedPath - Optional model-provided filename
+ * @param format - Selected image format
+ * @returns Safe filename with the correct extension, or `undefined` when no path was provided
+ */
+function resolveScreenshotArtifactName(
+  requestedPath: string | undefined,
+  format: 'jpeg' | 'png',
+): string | undefined {
+  if (!requestedPath) return undefined;
+  const fileName = basename(requestedPath);
+  const currentExtension = extname(fileName).toLowerCase();
+  const expectedExtension = format === 'jpeg' ? '.jpg' : '.png';
+
+  if (currentExtension === expectedExtension) {
+    return fileName;
+  }
+
+  if (currentExtension.length > 0) {
+    return `${fileName.slice(0, -currentExtension.length)}${expectedExtension}`;
+  }
+
+  return `${fileName}${expectedExtension}`;
+}
+
+/**
+ * Resolves an optional screenshot directory to a safe location within the eval evidence directory.
+ *
+ * @param evidenceDir - Base evidence directory for the current eval run
+ * @param requestedDirectory - Optional model-provided screenshot directory
+ * @returns Absolute screenshot directory constrained to the evidence directory tree
+ */
+function resolveScreenshotDirectory(
+  evidenceDir: string,
+  requestedDirectory: string | undefined,
+): string {
+  if (!requestedDirectory) {
+    return evidenceDir;
+  }
+
+  const resolvedDirectory = resolve(evidenceDir, requestedDirectory);
+  const relativeDirectory = relative(evidenceDir, resolvedDirectory);
+  const isOutsideEvidenceDirectory =
+    relativeDirectory === '..' || relativeDirectory.startsWith(`..${sep}`);
+
+  return isOutsideEvidenceDirectory ? evidenceDir : resolvedDirectory;
+}
+
+/**
+ * Builds a binary screenshot tool result that multimodal models can inspect directly.
+ *
+ * @param screenshotPath - Saved screenshot artifact path
+ * @param format - Image format used for the screenshot
+ * @returns Tool result object containing both text and inline image data
+ */
+function buildScreenshotResult(screenshotPath: string, format: 'jpeg' | 'png'): ToolResultObject {
+  return {
+    textResultForLlm: `Saved screenshot to ${screenshotPath}. Inspect the attached image for visual evidence.`,
+    binaryResultsForLlm: [
+      {
+        data: readFileSync(screenshotPath, 'base64'),
+        mimeType: format === 'jpeg' ? 'image/jpeg' : 'image/png',
+        type: 'image',
+        description: `Screenshot artifact: ${basename(screenshotPath)}`,
+      },
+    ],
+    resultType: 'success',
+    toolTelemetry: {},
+  };
+}
+
+/**
+ * Creates the screenshot capture tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for saving an image screenshot and returning it to the model
+ */
+export function createScreenshotTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_screenshot', {
+    description:
+      'Capture a screenshot of the current page and return the image to the model. Use this for visual assertions such as colors, spacing, and layout.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        path: {
+          type: 'string',
+          description:
+            'Optional filename for the screenshot artifact (for example: "landing-page.jpg")',
+        },
+        annotate: {
+          type: 'boolean',
+          description: 'Overlay numbered labels on interactive elements in the screenshot.',
+        },
+        format: {
+          type: 'string',
+          description: 'Image format: "jpeg" (default) or "png"',
+        },
+        quality: {
+          type: 'number',
+          description: 'JPEG quality from 0-100. Recommended/default is 60 when format is "jpeg".',
+        },
+        screenshotDir: {
+          type: 'string',
+          description:
+            'Default screenshot output directory, scoped inside the eval evidence folder.',
+        },
+      },
+    },
+    handler: async (args) => {
+      const { path, annotate, format, quality, screenshotDir } = args as {
+        path?: string;
+        annotate?: boolean;
+        format?: string;
+        quality?: number;
+        screenshotDir?: string;
+      };
+      const screenshotFormat = resolveScreenshotFormat(format);
+      const screenshotDirectory = resolveScreenshotDirectory(context.outDir, screenshotDir);
+      mkdirSync(screenshotDirectory, { recursive: true });
+      const artifactName = resolveScreenshotArtifactName(path, screenshotFormat);
+      const screenshotPath = join(
+        screenshotDirectory,
+        artifactName ?? `screenshot-${Date.now()}.${screenshotFormat === 'jpeg' ? 'jpg' : 'png'}`,
+      );
+      const command = [
+        'screenshot',
+        '--screenshot-format',
+        screenshotFormat,
+        '--screenshot-dir',
+        screenshotDirectory,
+      ];
+
+      if (annotate) {
+        command.push('--annotate');
+      }
+
+      if (screenshotFormat === 'jpeg') {
+        const screenshotQuality = Math.min(100, Math.max(0, quality ?? 60));
+        command.push('--screenshot-quality', String(screenshotQuality));
+      }
+
+      command.push(screenshotPath);
+      context.invoke(command, 60_000);
+
+      return buildScreenshotResult(screenshotPath, screenshotFormat);
+    },
+  });
+}

package/src/commands/app/tools/scroll.ts

@@ -0,0 +1,45 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the scroll tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for page scrolling or scrolling an element into view
+ */
+export function createScrollTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_scroll', {
+    description: 'Scroll the page or scroll an element into view.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        direction: {
+          type: 'string',
+          description: 'Scroll direction: "up" or "down"',
+        },
+        amount: {
+          type: 'number',
+          description: 'Pixels to scroll (default: one viewport height)',
+        },
+        target: {
+          type: 'string',
+          description: 'Element ref to scroll into view (overrides direction/amount)',
+        },
+      },
+    },
+    handler: async (args) => {
+      const { direction, amount, target } = args as {
+        direction?: string;
+        amount?: number;
+        target?: string;
+      };
+      if (target) {
+        return context.invoke(['scrollintoview', target]);
+      }
+      const scrollDirection = direction ?? 'down';
+      const command = ['scroll', scrollDirection];
+      if (amount) command.push(String(amount));
+      return context.invoke(command);
+    },
+  });
+}

package/src/commands/app/tools/select.ts

@@ -0,0 +1,26 @@
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the select interaction tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for choosing an option in a select element
+ */
+export function createSelectTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_select', {
+    description: 'Select an option from a dropdown/select element.',
+    parameters: {
+      type: 'object' as const,
+      properties: {
+        target: { type: 'string', description: 'Element ref or CSS selector for the select' },
+        value: { type: 'string', description: 'Value or label of the option to select' },
+      },
+      required: ['target', 'value'],
+    },
+    handler: async (args) => {
+      const { target, value } = args as { target: string; value: string };
+      return context.invoke(['select', target, value]);
+    },
+  });
+}

package/src/commands/app/tools/snapshot.ts

@@ -0,0 +1,27 @@
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import type { AgentBrowserToolContext, DefineTool } from './types.js';
+
+/**
+ * Creates the accessibility snapshot tool.
+ *
+ * @param context - Shared browser tool execution context
+ * @param defineTool - Copilot SDK helper used to declare tools
+ * @returns Copilot tool definition for capturing the current page snapshot
+ */
+export function createSnapshotTool(context: AgentBrowserToolContext, defineTool: DefineTool) {
+  return defineTool('browser_snapshot', {
+    description:
+      'Capture an accessibility snapshot of the current page. Returns a text representation of all visible elements with ref identifiers (e.g. @e1, @e2) that can be used in other commands.',
+    parameters: {
+      type: 'object' as const,
+      properties: {},
+    },
+    handler: async () => {
+      const output = context.invoke(['snapshot', '-ci']);
+      writeFileSync(join(context.outDir, 'snapshot.txt'), output, 'utf-8');
+      return output;
+    },
+  });
+}