@relauts/spotcheck-service 1.0.0

package/src/computer-use/actions.ts

@@ -0,0 +1,273 @@
+import { asBoolean, asIntent, asNonNegativeNumber, asPoint, asString, asStringList, formatIntentLabel } from "./args.js";
+import { isBrowserActionName } from "./history.js";
+import { mapGeminiKey, mapGeminiKeys, selectAllShortcut } from "./keys.js";
+import {
+  isSafetyBlocked,
+  readSafetyDecision,
+  requiresSafetyConfirmation,
+} from "./safety.js";
+import { EMPTY_USAGE } from "../shared/cost.js";
+import { logger } from "../shared/logger.js";
+import {
+  AgentRunError,
+  type ActionResult,
+  type BrowserActionName,
+  type ComputerUsePage,
+  type ExecuteTurnResult,
+  type FunctionCallStep,
+  type FunctionResultStep,
+  type IntentListener,
+  type SafetyConfirmer,
+  type ScreenshotSaver,
+  type Sleeper,
+} from "./types.js";
+
+export interface ExecuteActionsOptions {
+  page: ComputerUsePage;
+  screenWidth: number;
+  screenHeight: number;
+  saveScreenshot: ScreenshotSaver;
+  confirmSafety: SafetyConfirmer;
+  sleep?: Sleeper;
+  selectAllKey?: string;
+  onIntent?: IntentListener;
+  signal?: AbortSignal;
+}
+
+type ActionHandler = (
+  args: Record<string, unknown>,
+  options: ExecuteActionsOptions,
+) => Promise<void>;
+
+const defaultSleep: Sleeper = (ms) =>
+  new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function scrollDeltas(
+  direction: string,
+  magnitude: number,
+): { deltaX: number; deltaY: number } {
+  switch (direction.toLowerCase()) {
+    case "up":
+      return { deltaX: 0, deltaY: -magnitude };
+    case "down":
+      return { deltaX: 0, deltaY: magnitude };
+    case "left":
+      return { deltaX: -magnitude, deltaY: 0 };
+    case "right":
+      return { deltaX: magnitude, deltaY: 0 };
+    default:
+      throw new Error(`direction must be up, down, left, or right`);
+  }
+}
+
+async function holdHotkey(page: ComputerUsePage, keys: string[]): Promise<void> {
+  for (const key of keys) {
+    await page.keyboard.down(key);
+  }
+
+  for (let index = keys.length - 1; index >= 0; index -= 1) {
+    const key = keys[index];
+    if (key) {
+      await page.keyboard.up(key);
+    }
+  }
+}
+
+const ACTION_HANDLERS: Record<BrowserActionName, ActionHandler> = {
+  async click(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.click(point.x, point.y);
+  },
+  async double_click(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.dblclick(point.x, point.y);
+  },
+  async triple_click(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.click(point.x, point.y, { clickCount: 3 });
+  },
+  async middle_click(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.click(point.x, point.y, { button: "middle" });
+  },
+  async right_click(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.click(point.x, point.y, { button: "right" });
+  },
+  async mouse_down(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.move(point.x, point.y);
+    await page.mouse.down();
+  },
+  async mouse_up(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.move(point.x, point.y);
+    await page.mouse.up();
+  },
+  async move(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    await page.mouse.move(point.x, point.y);
+  },
+  async type(args, { page, selectAllKey }) {
+    const text = asString(args.text, "text");
+    await page.keyboard.press(selectAllKey ?? selectAllShortcut());
+    await page.keyboard.press("Backspace");
+    await page.keyboard.type(text);
+    if (asBoolean(args.press_enter, false)) {
+      await page.keyboard.press("Enter");
+    }
+  },
+  async drag_and_drop(args, { page, screenWidth, screenHeight }) {
+    const start = asPoint({ x: args.start_x, y: args.start_y }, screenWidth, screenHeight);
+    const end = asPoint({ x: args.end_x, y: args.end_y }, screenWidth, screenHeight);
+    await page.mouse.move(start.x, start.y);
+    await page.mouse.down();
+    await page.mouse.move(end.x, end.y);
+    await page.mouse.up();
+  },
+  async wait(args, { sleep }) {
+    const seconds = asNonNegativeNumber(args.seconds, "seconds", 1);
+    await (sleep ?? defaultSleep)(seconds * 1000);
+  },
+  async press_key(args, { page }) {
+    await page.keyboard.press(mapGeminiKey(asString(args.key, "key")));
+  },
+  async key_down(args, { page }) {
+    await page.keyboard.down(mapGeminiKey(asString(args.key, "key")));
+  },
+  async key_up(args, { page }) {
+    await page.keyboard.up(mapGeminiKey(asString(args.key, "key")));
+  },
+  async hotkey(args, { page }) {
+    await holdHotkey(page, mapGeminiKeys(asStringList(args.keys, "keys")));
+  },
+  async take_screenshot() {
+    // Screenshot is captured after every action.
+  },
+  async scroll(args, { page, screenWidth, screenHeight }) {
+    const point = asPoint(args, screenWidth, screenHeight);
+    const magnitude = asNonNegativeNumber(args.magnitude_in_pixels, "magnitude_in_pixels", 300);
+    const { deltaX, deltaY } = scrollDeltas(asString(args.direction, "direction"), magnitude);
+    await page.mouse.move(point.x, point.y);
+    await page.mouse.wheel(deltaX, deltaY);
+  },
+  async go_back(_args, { page }) {
+    await page.goBack();
+  },
+  async navigate(args, { page }) {
+    await page.goto(asString(args.url, "url"));
+  },
+  async go_forward(_args, { page }) {
+    await page.goForward();
+  },
+};
+
+export async function executeFunctionCalls(
+  calls: FunctionCallStep[],
+  options: ExecuteActionsOptions,
+): Promise<ExecuteTurnResult> {
+  const results: ActionResult[] = [];
+  let terminated = false;
+  let lastScreenshotPath: string | undefined;
+
+  for (const call of calls) {
+    if (options.signal?.aborted) {
+      throw new AgentRunError("Agent stopped", EMPTY_USAGE);
+    }
+
+    if (terminated) {
+      results.push({
+        name: call.name,
+        callId: call.id,
+        payload: { error: "Skipped after safety denial" },
+      });
+      continue;
+    }
+
+    const payload: Record<string, unknown> = {};
+    const args = call.arguments;
+    const intent = asIntent(args);
+    logger.info(`Executing ${call.name} (Intent: ${intent})`);
+    await options.onIntent?.(formatIntentLabel(call.name, intent));
+
+    try {
+      const safety = readSafetyDecision(args);
+      if (isSafetyBlocked(safety)) {
+        throw new Error("Action blocked by safety policy");
+      }
+
+      if (requiresSafetyConfirmation(safety)) {
+        const confirmed = await options.confirmSafety(safety?.explanation ?? "", call.name);
+        if (!confirmed) {
+          terminated = true;
+          payload.error = "User denied safety confirmation";
+          await options.onIntent?.(formatIntentLabel(call.name, intent, true));
+          results.push({ name: call.name, callId: call.id, payload });
+          continue;
+        }
+
+        payload.safety_acknowledgement = "true";
+      }
+
+      if (!isBrowserActionName(call.name)) {
+        throw new Error(`Unsupported browser action: ${call.name}`);
+      }
+
+      await ACTION_HANDLERS[call.name](args, options);
+    } catch (error: unknown) {
+      if (error instanceof AgentRunError) {
+        throw error;
+      }
+
+      payload.error = errorMessage(error);
+      logger.error(`Failed to execute ${call.name}`, error);
+      await options.onIntent?.(formatIntentLabel(call.name, intent, true));
+    }
+
+    try {
+      lastScreenshotPath = await options.saveScreenshot(call.name);
+    } catch (error: unknown) {
+      payload.error = payload.error
+        ? `${String(payload.error)}; screenshot failed: ${errorMessage(error)}`
+        : `screenshot failed: ${errorMessage(error)}`;
+      logger.error(`Failed to screenshot after ${call.name}`, error);
+    }
+
+    results.push({ name: call.name, callId: call.id, payload });
+  }
+
+  return { results, terminated, lastScreenshotPath };
+}
+
+export function buildFunctionResults(
+  results: ActionResult[],
+  currentUrl: string,
+  screenshotBase64: string,
+): FunctionResultStep[] {
+  return results.map((item) => {
+    const hasError = typeof item.payload.error === "string";
+    return {
+      type: "function_result" as const,
+      name: item.name,
+      call_id: item.callId,
+      ...(hasError ? { is_error: true } : {}),
+      result: [
+        {
+          type: "text" as const,
+          text: JSON.stringify({ url: currentUrl, ...item.payload }),
+        },
+        {
+          type: "image" as const,
+          data: screenshotBase64,
+          mime_type: "image/webp" as const,
+        },
+      ],
+    };
+  });
+}

package/src/computer-use/args.ts

@@ -0,0 +1,97 @@
+import { denormalizeX, denormalizeY } from "./coords.js";
+
+export function asFiniteNumber(value: unknown, field: string): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    throw new Error(`${field} must be a finite number`);
+  }
+
+  return value;
+}
+
+export function asNonNegativeNumber(value: unknown, field: string, fallback: number): number {
+  if (value === undefined) {
+    return fallback;
+  }
+
+  const parsed = asFiniteNumber(value, field);
+  if (parsed < 0) {
+    throw new Error(`${field} must be >= 0`);
+  }
+
+  return parsed;
+}
+
+export function asString(value: unknown, field: string): string {
+  if (typeof value !== "string") {
+    throw new Error(`${field} must be a string`);
+  }
+
+  return value;
+}
+
+export function asBoolean(value: unknown, fallback: boolean): boolean {
+  if (value === undefined) {
+    return fallback;
+  }
+
+  if (typeof value === "boolean") {
+    return value;
+  }
+
+  if (value === "true" || value === 1) {
+    return true;
+  }
+
+  if (value === "false" || value === 0) {
+    return false;
+  }
+
+  return fallback;
+}
+
+export function asStringList(value: unknown, field: string): string[] {
+  if (typeof value === "string") {
+    const keys = value
+      .split("+")
+      .map((key) => key.trim())
+      .filter((key) => key.length > 0);
+    if (keys.length === 0) {
+      throw new Error(`${field} must not be empty`);
+    }
+
+    return keys;
+  }
+
+  if (Array.isArray(value) && value.every((item) => typeof item === "string")) {
+    const keys = value.map((item) => item.trim()).filter((item) => item.length > 0);
+    if (keys.length === 0) {
+      throw new Error(`${field} must not be empty`);
+    }
+
+    return keys;
+  }
+
+  throw new Error(`${field} must be a string list`);
+}
+
+export function asPoint(
+  args: Record<string, unknown>,
+  screenWidth: number,
+  screenHeight: number,
+): { x: number; y: number } {
+  return {
+    x: denormalizeX(asFiniteNumber(args.x, "x"), screenWidth),
+    y: denormalizeY(asFiniteNumber(args.y, "y"), screenHeight),
+  };
+}
+
+export function asIntent(args: Record<string, unknown>): string {
+  return typeof args.intent === "string" && args.intent.trim() ? args.intent : "N/A";
+}
+
+export function formatIntentLabel(name: string, intent: string, failed = false): string {
+  const trimmed = intent.trim();
+  const hasIntent = trimmed.length > 0 && trimmed !== "N/A";
+  const base = hasIntent ? `${name} — ${trimmed}` : name;
+  return failed ? `${base} (failed)` : base;
+}

package/src/computer-use/cli.ts

@@ -0,0 +1,8 @@
+export function parseAgentTask(argv: readonly string[]): string {
+  const task = argv.slice(2).join(" ").trim();
+  if (!task) {
+    throw new Error('Usage: npm run agent -- "<task>"');
+  }
+
+  return task;
+}

package/src/computer-use/client.ts

@@ -0,0 +1,81 @@
+import { GoogleGenAI } from "@google/genai";
+import type { AppConfig } from "../shared/config.js";
+import { parseUsage } from "../shared/cost.js";
+import { cloneSteps } from "./history.js";
+import { AgentRunError, type InteractionClient, type InteractionResponse, type InteractionStep } from "./types.js";
+
+type GeminiInteractionInput = Parameters<GoogleGenAI["interactions"]["create"]>[0]["input"];
+
+const SUMMARY_RESPONSE_FORMAT = {
+  type: "text",
+  mime_type: "application/json",
+  schema: {
+    type: "object",
+    required: ["action", "summary", "validations"],
+    properties: {
+      action: { type: "string", enum: ["summary"] },
+      summary: { type: "string" },
+      validations: {
+        type: "array",
+        items: {
+          type: "object",
+          required: ["check", "result", "observed"],
+          properties: {
+            check: { type: "string" },
+            result: { type: "string", enum: ["pass", "fail"] },
+            observed: { type: "string" },
+          },
+        },
+      },
+    },
+  },
+} as const;
+
+export function createGeminiInteractionClient(
+  config: AppConfig,
+  model = config.geminiModel,
+  systemInstruction?: string,
+): InteractionClient {
+  if (!config.geminiApiKey) {
+    throw new Error("GEMINI_API_KEY is required");
+  }
+
+  const ai = new GoogleGenAI({ apiKey: config.geminiApiKey });
+
+  return {
+    async create(input: InteractionStep[]): Promise<InteractionResponse> {
+      const interaction = await ai.interactions.create({
+        model,
+        store: false,
+        ...(systemInstruction ? { system_instruction: systemInstruction } : {}),
+        input: input as GeminiInteractionInput,
+        tools: [
+          {
+            type: "computer_use",
+            environment: "browser",
+            enable_prompt_injection_detection: false,
+            ...(config.geminiDisabledSafetyPolicies.length > 0
+              ? { disabled_safety_policies: [...config.geminiDisabledSafetyPolicies] }
+              : {}),
+          },
+        ],
+        generation_config: {
+          thinking_level: config.geminiThinkingLevel,
+          ...(config.geminiSeed !== undefined ? { seed: config.geminiSeed } : {}),
+        },
+        response_format: SUMMARY_RESPONSE_FORMAT,
+      });
+
+      const usage = parseUsage(interaction.usage);
+
+      if (interaction.status === "failed") {
+        throw new AgentRunError("Gemini interaction failed", usage);
+      }
+
+      return {
+        steps: cloneSteps(interaction.steps),
+        usage,
+      };
+    },
+  };
+}

package/src/computer-use/coords.ts

@@ -0,0 +1,26 @@
+export const NORMALIZED_MAX = 1000;
+
+export function clampNormalized(value: number): number {
+  if (!Number.isFinite(value)) {
+    throw new Error("Coordinate must be a finite number");
+  }
+
+  return Math.min(NORMALIZED_MAX, Math.max(0, value));
+}
+
+export function denormalize(value: number, screenSize: number): number {
+  if (!Number.isInteger(screenSize) || screenSize <= 0) {
+    throw new Error("Screen size must be a positive integer");
+  }
+
+  const pixel = Math.floor((clampNormalized(value) / NORMALIZED_MAX) * screenSize);
+  return Math.min(screenSize - 1, Math.max(0, pixel));
+}
+
+export function denormalizeX(x: number, screenWidth: number): number {
+  return denormalize(x, screenWidth);
+}
+
+export function denormalizeY(y: number, screenHeight: number): number {
+  return denormalize(y, screenHeight);
+}

package/src/computer-use/history.ts

@@ -0,0 +1,80 @@
+import { BROWSER_ACTION_NAMES, type BrowserActionName, type InteractionStep } from "./types.js";
+
+const BROWSER_ACTION_SET = new Set<string>(BROWSER_ACTION_NAMES);
+const OLD_SCREENSHOT_PLACEHOLDER = [{ type: "text", text: "[old screenshot removed]" }];
+
+export function isBrowserActionName(name: string): name is BrowserActionName {
+  return BROWSER_ACTION_SET.has(name);
+}
+
+export function getFunctionCalls(steps: InteractionStep[]): Array<{
+  type: "function_call";
+  id: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}> {
+  return steps.flatMap((step) => {
+    if (step.type !== "function_call") {
+      return [];
+    }
+
+    if (!step.id || !step.name) {
+      return [];
+    }
+
+    return [
+      {
+        type: "function_call" as const,
+        id: step.id,
+        name: step.name,
+        arguments: step.arguments ?? {},
+      },
+    ];
+  });
+}
+
+export function extractModelText(steps: InteractionStep[]): string {
+  return steps
+    .filter((step) => step.type === "model_output")
+    .flatMap((step) => step.content ?? [])
+    .filter((block) => block.type === "text" && typeof block.text === "string")
+    .map((block) => block.text as string)
+    .join(" ")
+    .trim();
+}
+
+function hasImageResult(result: unknown): boolean {
+  return (
+    Array.isArray(result) &&
+    result.some((block) => typeof block === "object" && block !== null && (block as { type?: unknown }).type === "image")
+  );
+}
+
+export function cloneSteps(steps: unknown): InteractionStep[] {
+  return JSON.parse(JSON.stringify(steps)) as InteractionStep[];
+}
+
+export function compactOldScreenshots(history: InteractionStep[]): InteractionStep[] {
+  let keepLatest = true;
+
+  for (let index = history.length - 1; index >= 0; index -= 1) {
+    const item = history[index];
+    if (!item || item.type !== "function_result" || !hasImageResult(item.result)) {
+      continue;
+    }
+
+    if (keepLatest) {
+      keepLatest = false;
+      continue;
+    }
+
+    const textBlocks = Array.isArray(item.result)
+      ? item.result.filter(
+          (block) => typeof block === "object" && block !== null && (block as { type?: unknown }).type === "text",
+        )
+      : [];
+    item.result = textBlocks.length > 0 ? textBlocks : OLD_SCREENSHOT_PLACEHOLDER;
+  }
+
+  return history;
+}

package/src/computer-use/index.ts

@@ -0,0 +1,121 @@
+import { createGeminiInteractionClient } from "./client.js";
+import { parseAgentTask } from "./cli.js";
+import { runComputerUseLoop } from "./loop.js";
+import { confirmSafetyInTerminal } from "./safety.js";
+import { defaultSystemPromptVars, loadSystemPrompt } from "./system-prompt.js";
+import { AgentRunError, type IntentListener, type SafetyConfirmer } from "./types.js";
+import { createProcessedScreenshotSaver } from "../playwright/processed.js";
+import {
+  BROWSER_SESSION_RESET_MESSAGE,
+  createBrowserSession,
+  type BrowserSession,
+} from "../playwright/session.js";
+import { loadConfig, requireGeminiApiKey, type AppConfig } from "../shared/config.js";
+import { type TokenUsage } from "../shared/cost.js";
+import { logger } from "../shared/logger.js";
+import { isMainModule } from "../shared/main-module.js";
+
+export { AgentRunError, BROWSER_SESSION_RESET_MESSAGE };
+
+export interface AgentRunResult {
+  readonly text: string;
+  readonly sessionReset: boolean;
+  readonly usage: TokenUsage;
+}
+
+const agentSession = createBrowserSession();
+let shuttingDown = false;
+
+export async function closeAgentSession(): Promise<void> {
+  await agentSession.close();
+}
+
+async function shutdown(exitCode: number, reason: string): Promise<never> {
+  if (shuttingDown) {
+    process.exit(exitCode);
+  }
+
+  shuttingDown = true;
+  logger.info(reason);
+  await closeAgentSession();
+  process.exit(exitCode);
+}
+
+export interface AgentRunExtras {
+  readonly onIntent?: IntentListener;
+  readonly signal?: AbortSignal;
+  readonly confirmSafety?: SafetyConfirmer;
+}
+
+export async function runAgent(
+  task: string,
+  config: AppConfig = loadConfig(),
+  confirmSafety: SafetyConfirmer = confirmSafetyInTerminal,
+  session: BrowserSession = agentSession,
+  model = config.geminiModel,
+  extras: AgentRunExtras = {},
+): Promise<AgentRunResult> {
+  requireGeminiApiKey(config);
+
+  logger.info(
+    `Computer-use model=${model} thinking=${config.geminiThinkingLevel}` +
+      (config.geminiSeed !== undefined ? ` seed=${config.geminiSeed}` : ""),
+  );
+  const { page, sessionReset } = await session.ensurePage(config);
+
+  if (sessionReset) {
+    logger.info(BROWSER_SESSION_RESET_MESSAGE);
+  }
+
+  const systemInstruction = await loadSystemPrompt(
+    defaultSystemPromptVars(config.viewportWidth, config.viewportHeight),
+  );
+
+  const result = await runComputerUseLoop(
+    {
+      client: createGeminiInteractionClient(config, model, systemInstruction),
+      page,
+      saveScreenshot: createProcessedScreenshotSaver(page, config.processedDir),
+      confirmSafety: extras.confirmSafety ?? confirmSafety,
+      screenWidth: config.viewportWidth,
+      screenHeight: config.viewportHeight,
+      maxTurns: config.agentMaxTurns,
+      onIntent: extras.onIntent,
+      signal: extras.signal,
+    },
+    task,
+  );
+
+  logger.info(`Computer-use status=${result.status} turns=${result.turns}`);
+  return { text: result.finalText, sessionReset, usage: result.usage };
+}
+
+function registerShutdownHandlers(): void {
+  const onSignal = (signal: NodeJS.Signals): void => {
+    void shutdown(0, `Received ${signal}, shutting down`);
+  };
+
+  process.once("SIGINT", onSignal);
+  process.once("SIGTERM", onSignal);
+}
+
+async function waitUntilStopped(): Promise<void> {
+  logger.info("Task finished. Browser stays open until Ctrl+C");
+  await new Promise<void>(() => {
+    // Shutdown handlers call process.exit.
+  });
+}
+
+async function main(): Promise<void> {
+  registerShutdownHandlers();
+  const task = parseAgentTask(process.argv);
+  await runAgent(task);
+  await waitUntilStopped();
+}
+
+if (isMainModule(import.meta.url)) {
+  main().catch((error: unknown) => {
+    logger.error("Fatal error", error);
+    void shutdown(1, "Exiting after fatal error");
+  });
+}