@relauts/spotcheck-service 1.0.0

package/src/computer-use/keys.ts

@@ -0,0 +1,54 @@
+const KEY_ALIASES: Record<string, string> = {
+  return: "Enter",
+  enter: "Enter",
+  esc: "Escape",
+  escape: "Escape",
+  ctrl: "Control",
+  control: "Control",
+  cmd: "Meta",
+  command: "Meta",
+  super: "Meta",
+  meta: "Meta",
+  alt: "Alt",
+  option: "Alt",
+  space: "Space",
+  spacebar: "Space",
+  backspace: "Backspace",
+  tab: "Tab",
+  del: "Delete",
+  delete: "Delete",
+  up: "ArrowUp",
+  down: "ArrowDown",
+  left: "ArrowLeft",
+  right: "ArrowRight",
+  arrowup: "ArrowUp",
+  arrowdown: "ArrowDown",
+  arrowleft: "ArrowLeft",
+  arrowright: "ArrowRight",
+  pageup: "PageUp",
+  pagedown: "PageDown",
+  home: "Home",
+  end: "End",
+  shift: "Shift",
+};
+
+export function mapGeminiKey(key: string): string {
+  const trimmed = key.trim();
+  if (!trimmed) {
+    throw new Error("Key must be a non-empty string");
+  }
+
+  if (trimmed.length === 1) {
+    return trimmed;
+  }
+
+  return KEY_ALIASES[trimmed.toLowerCase()] ?? trimmed;
+}
+
+export function mapGeminiKeys(keys: string[]): string[] {
+  return keys.map(mapGeminiKey);
+}
+
+export function selectAllShortcut(platform = process.platform): string {
+  return platform === "darwin" ? "Meta+A" : "Control+A";
+}

package/src/computer-use/loop.ts

@@ -0,0 +1,140 @@
+import { buildFunctionResults, executeFunctionCalls } from "./actions.js";
+import { compactOldScreenshots, extractModelText, getFunctionCalls } from "./history.js";
+import { logger } from "../shared/logger.js";
+import { addUsage, EMPTY_USAGE, type TokenUsage } from "../shared/cost.js";
+import { readWebpAsBase64 } from "./screenshot.js";
+import {
+  AgentRunError,
+  type ComputerUsePage,
+  type IntentListener,
+  type InteractionClient,
+  type InteractionStep,
+  type SafetyConfirmer,
+  type ScreenshotSaver,
+  type Sleeper,
+} from "./types.js";
+
+export interface AgentLoopDeps {
+  client: InteractionClient;
+  page: ComputerUsePage;
+  saveScreenshot: ScreenshotSaver;
+  confirmSafety: SafetyConfirmer;
+  screenWidth: number;
+  screenHeight: number;
+  maxTurns: number;
+  sleep?: Sleeper;
+  readScreenshot?: (filePath: string) => Promise<string>;
+  onIntent?: IntentListener;
+  signal?: AbortSignal;
+}
+
+export interface AgentLoopResult {
+  status: "completed" | "max_turns" | "safety_denied";
+  turns: number;
+  finalText: string;
+  usage: TokenUsage;
+}
+
+async function screenshotBase64(
+  saveScreenshot: ScreenshotSaver,
+  actionName: string,
+  readScreenshot: (filePath: string) => Promise<string>,
+): Promise<string> {
+  const filePath = await saveScreenshot(actionName);
+  return readScreenshot(filePath);
+}
+
+async function throwIfAborted(signal: AbortSignal | undefined, usage: TokenUsage): Promise<void> {
+  if (signal?.aborted) {
+    throw new AgentRunError("Agent stopped", usage);
+  }
+}
+
+export async function runComputerUseLoop(
+  deps: AgentLoopDeps,
+  task: string,
+): Promise<AgentLoopResult> {
+  const readScreenshot = deps.readScreenshot ?? readWebpAsBase64;
+  await throwIfAborted(deps.signal, EMPTY_USAGE);
+  const initialImage = await screenshotBase64(deps.saveScreenshot, "start", readScreenshot);
+
+  const history: InteractionStep[] = [
+    {
+      type: "user_input",
+      content: [
+        { type: "text", text: task },
+        { type: "image", data: initialImage, mime_type: "image/webp" },
+      ],
+    },
+  ];
+
+  let finalText = "";
+  let usage = EMPTY_USAGE;
+
+  for (let turn = 1; turn <= deps.maxTurns; turn += 1) {
+    await throwIfAborted(deps.signal, usage);
+    logger.info(`Computer-use turn ${turn}/${deps.maxTurns}`);
+
+    let interaction;
+    try {
+      interaction = await deps.client.create([...history]);
+      usage = addUsage(usage, interaction.usage ?? EMPTY_USAGE);
+    } catch (error: unknown) {
+      if (deps.signal?.aborted) {
+        throw new AgentRunError("Agent stopped", usage);
+      }
+
+      logger.error("Gemini API call failed", error);
+      if (error instanceof AgentRunError) {
+        throw new AgentRunError(error.message, addUsage(usage, error.usage));
+      }
+
+      throw new AgentRunError(error instanceof Error ? error.message : String(error), usage);
+    }
+
+    await throwIfAborted(deps.signal, usage);
+    history.push(...interaction.steps);
+    const calls = getFunctionCalls(interaction.steps);
+
+    if (calls.length === 0) {
+      finalText = extractModelText(interaction.steps);
+      logger.info(`Agent finished: ${finalText || "(no text)"}`);
+      return { status: "completed", turns: turn, finalText, usage };
+    }
+
+    let results;
+    let terminated;
+    let lastScreenshotPath;
+    try {
+      ({ results, terminated, lastScreenshotPath } = await executeFunctionCalls(calls, {
+        page: deps.page,
+        screenWidth: deps.screenWidth,
+        screenHeight: deps.screenHeight,
+        saveScreenshot: deps.saveScreenshot,
+        confirmSafety: deps.confirmSafety,
+        sleep: deps.sleep,
+        onIntent: deps.onIntent,
+        signal: deps.signal,
+      }));
+    } catch (error: unknown) {
+      if (error instanceof AgentRunError && error.message === "Agent stopped") {
+        throw new AgentRunError("Agent stopped", usage);
+      }
+
+      throw error;
+    }
+
+    if (terminated) {
+      logger.info("Agent stopped after safety denial");
+      return { status: "safety_denied", turns: turn, finalText, usage };
+    }
+
+    const screenshotPath = lastScreenshotPath ?? (await deps.saveScreenshot("state"));
+    const image = await readScreenshot(screenshotPath);
+    history.push(...buildFunctionResults(results, deps.page.url(), image));
+    compactOldScreenshots(history);
+  }
+
+  logger.info(`Agent stopped at max turns (${deps.maxTurns})`);
+  return { status: "max_turns", turns: deps.maxTurns, finalText, usage };
+}

package/src/computer-use/safety.ts

@@ -0,0 +1,59 @@
+import readline from "node:readline/promises";
+import { stdin as input, stdout as output } from "node:process";
+import { logger } from "../shared/logger.js";
+
+export interface SafetyDecision {
+  readonly decision: string;
+  readonly explanation: string;
+}
+
+export function readSafetyDecision(args: Record<string, unknown>): SafetyDecision | undefined {
+  const raw = args.safety_decision;
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    return undefined;
+  }
+
+  const record = raw as { decision?: unknown; explanation?: unknown };
+  return {
+    decision: typeof record.decision === "string" ? record.decision : "",
+    explanation: typeof record.explanation === "string" ? record.explanation : "",
+  };
+}
+
+export function isSafetyBlocked(decision: SafetyDecision | undefined): boolean {
+  return decision?.decision === "blocked";
+}
+
+export function requiresSafetyConfirmation(decision: SafetyDecision | undefined): boolean {
+  return decision?.decision === "require_confirmation";
+}
+
+/**
+ * Fallback confirmer for the web UI when no interactive confirmer is wired up
+ * (e.g. tests, or callers that don't pass a per-request confirmer). Auto-approving
+ * here is NOT safe for real production traffic; startWebServer always supplies a
+ * real popup-backed confirmer per request, see createWebSafetyConfirmer in server.ts.
+ */
+export async function confirmSafetyForWeb(explanation: string, actionName: string): Promise<boolean> {
+  logger.info(
+    `Safety confirmation auto-approved (web fallback): ${actionName} - ${explanation || "No explanation provided"}`,
+  );
+  return true;
+}
+
+export async function confirmSafetyInTerminal(explanation: string, actionName: string): Promise<boolean> {
+  if (!input.isTTY) {
+    logger.error("Safety confirmation required but stdin is not a TTY");
+    return false;
+  }
+
+  const rl = readline.createInterface({ input, output });
+  try {
+    const answer = await rl.question(
+      `Safety confirmation required for ${actionName}: ${explanation || "No explanation provided"}\nProceed? [y/N] `,
+    );
+    return /^y(es)?$/i.test(answer.trim());
+  } finally {
+    rl.close();
+  }
+}

package/src/computer-use/screenshot.ts

@@ -0,0 +1,20 @@
+import fs from "node:fs/promises";
+import { getLatestWebpPath } from "../playwright/processed.js";
+
+export async function readWebpAsBase64(filePath: string): Promise<string> {
+  return fs.readFile(filePath, { encoding: "base64" });
+}
+
+export async function loadLatestWebpBase64(
+  directory?: string,
+): Promise<{ path: string; data: string }> {
+  const filePath = await getLatestWebpPath(directory);
+  if (!filePath) {
+    throw new Error("No processed webp screenshot found");
+  }
+
+  return {
+    path: filePath,
+    data: await readWebpAsBase64(filePath),
+  };
+}

package/src/computer-use/system-prompt.ts

@@ -0,0 +1,65 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const SYSTEM_PROMPT_PATH = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  "../../files/system_prompt.txt",
+);
+
+export interface SystemPromptVars {
+  readonly os: string;
+  readonly displayWidth: number;
+  readonly displayHeight: number;
+  readonly keyboardLayout: string;
+}
+
+export function osLabel(platform = process.platform): string {
+  if (platform === "darwin") {
+    return "Chromium on macOS";
+  }
+  if (platform === "win32") {
+    return "Chromium on Windows";
+  }
+  if (platform === "linux") {
+    return "Chromium on Linux";
+  }
+
+  return `Chromium on ${platform}`;
+}
+
+export function defaultSystemPromptVars(
+  displayWidth: number,
+  displayHeight: number,
+  platform = process.platform,
+  locale = Intl.DateTimeFormat().resolvedOptions().locale,
+): SystemPromptVars {
+  return {
+    os: osLabel(platform),
+    displayWidth,
+    displayHeight,
+    keyboardLayout: locale,
+  };
+}
+
+export function renderSystemPrompt(template: string, vars: SystemPromptVars): string {
+  return template
+    .replaceAll("{{os}}", vars.os)
+    .replaceAll("{{display_width}}", String(vars.displayWidth))
+    .replaceAll("{{display_height}}", String(vars.displayHeight))
+    .replaceAll("{{keyboard_layout}}", vars.keyboardLayout)
+    .trim();
+}
+
+export async function loadSystemPrompt(
+  vars: SystemPromptVars,
+  filePath = SYSTEM_PROMPT_PATH,
+): Promise<string> {
+  const template = await fs.readFile(filePath, "utf8");
+  const rendered = renderSystemPrompt(template, vars);
+  if (!rendered) {
+    throw new Error("system_prompt.txt is empty");
+  }
+
+  return rendered;
+}

package/src/computer-use/types.ts

@@ -0,0 +1,126 @@
+import type { TokenUsage } from "../shared/cost.js";
+
+export const BROWSER_ACTION_NAMES = [
+  "click",
+  "double_click",
+  "triple_click",
+  "middle_click",
+  "right_click",
+  "mouse_down",
+  "mouse_up",
+  "move",
+  "type",
+  "drag_and_drop",
+  "wait",
+  "press_key",
+  "key_down",
+  "key_up",
+  "hotkey",
+  "take_screenshot",
+  "scroll",
+  "go_back",
+  "navigate",
+  "go_forward",
+] as const;
+
+export type BrowserActionName = (typeof BROWSER_ACTION_NAMES)[number];
+
+export type MouseButton = "left" | "right" | "middle";
+
+export interface ComputerUseMouse {
+  click(
+    x: number,
+    y: number,
+    options?: { button?: MouseButton; clickCount?: number },
+  ): Promise<void>;
+  dblclick(x: number, y: number): Promise<void>;
+  move(x: number, y: number): Promise<void>;
+  down(): Promise<void>;
+  up(): Promise<void>;
+  wheel(deltaX: number, deltaY: number): Promise<void>;
+}
+
+export interface ComputerUseKeyboard {
+  type(text: string): Promise<void>;
+  press(key: string): Promise<void>;
+  down(key: string): Promise<void>;
+  up(key: string): Promise<void>;
+}
+
+export interface ComputerUsePage {
+  url(): string;
+  goto(url: string): Promise<unknown>;
+  goBack(): Promise<unknown>;
+  goForward(): Promise<unknown>;
+  mouse: ComputerUseMouse;
+  keyboard: ComputerUseKeyboard;
+}
+
+export interface ContentBlock {
+  type: string;
+  text?: string;
+  data?: string;
+  mime_type?: string;
+}
+
+export interface InteractionStep {
+  type: string;
+  id?: string;
+  name?: string;
+  call_id?: string;
+  arguments?: Record<string, unknown>;
+  content?: ContentBlock[];
+  result?: unknown;
+  is_error?: boolean;
+}
+
+export interface FunctionCallStep extends InteractionStep {
+  type: "function_call";
+  id: string;
+  name: string;
+  arguments: Record<string, unknown>;
+}
+
+export interface FunctionResultStep extends InteractionStep {
+  type: "function_result";
+  call_id: string;
+  name: string;
+  is_error?: boolean;
+  result: ContentBlock[];
+}
+
+export interface InteractionResponse {
+  steps: InteractionStep[];
+  usage?: TokenUsage;
+}
+
+export class AgentRunError extends Error {
+  constructor(
+    message: string,
+    readonly usage: TokenUsage,
+  ) {
+    super(message);
+    this.name = "AgentRunError";
+  }
+}
+
+export interface InteractionClient {
+  create(input: InteractionStep[]): Promise<InteractionResponse>;
+}
+
+export type ScreenshotSaver = (actionName: string) => Promise<string>;
+export type SafetyConfirmer = (explanation: string, actionName: string) => Promise<boolean>;
+export type Sleeper = (ms: number) => Promise<void>;
+export type IntentListener = (label: string) => void | Promise<void>;
+
+export interface ActionResult {
+  name: string;
+  callId: string;
+  payload: Record<string, unknown>;
+}
+
+export interface ExecuteTurnResult {
+  results: ActionResult[];
+  terminated: boolean;
+  lastScreenshotPath: string | undefined;
+}