minimal-agent 0.10.0 → 0.12.0

package/src/tools/computeruse/diagnosticTrace.js

@@ -0,0 +1,154 @@
+import { createHash } from 'node:crypto';
+import { appendFile, chmod, copyFile, mkdir, writeFile } from 'node:fs/promises';
+import { extname, join } from 'node:path';
+const traces = new Map();
+function runKey(context) {
+    return `${context.ownerId}\0${context.conversationKey}\0${context.runId}`;
+}
+function safeRunName(context) {
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const suffix = context.runId.replace(/[^a-zA-Z0-9_-]/g, '').slice(0, 12) || 'run';
+    return `${stamp}-${suffix}`;
+}
+function conversationHash(context) {
+    return createHash('sha256').update(context.conversationKey).digest('hex').slice(0, 12);
+}
+function traceState(context) {
+    if (!context.diagnosticTraceDir)
+        return null;
+    const key = runKey(context);
+    let state = traces.get(key);
+    if (!state) {
+        state = {
+            dir: join(context.diagnosticTraceDir, safeRunName(context)),
+            step: 0,
+            queue: Promise.resolve(),
+        };
+        traces.set(key, state);
+    }
+    return state;
+}
+function traceFullTextEnabled() {
+    return process.env.COMPUTER_USE_TRACE_FULL === '1';
+}
+function actionMetadata(input) {
+    const fullText = traceFullTextEnabled();
+    return {
+        action: input.action,
+        ...(input.appId ? { appId: input.appId } : {}),
+        ...(input.displayId ? { displayId: input.displayId } : {}),
+        ...(input.windowId ? { windowId: input.windowId } : {}),
+        ...(input.x !== undefined ? { x: input.x } : {}),
+        ...(input.y !== undefined ? { y: input.y } : {}),
+        ...(input.x2 !== undefined ? { x2: input.x2 } : {}),
+        ...(input.y2 !== undefined ? { y2: input.y2 } : {}),
+        ...(input.toX !== undefined ? { toX: input.toX } : {}),
+        ...(input.toY !== undefined ? { toY: input.toY } : {}),
+        ...(input.button ? { button: input.button } : {}),
+        ...(input.key ? { key: input.key } : {}),
+        // ⚠️ `repeat` / `repeatFrames` 必须记 —— 它们把一次调用放大成最多 8 次实际动作。
+        //    漏记的代价是**诊断假阴性**：实测排查「模型到底有没有用批量翻页」时，trace 里
+        //    只看得到 7 次 key 事件，据此判定「一次批量都没用」，而真相是那 7 次调用每次
+        //    都带着 repeat:8（7×8=56 次按键，正好覆盖需要翻的 97 页）。
+        //    trace 是排查时唯一的证据通道，它漏记什么，什么就会被误判成没发生。
+        ...(input.repeat !== undefined ? { repeat: input.repeat } : {}),
+        ...(input.repeatFrames ? { repeatFrames: input.repeatFrames } : {}),
+        ...(input.text !== undefined
+            ? fullText
+                ? { text: input.text, textLength: input.text.length }
+                : { textLength: input.text.length }
+            : {}),
+        ...(input.filePaths?.length
+            ? fullText
+                ? { filePaths: input.filePaths, fileCount: input.filePaths.length }
+                : { fileCount: input.filePaths.length }
+            : {}),
+    };
+}
+async function initialize(state, context) {
+    const screenshotDir = join(state.dir, 'screenshots');
+    await mkdir(screenshotDir, { recursive: true, mode: 0o700 });
+    await chmod(state.dir, 0o700).catch(() => { });
+    await chmod(screenshotDir, 0o700).catch(() => { });
+    const manifestPath = join(state.dir, 'manifest.json');
+    await writeFile(manifestPath, `${JSON.stringify({
+        schemaVersion: 1,
+        runId: context.runId,
+        conversationHash: conversationHash(context),
+        surface: context.surface,
+        createdAt: new Date().toISOString(),
+        privacy: traceFullTextEnabled()
+            ? 'COMPUTER_USE_TRACE_FULL=1 is set: typed text and file paths are included in events.jsonl. Screenshots may contain sensitive on-screen content.'
+            : 'Screenshots may contain sensitive on-screen content. Typed text, clipboard content and source file paths are omitted from events.jsonl. Set COMPUTER_USE_TRACE_FULL=1 to include them.',
+    }, null, 2)}\n`, { mode: 0o600, flag: 'wx' }).catch((error) => {
+        if (error.code !== 'EEXIST')
+            throw error;
+    });
+}
+function redactedError(result, request) {
+    if (result.ok)
+        return undefined;
+    let message = result.error.slice(0, 500);
+    const fullText = traceFullTextEnabled();
+    if (!fullText && request.text)
+        message = message.split(request.text).join('[REDACTED_TEXT]');
+    for (const path of request.filePaths ?? []) {
+        if (!fullText)
+            message = message.split(path).join('[REDACTED_PATH]');
+    }
+    for (const image of result.visionImages ?? []) {
+        if (image.source && !image.source.startsWith('data:')) {
+            message = message.split(image.source).join('[REDACTED_PATH]');
+        }
+    }
+    return message;
+}
+async function copyImages(state, step, result) {
+    const copied = [];
+    for (const [index, image] of (result.visionImages ?? []).entries()) {
+        if (!image.source || image.source.startsWith('data:') || /^https?:/i.test(image.source))
+            continue;
+        const extension = extname(image.source).toLowerCase() || '.png';
+        const name = `${String(step).padStart(3, '0')}-${index + 1}${extension}`;
+        const destination = join(state.dir, 'screenshots', name);
+        await copyFile(image.source, destination);
+        await chmod(destination, 0o600).catch(() => { });
+        copied.push(`screenshots/${name}`);
+    }
+    return copied;
+}
+/**
+ * Persist one Computer Use tool step when the caller explicitly opted into a
+ * trace directory. Failures are swallowed by the caller: diagnostics must not
+ * turn a successful desktop action into a failed task.
+ */
+export async function recordComputerUseDiagnosticStep(input) {
+    const state = traceState(input.context);
+    if (!state)
+        return;
+    const operation = async () => {
+        await initialize(state, input.context);
+        const step = ++state.step;
+        const screenshots = await copyImages(state, step, input.result);
+        const event = {
+            schemaVersion: 1,
+            step,
+            at: new Date().toISOString(),
+            elapsedMs: input.elapsedMs,
+            ...actionMetadata(input.request),
+            ok: input.result.ok,
+            ...(redactedError(input.result, input.request)
+                ? { error: redactedError(input.result, input.request) }
+                : {}),
+            screenshots,
+            diagnostics: input.diagnostics ?? {},
+        };
+        await appendFile(join(state.dir, 'events.jsonl'), `${JSON.stringify(event)}\n`, { mode: 0o600 });
+        await chmod(join(state.dir, 'events.jsonl'), 0o600).catch(() => { });
+    };
+    state.queue = state.queue.then(operation, operation);
+    await state.queue;
+}
+export function diagnosticTraceDirectory(context) {
+    return traceState(context)?.dir;
+}

package/src/tools/computeruse/diagnosticTrace.ts

@@ -0,0 +1,195 @@
+import { createHash } from 'node:crypto';
+import { appendFile, chmod, copyFile, mkdir, writeFile } from 'node:fs/promises';
+import { extname, join } from 'node:path';
+
+import type { ToolResult } from '../../types.js';
+import type { ComputerUseRequest, ComputerUseRunContext } from './protocol.js';
+
+/**
+ * trace 记录的是**工具层的入参**（一次工具调用记一条），不是发给守护进程的线路请求。
+ *
+ * ⚠️ 这里曾经直接复用 `ComputerUseRequest`，把 `repeat` / `repeatFrames` 挡在了类型外面 ——
+ * 它们是纯 TS 层的循环参数，线路协议里没有。后果是**诊断假阴性**：排查「模型到底有没有
+ * 用批量翻页」时，trace 里只看得到 7 次 key 事件，据此判定「一次批量都没用」，
+ * 而真相是那 7 次调用每次都带 repeat（7×8=56 次按键，正好覆盖需要翻的 97 页）。
+ * trace 是排查时唯一的证据通道 —— 它漏记什么，什么就会被误判成没发生过。
+ */
+type TraceInput = Omit<
+  ComputerUseRequest,
+  'protocolVersion' | 'requestId' | 'context'
+> & {
+  repeat?: number;
+  repeatFrames?: 'all' | 'last';
+};
+
+interface TraceState {
+  dir: string;
+  step: number;
+  queue: Promise<void>;
+}
+
+const traces = new Map<string, TraceState>();
+
+function runKey(context: ComputerUseRunContext): string {
+  return `${context.ownerId}\0${context.conversationKey}\0${context.runId}`;
+}
+
+function safeRunName(context: ComputerUseRunContext): string {
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const suffix = context.runId.replace(/[^a-zA-Z0-9_-]/g, '').slice(0, 12) || 'run';
+  return `${stamp}-${suffix}`;
+}
+
+function conversationHash(context: ComputerUseRunContext): string {
+  return createHash('sha256').update(context.conversationKey).digest('hex').slice(0, 12);
+}
+
+function traceState(context: ComputerUseRunContext): TraceState | null {
+  if (!context.diagnosticTraceDir) return null;
+  const key = runKey(context);
+  let state = traces.get(key);
+  if (!state) {
+    state = {
+      dir: join(context.diagnosticTraceDir, safeRunName(context)),
+      step: 0,
+      queue: Promise.resolve(),
+    };
+    traces.set(key, state);
+  }
+  return state;
+}
+
+function traceFullTextEnabled(): boolean {
+  return process.env.COMPUTER_USE_TRACE_FULL === '1';
+}
+
+function actionMetadata(input: TraceInput): Record<string, unknown> {
+  const fullText = traceFullTextEnabled();
+  return {
+    action: input.action,
+    ...(input.appId ? { appId: input.appId } : {}),
+    ...(input.displayId ? { displayId: input.displayId } : {}),
+    ...(input.windowId ? { windowId: input.windowId } : {}),
+    ...(input.x !== undefined ? { x: input.x } : {}),
+    ...(input.y !== undefined ? { y: input.y } : {}),
+    ...(input.x2 !== undefined ? { x2: input.x2 } : {}),
+    ...(input.y2 !== undefined ? { y2: input.y2 } : {}),
+    ...(input.toX !== undefined ? { toX: input.toX } : {}),
+    ...(input.toY !== undefined ? { toY: input.toY } : {}),
+    ...(input.button ? { button: input.button } : {}),
+    ...(input.key ? { key: input.key } : {}),
+    // ⚠️ `repeat` / `repeatFrames` 必须记 —— 它们把一次调用放大成最多 8 次实际动作。
+    //    漏记的代价是**诊断假阴性**：实测排查「模型到底有没有用批量翻页」时，trace 里
+    //    只看得到 7 次 key 事件，据此判定「一次批量都没用」，而真相是那 7 次调用每次
+    //    都带着 repeat:8（7×8=56 次按键，正好覆盖需要翻的 97 页）。
+    //    trace 是排查时唯一的证据通道，它漏记什么，什么就会被误判成没发生。
+    ...(input.repeat !== undefined ? { repeat: input.repeat } : {}),
+    ...(input.repeatFrames ? { repeatFrames: input.repeatFrames } : {}),
+    ...(input.text !== undefined
+      ? fullText
+        ? { text: input.text, textLength: input.text.length }
+        : { textLength: input.text.length }
+      : {}),
+    ...(input.filePaths?.length
+      ? fullText
+        ? { filePaths: input.filePaths, fileCount: input.filePaths.length }
+        : { fileCount: input.filePaths.length }
+      : {}),
+  };
+}
+
+async function initialize(state: TraceState, context: ComputerUseRunContext): Promise<void> {
+  const screenshotDir = join(state.dir, 'screenshots');
+  await mkdir(screenshotDir, { recursive: true, mode: 0o700 });
+  await chmod(state.dir, 0o700).catch(() => {});
+  await chmod(screenshotDir, 0o700).catch(() => {});
+  const manifestPath = join(state.dir, 'manifest.json');
+  await writeFile(manifestPath, `${JSON.stringify({
+    schemaVersion: 1,
+    runId: context.runId,
+    conversationHash: conversationHash(context),
+    surface: context.surface,
+    createdAt: new Date().toISOString(),
+    privacy: traceFullTextEnabled()
+      ? 'COMPUTER_USE_TRACE_FULL=1 is set: typed text and file paths are included in events.jsonl. Screenshots may contain sensitive on-screen content.'
+      : 'Screenshots may contain sensitive on-screen content. Typed text, clipboard content and source file paths are omitted from events.jsonl. Set COMPUTER_USE_TRACE_FULL=1 to include them.',
+  }, null, 2)}\n`, { mode: 0o600, flag: 'wx' }).catch((error: NodeJS.ErrnoException) => {
+    if (error.code !== 'EEXIST') throw error;
+  });
+}
+
+function redactedError(result: ToolResult, request: TraceInput): string | undefined {
+  if (result.ok) return undefined;
+  let message = result.error.slice(0, 500);
+  const fullText = traceFullTextEnabled();
+  if (!fullText && request.text) message = message.split(request.text).join('[REDACTED_TEXT]');
+  for (const path of request.filePaths ?? []) {
+    if (!fullText) message = message.split(path).join('[REDACTED_PATH]');
+  }
+  for (const image of result.visionImages ?? []) {
+    if (image.source && !image.source.startsWith('data:')) {
+      message = message.split(image.source).join('[REDACTED_PATH]');
+    }
+  }
+  return message;
+}
+
+async function copyImages(
+  state: TraceState,
+  step: number,
+  result: ToolResult,
+): Promise<string[]> {
+  const copied: string[] = [];
+  for (const [index, image] of (result.visionImages ?? []).entries()) {
+    if (!image.source || image.source.startsWith('data:') || /^https?:/i.test(image.source)) continue;
+    const extension = extname(image.source).toLowerCase() || '.png';
+    const name = `${String(step).padStart(3, '0')}-${index + 1}${extension}`;
+    const destination = join(state.dir, 'screenshots', name);
+    await copyFile(image.source, destination);
+    await chmod(destination, 0o600).catch(() => {});
+    copied.push(`screenshots/${name}`);
+  }
+  return copied;
+}
+
+/**
+ * Persist one Computer Use tool step when the caller explicitly opted into a
+ * trace directory. Failures are swallowed by the caller: diagnostics must not
+ * turn a successful desktop action into a failed task.
+ */
+export async function recordComputerUseDiagnosticStep(input: {
+  context: ComputerUseRunContext;
+  request: TraceInput;
+  result: ToolResult;
+  elapsedMs: number;
+  diagnostics?: Record<string, unknown>;
+}): Promise<void> {
+  const state = traceState(input.context);
+  if (!state) return;
+  const operation = async (): Promise<void> => {
+    await initialize(state, input.context);
+    const step = ++state.step;
+    const screenshots = await copyImages(state, step, input.result);
+    const event = {
+      schemaVersion: 1,
+      step,
+      at: new Date().toISOString(),
+      elapsedMs: input.elapsedMs,
+      ...actionMetadata(input.request),
+      ok: input.result.ok,
+      ...(redactedError(input.result, input.request)
+        ? { error: redactedError(input.result, input.request) }
+        : {}),
+      screenshots,
+      diagnostics: input.diagnostics ?? {},
+    };
+    await appendFile(join(state.dir, 'events.jsonl'), `${JSON.stringify(event)}\n`, { mode: 0o600 });
+    await chmod(join(state.dir, 'events.jsonl'), 0o600).catch(() => {});
+  };
+  state.queue = state.queue.then(operation, operation);
+  await state.queue;
+}
+
+export function diagnosticTraceDirectory(context: ComputerUseRunContext): string | undefined {
+  return traceState(context)?.dir;
+}