@frockbot/plugin-computer 0.3.15 → 0.3.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@frockbot/plugin-computer",
-  "version": "0.3.15",
+  "version": "0.3.17",
   "private": false,
   "type": "module",
   "exports": {
@@ -29,21 +29,21 @@
   },
   "dependencies": {
     "@cordisjs/client": "0.8.2",
-    "@frockbot/client-core": "0.3.15",
-    "@frockbot/client-ui": "0.3.15",
-    "@frockbot/computer-core": "0.3.15",
-    "@frockbot/computer-host-runtime": "0.3.15",
-    "@frockbot/kernel-agent-loop": "0.3.15",
-    "@frockbot/kernel-contracts": "0.3.15",
-    "@frockbot/plugin-prompt": "0.3.15",
-    "@frockbot/plugin-shell": "0.3.15",
-    "@frockbot/plugin-tools": "0.3.15",
+    "@frockbot/client-core": "0.3.17",
+    "@frockbot/client-ui": "0.3.17",
+    "@frockbot/computer-core": "0.3.17",
+    "@frockbot/computer-host-runtime": "0.3.17",
+    "@frockbot/kernel-agent-loop": "0.3.17",
+    "@frockbot/kernel-contracts": "0.3.17",
+    "@frockbot/plugin-prompt": "0.3.17",
+    "@frockbot/plugin-shell": "0.3.17",
+    "@frockbot/plugin-tools": "0.3.17",
     "cordis": "4.0.0-rc.8",
     "vue": "3.5.41"
   },
   "devDependencies": {
-    "@frockbot/plugin-models": "0.3.15",
-    "@frockbot/plugin-testkit": "0.3.15",
+    "@frockbot/plugin-models": "0.3.17",
+    "@frockbot/plugin-testkit": "0.3.17",
     "@types/bun": "1.4.0",
     "@types/node": "26.2.0",
     "@vitejs/plugin-vue": "6.0.8",

package/src/agent.test.ts

@@ -108,6 +108,78 @@ describe("computer agent contribution", () => {
     await harness.dispose();
   });
 
+  test("computer_browser says which field a click is missing and takes label as name", async () => {
+    const calls: string[] = [];
+    const provider: ComputerProvider = {
+      id: "fixture",
+      open: async (identity, tenant, assignment) => ({
+        assignment,
+        identity,
+        tenant,
+        exec: {
+          execute: async () => ({
+            exitCode: 0,
+            stdout: new Uint8Array(),
+            stderr: new Uint8Array(),
+            outputTruncated: false,
+          }),
+        },
+        browser: {
+          perform: async (action) => {
+            calls.push(JSON.stringify(action));
+            return { accessibilitySnapshot: 'checkbox "Mark done"' };
+          },
+        },
+        close: () => Promise.resolve(),
+      }),
+    };
+    const harness = await createPluginHarness([
+      ComputerRegistry,
+      ToolRegistry,
+      SystemPromptRegistry,
+      SessionStore,
+    ]);
+    harness.root.computers.register(provider);
+    await harness.mount(
+      createComputerAgentPlugin({
+        userId: "user-1",
+        defaultProviderId: "fixture",
+      }),
+    );
+
+    // Bob's first three attempts on production, in order.
+    const onlyName = await execute(harness, "computer_browser", {
+      action: "click",
+      name: "Add",
+    });
+    expect(onlyName.isError).toBe(true);
+    expect(onlyName.content).toContain("role and name are both required");
+    expect(onlyName.content).toContain('"role":"button"');
+
+    const onlyLabel = await execute(harness, "computer_browser", {
+      action: "click",
+      label: "Add",
+    });
+    expect(onlyLabel.isError).toBe(true);
+
+    const labelWithRole = await execute(harness, "computer_browser", {
+      action: "click",
+      label: "Mark done",
+      role: "checkbox",
+    });
+    expect(labelWithRole.isError).toBe(false);
+    expect(calls).toEqual([
+      JSON.stringify({ type: "click", role: "checkbox", name: "Mark done" }),
+    ]);
+
+    const unknown = await execute(harness, "computer_browser", {
+      action: "hover",
+    });
+    expect(unknown.isError).toBe(true);
+    expect(unknown.content).toContain('"action" must be one of');
+    await harness.dispose();
+  });
+
   test("computer_exec during an update returns an actionable tool failure", async () => {
     const provider: ComputerProvider = {
       id: "fixture",

package/src/agent.ts

@@ -256,6 +256,46 @@ function decodeProcessId(input: unknown): string | undefined {
   return isComputerProcessIdV1(value) ? value : undefined;
 }
 
+/**
+ * What each `computer_browser` action needs, in the words the refusal uses.
+ *
+ * Bob on production (2026-09-04) clicked a button with `{name}`, then
+ * `{label}`, then `{label, role}` before landing on `{role, name}` — three
+ * wasted steps per click, because the loop's generic "Invalid input for tool"
+ * names no field. The snapshot lists elements as `button "Add"` and
+ * `checkbox "Mark done"`, so a click takes that role and that accessible name;
+ * `label` is accepted as a synonym for `name` (and `name` for `label` on
+ * `fill`) since the model reaches for both.
+ */
+export const BROWSER_ACTION_SHAPES_V1: Readonly<Record<string, string>> = {
+  snapshot: '{"action":"snapshot"}',
+  navigate: '{"action":"navigate","url":"http://127.0.0.1:8944/"}',
+  click:
+    '{"action":"click","role":"button","name":"Add"} — role and name are both required; take them from the snapshot line (button "Add" → role "button", name "Add"; a checkbox line → role "checkbox")',
+  fill: '{"action":"fill","label":"New todo","text":"Buy milk"} — label is the field\'s accessible label from the snapshot',
+  press: '{"action":"press","key":"Enter"}',
+  wait: '{"action":"wait","milliseconds":500} (0 to 30000)',
+};
+
+/** Why a `computer_browser` input could not be used, and what to send. */
+export function browserInputRefusalV1(input: unknown): string {
+  const value = record(input);
+  const action = typeof value?.action === "string" ? value.action : undefined;
+  const shape = action ? BROWSER_ACTION_SHAPES_V1[action] : undefined;
+  if (!shape) {
+    return `computer_browser input is invalid: "action" must be one of ${Object.keys(
+      BROWSER_ACTION_SHAPES_V1,
+    )
+      .map((name) => `"${name}"`)
+      .join(", ")}. For example ${BROWSER_ACTION_SHAPES_V1.click}.`;
+  }
+  return `computer_browser input is invalid for "${action}". Expected ${shape}.`;
+}
+
+function optionalString(value: unknown): string | undefined {
+  return typeof value === "string" && value.length > 0 ? value : undefined;
+}
+
 function decodeBrowser(input: unknown): ComputerBrowserAction | undefined {
   const value = record(input);
   switch (value?.action) {
@@ -265,24 +305,28 @@ function decodeBrowser(input: unknown): ComputerBrowserAction | undefined {
       return typeof value.url === "string" && value.url
         ? { type: "navigate", url: value.url }
         : undefined;
-    case "click":
-      return typeof value.role === "string" && typeof value.name === "string"
+    case "click": {
+      const name = optionalString(value.name) ?? optionalString(value.label);
+      return typeof value.role === "string" && name !== undefined
         ? {
             type: "click",
             role: value.role,
-            name: value.name,
+            name,
             exact: typeof value.exact === "boolean" ? value.exact : undefined,
           }
         : undefined;
-    case "fill":
-      return typeof value.label === "string" && typeof value.text === "string"
+    }
+    case "fill": {
+      const label = optionalString(value.label) ?? optionalString(value.name);
+      return label !== undefined && typeof value.text === "string"
         ? {
             type: "fill",
-            label: value.label,
+            label,
             text: value.text,
             exact: typeof value.exact === "boolean" ? value.exact : undefined,
           }
         : undefined;
+    }
     case "press":
       return typeof value.key === "string"
         ? { type: "press", key: value.key }
@@ -394,9 +438,11 @@ export async function syncWorkspaceRootNowV1(request: {
   sessionId: string;
   turn: number;
   root: WorkspaceRootV1;
+  requiredPaths?: readonly string[];
   signal?: AbortSignal;
 }): Promise<ComputerSyncSummaryV1> {
-  const { computer, sessions, sessionId, turn, root, signal } = request;
+  const { computer, sessions, sessionId, turn, root, requiredPaths, signal } =
+    request;
   const sync = computer.sync;
   let summary: ComputerSyncSummaryV1;
   if (!sync?.reconcileRoot) {
@@ -409,7 +455,12 @@ export async function syncWorkspaceRootNowV1(request: {
       summary = await sync.reconcileRoot(
         root,
         "publish",
-        signal ? { signal } : undefined,
+        signal || requiredPaths
+          ? {
+              ...(signal ? { signal } : {}),
+              ...(requiredPaths ? { requiredPaths } : {}),
+            }
+          : undefined,
       );
     } catch (error) {
       summary = computerSyncSummaryV1(
@@ -1470,7 +1521,7 @@ export function createComputerAgentPlugin(
       },
       idempotent: config.idempotentEffects === true,
       description:
-        "Control the browser in the Bot's selected Computer and return an accessibility snapshot.",
+        'Control the browser in the Bot\'s selected Computer and return an accessibility snapshot. Shapes: {"action":"snapshot"}; {"action":"navigate","url":...}; {"action":"click","role":"button","name":"Add"} (role AND name, both from the snapshot line, e.g. checkbox "Mark done"); {"action":"fill","label":"New todo","text":...}; {"action":"press","key":"Enter"}; {"action":"wait","milliseconds":500}.',
       inputSchema: {
         type: "object",
         properties: {
@@ -1479,9 +1530,20 @@ export function createComputerAgentPlugin(
             enum: ["snapshot", "navigate", "click", "fill", "press", "wait"],
           },
           url: { type: "string" },
-          role: { type: "string" },
-          name: { type: "string" },
-          label: { type: "string" },
+          role: {
+            type: "string",
+            description:
+              "click: the element's role from the snapshot (button, checkbox, link, textbox…)",
+          },
+          name: {
+            type: "string",
+            description:
+              "click: the element's accessible name from the snapshot",
+          },
+          label: {
+            type: "string",
+            description: "fill: the field's accessible label from the snapshot",
+          },
           text: { type: "string" },
           key: { type: "string" },
           exact: { type: "boolean" },
@@ -1490,11 +1552,15 @@ export function createComputerAgentPlugin(
         required: ["action"],
         additionalProperties: false,
       },
-      validate: (input) => decodeBrowser(input) !== undefined,
+      // Deliberately permissive: a wrong shape reaches `execute`, which says
+      // which field is missing and shows the shape. A bare `false` here becomes
+      // the loop's generic "Invalid input for tool", which cost Bob three
+      // steps per click. Same reasoning as `skill_load`.
+      validate: (input) => !!record(input),
       execute: async (input, context) => {
         const action = decodeBrowser(input);
         if (!action)
-          return { content: "Invalid browser action", isError: true };
+          return { content: browserInputRefusalV1(input), isError: true };
         try {
           return await useComputer(
             await open(context.botId, context.sessionId, context.signal),

package/src/sync.test.ts

@@ -254,6 +254,34 @@ describe("the Computer Package as the sync's caller", () => {
     ]);
     expect(syncEvents(events)[0]?.detail).toContain("paused");
   });
+
+  test("records every incomplete sync operation in one Turn", async () => {
+    const { provider, signal } = fixture(() => ({
+      ...computerSyncSummaryV1(
+        "degraded",
+        "Excluded 1 reproducible Workspace item from sync.",
+      ),
+      ignored: 1,
+    }));
+    const model = modelRunning(["first", "second"], (step) => {
+      if (step === 2) signal.value = "signal-2";
+    });
+
+    const events = await runTurn(provider, model);
+
+    expect(
+      syncEvents(events).map((event) => ({
+        reason: event.reason,
+        status: event.status,
+        ignored: event.ignored,
+        omitted: event.omitted,
+      })),
+    ).toEqual([
+      { reason: "open", status: "degraded", ignored: 1, omitted: 0 },
+      { reason: "signal", status: "degraded", ignored: 1, omitted: 0 },
+      { reason: "turn-end", status: "degraded", ignored: 1, omitted: 0 },
+    ]);
+  });
 });
 
 /**
@@ -298,8 +326,9 @@ describe("syncWorkspaceRootNowV1", () => {
         calls.push("reconcile");
         return Promise.resolve(computerSyncSummaryV1("ok"));
       },
-      reconcileRoot: (asked, reason) => {
+      reconcileRoot: (asked, reason, options) => {
         calls.push(`reconcileRoot:${reason}:${asked.kind}`);
+        calls.push(`required:${options?.requiredPaths?.join(",") ?? ""}`);
         return Promise.resolve({ ...computerSyncSummaryV1("ok"), pushed: 1 });
       },
       signal: () => Promise.resolve(undefined),
@@ -312,12 +341,16 @@ describe("syncWorkspaceRootNowV1", () => {
       sessionId: "session-1",
       turn: 3,
       root: appletsRoot,
+      requiredPaths: ["todo/dist/server.js", "todo/dist/ui.html"],
     });
 
     expect(summary.status).toBe("ok");
     // One root, never the whole Workspace: the Turn's own policy still owns
     // `open`, `signal`, and `turn-end`, and this borrows none of them.
-    expect(calls).toEqual(["reconcileRoot:publish:package-declared"]);
+    expect(calls).toEqual([
+      "reconcileRoot:publish:package-declared",
+      "required:todo/dist/server.js,todo/dist/ui.html",
+    ]);
     const recorded = syncEvents([...harness.session.events]);
     expect(recorded).toHaveLength(1);
     expect(recorded[0]).toMatchObject({