@namzu/sdk 61.2.1 → 61.4.0

package/dist/tools/builtins/computer-use.js

@@ -11,11 +11,37 @@ const DEFAULT_SETTLE_MS = 500;
 const DEFAULT_MAX_BATCH_ACTIONS = 20;
 const DEFAULT_MAX_WAIT_MS = 10_000;
 const DEFAULT_WAIT_FOR_MS = 10_000;
+/**
+ * The wait between the last action and the screenshot that follows it. A host
+ * that can tell when the window stopped changing (see
+ * {@link ComputerUseHost.settle}) is asked first and ends the wait early; a
+ * host without it, one that cannot say, or one that fails keeps the fixed
+ * `settleMs` it always had. Cancellation is never swallowed.
+ */
+async function settleAfterActions(host, windowId, settleMs, signal) {
+    if (settleMs > 0 && typeof host.settle === 'function') {
+        try {
+            const settled = await host.settle({ budgetMs: settleMs, ...(windowId !== undefined ? { windowId } : {}) }, signal);
+            if (settled !== undefined)
+                return;
+        }
+        catch {
+            // Fall through to the fixed wait; a cancelled signal makes it reject at once.
+        }
+    }
+    await sleep(settleMs, signal);
+}
 const MAX_LISTED_WINDOWS = 50;
 /** The most of a ui_snapshot the model is shown, in characters (about 4 000 tokens). */
 const MAX_UI_SNAPSHOT_CHARS = 14_000;
 /** The refs of earlier snapshots the tool still resolves for the host. */
 const MAX_HELD_UI_REFS = 4_000;
+/** The least confidence in the best control for the tool to act on it. */
+const ADVICE_MIN_CONFIDENCE = 0.85;
+/** How far the best control must lead the next one. */
+const ADVICE_MIN_MARGIN = 0.4;
+/** The most controls an advisor is asked about at once. */
+const MAX_ADVISED_CANDIDATES = 255;
 /**
  * Why `provider` cannot drive `computer_use`, or undefined when it can.
  *
@@ -149,6 +175,16 @@ const uiActSchema = z.object({
     ref: z.string().min(1),
     action: z.enum(UI_ACTIONS),
     value: z.string().optional(),
+    /** What this is for, in a few words; only read when the tool has an advisor. */
+    intent: z.string().min(1).max(200).optional(),
+});
+const uiDoSchema = z.object({
+    type: z.literal('ui_do'),
+    /** The control, in words: "the Save button", "the name field". */
+    target: z.string().min(1).max(200),
+    action: z.enum(UI_ACTIONS),
+    value: z.string().optional(),
+    window_id: z.string().min(1).optional(),
 });
 /** What a batch may carry: everything but the image-returning actions and another batch. */
 const batchItemSchema = z.discriminatedUnion('type', [
@@ -163,6 +199,7 @@ const batchItemSchema = z.discriminatedUnion('type', [
     listWindowsSchema,
     focusWindowSchema,
     uiActSchema,
+    uiDoSchema,
 ]);
 const withFrame = { screenshot_id: screenshotIdSchema };
 const actionSchema = z.discriminatedUnion('type', [
@@ -184,6 +221,7 @@ const actionSchema = z.discriminatedUnion('type', [
     uiSnapshotSchema,
     waitForSchema,
     uiActSchema.extend(withFrame),
+    uiDoSchema.extend(withFrame),
     z.object({
         type: z.literal('batch'),
         actions: z.array(batchItemSchema).min(1),
@@ -216,6 +254,7 @@ const ALL_ACTIONS = [
     'ui_snapshot',
     'wait_for',
     'ui_act',
+    'ui_do',
     'batch',
 ];
 /** Observations: they change nothing on the desktop. */
@@ -235,6 +274,7 @@ const DESTRUCTIVE_ACTIONS = new Set([
     'key',
     'scroll',
     'ui_act',
+    'ui_do',
 ]);
 /** Observations that show the model what is on the screen, on their own. */
 const SCREEN_OBSERVATIONS = new Set([
@@ -300,6 +340,7 @@ function requiredCapability(type) {
         case 'ui_snapshot':
         case 'wait_for':
         case 'ui_act':
+        case 'ui_do':
             return 'uiTree';
         default:
             return null;
@@ -317,7 +358,7 @@ function windowCaptureAvailable(host, caps) {
         typeof host.captureWindow === 'function' &&
         typeof host.executeWindow === 'function');
 }
-function availableActions(host, caps) {
+function availableActions(host, caps, advisor) {
     const displayScreenshot = hostActionAvailable(caps, 'screenshot');
     const window = windowCaptureAvailable(host, caps);
     const screenshot = displayScreenshot || window;
@@ -350,6 +391,8 @@ function availableActions(host, caps) {
             case 'wait_for':
             case 'ui_act':
                 return uiTree;
+            case 'ui_do':
+                return uiTree && advisor?.acts === true;
             case 'batch':
                 return false;
             default:
@@ -383,8 +426,8 @@ function unavailableHostActions(host, caps) {
 // ---------------------------------------------------------------------------
 // Model-facing description and schema
 // ---------------------------------------------------------------------------
-function buildDescription(host, caps, settings) {
-    const available = availableActions(host, caps);
+function buildDescription(host, caps, settings, advisor) {
+    const available = availableActions(host, caps, advisor);
     const unavailable = unavailableHostActions(host, caps);
     const intro = hostActionAvailable(caps, 'screenshot')
         ? `Controls the user's desktop on a ${caps.displayServer} host: screenshots, mouse and keyboard, for GUI tasks.`
@@ -413,6 +456,8 @@ function buildDescription(host, caps, settings) {
         available.includes('key') &&
         available.includes('type_text'))
         lines.push('To start a Windows program, a shell command (Start-Process <program>, or cmd.exe /c start <program>) is surest when you have a shell tool. The Start menu searches by display names in the system language, and ENTER on a name it does not find opens a web search in the browser.');
+    if (available.includes('ui_do'))
+        lines.push('ui_do {target, action, value, window_id} acts on a control named in words ("the Save button") among those the latest ui_snapshot showed: a second model chooses it and the tool says which control it acted on. When it is not sure it acts on nothing and lists the closest controls: then use ui_act on one of their refs. Use ui_act with a ref whenever you already have one.');
     if (available.includes('list_windows'))
         lines.push('list_windows names the open windows with their ids; focus_window brings one to the front.');
     if (windowCaptureAvailable(host, caps))
@@ -487,6 +532,7 @@ const ACTION_REQUIREMENTS = {
     ui_snapshot: 'ui_snapshot takes an optional window_id, root (a ref to open) and query (text to find)',
     wait_for: 'wait_for needs until and match; role, window_id and timeout_ms are optional',
     ui_act: 'ui_act needs ref and action, and value for set_value',
+    ui_do: 'ui_do needs target (the control, in words) and action, and value for set_value',
     batch: 'batch needs actions',
 };
 function actionRequirement(action, host, caps) {
@@ -494,10 +540,10 @@ function actionRequirement(action, host, caps) {
         ? `screenshot ${hostActionAvailable(caps, 'screenshot') ? 'takes an optional' : 'requires'} window_id to capture pixels for that window`
         : ACTION_REQUIREMENTS[action];
 }
-function hostModelSchema(host, caps, settings) {
+function hostModelSchema(host, caps, settings, advisor) {
     // An unavailable host remains a diagnostic tool. Avoid invalid empty enums
     // on provider wires; every execution is still refused before host access.
-    const offered = availableActions(host, caps);
+    const offered = availableActions(host, caps, advisor);
     const actions = offered.length > 0 ? offered : ALL_ACTIONS.filter((a) => a !== 'batch');
     const items = actions.filter((action) => !OUTSIDE_BATCH.has(action));
     const buttons = new Set();
@@ -613,6 +659,28 @@ function hostModelSchema(host, caps, settings) {
                 type: 'string',
                 description: 'The text set_value puts in the control.',
             };
+            if (advisor)
+                fields.intent = {
+                    type: 'string',
+                    description: 'For ui_act: what this is for, in a few words ("save the file"). Optional; it only helps check the choice.',
+                };
+        }
+        if (present.includes('ui_do')) {
+            fields.target = {
+                type: 'string',
+                description: 'For ui_do: the control in words, such as "the Save button" or "the name field". The tool picks the control from the latest ui_snapshot.',
+            };
+            if (!present.includes('ui_act')) {
+                fields.action = {
+                    type: 'string',
+                    enum: [...UI_ACTIONS],
+                    description: 'What ui_do does to the control.',
+                };
+                fields.value = {
+                    type: 'string',
+                    description: 'The text set_value puts in the control.',
+                };
+            }
         }
         if (!forItems && present.includes('zoom'))
             fields.region = {
@@ -719,6 +787,28 @@ function uiActLabel(input, describe) {
             return `Scroll to ${target}`;
     }
 }
+/** What a ui_do does, over the words the model named the control with. */
+function uiDoLabel(input) {
+    const target = quotedText(input.target);
+    switch (input.action) {
+        case 'invoke':
+            return `Press the control for ${target}`;
+        case 'set_value':
+            return `Set the control for ${target} to ${quotedText(input.value ?? '')}`;
+        case 'toggle':
+            return `Toggle the control for ${target}`;
+        case 'select':
+            return `Select the control for ${target}`;
+        case 'expand':
+            return `Expand the control for ${target}`;
+        case 'collapse':
+            return `Collapse the control for ${target}`;
+        case 'focus':
+            return `Focus the control for ${target}`;
+        case 'scroll_into_view':
+            return `Scroll to the control for ${target}`;
+    }
+}
 /**
  * Human activity text for one action; the raw input remains the model-facing
  * record. `describe` names a ui_act ref's control when the tool knows it.
@@ -759,6 +849,8 @@ function itemLabel(input, describe) {
             return `Wait until ${input.until.replace('_', ' ')} ${quotedText(input.match)}`;
         case 'ui_act':
             return uiActLabel(input, describe);
+        case 'ui_do':
+            return uiDoLabel(input);
         case 'batch':
             return batchLabel(input.actions, describe);
     }
@@ -973,13 +1065,16 @@ export function createComputerUseTool(host, options = {}) {
         }
         : host.capabilities;
     const frames = new ScreenshotFrames();
-    const available = new Set(availableActions(host, caps));
+    const available = new Set(availableActions(host, caps, options.advisor));
+    const advisor = options.advisor;
     // The controls of the latest ui_snapshot, by the ref the model was shown.
     // A ref names a control, not one reading of it: the same host ref keeps the
     // same number in every snapshot, and a number is never reused for another
     // control. Whether the control is still there is the host's answer.
     const uiRefs = new Map();
     const uiRefByHost = new Map();
+    // The refs the latest reading of each window showed, so an advisor is asked about what is there now.
+    const shownRefs = new Map();
     let uiSnapshots = 0;
     let uiRefCount = 0;
     let uiSnapshotWindowId;
@@ -993,6 +1088,31 @@ export function createComputerUseTool(host, options = {}) {
         return entry ? `${uiElementText(entry.element)} (${ref})` : undefined;
     };
     const label = (input) => itemLabel(input, describeUiRef);
+    /** The controls of a window's latest reading that offer an action (any, when none is named), as an advisor is shown them. */
+    const advisedCandidates = (windowId, action) => {
+        const shown = shownRefs.get(windowId ?? '');
+        const out = [];
+        if (!shown)
+            return out;
+        for (const ref of shown) {
+            const entry = uiRefs.get(ref);
+            if (!entry)
+                continue;
+            const offered = entry.element.actions ?? [];
+            if (offered.length === 0)
+                continue;
+            if (action !== undefined && !offered.includes(action))
+                continue;
+            const text = uiElementText(entry.element);
+            out.push({
+                ref,
+                description: text.length > 80 ? `${text.slice(0, 79)}…` : text,
+            });
+            if (out.length >= MAX_ADVISED_CANDIDATES)
+                break;
+        }
+        return out;
+    };
     const refusal = (type) => {
         if (available.has(type))
             return null;
@@ -1242,6 +1362,14 @@ export function createComputerUseTool(host, options = {}) {
                 if (item.action === 'set_value' && item.value === undefined)
                     throw new StepFailure('set_value needs value, the text to put in the control');
                 return step(async () => {
+                    // A second opinion on the model's own choice: asked beside the action, never waited for.
+                    if (advisor && item.intent !== undefined) {
+                        const candidates = advisedCandidates(entry.windowId);
+                        if (candidates.length >= 2)
+                            void advisor
+                                .pick({ intent: item.intent, candidates, chosen: item.ref })
+                                .catch(() => undefined);
+                    }
                     let outcome;
                     try {
                         outcome = await host.uiAct(entry.hostRef, item.action, item.value, callSignal);
@@ -1260,6 +1388,56 @@ export function createComputerUseTool(host, options = {}) {
                     return deliveryOutcome(outcome);
                 });
             }
+            case 'ui_do': {
+                if (!advisor?.acts)
+                    throw new StepFailure('ui_do is not available here; use ui_act with a ref');
+                if (item.action === 'set_value' && item.value === undefined)
+                    throw new StepFailure('set_value needs value, the text to put in the control');
+                if (uiSnapshots === 0)
+                    throw new StepFailure('no ui_snapshot has been taken yet; take one, because ui_do chooses among the controls it showed');
+                const windowId = item.window_id ?? uiSnapshotWindowId;
+                const candidates = advisedCandidates(windowId, item.action);
+                if (candidates.length === 0)
+                    throw new StepFailure(`no control in the latest ui_snapshot${windowId ? ` of window ${windowId}` : ''} offers ${item.action}; take a ui_snapshot and act with ui_act on a ref`);
+                const target = item.target;
+                return step(async (signal) => {
+                    let advice;
+                    try {
+                        advice = await advisor.pick({ intent: target, candidates }, signal);
+                    }
+                    catch {
+                        if (signal?.aborted)
+                            throw new StepFailure('cancelled');
+                    }
+                    const [best, next] = advice?.ranked ?? [];
+                    const sure = best !== undefined &&
+                        best.confidence >= ADVICE_MIN_CONFIDENCE &&
+                        best.confidence - (next?.confidence ?? 0) >= ADVICE_MIN_MARGIN &&
+                        uiRefs.has(best.ref);
+                    if (!best || !sure) {
+                        const close = (advice?.ranked ?? [])
+                            .filter((entry) => uiRefs.has(entry.ref))
+                            .slice(0, 3)
+                            .map((entry) => describeUiRef(entry.ref));
+                        throw deliveryFailure({
+                            delivery: 'not_delivered',
+                            retrySafety: 'safe',
+                            detail: `ui_do was not sure which control is ${quotedText(target)}, so it acted on nothing.${close.length > 0 ? ` Closest: ${close.join('; ')}.` : ''} Use ui_act on one of the refs, or ui_snapshot {query: ...} to find it.`,
+                        });
+                    }
+                    const chosen = describeUiRef(best.ref) ?? best.ref;
+                    const ran = await plan({
+                        type: 'ui_act',
+                        ref: best.ref,
+                        action: item.action,
+                        ...(item.value !== undefined ? { value: item.value } : {}),
+                    }, undefined).run(signal);
+                    const prefix = `chose ${chosen} for ${quotedText(target)} (${Math.round(best.confidence * 100)}% sure)`;
+                    return typeof ran === 'string'
+                        ? `${prefix}: ${ran}`
+                        : { ...ran, note: `${prefix}: ${ran.note}` };
+                });
+            }
         }
     };
     /** Read one window's controls, number the ones the model can act on, and show them. */
@@ -1344,6 +1522,11 @@ export function createComputerUseTool(host, options = {}) {
             }
         }
         uiSnapshotWindowId = snapshot.windowId ?? requestedWindowId;
+        {
+            const key = uiSnapshotWindowId ?? '';
+            const now = new Set(refs.keys());
+            shownRefs.set(key, snapshot.region ? new Set([...(shownRefs.get(key) ?? []), ...now]) : now);
+        }
         if (uiSnapshotWindowId !== undefined)
             treeUsable.set(uiSnapshotWindowId, refs.size > 0);
         uiSnapshotFrame = frames.latest();
@@ -1815,7 +1998,7 @@ export function createComputerUseTool(host, options = {}) {
                 const outcome = typeof ran === 'string' ? { note: ran } : ran;
                 if (step.item.type === 'focus_window' && windowCaptureAvailable(host, caps))
                     postWindowId = step.item.window_id;
-                else if (step.item.type === 'ui_act' &&
+                else if ((step.item.type === 'ui_act' || step.item.type === 'ui_do') &&
                     uiSnapshotWindowId &&
                     !explicitDisplayFrame &&
                     windowCaptureAvailable(host, caps))
@@ -1869,7 +2052,7 @@ export function createComputerUseTool(host, options = {}) {
         let shotError;
         if (looks) {
             try {
-                await sleep(settings.settleMs, signal);
+                await settleAfterActions(host, postWindowId, settings.settleMs, signal);
                 shot = await capture(postWindowId);
             }
             catch (error) {
@@ -1964,9 +2147,9 @@ export function createComputerUseTool(host, options = {}) {
     const capturesScreen = (input) => observes && capturesScreenInput(input, settings.screenshotAfterActions);
     const tool = defineTool({
         name: COMPUTER_USE_TOOL_NAME,
-        description: buildDescription(host, caps, settings),
+        description: buildDescription(host, caps, settings, options.advisor),
         inputSchema: actionSchema,
-        modelInputSchema: hostModelSchema(host, caps, settings),
+        modelInputSchema: hostModelSchema(host, caps, settings, options.advisor),
         validationErrorHint: `Action requirements: ${ALL_ACTIONS.map((action) => actionRequirement(action, host, caps)).join('; ')}. A batch is {"type":"batch","actions":[...]} with at most ${settings.maxBatchActions} actions and no screenshot, zoom, ui_snapshot or batch inside.`,
         category: 'custom',
         permissions: [],