@namzu/sdk 61.3.0 → 62.0.0

package/src/runtime/query/turn/stall-monitor.ts

@@ -0,0 +1,158 @@
+/**
+ * Watches the steps of one turn for a run that keeps thinking without
+ * changing anything, and the per-step reasoning cap that goes with it.
+ *
+ * Both judge what the turn did, not what the model says it will do. A step
+ * changed something when it made a call that was not read-only and did not
+ * fail, and the task list does not count: keeping a list is not progress on
+ * the task.
+ */
+
+import type { ReasoningEffort } from '../../../types/provider/index.js'
+
+/** What the model is told when a step spent its whole reasoning allowance before acting. */
+export const REASONING_CAPPED_MESSAGE =
+	'Your thinking for that step reached its limit before you acted, so it was cut short. Act now on what you already know: make the first change, or run the one command that tells you what you still need. Refine as you go; do not plan the whole task again.'
+
+/** What the model is told when a reply held thinking but no answer and no action. */
+export const REASONING_ONLY_MESSAGE =
+	'Your last reply held thinking only: no answer and no action. Continue now: give your answer, or make the call that moves the task forward.'
+
+/** What the model is told when several steps passed with nothing changed. */
+export const STALL_NUDGE_MESSAGE =
+	'Several steps have passed and nothing has changed yet. Begin changing things now: make the first edit you are sure of, or run the one check that settles what you are unsure of. Reading more or planning again will not move the task.'
+
+/**
+ * Said again, on one request only, when a step must act but the provider
+ * refuses a forced tool choice: the instruction travels in words instead.
+ */
+export const FORCED_STEP_MESSAGE =
+	'This reply must be a tool call, not more planning. Make the call that moves the task forward now.'
+
+/** The default per-step reasoning allowance, in tokens. */
+export const DEFAULT_MAX_REASONING_TOKENS = 24_000
+
+/** Characters a token stands for when counting streamed reasoning. */
+const CHARS_PER_TOKEN = 4
+
+/** Efforts at which the per-step reasoning cap applies unless a host says otherwise. */
+const CAPPED_EFFORTS: ReadonlySet<ReasoningEffort> = new Set(['medium', 'high', 'xhigh', 'max'])
+
+/** How a turn bounds its reasoning. */
+export interface ReasoningPaceConfig {
+	/**
+	 * The most reasoning one step may stream, in tokens. `0` removes the cap.
+	 * When absent the cap is {@link DEFAULT_MAX_REASONING_TOKENS} at medium and
+	 * higher effort and absent at lower effort or none.
+	 */
+	readonly maxTokens?: number
+	/** Tunes the stall monitor, or `false` turns it off. */
+	readonly stall?: false | StallConfig
+}
+
+/** The characters of reasoning a step may stream before it is cut, or `undefined` for no cap. */
+export function reasoningCapChars(
+	config: ReasoningPaceConfig | undefined,
+	effort: ReasoningEffort | undefined,
+): number | undefined {
+	const configured = config?.maxTokens
+	if (configured !== undefined) {
+		return configured > 0 ? configured * CHARS_PER_TOKEN : undefined
+	}
+	if (effort === undefined || !CAPPED_EFFORTS.has(effort)) return undefined
+	return DEFAULT_MAX_REASONING_TOKENS * CHARS_PER_TOKEN
+}
+
+/** Tools whose calls keep the task list; a turn that only does these has changed nothing. */
+const TASK_LIST_TOOLS = new Set([
+	'task_create',
+	'task_update',
+	'task_list',
+	'task_get',
+	'task_delete',
+	'task_output',
+	'task_stop',
+])
+
+/**
+ * Whether a finished call changed something. A failed call changed nothing, a
+ * task-list call is not progress, and a call the tool declares read-only only
+ * looked. A tool that does not say is taken to have changed things.
+ */
+export function callChangedState(
+	toolName: string,
+	readOnly: boolean | undefined,
+	isError: boolean | undefined,
+): boolean {
+	if (isError) return false
+	if (TASK_LIST_TOOLS.has(toolName)) return false
+	return readOnly !== true
+}
+
+/** When the monitor speaks. All counts are of consecutive steps that changed nothing. */
+export interface StallConfig {
+	/** Steps without a change after which the model is nudged. Default 12. */
+	readonly nudgeAfterSteps?: number
+	/** Reasoning characters streamed without a change after which the model is nudged. Default 80,000. */
+	readonly nudgeAfterReasoningChars?: number
+	/** Further steps without a change after the nudge that make the next step an action step. Default 5. */
+	readonly forceAfterSteps?: number
+}
+
+/** What the monitor wants done after a step. */
+export type StallDirective =
+	| { readonly kind: 'none' }
+	| { readonly kind: 'nudge'; readonly text: string }
+	| { readonly kind: 'force' }
+
+/** One finished step as the monitor sees it. */
+export interface StallObservation {
+	readonly reasoningChars: number
+	readonly changedState: boolean
+}
+
+export class StallMonitor {
+	private readonly nudgeAfterSteps: number
+	private readonly nudgeAfterChars: number
+	private readonly forceAfterSteps: number
+	private idleSteps = 0
+	private idleChars = 0
+	private nudged = false
+	private stepsSinceNudge = 0
+
+	constructor(config?: StallConfig) {
+		this.nudgeAfterSteps = config?.nudgeAfterSteps ?? 12
+		this.nudgeAfterChars = config?.nudgeAfterReasoningChars ?? 80_000
+		this.forceAfterSteps = config?.forceAfterSteps ?? 5
+	}
+
+	observe(step: StallObservation): StallDirective {
+		if (step.changedState) {
+			this.reset()
+			return { kind: 'none' }
+		}
+		this.idleSteps += 1
+		this.idleChars += step.reasoningChars
+		if (!this.nudged) {
+			if (this.idleSteps >= this.nudgeAfterSteps || this.idleChars >= this.nudgeAfterChars) {
+				this.nudged = true
+				this.stepsSinceNudge = 0
+				return { kind: 'nudge', text: STALL_NUDGE_MESSAGE }
+			}
+			return { kind: 'none' }
+		}
+		this.stepsSinceNudge += 1
+		if (this.stepsSinceNudge >= this.forceAfterSteps) {
+			this.reset()
+			return { kind: 'force' }
+		}
+		return { kind: 'none' }
+	}
+
+	private reset(): void {
+		this.idleSteps = 0
+		this.idleChars = 0
+		this.nudged = false
+		this.stepsSinceNudge = 0
+	}
+}

package/src/session/paths.ts

@@ -28,7 +28,7 @@ import { uuidv7 } from '../utils/uuidv7.js'
  *   index.sqlite                     rebuildable index over every log
  *   projects/<slug>/
  *     project.json                   the project id, minted once
- *     memory/  residents/  worktrees/<label>/
+ *     memory/  worktrees/<label>/
  *     <session-id>.jsonl             the session log (source of truth)
  *     <session-id>/
  *       subagents/<child-id>.jsonl, <child-id>.meta.json, <child-id>/subagents/…
@@ -96,7 +96,7 @@ export class SessionPathError extends Error {
 }
 
 const SLUG = /^[A-Za-z0-9-]+$/
-/** Worktree labels and resident agent keys: one plain segment, never `.` or `..`. */
+/** Worktree labels: one plain segment, never `.` or `..`. */
 const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
 /** Longest slug before a hash suffix replaces the tail; well under the 255-byte segment limit. */
 const MAX_SLUG = 200
@@ -301,10 +301,6 @@ export class SessionPaths {
 		return join(this.projectDir(), 'memory')
 	}
 
-	residentDir(agentKey: string): string {
-		return join(this.projectDir(), 'residents', checkName(agentKey, 'resident agent key'))
-	}
-
 	/** The `GitWorktreeManager` default root, or one labelled worktree inside it. */
 	worktrees(label?: string): string {
 		const root = join(this.projectDir(), 'worktrees')

package/src/tools/builtins/bash.ts

@@ -2,7 +2,7 @@ import { rm } from 'node:fs/promises'
 import { z } from 'zod'
 import { isReadOnlyCommandLine } from '../../authorization/readonly-commands.js'
 import { SANDBOX_KILL_GRACE_MS } from '../../constants/sandbox/index.js'
-import { DANGEROUS_PATTERNS } from '../../constants/tools/index.js'
+import { DANGEROUS_PATTERNS, dangerousCommandReason } from '../../constants/tools/index.js'
 import { killTree } from '../../process/kill-tree.js'
 import { genericLabel } from '../../registry/tool/presentation.js'
 import { cancelCauseOf } from '../../types/session/cancel-cause.js'
@@ -33,6 +33,7 @@ import {
 } from './bash-cwd.js'
 import { judgeExit } from './exit-semantics.js'
 import { jsonStringEscapes } from './json-string-hint.js'
+import { creditShellReads } from './shell-read-credit.js'
 import { leadingLongSleep, longSleepRefusal } from './sleep-guard.js'
 
 // Namzu owns its own bash timeout knob — `NAMZU_BASH_TIMEOUT_MS` — and
@@ -517,6 +518,8 @@ export function execHostShell(
 	})
 }
 
+export { dangerousCommandReason }
+
 /** The dangerous-pattern check `bash` applies, shared so the `monitor` tool refuses the same lines. */
 export function isDangerousCommand(command: string): boolean {
 	return DANGEROUS_PATTERNS.some((pattern) => pattern.test(command))
@@ -592,7 +595,7 @@ const BashToolBase = defineTool({
 			return {
 				success: false,
 				output: '',
-				error: `Dangerous command blocked: "${input.command}"`,
+				error: `Dangerous command blocked: "${input.command}". Why: ${dangerousCommandReason(input.command)}. Nothing ran. Use a narrower command that avoids it, or ask the user to run this one.`,
 				facts: { kind: 'failure', reason: 'blocked', subject: 'bash' },
 			}
 		}
@@ -898,6 +901,7 @@ const BashToolBase = defineTool({
 				}
 			}
 			const reset = await settle()
+			await creditShellReads(input.command, context, startDirectory)
 			return {
 				success: true,
 				output: withResetNotice(formatShellOutput(stdout, stderr) || '(no output)', reset),

package/src/tools/builtins/computer-use.ts

@@ -91,11 +91,72 @@ const MAX_UI_SNAPSHOT_CHARS = 14_000
 /** The refs of earlier snapshots the tool still resolves for the host. */
 const MAX_HELD_UI_REFS = 4_000
 
+/** One control an advisor may be asked to choose among. */
+export interface ComputerUseAdvisorCandidate {
+	/** The ref the model was shown for it. */
+	readonly ref: string
+	/** The control as the model reads it, such as `Button "Save"`. */
+	readonly description: string
+}
+
+/** What an advisor is asked: which of these controls does this intent name? */
+export interface ComputerUseAdvisorRequest {
+	/** What the model wants done, in its own words. */
+	readonly intent: string
+	readonly candidates: readonly ComputerUseAdvisorCandidate[]
+	/**
+	 * The ref the model itself chose for this intent, when it chose one. Then the
+	 * advisor is only watching: the answer is not acted on.
+	 */
+	readonly chosen?: string
+}
+
+/** An advisor's answer: the candidates in order of fit, best first, each with how sure it is (0 to 1). */
+export interface ComputerUseAdvice {
+	readonly ranked: readonly {
+		readonly ref: string
+		readonly confidence: number
+	}[]
+}
+
+/**
+ * A fast second opinion on which control an intent names, for hosts that run a
+ * small decision model beside the main one. The tool never depends on it: an
+ * advisor that fails, is slow or is unsure is the same as none.
+ */
+export interface ComputerUseAdvisor {
+	/**
+	 * Whether the tool may act on the answer. Then the model can name a control
+	 * by words (`ui_do`) and the tool chooses; when false the advisor is asked
+	 * about the controls the model chose by ref (`ui_act` with `intent`) and its
+	 * answer is not used.
+	 */
+	readonly acts: boolean
+	pick(
+		request: ComputerUseAdvisorRequest,
+		signal?: AbortSignal,
+	): Promise<ComputerUseAdvice | undefined>
+}
+
+/** The least confidence in the best control for the tool to act on it. */
+const ADVICE_MIN_CONFIDENCE = 0.85
+/** How far the best control must lead the next one. */
+const ADVICE_MIN_MARGIN = 0.4
+/** The most controls an advisor is asked about at once. */
+const MAX_ADVISED_CANDIDATES = 255
+
 /**
  * How `createComputerUseTool` sizes screenshots, paces actions and bounds a
  * batch. Every field is optional.
  */
 export interface ComputerUseToolOptions {
+	/**
+	 * A second opinion on which control an intent names; see
+	 * {@link ComputerUseAdvisor}. With one that `acts`, the tool offers `ui_do`
+	 * (name a control by words); with any, `ui_act` accepts an `intent`. Absent,
+	 * neither exists and the tool is exactly what it was.
+	 */
+	readonly advisor?: ComputerUseAdvisor
 	/**
 	 * What every screenshot and zoom image is fitted to before the model sees
 	 * it. Default {@link STANDARD_SCREENSHOT_LIMITS}, which every current vision
@@ -270,6 +331,16 @@ const uiActSchema = z.object({
 	ref: z.string().min(1),
 	action: z.enum(UI_ACTIONS),
 	value: z.string().optional(),
+	/** What this is for, in a few words; only read when the tool has an advisor. */
+	intent: z.string().min(1).max(200).optional(),
+})
+const uiDoSchema = z.object({
+	type: z.literal('ui_do'),
+	/** The control, in words: "the Save button", "the name field". */
+	target: z.string().min(1).max(200),
+	action: z.enum(UI_ACTIONS),
+	value: z.string().optional(),
+	window_id: z.string().min(1).optional(),
 })
 
 /** What a batch may carry: everything but the image-returning actions and another batch. */
@@ -285,6 +356,7 @@ const batchItemSchema = z.discriminatedUnion('type', [
 	listWindowsSchema,
 	focusWindowSchema,
 	uiActSchema,
+	uiDoSchema,
 ])
 
 const withFrame = { screenshot_id: screenshotIdSchema }
@@ -308,6 +380,7 @@ const actionSchema = z.discriminatedUnion('type', [
 	uiSnapshotSchema,
 	waitForSchema,
 	uiActSchema.extend(withFrame),
+	uiDoSchema.extend(withFrame),
 	z.object({
 		type: z.literal('batch'),
 		actions: z.array(batchItemSchema).min(1),
@@ -355,6 +428,7 @@ const ALL_ACTIONS: readonly ToolActionType[] = [
 	'ui_snapshot',
 	'wait_for',
 	'ui_act',
+	'ui_do',
 	'batch',
 ]
 
@@ -376,6 +450,7 @@ const DESTRUCTIVE_ACTIONS = new Set<string>([
 	'key',
 	'scroll',
 	'ui_act',
+	'ui_do',
 ])
 
 /** Observations that show the model what is on the screen, on their own. */
@@ -454,6 +529,7 @@ function requiredCapability(type: string): keyof ComputerUseCapabilities | null
 		case 'ui_snapshot':
 		case 'wait_for':
 		case 'ui_act':
+		case 'ui_do':
 			return 'uiTree'
 		default:
 			return null
@@ -481,7 +557,11 @@ function windowCaptureAvailable(host: ComputerUseHost, caps: ComputerUseCapabili
 	)
 }
 
-function availableActions(host: ComputerUseHost, caps: ComputerUseCapabilities): ToolActionType[] {
+function availableActions(
+	host: ComputerUseHost,
+	caps: ComputerUseCapabilities,
+	advisor?: ComputerUseAdvisor,
+): ToolActionType[] {
 	const displayScreenshot = hostActionAvailable(caps, 'screenshot')
 	const window = windowCaptureAvailable(host, caps)
 	const screenshot = displayScreenshot || window
@@ -516,6 +596,8 @@ function availableActions(host: ComputerUseHost, caps: ComputerUseCapabilities):
 			case 'wait_for':
 			case 'ui_act':
 				return uiTree
+			case 'ui_do':
+				return uiTree && advisor?.acts === true
 			case 'batch':
 				return false
 			default:
@@ -550,8 +632,9 @@ function buildDescription(
 	host: ComputerUseHost,
 	caps: ComputerUseCapabilities,
 	settings: ResolvedSettings,
+	advisor?: ComputerUseAdvisor,
 ): string {
-	const available = availableActions(host, caps)
+	const available = availableActions(host, caps, advisor)
 	const unavailable = unavailableHostActions(host, caps)
 	const intro = hostActionAvailable(caps, 'screenshot')
 		? `Controls the user's desktop on a ${caps.displayServer} host: screenshots, mouse and keyboard, for GUI tasks.`
@@ -597,6 +680,10 @@ function buildDescription(
 		lines.push(
 			'To start a Windows program, a shell command (Start-Process <program>, or cmd.exe /c start <program>) is surest when you have a shell tool. The Start menu searches by display names in the system language, and ENTER on a name it does not find opens a web search in the browser.',
 		)
+	if (available.includes('ui_do'))
+		lines.push(
+			'ui_do {target, action, value, window_id} acts on a control named in words ("the Save button") among those the latest ui_snapshot showed: a second model chooses it and the tool says which control it acted on. When it is not sure it acts on nothing and lists the closest controls: then use ui_act on one of their refs. Use ui_act with a ref whenever you already have one.',
+		)
 	if (available.includes('list_windows'))
 		lines.push(
 			'list_windows names the open windows with their ids; focus_window brings one to the front.',
@@ -695,6 +782,7 @@ const ACTION_REQUIREMENTS: Readonly<Record<ToolActionType, string>> = {
 		'ui_snapshot takes an optional window_id, root (a ref to open) and query (text to find)',
 	wait_for: 'wait_for needs until and match; role, window_id and timeout_ms are optional',
 	ui_act: 'ui_act needs ref and action, and value for set_value',
+	ui_do: 'ui_do needs target (the control, in words) and action, and value for set_value',
 	batch: 'batch needs actions',
 }
 
@@ -712,10 +800,11 @@ function hostModelSchema(
 	host: ComputerUseHost,
 	caps: ComputerUseCapabilities,
 	settings: ResolvedSettings,
+	advisor?: ComputerUseAdvisor,
 ): Record<string, unknown> {
 	// An unavailable host remains a diagnostic tool. Avoid invalid empty enums
 	// on provider wires; every execution is still refused before host access.
-	const offered = availableActions(host, caps)
+	const offered = availableActions(host, caps, advisor)
 	const actions = offered.length > 0 ? offered : ALL_ACTIONS.filter((a) => a !== 'batch')
 	const items = actions.filter((action) => !OUTSIDE_BATCH.has(action))
 	const buttons = new Set<string>()
@@ -841,6 +930,30 @@ function hostModelSchema(
 				type: 'string',
 				description: 'The text set_value puts in the control.',
 			}
+			if (advisor)
+				fields.intent = {
+					type: 'string',
+					description:
+						'For ui_act: what this is for, in a few words ("save the file"). Optional; it only helps check the choice.',
+				}
+		}
+		if (present.includes('ui_do')) {
+			fields.target = {
+				type: 'string',
+				description:
+					'For ui_do: the control in words, such as "the Save button" or "the name field". The tool picks the control from the latest ui_snapshot.',
+			}
+			if (!present.includes('ui_act')) {
+				fields.action = {
+					type: 'string',
+					enum: [...UI_ACTIONS],
+					description: 'What ui_do does to the control.',
+				}
+				fields.value = {
+					type: 'string',
+					description: 'The text set_value puts in the control.',
+				}
+			}
 		}
 		if (!forItems && present.includes('zoom'))
 			fields.region = {
@@ -963,6 +1076,33 @@ function uiActLabel(
 	}
 }
 
+/** What a ui_do does, over the words the model named the control with. */
+function uiDoLabel(input: {
+	readonly target: string
+	readonly action: UiElementAction
+	readonly value?: string
+}): string {
+	const target = quotedText(input.target)
+	switch (input.action) {
+		case 'invoke':
+			return `Press the control for ${target}`
+		case 'set_value':
+			return `Set the control for ${target} to ${quotedText(input.value ?? '')}`
+		case 'toggle':
+			return `Toggle the control for ${target}`
+		case 'select':
+			return `Select the control for ${target}`
+		case 'expand':
+			return `Expand the control for ${target}`
+		case 'collapse':
+			return `Collapse the control for ${target}`
+		case 'focus':
+			return `Focus the control for ${target}`
+		case 'scroll_into_view':
+			return `Scroll to the control for ${target}`
+	}
+}
+
 /**
  * Human activity text for one action; the raw input remains the model-facing
  * record. `describe` names a ui_act ref's control when the tool knows it.
@@ -1008,6 +1148,8 @@ function itemLabel(
 			return `Wait until ${input.until.replace('_', ' ')} ${quotedText(input.match)}`
 		case 'ui_act':
 			return uiActLabel(input, describe)
+		case 'ui_do':
+			return uiDoLabel(input)
 		case 'batch':
 			return batchLabel(input.actions, describe)
 	}
@@ -1349,7 +1491,8 @@ export function createComputerUseTool(
 			}
 		: host.capabilities
 	const frames = new ScreenshotFrames()
-	const available = new Set<string>(availableActions(host, caps))
+	const available = new Set<string>(availableActions(host, caps, options.advisor))
+	const advisor = options.advisor
 
 	// The controls of the latest ui_snapshot, by the ref the model was shown.
 	// A ref names a control, not one reading of it: the same host ref keeps the
@@ -1357,6 +1500,8 @@ export function createComputerUseTool(
 	// control. Whether the control is still there is the host's answer.
 	const uiRefs = new Map<string, UiRef>()
 	const uiRefByHost = new Map<string, string>()
+	// The refs the latest reading of each window showed, so an advisor is asked about what is there now.
+	const shownRefs = new Map<string, Set<string>>()
 	let uiSnapshots = 0
 	let uiRefCount = 0
 	let uiSnapshotWindowId: string | undefined
@@ -1371,6 +1516,30 @@ export function createComputerUseTool(
 	}
 	const label = (input: BatchItem | ActionInput): string => itemLabel(input, describeUiRef)
 
+	/** The controls of a window's latest reading that offer an action (any, when none is named), as an advisor is shown them. */
+	const advisedCandidates = (
+		windowId: string | undefined,
+		action?: UiElementAction,
+	): ComputerUseAdvisorCandidate[] => {
+		const shown = shownRefs.get(windowId ?? '')
+		const out: ComputerUseAdvisorCandidate[] = []
+		if (!shown) return out
+		for (const ref of shown) {
+			const entry = uiRefs.get(ref)
+			if (!entry) continue
+			const offered = entry.element.actions ?? []
+			if (offered.length === 0) continue
+			if (action !== undefined && !offered.includes(action)) continue
+			const text = uiElementText(entry.element)
+			out.push({
+				ref,
+				description: text.length > 80 ? `${text.slice(0, 79)}…` : text,
+			})
+			if (out.length >= MAX_ADVISED_CANDIDATES) break
+		}
+		return out
+	}
+
 	const refusal = (type: string): string | null => {
 		if (available.has(type)) return null
 		const required = requiredCapability(type)
@@ -1705,6 +1874,14 @@ export function createComputerUseTool(
 				if (item.action === 'set_value' && item.value === undefined)
 					throw new StepFailure('set_value needs value, the text to put in the control')
 				return step(async () => {
+					// A second opinion on the model's own choice: asked beside the action, never waited for.
+					if (advisor && item.intent !== undefined) {
+						const candidates = advisedCandidates(entry.windowId)
+						if (candidates.length >= 2)
+							void advisor
+								.pick({ intent: item.intent, candidates, chosen: item.ref })
+								.catch(() => undefined)
+					}
 					let outcome: UiActResult
 					try {
 						outcome = await (host.uiAct as NonNullable<ComputerUseHost['uiAct']>)(
@@ -1728,6 +1905,62 @@ export function createComputerUseTool(
 					return deliveryOutcome(outcome)
 				})
 			}
+			case 'ui_do': {
+				if (!advisor?.acts)
+					throw new StepFailure('ui_do is not available here; use ui_act with a ref')
+				if (item.action === 'set_value' && item.value === undefined)
+					throw new StepFailure('set_value needs value, the text to put in the control')
+				if (uiSnapshots === 0)
+					throw new StepFailure(
+						'no ui_snapshot has been taken yet; take one, because ui_do chooses among the controls it showed',
+					)
+				const windowId = item.window_id ?? uiSnapshotWindowId
+				const candidates = advisedCandidates(windowId, item.action)
+				if (candidates.length === 0)
+					throw new StepFailure(
+						`no control in the latest ui_snapshot${windowId ? ` of window ${windowId}` : ''} offers ${item.action}; take a ui_snapshot and act with ui_act on a ref`,
+					)
+				const target = item.target
+				return step(async (signal) => {
+					let advice: ComputerUseAdvice | undefined
+					try {
+						advice = await advisor.pick({ intent: target, candidates }, signal)
+					} catch {
+						if (signal?.aborted) throw new StepFailure('cancelled')
+					}
+					const [best, next] = advice?.ranked ?? []
+					const sure =
+						best !== undefined &&
+						best.confidence >= ADVICE_MIN_CONFIDENCE &&
+						best.confidence - (next?.confidence ?? 0) >= ADVICE_MIN_MARGIN &&
+						uiRefs.has(best.ref)
+					if (!best || !sure) {
+						const close = (advice?.ranked ?? [])
+							.filter((entry) => uiRefs.has(entry.ref))
+							.slice(0, 3)
+							.map((entry) => describeUiRef(entry.ref))
+						throw deliveryFailure({
+							delivery: 'not_delivered',
+							retrySafety: 'safe',
+							detail: `ui_do was not sure which control is ${quotedText(target)}, so it acted on nothing.${close.length > 0 ? ` Closest: ${close.join('; ')}.` : ''} Use ui_act on one of the refs, or ui_snapshot {query: ...} to find it.`,
+						})
+					}
+					const chosen = describeUiRef(best.ref) ?? best.ref
+					const ran = await plan(
+						{
+							type: 'ui_act',
+							ref: best.ref,
+							action: item.action,
+							...(item.value !== undefined ? { value: item.value } : {}),
+						},
+						undefined,
+					).run(signal)
+					const prefix = `chose ${chosen} for ${quotedText(target)} (${Math.round(best.confidence * 100)}% sure)`
+					return typeof ran === 'string'
+						? `${prefix}: ${ran}`
+						: { ...ran, note: `${prefix}: ${ran.note}` }
+				})
+			}
 		}
 	}
 
@@ -1822,6 +2055,11 @@ export function createComputerUseTool(
 			}
 		}
 		uiSnapshotWindowId = snapshot.windowId ?? requestedWindowId
+		{
+			const key = uiSnapshotWindowId ?? ''
+			const now = new Set(refs.keys())
+			shownRefs.set(key, snapshot.region ? new Set([...(shownRefs.get(key) ?? []), ...now]) : now)
+		}
 		if (uiSnapshotWindowId !== undefined) treeUsable.set(uiSnapshotWindowId, refs.size > 0)
 		uiSnapshotFrame = frames.latest()
 		const window =
@@ -2347,7 +2585,7 @@ export function createComputerUseTool(
 				if (step.item.type === 'focus_window' && windowCaptureAvailable(host, caps))
 					postWindowId = step.item.window_id
 				else if (
-					step.item.type === 'ui_act' &&
+					(step.item.type === 'ui_act' || step.item.type === 'ui_do') &&
 					uiSnapshotWindowId &&
 					!explicitDisplayFrame &&
 					windowCaptureAvailable(host, caps)
@@ -2504,9 +2742,9 @@ export function createComputerUseTool(
 
 	const tool = defineTool({
 		name: COMPUTER_USE_TOOL_NAME,
-		description: buildDescription(host, caps, settings),
+		description: buildDescription(host, caps, settings, options.advisor),
 		inputSchema: actionSchema,
-		modelInputSchema: hostModelSchema(host, caps, settings),
+		modelInputSchema: hostModelSchema(host, caps, settings, options.advisor),
 		validationErrorHint: `Action requirements: ${ALL_ACTIONS.map((action) => actionRequirement(action, host, caps)).join('; ')}. A batch is {"type":"batch","actions":[...]} with at most ${settings.maxBatchActions} actions and no screenshot, zoom, ui_snapshot or batch inside.`,
 		category: 'custom',
 		permissions: [],

package/src/tools/builtins/content-fingerprint.ts

@@ -34,6 +34,31 @@ export function unreadFileError(path: string, operation: 'edit' | 'write' = 'edi
 	return `${path} has not been read yet. Read it first, then redo the ${operation}. Nothing was written.`
 }
 
+/** The longest file `edit` hands back when it refuses one the session never read, in lines. */
+export const UNREAD_EDIT_SHOWN_MAX_LINES = 600
+/** The longest file `edit` hands back when it refuses one the session never read, in characters. */
+export const UNREAD_EDIT_SHOWN_MAX_CHARS = 24_000
+
+/**
+ * What `edit` tells the agent when the session never read the file and the
+ * file is small enough to hand over: the refusal and the numbered contents.
+ * The caller records the read, so the repeated edit is admitted. `undefined`
+ * when the file is too large or is not text.
+ */
+export function unreadEditShownError(path: string, content: string): string | undefined {
+	if (content.length > UNREAD_EDIT_SHOWN_MAX_CHARS || content.includes('\u0000')) return undefined
+	const lines = content.split('\n')
+	if (lines.length > 1 && lines[lines.length - 1] === '') lines.pop()
+	if (lines.length > UNREAD_EDIT_SHOWN_MAX_LINES) return undefined
+	const numbered = lines.map((line, i) => `${i + 1}\t${line}`).join('\n')
+	return `${path} had not been read in this session, so this edit was not applied. It has now been read for you; its current contents follow. Repeat the edit against them. Nothing was written.\n\n${numbered}`
+}
+
+/** What `edit` tells the agent when the file was never read and is too large to hand over. */
+export function unreadEditTooLargeError(path: string): string {
+	return `${path} has not been read in this session, so the edit was not applied. Read it with the read tool (one call, no range), then repeat the edit.`
+}
+
 /**
  * Whether the file at `key` has moved on disk since the session read it.
  *