@anionex/dsh-computer-use 0.1.0 → 0.2.0

package/src/providers/macos.ts

@@ -14,6 +14,7 @@ import type {
   BackendObservation,
   BackendObserveOptions,
   ComputerUseBackend,
+  CursorVisibility,
 } from '../backend.ts'
 import {
   Config,
@@ -22,10 +23,10 @@ import {
   type ComputerUseConfig,
   type ResolvedComputerUseConfig,
 } from '../config.ts'
-import { ComputerUseError } from '../errors.ts'
 import { ComputerUseService } from '../service.ts'
 import type { ComputerAppIdentity, ComputerAppSelector, ComputerAppSummary } from '../types.ts'
 import { NativeHelperClient } from './native-helper.ts'
+import { UnsupportedPlatformBackend } from './unsupported.ts'
 
 interface NativeHealth {
   helperVersion: string
@@ -35,8 +36,14 @@ interface NativeHealth {
 
 interface NativeObservation extends BackendObservation {}
 
+function createBackend(ctx: Context, config: ResolvedComputerUseConfig): ComputerUseBackend {
+  return process.platform === 'darwin'
+    ? new MacOSBackend(ctx, config)
+    : new UnsupportedPlatformBackend(process.platform)
+}
+
 /** Fixed-command native backend. */
-class MacOSBackend implements ComputerUseBackend {
+export class MacOSBackend implements ComputerUseBackend {
   readonly name = 'macos-ax' as const
   readonly client: NativeHelperClient
 
@@ -79,11 +86,17 @@ class MacOSBackend implements ComputerUseBackend {
     }, signal)
   }
 
-  async visualizeCursor(action: BackendCursorAction, phase: 'before' | 'after', signal: AbortSignal): Promise<void> {
-    if (this.config.interaction.cursorVisualization !== 'visible') return
+  async visualizeCursor(action: BackendCursorAction, phase: 'before' | 'after', signal: AbortSignal): Promise<CursorVisibility> {
+    if (this.config.interaction.cursorVisualization !== 'visible') return { visible: false, reason: 'the agent cursor is disabled by configuration' }
+    // The overlay answers per command; the least visible outcome wins, because
+    // a cursor that vanished partway through is a cursor the user cannot follow.
+    let outcome: CursorVisibility = { visible: true }
+    const record = (response: CursorVisibility): void => {
+      if (!response.visible && outcome.visible) outcome = response
+    }
     const autoHideMs = this.config.interaction.cursorAutoHideMs
     const move = async (point: { x: number; y: number }, durationMs: number): Promise<void> => {
-      await this.client.cursorCommand({
+      record(await this.client.cursorCommand({
         op: 'move',
         x: point.x,
         y: point.y,
@@ -92,36 +105,39 @@ class MacOSBackend implements ComputerUseBackend {
         targetPid: action.targetPid,
         targetWindowNumber: action.targetWindowNumber,
         targetWindowFrame: action.targetWindowFrame,
-      }, signal)
+      }, signal))
     }
     if (phase === 'after') {
-      if (action.kind === 'drag') await this.client.cursorCommand({
-        op: 'release',
+      // Every action validates the bound target after native input. Only drag
+      // needs release semantics; click and scroll use a side-effect-free check.
+      record(await this.client.cursorCommand({
+        op: action.kind === 'drag' ? 'release' : 'validate',
         autoHideMs,
         targetPid: action.targetPid,
         targetWindowNumber: action.targetWindowNumber,
         targetWindowFrame: action.targetWindowFrame,
-      }, signal)
-      return
+      }, signal))
+      return outcome
     }
     const start = action.kind === 'drag' ? action.from : action.to
-    if (start === undefined) return
+    if (start === undefined) return { visible: false, reason: 'this action has no cursor position to show' }
     await move(start, this.config.interaction.cursorMotionMs)
     if (this.config.interaction.cursorMotionMs > 0) {
       await delay(this.config.interaction.cursorMotionMs, undefined, { signal })
     }
-    if (action.kind === 'scroll') return
-    await this.client.cursorCommand({
+    if (action.kind === 'scroll') return outcome
+    record(await this.client.cursorCommand({
       op: 'press',
       autoHideMs,
       targetPid: action.targetPid,
       targetWindowNumber: action.targetWindowNumber,
       targetWindowFrame: action.targetWindowFrame,
       sustainedPress: action.kind === 'drag',
-    }, signal)
+    }, signal))
     if (action.kind === 'drag') {
       await move(action.to, Math.max(this.config.interaction.cursorMotionMs, 240))
     }
+    return outcome
   }
 
   async dispose(): Promise<void> {
@@ -152,20 +168,20 @@ export class MacOSComputerUseProvider extends ComputerUseService {
   private readonly settings
 
   constructor(ctx: Context, config: ComputerUseConfig = {}) {
-    if (process.platform !== 'darwin') {
-      throw new ComputerUseError('COMPUTER_UNSUPPORTED_PLATFORM', `dsh-computer-use 0.1.0 supports macOS only; current platform is ${process.platform}`)
-    }
     const settings = ctx.settings.register(COMPUTER_USE_SETTINGS_NAMESPACE, Config, {
       base: config,
       applies: 'live',
       validate: (value) => { resolveConfig(value) },
     })
     const resolved = resolveConfig(settings.get())
-    super(ctx, new MacOSBackend(ctx, resolved), resolved)
+    super(ctx, createBackend(ctx, resolved), resolved)
     this.settings = settings
+    if (process.platform !== 'darwin') {
+      ctx.logger.warn('dsh-computer-use: supports macOS only; Computer Use Tools are disabled on %s', process.platform)
+    }
     ctx.effect(() => this.settings.watch(async (next) => {
       const candidate = resolveConfig(next)
-      const backend = new MacOSBackend(ctx, candidate)
+      const backend = createBackend(ctx, candidate)
       try {
         await this.reconfigure(backend, candidate)
       } catch (error) {

package/src/providers/native-helper.ts

@@ -8,6 +8,7 @@ import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import type { Context } from '@deepseek-ai/cordis'
 import type { SubprocessHandle, SubprocessOutcome, SubprocessOutputReader } from '@deepseek-ai/dsh-subprocess'
+import type { CursorVisibility } from '../backend.ts'
 import type { ResolvedComputerUseConfig } from '../config.ts'
 import { ComputerUseError, computerUseError, type ComputerUseErrorCode } from '../errors.ts'
 
@@ -38,14 +39,26 @@ interface HelperSuccess<T> {
 
 type HelperEnvelope<T> = HelperFailure | HelperSuccess<T>
 
+/** The overlay normally answers in under a millisecond; silence is a visibility failure. */
+// Keep this off the native action's critical path. A timed-out generation is
+// discarded before the next serialized command, so a late frame cannot be
+// mistaken for the next command's response.
+const CURSOR_RESPONSE_TIMEOUT_MS = 120
+
 const CURSOR_READY_TIMEOUT_MS = 2_000
 const CURSOR_PROTOCOL_MAX_BYTES = 64 * 1024
 
+type CursorProtocolFrame =
+  | { kind: 'response'; response: Record<string, unknown> }
+  | { kind: 'failure'; reason: string }
+
 interface CursorProcess {
   stdin: Writable
   done: Promise<SubprocessOutcome>
   terminate: () => void
   waitForExit: SubprocessHandle['waitForExit']
+  /** Next response line from this process generation, in command order. */
+  nextResponse: () => Promise<CursorProtocolFrame>
 }
 
 function collected(reader: SubprocessOutputReader | undefined): string {
@@ -63,6 +76,59 @@ function nativeRoot(): string {
   return fileURLToPath(new URL('../../native/macos/', import.meta.url))
 }
 
+function cursorErrorMessage(error: unknown): string | undefined {
+  if (typeof error === 'string' && error.length > 0) return error.slice(0, 1000)
+  if (typeof error !== 'object' || error === null) return undefined
+  const message = (error as { message?: unknown }).message
+  return typeof message === 'string' && message.length > 0 ? message.slice(0, 1000) : undefined
+}
+
+function normalizeCursorResponse(
+  command: Record<string, unknown>,
+  response: Record<string, unknown>,
+): { result: CursorVisibility; discardGeneration: boolean } {
+  if (response.ok === false) {
+    const detail = cursorErrorMessage(response.error)
+    return {
+      result: {
+        visible: false,
+        reason: detail === undefined
+          ? 'the native cursor overlay rejected its command'
+          : `the native cursor overlay rejected its command: ${detail}`,
+      },
+      discardGeneration: false,
+    }
+  }
+  if (response.ok !== true) {
+    return {
+      result: { visible: false, reason: 'the native cursor overlay returned a malformed response' },
+      discardGeneration: true,
+    }
+  }
+  if (typeof command.op !== 'string' || response.op !== command.op) {
+    return {
+      result: { visible: false, reason: 'the native cursor overlay response did not match its command' },
+      discardGeneration: true,
+    }
+  }
+  if (response.visible === true) return { result: { visible: true }, discardGeneration: false }
+  if (response.visible === false) {
+    return {
+      result: {
+        visible: false,
+        ...(typeof response.reason === 'string' && response.reason.length > 0
+          ? { reason: response.reason.slice(0, 1000) }
+          : { reason: 'the native cursor overlay reported that the cursor is not visible' }),
+      },
+      discardGeneration: false,
+    }
+  }
+  return {
+    result: { visible: false, reason: 'the native cursor overlay did not report boolean visibility' },
+    discardGeneration: true,
+  }
+}
+
 /** Exact helper paths and integrity data for one active generation. */
 export interface PreparedNativeHelper {
   path: string
@@ -75,6 +141,7 @@ export class NativeHelperClient {
   private prepared?: PreparedNativeHelper
   private cursor: CursorProcess | undefined
   private cursorStart: { promise: Promise<CursorProcess> } | undefined
+  private cursorCommandTail: Promise<void> = Promise.resolve()
 
   constructor(
     private readonly ctx: Context,
@@ -186,8 +253,14 @@ export class NativeHelperClient {
     return envelope.value
   }
 
-  /** Send one best-effort command to the persistent, click-through Agent cursor overlay. */
-  async cursorCommand(command: Record<string, unknown>, signal: AbortSignal): Promise<void> {
+  /** Send one serialized command to the persistent, click-through Agent cursor overlay. */
+  cursorCommand(command: Record<string, unknown>, signal: AbortSignal): Promise<CursorVisibility> {
+    const run = this.cursorCommandTail.then(async () => await this.executeCursorCommand(command, signal))
+    this.cursorCommandTail = run.then(() => undefined, () => undefined)
+    return run
+  }
+
+  private async executeCursorCommand(command: Record<string, unknown>, signal: AbortSignal): Promise<CursorVisibility> {
     const prepared = this.prepared ?? await this.prepare(signal)
     const cursor = await this.getCursor(prepared, signal)
     signal.throwIfAborted()
@@ -198,13 +271,25 @@ export class NativeHelperClient {
           else rejectWrite(error)
         })
       })
+      const frame = await cursor.nextResponse()
+      if (frame.kind === 'failure') {
+        this.discardCursor(cursor)
+        return { visible: false, reason: frame.reason }
+      }
+      const normalized = normalizeCursorResponse(command, frame.response)
+      if (normalized.discardGeneration) this.discardCursor(cursor)
+      return normalized.result
     } catch (error) {
-      if (this.cursor === cursor) this.cursor = undefined
-      cursor.terminate()
+      this.discardCursor(cursor)
       throw computerUseError(error, 'native cursor overlay command failed')
     }
   }
 
+  private discardCursor(cursor: CursorProcess): void {
+    if (this.cursor === cursor) this.cursor = undefined
+    cursor.terminate()
+  }
+
   /** Stop the cursor process before a provider generation is replaced or disposed. */
   async dispose(): Promise<void> {
     const cursor = this.cursor ?? await this.cursorStart?.promise.catch(() => undefined)
@@ -308,10 +393,74 @@ export class NativeHelperClient {
         { cause: error },
       )
     }
-    handle.stdout.resume()
+    // Commands are serialized by the client and the overlay answers one line
+    // per command. A protocol failure invalidates this entire process
+    // generation; its buffered or late frames can never reach a replacement.
+    const pending: Array<(frame: CursorProtocolFrame) => void> = []
+    const buffered: CursorProtocolFrame[] = []
+    const dispatch = (frame: CursorProtocolFrame): void => {
+      const waiter = pending.shift()
+      if (waiter === undefined) buffered.push(frame)
+      else waiter(frame)
+    }
+    let residue = ''
+    handle.stdout.setEncoding('utf8').on('data', (chunk: string) => {
+      residue += chunk
+      if (Buffer.byteLength(residue) > CURSOR_PROTOCOL_MAX_BYTES && !residue.includes('\n')) {
+        residue = ''
+        dispatch({ kind: 'failure', reason: 'the native cursor overlay response exceeded its protocol limit' })
+        return
+      }
+      while (true) {
+        const newline = residue.indexOf('\n')
+        if (newline < 0) break
+        const line = residue.slice(0, newline).trim()
+        residue = residue.slice(newline + 1)
+        if (line.length === 0) continue
+        if (Buffer.byteLength(line) > CURSOR_PROTOCOL_MAX_BYTES) {
+          dispatch({ kind: 'failure', reason: 'the native cursor overlay response exceeded its protocol limit' })
+          continue
+        }
+        let parsed: unknown
+        try {
+          parsed = JSON.parse(line)
+        } catch {
+          dispatch({ kind: 'failure', reason: 'the native cursor overlay returned invalid JSON' })
+          continue
+        }
+        if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+          dispatch({ kind: 'failure', reason: 'the native cursor overlay returned a malformed response' })
+          continue
+        }
+        dispatch({ kind: 'response', response: parsed as Record<string, unknown> })
+      }
+    })
+    handle.stdout.once('end', () => {
+      dispatch({ kind: 'failure', reason: 'the native cursor overlay closed before replying' })
+    })
+    handle.stdout.once('error', error => {
+      dispatch({ kind: 'failure', reason: `the native cursor overlay response stream failed: ${error.message}` })
+    })
     return {
       stdin: handle.stdin,
       done: handle.done,
+      nextResponse: async () => {
+        const ready = buffered.shift()
+        if (ready !== undefined) return ready
+        return await new Promise<CursorProtocolFrame>((resolveResponse) => {
+          const settle = (frame: CursorProtocolFrame): void => {
+            const index = pending.indexOf(settle)
+            if (index >= 0) pending.splice(index, 1)
+            clearTimeout(timer)
+            resolveResponse(frame)
+          }
+          const timer = setTimeout(() => settle({
+            kind: 'failure',
+            reason: `the native cursor overlay did not respond within ${CURSOR_RESPONSE_TIMEOUT_MS} milliseconds`,
+          }), CURSOR_RESPONSE_TIMEOUT_MS)
+          pending.push(settle)
+        })
+      },
       terminate: () => {
         cursorSignal.abort()
         handle.terminate()

package/src/providers/unsupported.ts

@@ -0,0 +1,67 @@
+/** Non-macOS fallback backend: keeps the Service injectable, fails closed, and reports an unavailable health state. */
+
+import type {
+  BackendActionRequest,
+  BackendActionResult,
+  BackendCursorAction,
+  BackendHealth,
+  BackendObservation,
+  BackendObserveOptions,
+  ComputerUseBackend,
+  CursorVisibility,
+} from '../backend.ts'
+import { ComputerUseError } from '../errors.ts'
+import type { ComputerAppIdentity, ComputerAppSelector, ComputerAppSummary } from '../types.ts'
+
+function unsupported(platform: NodeJS.Platform): ComputerUseError {
+  return new ComputerUseError(
+    'COMPUTER_UNSUPPORTED_PLATFORM',
+    `dsh-computer-use supports macOS only; Computer Use is disabled on ${platform}`,
+  )
+}
+
+/** Backend that reports a clear unavailable state instead of failing profile startup on non-macOS hosts. */
+export class UnsupportedPlatformBackend implements ComputerUseBackend {
+  readonly name = 'unsupported' as const
+  readonly helperPath = ''
+
+  constructor(private readonly platform: NodeJS.Platform) {}
+
+  async health(): Promise<BackendHealth> {
+    const failure = unsupported(this.platform)
+    return {
+      ready: false,
+      error: failure.message,
+      helperVersion: 'unsupported',
+      helperSha256: '',
+      accessibility: 'unavailable',
+      screenRecording: 'unavailable',
+    }
+  }
+
+  resolveApp(_selector: ComputerAppSelector): Promise<ComputerAppIdentity> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  listApps(): Promise<ComputerAppSummary[]> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  observe(_app: ComputerAppIdentity, _options: BackendObserveOptions): Promise<BackendObservation> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  act(_request: BackendActionRequest): Promise<BackendActionResult> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  visualizeCursor(_action: BackendCursorAction, _phase: 'before' | 'after'): Promise<CursorVisibility> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  openSettings(_kind: 'accessibility' | 'screen-recording'): Promise<void> {
+    return Promise.reject(unsupported(this.platform))
+  }
+
+  async dispose(): Promise<void> {}
+}

package/src/service.ts

@@ -4,7 +4,7 @@ import { randomUUID } from 'node:crypto'
 import { setTimeout as delay } from 'node:timers/promises'
 import { Context, Service } from '@deepseek-ai/cordis'
 import type { Agent } from '@deepseek-ai/dsh-agent'
-import type { BackendCursorAction, BackendObservation, ComputerUseBackend } from './backend.ts'
+import type { BackendCursorAction, BackendHealth, BackendObservation, ComputerUseBackend, CursorVisibility } from './backend.ts'
 import { allocateScreenshotPath, describeScreenshot } from './artifacts.ts'
 import type { ResolvedComputerUseConfig } from './config.ts'
 import { ComputerConfirmationManager } from './confirmations.ts'
@@ -188,6 +188,7 @@ export class ComputerUseService extends Service {
   private config: ResolvedComputerUseConfig
   private generation = 1
   private readonly agents = new Map<Agent, AgentState>()
+  private readonly actionTails = new Map<string, Promise<void>>()
   private readonly leases: ComputerLeaseManager
   private readonly confirmations: ComputerConfirmationManager
   private readonly lifecycle = new AbortController()
@@ -197,6 +198,18 @@ export class ComputerUseService extends Service {
     screenRecording: 'unavailable',
   }
 
+  /** Persist backend health facts while allowing a disabled provider to stay ready=false with a visible reason. */
+  private applyHealth(health: BackendHealth): void {
+    this.healthState = {
+      ready: health.ready ?? true,
+      helperVersion: health.helperVersion,
+      helperSha256: health.helperSha256,
+      accessibility: health.accessibility,
+      screenRecording: health.screenRecording,
+      ...(health.error === undefined ? {} : { lastError: health.error }),
+    }
+  }
+
   /** Register `ctx.computerUse` using one validated backend and configuration generation. */
   constructor(ctx: Context, backend: ComputerUseBackend, config: ResolvedComputerUseConfig) {
     super(ctx, 'computerUse')
@@ -215,8 +228,7 @@ export class ComputerUseService extends Service {
   protected async initialize(): Promise<void> {
     try {
       await this.leases.initialize()
-      const health = await this.backend.health(this.lifecycle.signal)
-      this.healthState = { ready: true, ...health }
+      this.applyHealth(await this.backend.health(this.lifecycle.signal))
     } catch (error) {
       const failure = computerUseError(error, 'Computer Use provider initialization failed')
       this.healthState = {
@@ -237,7 +249,7 @@ export class ComputerUseService extends Service {
     this.config = config
     this.generation += 1
     this.clearState()
-    this.healthState = { ready: true, ...health }
+    this.applyHealth(health)
     await previous.dispose()
   }
 
@@ -245,7 +257,7 @@ export class ComputerUseService extends Service {
   status(): ComputerUseStatus {
     return {
       platform: process.platform,
-      provider: 'macos-ax',
+      provider: this.backend.name,
       generation: this.generation,
       helperPath: this.backend.helperPath,
       ...this.healthState,
@@ -255,8 +267,7 @@ export class ComputerUseService extends Service {
   /** Re-run non-mutating provider health checks. */
   async health(signal: AbortSignal): Promise<ComputerUseStatus> {
     try {
-      const health = await this.backend.health(AbortSignal.any([signal, this.lifecycle.signal]))
-      this.healthState = { ready: true, ...health }
+      this.applyHealth(await this.backend.health(AbortSignal.any([signal, this.lifecycle.signal])))
     } catch (error) {
       const failure = computerUseError(error, 'Computer Use health check failed')
       this.healthState = { ...this.healthState, ready: false, lastError: failure.message }
@@ -300,6 +311,19 @@ export class ComputerUseService extends Service {
     const signal = AbortSignal.any([context.signal, this.lifecycle.signal])
     const stored = this.requireObservation(action.observationId, context.agent)
     if (action.kind === 'wait') return await this.wait(stored, action, context, signal)
+    return await this.enqueueAction(stored.backend.app, async () => {
+      signal.throwIfAborted()
+      return await this.actNow(action, context, signal)
+    })
+  }
+
+  /** Keep this service's actions for one process ordered through post-action observation. */
+  private async actNow(
+    action: Exclude<ComputerActionRequest, { kind: 'wait' }>,
+    context: ComputerUseContext,
+    signal: AbortSignal,
+  ): Promise<ComputerActionResult> {
+    const stored = this.requireObservation(action.observationId, context.agent)
     const index = targetIndex(action)
     const handle = targetHandle(action)
     const originalElement = index === undefined ? undefined : stored.backend.elements.find(candidate => candidate.index === index)
@@ -366,13 +390,31 @@ export class ComputerUseService extends Service {
     }
     this.confirmations.consume(context.agent, stored.backend.app, action)
     const visualization = cursorAction(action, element, actionObservation.window, actionObservation.app)
+    const cursorRequested = this.config.interaction.cursorVisualization === 'visible'
+      && (action.kind === 'click' || action.kind === 'scroll' || action.kind === 'drag')
     let cursorStarted = false
-    if (visualization !== undefined && this.config.interaction.cursorVisualization === 'visible') {
+    // The overlay is presentation-only and never blocks native input, but its
+    // least-visible outcome is reported across both phases of the action.
+    let cursorState: CursorVisibility | undefined
+    const recordCursor = (next: CursorVisibility): void => {
+      if (cursorState === undefined || (cursorState.visible && !next.visible)) cursorState = next
+    }
+    if (cursorRequested && visualization === undefined) {
+      recordCursor({
+        visible: false,
+        reason: actionObservation.window?.id === undefined
+          ? 'the agent cursor could not be bound because the target window has no stable window id'
+          : 'the agent cursor could not be placed because this action has no observable cursor position',
+      })
+    } else if (visualization !== undefined && cursorRequested) {
       try {
-        await this.backend.visualizeCursor(visualization, 'before', signal)
+        recordCursor(await this.backend.visualizeCursor(visualization, 'before', signal))
         cursorStarted = true
-      } catch {
-        // The overlay is presentation-only; native input remains authoritative.
+      } catch (error) {
+        recordCursor({
+          visible: false,
+          reason: `the agent cursor could not be driven before the action: ${error instanceof Error ? error.message : String(error)}`,
+        })
       }
     }
     let outcome
@@ -390,14 +432,21 @@ export class ComputerUseService extends Service {
     } finally {
       if (cursorStarted && visualization !== undefined) {
         try {
-          await this.backend.visualizeCursor(visualization, 'after', signal)
-        } catch {
-          // The overlay is presentation-only; native input remains authoritative.
+          recordCursor(await this.backend.visualizeCursor(visualization, 'after', signal))
+        } catch (error) {
+          recordCursor({
+            visible: false,
+            reason: `the agent cursor could not be validated after the action: ${error instanceof Error ? error.message : String(error)}`,
+          })
         }
       }
     }
+    // The settle loop reports whether the bounded structural observation
+    // changed. It complements routing facts without claiming causal proof or
+    // visibility into pixel-only, transient, or remote effects.
     const started = Date.now()
     let latest: BackendObservation | undefined
+    let settled = false
     do {
       if (this.config.settleMs > 0) await delay(this.config.settleMs, undefined, { signal })
       latest = await this.backend.observe(stored.backend.app, {
@@ -406,7 +455,7 @@ export class ComputerUseService extends Service {
         maxDepth: this.config.maxDepth,
         maxTextBytes: this.config.maxTextBytes,
       }, signal)
-      if (latest.stateHash !== actionObservation.stateHash) break
+      if (latest.stateHash !== actionObservation.stateHash) { settled = true; break }
     } while (Date.now() - started < this.config.maxSettleMs)
     const observation = await this.capture(
       stored.backend.app,
@@ -421,11 +470,41 @@ export class ComputerUseService extends Service {
       activation: outcome.activation,
       pointerInput: outcome.pointerInput,
       pointerRouting: outcome.pointerRouting,
+      // Only reported when the cursor is meant to be showing and is not, so a
+      // normal result stays unchanged and a lost cursor becomes visible to the
+      // caller instead of to nobody.
+      ...(cursorState === undefined || cursorState.visible ? {} : {
+        agentCursor: { visible: false, ...(cursorState.reason === undefined ? {} : { reason: cursorState.reason }) },
+      }),
+      // This reports only what the bounded structural observation can prove.
+      // Pixel-only, remote, or transient effects remain outside this hash and
+      // must not be described as action failure.
+      effect: {
+        observedStateChanged: settled,
+        observedForMs: Date.now() - started,
+        ...(settled ? {} : {
+          note: 'no change was observed in the window title, id, frame, or accessibility element tree;'
+            + ' pixel-only, remote, or transient effects may still have occurred',
+        }),
+      },
       ...(resolution === undefined ? {} : { resolution }),
       observation,
     }
   }
 
+  private async enqueueAction<T>(app: ComputerAppIdentity, operation: () => Promise<T>): Promise<T> {
+    const key = `${app.bundleId}:${app.pid}`
+    const previous = this.actionTails.get(key) ?? Promise.resolve()
+    const run = previous.catch(() => undefined).then(operation)
+    const tail = run.then(() => undefined, () => undefined)
+    this.actionTails.set(key, tail)
+    try {
+      return await run
+    } finally {
+      if (this.actionTails.get(key) === tail) this.actionTails.delete(key)
+    }
+  }
+
   /** Release all scoped observations and confirmations for one disposed Agent. */
   releaseAgent(agent: Agent): void {
     this.agents.delete(agent)
@@ -545,7 +624,8 @@ export class ComputerUseService extends Service {
     if (!Number.isInteger(timeoutMs) || timeoutMs < 100 || timeoutMs > this.config.maxSettleMs) {
       throw new ComputerUseError('COMPUTER_TIMEOUT', `wait timeout must be between 100 and ${this.config.maxSettleMs} milliseconds`)
     }
-    const deadline = Date.now() + timeoutMs
+    const started = Date.now()
+    const deadline = started + timeoutMs
     let latest = stored.backend
     while (!matchesWait(latest, action)) {
       if (Date.now() >= deadline) throw new ComputerUseError('COMPUTER_TIMEOUT', 'wait condition was not met before the configured deadline')
@@ -568,6 +648,13 @@ export class ComputerUseService extends Service {
       action: 'wait',
       channel: 'wait',
       activation: 'not-requested',
+      effect: {
+        observedStateChanged: latest.stateHash !== stored.backend.stateHash,
+        observedForMs: Date.now() - started,
+        ...(latest.stateHash === stored.backend.stateHash
+          ? { note: 'the wait condition was already satisfied by the referenced observation' }
+          : {}),
+      },
       pointerInput: false,
       pointerRouting: 'none',
       observation,

package/src/skill.ts

@@ -29,7 +29,13 @@ plugin; an API or CLI; browser automation for browser tasks; then Computer Use.
    carries the same observationId.
 5. Every successful action returns the fresh post-action observation. Read it
    before deciding the next step; do not add a redundant observe unless you need
-   a full tree or screenshot that the returned state omitted.
+   a full tree or screenshot that the returned state omitted. The accompanying
+   effect.observedStateChanged reports only changes in the window metadata and
+   Accessibility element tree. False does not prove the action failed: pixel-only,
+   transient, remote, or otherwise external effects may still have occurred.
+   Verify those effects visually before retrying, especially for sensitive actions.
+6. When agentCursor is present, the separate Agent cursor was expected but is not
+   visible. Read its reason and do not claim the user can see where the action landed.
 
 ## Visual evidence handoff