newmark-agent 0.6.4 → 0.6.8

package/dist/tools/computerUse.js

@@ -68,6 +68,7 @@ const DEFAULT_CAPTURE_MAX_HEIGHT = 960;
 const MIN_CAPTURE_MAX_WIDTH = 320;
 const MIN_CAPTURE_MAX_HEIGHT = 240;
 const MAX_CAPTURE_DIMENSION = 2048;
+const MAX_SPARSE_WAIT_MS = 5000;
 let lastScreenshotCleanupAt = 0;
 function ephemeralScreenshotPath(kind, directory = tempScreenshotDir(), createdAt = Date.now(), ownerPid = process.pid, suffix = crypto.randomBytes(4).toString('hex')) {
     const pid = Math.max(1, Math.floor(Number(ownerPid) || process.pid));
@@ -210,6 +211,172 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
         'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)',
     ].join('\r\n');
 }
+function boundedSparseWait(value) {
+    const requested = Number(value);
+    if (!Number.isFinite(requested) || requested <= 0)
+        return 0;
+    return Math.min(MAX_SPARSE_WAIT_MS, Math.floor(requested));
+}
+function sparseMeasurementSampleBudget(waitMs) {
+    const boundedWait = Math.max(1, Math.floor(Number(waitMs) || 1));
+    // A live PowerShell process handshake does not mean its first GDI capture,
+    // script execution and IPC response are warm. Reserve completion time on
+    // every call so the lane's own wait-elapsed result beats the outer fallback.
+    const completionMarginMs = Math.min(Math.max(0, boundedWait - 1), Math.min(1500, Math.max(500, Math.floor(boundedWait * 0.3))));
+    return Math.max(1, boundedWait - completionMarginMs);
+}
+function sparseMeasurementScript(waitMs) {
+    const boundedWait = Math.min(MAX_SPARSE_WAIT_MS, Math.max(1, Math.floor(waitMs)));
+    return [
+        '$ErrorActionPreference = "Stop"',
+        '$bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen',
+        '$x=[int]$bounds.Left; $y=[int]$bounds.Top; $w=[int]$bounds.Width; $h=[int]$bounds.Height',
+        'if ($w -le 0 -or $h -le 0) { throw "Virtual screen geometry is unavailable." }',
+        '$gridWidth=32; $gridHeight=18; $pollMs=200; $cellThreshold=24; $strongThreshold=64',
+        '$watch = [System.Diagnostics.Stopwatch]::StartNew(); $sampleCount=0; $changedCells=0; $triggered=$false; $geometryChanged=$false',
+        '$thumbnail = [NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight)',
+        '$baseline = [byte[]]::new($gridWidth * $gridHeight)',
+        'try {',
+        '  for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $baseline[$index]=$thumbnail.GetLuma($column,$row) } }',
+        '  $sampleCount=1',
+        `  while ($watch.ElapsedMilliseconds -lt ${boundedWait}) {`,
+        `    $remaining=[Math]::Max(1, ${boundedWait} - [int]$watch.ElapsedMilliseconds); Start-Sleep -Milliseconds ([Math]::Min($pollMs,$remaining))`,
+        '    $nextBounds=[System.Windows.Forms.SystemInformation]::VirtualScreen',
+        '    if ($nextBounds.Left -ne $x -or $nextBounds.Top -ne $y -or $nextBounds.Width -ne $w -or $nextBounds.Height -ne $h) { $geometryChanged=$true; break }',
+        '    $nextThumbnail=[NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight); $thumbnail.Dispose(); $thumbnail=$nextThumbnail',
+        '    $changedCells=0; $strongCells=0',
+        '    for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $luma=[int]$thumbnail.GetLuma($column,$row); $delta=[Math]::Abs($luma-$baseline[$index]); if ($delta -ge $cellThreshold) { $changedCells++ }; if ($delta -ge $strongThreshold) { $strongCells++ } } }',
+        '    $sampleCount++',
+        '    if ($changedCells -ge 4 -or $strongCells -ge 2) { $triggered=$true; break }',
+        '  }',
+    ].join('\r\n') + [
+        '} finally { $watch.Stop(); $thumbnail.Dispose() }',
+        '$reason = if ($geometryChanged) { "screen_geometry_changed" } elseif ($triggered) { "coarse_change" } else { "wait_elapsed" }',
+        'Write-Output (@{ ok=$true; triggered=$triggered; geometry_changed=$geometryChanged; sample_count=$sampleCount; changed_cells=$changedCells; elapsed_ms=[int]$watch.ElapsedMilliseconds; reason=$reason } | ConvertTo-Json -Compress)',
+    ].join('\r\n');
+}
+function sparseAbortError(signal) {
+    if (signal.reason instanceof Error)
+        return signal.reason;
+    return new Error(signal.reason ? String(signal.reason) : 'Computer Use sparse observation was cancelled.');
+}
+async function waitForSparseChange(waitMs, signal) {
+    if (signal?.aborted)
+        throw sparseAbortError(signal);
+    return await new Promise((resolve, reject) => {
+        let settled = false;
+        let phase = 'preparing';
+        const deadlineAt = Date.now() + waitMs;
+        let deadlineTimer;
+        const cleanup = () => {
+            if (deadlineTimer)
+                clearTimeout(deadlineTimer);
+            signal?.removeEventListener('abort', onAbort);
+        };
+        const onAbort = () => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            (0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
+            reject(sparseAbortError(signal));
+        };
+        deadlineTimer = setTimeout(() => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            // A normal wait deadline during helper startup must not kill the shared
+            // lane. Its initialization can finish in the background and serve the
+            // next observation. Once a sample has actually been submitted, however,
+            // stop a worker that failed to return before its bounded sampling window.
+            if (phase === 'sampling')
+                (0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
+            const error = phase === 'preparing'
+                ? 'Sparse helper did not become ready before the observation deadline; its startup continues for the next observation.'
+                : 'Sparse sample did not return before the observation deadline; continuing with a fresh full observation.';
+            resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error });
+        }, waitMs);
+        signal?.addEventListener('abort', onAbort, { once: true });
+        if (signal?.aborted) {
+            onAbort();
+            return;
+        }
+        const parseResult = (result) => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            if (!result.ok) {
+                resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: result.output.slice(0, 500) });
+                return;
+            }
+            try {
+                const parsed = JSON.parse(result.output);
+                resolve({
+                    ok: parsed.ok === true,
+                    triggered: parsed.triggered === true,
+                    geometryChanged: parsed.geometry_changed === true,
+                    sampleCount: Math.max(0, Number(parsed.sample_count) || 0),
+                    changedCells: Math.max(0, Number(parsed.changed_cells) || 0),
+                    elapsedMs: Math.max(0, Number(parsed.elapsed_ms) || result.elapsedMs),
+                    reason: String(parsed.reason || 'sampling_failed'),
+                    ...(parsed.ok === true ? {} : { error: 'Sparse measurement returned an invalid result.' }),
+                });
+            }
+            catch {
+                resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: 'Sparse measurement result was not valid JSON.' });
+            }
+        };
+        const prepareAndSample = async () => {
+            const preparation = await (0, computerUsePowerShellHost_1.prepareComputerUsePowerShellLane)('sparse', waitMs);
+            if (settled)
+                return;
+            if (!preparation.ready) {
+                settled = true;
+                cleanup();
+                resolve({
+                    ok: false,
+                    triggered: false,
+                    geometryChanged: false,
+                    sampleCount: null,
+                    changedCells: null,
+                    elapsedMs: Math.max(preparation.elapsedMs, Date.now() - (deadlineAt - waitMs)),
+                    reason: 'wait_elapsed',
+                    error: 'Sparse helper initialization exceeded the observation deadline; the lane remains available to finish startup.',
+                });
+                return;
+            }
+            const remainingMs = Math.max(0, deadlineAt - Date.now());
+            if (remainingMs < 100) {
+                settled = true;
+                cleanup();
+                resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error: 'Sparse helper became ready without enough time left to capture a bounded sample.' });
+                return;
+            }
+            const sampleBudgetMs = sparseMeasurementSampleBudget(remainingMs);
+            phase = 'sampling';
+            try {
+                const result = await (0, computerUsePowerShellHost_1.runPersistentPowerShell)(sparseMeasurementScript(sampleBudgetMs), remainingMs, 'sparse');
+                parseResult(result);
+            }
+            catch (error) {
+                if (settled)
+                    return;
+                settled = true;
+                cleanup();
+                resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
+            }
+        };
+        void prepareAndSample().catch(error => {
+            if (settled)
+                return;
+            settled = true;
+            cleanup();
+            resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
+        });
+    });
+}
 function gradientPalette(_input) {
     return ['#000000', '#ffffff', '#000000', '#ffffff'];
 }
@@ -438,11 +605,15 @@ function sceneGeneration(apps, elements) {
 async function observeUiAutomation(maxChars, rootHandle, timeoutMs = 30000, lane = 'uia') {
     const limit = Math.min(Math.max(Math.floor(maxChars || 30000), 1000), 200000);
     const handleHex = String(rootHandle || '').trim().replace(/^0x/i, '').replace(/[^0-9a-f]/gi, '');
+    if (!/[1-9a-f]/i.test(handleHex)) {
+        return { elements: [], visited: 0, error: 'Window-scoped UI Automation requires a non-zero native window handle.', elapsedMs: 0 };
+    }
+    const hostProcessId = Math.max(1, Math.floor(process.pid));
     const script = [
-        handleHex
-            ? `$nativeHandle = [System.IntPtr]::new([Convert]::ToInt64(${psQuote(handleHex)}, 16)); $root = [System.Windows.Automation.AutomationElement]::FromHandle($nativeHandle)`
-            : '$root = [System.Windows.Automation.AutomationElement]::RootElement',
+        `$nativeHandle = [System.IntPtr]::new([Convert]::ToInt64(${psQuote(handleHex)}, 16)); $root = [System.Windows.Automation.AutomationElement]::FromHandle($nativeHandle)`,
         'if ($null -eq $root) { throw "UI Automation root is unavailable." }',
+        '$rootProcessId = 0; try { $rootProcessId = [int]$root.Current.ProcessId } catch {}',
+        `if ($rootProcessId -eq ${hostProcessId}) { throw "UI Automation of the Newmark host window is skipped to avoid Electron self-accessibility hangs." }`,
         '$cache = New-Object System.Windows.Automation.CacheRequest',
         '$cache.TreeScope = [System.Windows.Automation.TreeScope]::Element',
         '$cache.TreeFilter = [System.Windows.Automation.Automation]::ControlViewCondition',
@@ -496,6 +667,9 @@ async function observeUiAutomation(maxChars, rootHandle, timeoutMs = 30000, lane
         return { elements: parseJsonArray(result.output), visited: 0, error: 'UI Automation returned invalid JSON.', elapsedMs: result.elapsedMs };
     }
 }
+function skippedUiAutomation(error) {
+    return { elements: [], visited: 0, ...(error ? { error } : {}), elapsedMs: 0 };
+}
 async function observeAppWindows(maxChars, timeoutMs = 30000, lane = 'windows') {
     const limit = Math.min(Math.max(Math.floor(maxChars || 30000), 1000), 200000);
     const script = [
@@ -597,7 +771,162 @@ async function selectAppWindow(target, handle) {
         return { apps, error: `No visible application window matched: ${target || handle}.`, elapsedMs: observed.elapsedMs };
     return { app, apps, elapsedMs: observed.elapsedMs };
 }
-async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage, app, includeRawUi = false, captureMaxWidth, captureMaxHeight) {
+const MAX_WAIT_FOR_TIMEOUT_MS = 30_000;
+const DEFAULT_WAIT_FOR_TIMEOUT_MS = 5_000;
+const WAIT_FOR_POLL_INTERVAL_MS = 250;
+function boundedWaitForTimeout(value) {
+    const requested = Number(value);
+    if (!Number.isFinite(requested) || requested <= 0)
+        return DEFAULT_WAIT_FOR_TIMEOUT_MS;
+    return Math.min(MAX_WAIT_FOR_TIMEOUT_MS, Math.max(WAIT_FOR_POLL_INTERVAL_MS, Math.floor(requested)));
+}
+async function waitForUi(options, ownerId) {
+    const startedAt = Date.now();
+    const timeoutMs = boundedWaitForTimeout(options.timeoutMs);
+    const deadline = startedAt + timeoutMs;
+    const workspacePath = options.workspacePath;
+    const cache = observations.get(observationKey(workspacePath, ownerId));
+    const requestedText = String(options.text || '').trim();
+    const normalizedQuery = normalizeText(requestedText);
+    const targetId = String(options.targetId || '').trim();
+    const appTarget = String(options.appTarget || '').trim();
+    const requestedHandle = String(options.windowHandle || '').trim();
+    const cachedTarget = targetId ? cache?.objects.find(object => object.target_id === targetId) : undefined;
+    let app;
+    let lastWindowError;
+    let attempts = 0;
+    let nextWindowPollAt = 0;
+    if (!normalizedQuery && !targetId && !appTarget && !requestedHandle) {
+        return { ok: false, action: 'wait_for', error: 'Provide text, target_id, app_target, or window_handle to wait for.' };
+    }
+    if (targetId && !cachedTarget) {
+        return { ok: false, action: 'wait_for', error: `target_id not found in latest observation: ${targetId}. Call computer_use observe first.` };
+    }
+    const directHandle = requestedHandle || (!appTarget ? String(cache?.windowHandle || '') : '');
+    if (directHandle) {
+        const direct = await observeAppWindowByHandle(directHandle);
+        attempts += 1;
+        if (!direct.app) {
+            return { ok: false, action: 'wait_for', status: 'window_unavailable', error: direct.error || 'The selected window is no longer available.', elapsed_ms: Date.now() - startedAt, attempts };
+        }
+        app = direct.app;
+    }
+    if (!app && !appTarget) {
+        return { ok: false, action: 'wait_for', error: 'Call computer_use observe/app_observe first, or provide app_target/window_handle.' };
+    }
+    if (targetId && cache?.windowHandle && app && cache.windowHandle.toLowerCase() !== app.handle.toLowerCase()) {
+        return { ok: false, action: 'wait_for', error: `target_id=${targetId} belongs to a different window than ${app.handle}. Observe that window before waiting on the target.` };
+    }
+    while (Date.now() <= deadline) {
+        if (options.signal?.aborted) {
+            return { ok: false, action: 'wait_for', status: 'cancelled', error: 'wait_for was cancelled.', elapsed_ms: Date.now() - startedAt, attempts };
+        }
+        if (!app && appTarget) {
+            if (Date.now() >= nextWindowPollAt) {
+                const observed = await observeAppWindows(30000, 750, 'windows_advisory');
+                attempts += 1;
+                lastWindowError = observed.error;
+                app = observed.apps.find(candidate => appMatches(candidate, appTarget));
+                nextWindowPollAt = Date.now() + 1000;
+            }
+            if (!app) {
+                const remaining = deadline - Date.now();
+                if (remaining <= 0)
+                    break;
+                await new Promise(resolve => setTimeout(resolve, Math.min(Math.max(1, nextWindowPollAt - Date.now()), remaining)));
+                continue;
+            }
+        }
+        if (targetId && cache?.windowHandle && app && cache.windowHandle.toLowerCase() !== app.handle.toLowerCase()) {
+            return { ok: false, action: 'wait_for', error: `target_id=${targetId} belongs to a different window than ${app.handle}. Observe that window before waiting on the target.`, elapsed_ms: Date.now() - startedAt, attempts };
+        }
+        if (app && !normalizedQuery && !targetId) {
+            return {
+                ok: true,
+                action: 'wait_for',
+                status: 'matched',
+                found: true,
+                condition: { app_target: appTarget || undefined, window_handle: app.handle },
+                app,
+                elapsed_ms: Date.now() - startedAt,
+                attempts,
+            };
+        }
+        if (app) {
+            const ui = await observeUiAutomation(Number(options.maxChars || 30000), app.handle, 750, 'uia_advisory');
+            attempts += 1;
+            if (ui.error) {
+                return { ok: false, action: 'wait_for', status: 'uia_unavailable', error: ui.error, app, elapsed_ms: Date.now() - startedAt, attempts };
+            }
+            const objects = semanticObjects(ui.elements, app.bbox.width, app.bbox.height, workspacePath, ownerId);
+            const match = objects.find(object => {
+                if (targetId && object.target_id !== targetId)
+                    return false;
+                if (!normalizedQuery)
+                    return true;
+                return normalizeText(object.name).includes(normalizedQuery) || normalizeText(object.automation_id).includes(normalizedQuery);
+            });
+            if (match) {
+                const generation = sceneGeneration([app], ui.elements);
+                observations.set(observationKey(workspacePath, ownerId), {
+                    workspacePath,
+                    objects,
+                    windowHandle: app.handle,
+                    width: Math.max(1, app.bbox.width),
+                    height: Math.max(1, app.bbox.height),
+                    at: new Date().toISOString(),
+                    sceneGeneration: generation,
+                });
+                return {
+                    ok: true,
+                    action: 'wait_for',
+                    status: 'matched',
+                    found: true,
+                    condition: {
+                        ...(requestedText ? { text: requestedText } : {}),
+                        ...(targetId ? { target_id: targetId } : {}),
+                        ...(appTarget ? { app_target: appTarget } : {}),
+                    },
+                    app,
+                    scene_generation: generation,
+                    element: {
+                        target_id: match.target_id,
+                        label: match.label,
+                        role: match.role,
+                        control_type: match.control_type,
+                        automation_id: match.automation_id,
+                        bbox: match.bbox,
+                        center: match.center,
+                    },
+                    elapsed_ms: Date.now() - startedAt,
+                    attempts,
+                };
+            }
+        }
+        const remaining = deadline - Date.now();
+        if (remaining <= 0)
+            break;
+        await new Promise(resolve => setTimeout(resolve, Math.min(WAIT_FOR_POLL_INTERVAL_MS, remaining)));
+    }
+    return {
+        ok: true,
+        action: 'wait_for',
+        status: 'timed_out',
+        found: false,
+        condition: {
+            ...(requestedText ? { text: requestedText } : {}),
+            ...(targetId ? { target_id: targetId } : {}),
+            ...(appTarget ? { app_target: appTarget } : {}),
+            ...(requestedHandle ? { window_handle: requestedHandle } : {}),
+        },
+        ...(app ? { app } : {}),
+        ...(lastWindowError ? { warning: lastWindowError } : {}),
+        timeout_ms: timeoutMs,
+        elapsed_ms: Date.now() - startedAt,
+        attempts,
+    };
+}
+async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage, app, includeRawUi = false, captureMaxWidth, captureMaxHeight, includeUiTree = true) {
     const totalStarted = Date.now();
     const outPath = ephemeralScreenshotPath('app');
     let retainedForVision = false;
@@ -608,7 +937,9 @@ async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage,
         ], captureMaxWidth, captureMaxHeight);
         const [result, ui] = await Promise.all([
             runPowerShell(script, 30000),
-            observeUiAutomation(30000, app.handle, 750, 'uia_advisory'),
+            includeUiTree
+                ? observeUiAutomation(30000, app.handle, 750, 'uia_advisory')
+                : Promise.resolve(skippedUiAutomation()),
         ]);
         if (!result.ok)
             return captureFailure('app_observe', app);
@@ -631,7 +962,7 @@ async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage,
         }
         const cacheKey = observationKey(workspacePath, ownerId);
         const generation = sceneGeneration([app], elements);
-        observations.set(cacheKey, { workspacePath, objects, width: Number(parsed.width || b.width || 1), height: Number(parsed.height || b.height || 1), at: new Date().toISOString(), sceneGeneration: generation });
+        observations.set(cacheKey, { workspacePath, objects, windowHandle: app.handle, width: Number(parsed.width || b.width || 1), height: Number(parsed.height || b.height || 1), at: new Date().toISOString(), sceneGeneration: generation });
         const payload = {
             ok: true,
             action: 'app_observe',
@@ -640,8 +971,10 @@ async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage,
             screenshot_retention: allowEphemeralVisionImage ? 'ephemeral-delete-after-vision-input' : 'ephemeral-deleted-before-tool-return',
             coordinate_system: 'virtual-screen-pixels',
             perception: {
-                mode: 'native-application-screenshot-plus-windows-ui-automation',
+                mode: includeUiTree ? 'native-application-screenshot-plus-windows-ui-automation' : 'native-application-screenshot-only',
                 application_scope: 'single-visible-window',
+                ui_tree_status: includeUiTree ? (ui.error ? 'unavailable' : 'captured') : 'skipped',
+                ui_tree_skipped: !includeUiTree,
                 element_count: elements.length,
                 visited_count: ui.visited,
                 scene_generation: generation,
@@ -652,6 +985,7 @@ async function cropScreenshot(workspacePath, ownerId, allowEphemeralVisionImage,
             telemetry: {
                 capture_ms: result.elapsedMs,
                 uia_ms: ui.elapsedMs,
+                ui_tree_skipped: !includeUiTree,
                 total_ms: Date.now() - totalStarted,
                 element_count: elements.length,
                 visited_count: ui.visited,
@@ -901,8 +1235,28 @@ function resolveTarget(targetId, workspacePath, ownerId) {
         return { x: 0, y: 0, error: `target_id not found in latest observation: ${id}.` };
     return { x: target.center.x, y: target.center.y, target };
 }
-async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight) {
+async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight, sparseWaitMs, signal, includeUiTree = true) {
     const totalStarted = Date.now();
+    const sparseDurationMs = boundedSparseWait(sparseWaitMs);
+    const sparseStarted = Date.now();
+    const sparse = sparseDurationMs > 0 ? await waitForSparseChange(sparseDurationMs, signal) : undefined;
+    const sparseWallMs = sparse ? Date.now() - sparseStarted : 0;
+    if (signal?.aborted)
+        throw sparseAbortError(signal);
+    const sparseDetails = sparse ? {
+        method: 'coarse-grid-luminance-change-wakeup',
+        grid_width: 32,
+        grid_height: 18,
+        sample_count: sparse.sampleCount,
+        changed_cells: sparse.changedCells,
+        triggered: sparse.triggered,
+        reason: sparse.reason,
+        elapsed_ms: sparse.elapsedMs,
+        wall_elapsed_ms: sparseWallMs,
+        fresh_full_observation_follows: true,
+        interpretation: 'Wake signal only. A timeout or no detected change is not evidence that the screen stayed unchanged; small, brief, or color-only changes can be missed.',
+        ...(sparse.error ? { warning: sparse.error } : {}),
+    } : undefined;
     const outPath = ephemeralScreenshotPath('observe');
     let retainedForVision = false;
     try {
@@ -913,23 +1267,29 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
         const foreground = await foregroundWindowHandle();
         const [result, ui, apps] = await Promise.all([
             runPowerShell(script, 30000),
-            observeUiAutomation(30000, foreground.handle, 750, 'uia_advisory'),
-            observeAppWindows(30000, 750, 'windows_advisory'),
+            includeUiTree && foreground.handle
+                ? observeUiAutomation(30000, foreground.handle, 750, 'uia_advisory')
+                : Promise.resolve(skippedUiAutomation(includeUiTree ? 'Foreground window handle is unavailable; UI Automation tree capture was skipped.' : undefined)),
+            includeUiTree
+                ? observeAppWindows(30000, 750, 'windows_advisory')
+                : Promise.resolve({ apps: [], error: undefined, elapsedMs: 0 }),
         ]);
+        if (signal?.aborted)
+            throw sparseAbortError(signal);
         if (!result.ok)
-            return captureFailure('observe');
+            return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
         let parsed = {};
         try {
             parsed = JSON.parse(result.output);
         }
         catch {
-            return captureFailure('observe');
+            return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
         }
         const screenWidth = Number(parsed.width || 1);
         const screenHeight = Number(parsed.height || 1);
         const objects = semanticObjects(ui.elements, screenWidth, screenHeight, workspacePath, ownerId);
         const generation = sceneGeneration(apps.apps, ui.elements);
-        observations.set(observationKey(workspacePath, ownerId), { workspacePath, objects, width: screenWidth, height: screenHeight, at: new Date().toISOString(), sceneGeneration: generation });
+        observations.set(observationKey(workspacePath, ownerId), { workspacePath, objects, windowHandle: foreground.handle || undefined, width: screenWidth, height: screenHeight, at: new Date().toISOString(), sceneGeneration: generation });
         const payload = {
             ok: true,
             action: 'observe',
@@ -937,28 +1297,47 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
             screenshot_retention: allowEphemeralVisionImage ? 'ephemeral-delete-after-vision-input' : 'ephemeral-deleted-before-tool-return',
             coordinate_system: 'virtual-screen-pixels',
             perception: {
-                mode: 'native-screenshot-plus-windows-ui-automation',
+                mode: includeUiTree ? 'native-screenshot-plus-windows-ui-automation' : 'native-screenshot-only',
+                ui_tree_status: includeUiTree ? (ui.error ? 'unavailable' : 'captured') : 'skipped',
+                ui_tree_skipped: !includeUiTree,
                 vision_assist: {
-                    when_model_supports_vision: 'The Agent sends this screenshot as image input together with the UI Automation element tree, so visual recognition and control bounding boxes can be used in the same decision step.',
-                    fallback_without_vision: 'Use the UI Automation element tree, bbox, center coordinates, and semantic target IDs only; no screenshot file path is retained or exposed.',
+                    when_model_supports_vision: includeUiTree
+                        ? 'The Agent sends this screenshot as image input together with the UI Automation element tree, so visual recognition and control bounding boxes can be used in the same decision step.'
+                        : 'The Agent receives this screenshot as image input without UI Automation traversal or semantic target IDs.',
+                    fallback_without_vision: includeUiTree
+                        ? 'Use the UI Automation element tree, bbox, center coordinates, and semantic target IDs only; no screenshot file path is retained or exposed.'
+                        : 'This result has no semantic target IDs; request include_ui_tree=true for a UI Automation element tree.',
                 },
                 element_count: ui.elements.length,
                 visited_count: ui.visited,
                 scene_generation: generation,
                 scene_summary: buildSceneSummary(objects, screenWidth, screenHeight),
                 objects: compactSemanticObjects(objects),
-                object_usage: 'Prefer target_id from scene_summary.high_priority_objects or objects for click/move/scroll when available; it resolves to the latest observed center coordinate and keeps actions tied to a semantic UI object.',
+                ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}),
+                object_usage: includeUiTree
+                    ? 'Prefer target_id from scene_summary.high_priority_objects or objects for click/move/scroll when available; it resolves to the latest observed center coordinate and keeps actions tied to a semantic UI object.'
+                    : 'This screenshot-only observation has no semantic target IDs. Use visual coordinates or request another observe with include_ui_tree=true before using target_id.',
                 warning: foreground.error || ui.error || apps.error || undefined,
-                note: 'Inspired by capture-parse-decide-act and MCP-style Computer Control agents: screenshot capture plus native Windows UI Automation text/control bounding boxes, semantic target objects, stable target IDs, high-priority object summaries, normalized bboxes, allowed actions, and risk hints. It does not copy third-party code.',
+                note: includeUiTree
+                    ? 'Native screenshot plus a bounded, foreground-window-scoped Windows UI Automation tree. No third-party server or code is loaded.'
+                    : 'Fast native screenshot path; UI Automation traversal was intentionally skipped. No third-party server or code is loaded.',
             },
             telemetry: {
                 capture_ms: result.elapsedMs,
                 uia_ms: ui.elapsedMs,
+                ui_tree_skipped: !includeUiTree,
                 app_list_ms: apps.elapsedMs,
                 foreground_ms: foreground.elapsedMs,
                 total_ms: Date.now() - totalStarted,
                 element_count: ui.elements.length,
                 visited_count: ui.visited,
+                ...(sparse ? {
+                    sparse_sampling_ms: sparse.elapsedMs,
+                    sparse_wall_ms: sparseWallMs,
+                    sparse_configured_wait_ms: sparseDurationMs,
+                    sparse_sample_count: sparse.sampleCount,
+                    sparse_triggered: sparse.triggered,
+                } : {}),
             },
             ...parsed,
         };
@@ -1093,7 +1472,9 @@ async function foregroundSceneGeneration() {
     const handle = handleResult.handle;
     const [apps, ui] = await Promise.all([
         observeAppWindows(30000),
-        observeUiAutomation(30000, handle),
+        handle
+            ? observeUiAutomation(30000, handle)
+            : Promise.resolve(skippedUiAutomation('Foreground window handle is unavailable; UI Automation tree capture was skipped.')),
     ]);
     return {
         generation: sceneGeneration(apps.apps, ui.elements),
@@ -1207,6 +1588,7 @@ function stringifyComputerUseResult(result) {
 exports.computerUseInternals = {
     cleanupStaleScreenshots,
     ephemeralScreenshotPath,
+    sparseMeasurementSampleBudget,
     stringifyComputerUseResult,
 };
 async function runComputerUse(options) {
@@ -1230,7 +1612,7 @@ async function runComputerUse(options) {
         result = { ok: true, action: 'takeover_stop', takeover: false };
     }
     else if (action === 'observe') {
-        result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight);
+        result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight, options.sparseWaitMs, options.signal, options.includeUiTree !== false);
     }
     else if (action === 'app_list') {
         const first = await observeAppWindows(Number(options.maxChars || 60000));
@@ -1247,7 +1629,7 @@ async function runComputerUse(options) {
     else if (action === 'app_observe') {
         const selected = await selectAppWindow(options.appTarget, options.windowHandle);
         result = selected.app
-            ? await cropScreenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, selected.app, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight)
+            ? await cropScreenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, selected.app, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight, options.includeUiTree !== false)
             : { ok: false, action, error: selected.error, applications: selected.apps.slice(0, 20) };
     }
     else if (action === 'app_activate') {
@@ -1257,6 +1639,9 @@ async function runComputerUse(options) {
     else if (action === 'wait') {
         result = await wait(Number(options.durationMs || 1000));
     }
+    else if (action === 'wait_for') {
+        result = await waitForUi(options, ownerId);
+    }
     else if (action === 'sequence') {
         result = await executeSequence(options, ownerId);
     }

package/dist/tools/computerUsePowerShellHost.d.ts

@@ -1,4 +1,4 @@
-type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory';
+type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory' | 'sparse';
 export interface PowerShellResult {
     ok: boolean;
     output: string;
@@ -6,6 +6,11 @@ export interface PowerShellResult {
 }
 export declare function runPersistentPowerShell(script: string, timeoutMs?: number, lane?: HelperLane): Promise<PowerShellResult>;
 export declare function computerUsePowerShellLaneReady(lane: HelperLane): boolean;
+export declare function prepareComputerUsePowerShellLane(lane: HelperLane, timeoutMs: number): Promise<{
+    ready: boolean;
+    elapsedMs: number;
+}>;
+export declare function stopComputerUsePowerShellLane(lane: HelperLane): void;
 export declare function stopComputerUsePowerShellHost(): void;
 export {};
 //# sourceMappingURL=computerUsePowerShellHost.d.ts.map