newmark-agent 0.6.3 → 0.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/config.example.json +13 -3
  2. package/dist/cli-commands.d.ts +1 -1
  3. package/dist/cli-commands.js +53 -2
  4. package/dist/cli-discovery.js +3 -0
  5. package/dist/conversation-utility-host.bundle.cjs +3046 -1038
  6. package/dist/conversation-utility-host.js +94 -7
  7. package/dist/core/agent.d.ts +149 -12
  8. package/dist/core/agent.js +843 -176
  9. package/dist/core/agentKernelRunner.d.ts +7 -1
  10. package/dist/core/agentKernelRunner.js +171 -22
  11. package/dist/core/autoRouter.d.ts +49 -44
  12. package/dist/core/autoRouter.js +117 -302
  13. package/dist/core/branchIdentity.d.ts +29 -0
  14. package/dist/core/branchIdentity.js +54 -0
  15. package/dist/core/config.js +27 -11
  16. package/dist/core/continuation/contracts.d.ts +1 -1
  17. package/dist/core/continuation/store.d.ts +119 -1
  18. package/dist/core/continuation/store.js +337 -19
  19. package/dist/core/conversationKernel.d.ts +62 -8
  20. package/dist/core/conversationKernel.js +361 -47
  21. package/dist/core/conversationStateDocument.d.ts +18 -0
  22. package/dist/core/conversationStateDocument.js +88 -0
  23. package/dist/core/conversationVisualBudget.d.ts +74 -0
  24. package/dist/core/conversationVisualBudget.js +153 -0
  25. package/dist/core/electronUtilityAgentClient.d.ts +3 -2
  26. package/dist/core/electronUtilityAgentClient.js +32 -8
  27. package/dist/core/electronUtilityRuntimePool.d.ts +28 -3
  28. package/dist/core/electronUtilityRuntimePool.js +93 -24
  29. package/dist/core/hostRuntimeHooks.d.ts +23 -0
  30. package/dist/core/hostRuntimeHooks.js +17 -0
  31. package/dist/core/jevDecision.d.ts +144 -0
  32. package/dist/core/jevDecision.js +226 -0
  33. package/dist/core/memoryProbe.d.ts +24 -0
  34. package/dist/core/memoryProbe.js +104 -0
  35. package/dist/core/mobilePairing.d.ts +7 -1
  36. package/dist/core/mobilePairing.js +9 -1
  37. package/dist/core/performanceDiagnostics.d.ts +1 -1
  38. package/dist/core/routeDecisionValidator.d.ts +86 -0
  39. package/dist/core/routeDecisionValidator.js +249 -0
  40. package/dist/core/routeEligibility.d.ts +52 -0
  41. package/dist/core/routeEligibility.js +108 -0
  42. package/dist/core/runtimeMemoryBudget.d.ts +6 -0
  43. package/dist/core/runtimeMemoryBudget.js +12 -0
  44. package/dist/core/utilityAgentProtocol.d.ts +8 -10
  45. package/dist/core/utilityHostToolRouter.js +2 -0
  46. package/dist/core/visualDownscale.d.ts +6 -0
  47. package/dist/core/visualDownscale.js +137 -0
  48. package/dist/core/wslAgentClient.d.ts +1 -2
  49. package/dist/core/wslAgentClient.js +0 -8
  50. package/dist/core/wslAgentProtocol.d.ts +1 -10
  51. package/dist/core/wslAgentRuntimePool.d.ts +1 -3
  52. package/dist/core/wslAgentRuntimePool.js +17 -22
  53. package/dist/llm/provider.d.ts +3 -1
  54. package/dist/llm/provider.js +53 -9
  55. package/dist/main.js +108 -31
  56. package/dist/preload.js +3 -1
  57. package/dist/server.js +30 -18
  58. package/dist/tools/computerUse.d.ts +4 -0
  59. package/dist/tools/computerUse.js +202 -4
  60. package/dist/tools/computerUsePowerShellHost.d.ts +6 -1
  61. package/dist/tools/computerUsePowerShellHost.js +105 -31
  62. package/dist/tools/index.js +4 -1
  63. package/dist/tui/src/adapters/core-runtime-adapter.js +13 -1
  64. package/dist/tui/src/i18n.js +12 -1
  65. package/dist/tui/src/render.js +9 -2
  66. package/dist/tui/src/settings-schema.js +19 -0
  67. package/dist/tui/src/state.js +69 -1
  68. package/dist/ui/index.html +994 -277
  69. package/dist/ui/lucide-sprite.svg +0 -8
  70. package/dist/wsl-agent-host.bundle.cjs +2851 -1013
  71. package/dist/wsl-agent-host.js +0 -3
  72. package/package.json +38 -9
package/dist/server.js CHANGED
@@ -59,6 +59,17 @@ const workEventCoalescer_1 = require("./core/workEventCoalescer");
59
59
  const conversationListEvent_1 = require("./core/conversationListEvent");
60
60
  const searchMcpPool_1 = require("./core/searchMcpPool");
61
61
  const PORT = 47890;
62
+ /**
63
+ * Effective listen/advertised port. Production and every existing harness use
64
+ * `PORT` untouched (the isolated-server gates patch that literal); the
65
+ * `NEWMARK_MOBILE_PORT` override exists so a second, isolated dev/test instance
66
+ * can host a paired client without taking the production port away from the
67
+ * instance the user is already running. It stays in sync with the pairing
68
+ * URL/QR, which resolves the same override through `MOBILE_PORT`.
69
+ */
70
+ const HOSTED_PORT = Number(process.env.NEWMARK_MOBILE_PORT) > 0
71
+ ? Math.trunc(Number(process.env.NEWMARK_MOBILE_PORT))
72
+ : PORT;
62
73
  let agent = null;
63
74
  let automation = null;
64
75
  let workspaceFileRouter = null;
@@ -104,7 +115,7 @@ function mobileAuthorized(req) {
104
115
  if (bearer.startsWith('Bearer '))
105
116
  return bearer.slice('Bearer '.length).trim() === mobileToken;
106
117
  try {
107
- const url = new URL(req.url || '/', `http://localhost:${PORT}`);
118
+ const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
108
119
  const queryToken = url.searchParams.get('token') || '';
109
120
  return queryToken === mobileToken;
110
121
  }
@@ -223,7 +234,7 @@ async function handleMobileWorkspaceUpload(req, res) {
223
234
  req.resume();
224
235
  return;
225
236
  }
226
- const url = new URL(req.url || '/', `http://localhost:${PORT}`);
237
+ const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
227
238
  const workspaceId = url.searchParams.get('workspaceId') || '';
228
239
  const directory = url.searchParams.get('directory') || '';
229
240
  const fileName = url.searchParams.get('fileName') || '';
@@ -392,13 +403,17 @@ function mobileScopedAgent(ws, conversationId) {
392
403
  }
393
404
  function mobileConversationRuntimeBusy(current, ws, conversationId) {
394
405
  const scoped = mobileScopedRuntimes.get(mobileRuntimeKey(String(ws.id || ''), conversationId));
395
- if (scoped && scoped.agent.status !== 'idle')
406
+ // Only an in-flight run blocks a mobile mutation. `goal_paused` (a goal the
407
+ // user paused) and `error` (a terminal failure) are not running work, yet
408
+ // treating them as busy refused archiving the conversation the user is
409
+ // actually looking at, which is the common case for a paused goal.
410
+ if (scoped && scoped.agent.status === 'working')
396
411
  return true;
397
412
  const currentWs = current.workspace.current;
398
413
  return !!currentWs
399
414
  && path.resolve(currentWs.path) === path.resolve(ws.path)
400
415
  && current.activeConversationId === conversationId
401
- && current.status !== 'idle';
416
+ && current.status === 'working';
402
417
  }
403
418
  function handleMobileEvents(req, res) {
404
419
  if (agent && !agent.config.getBool('remote', 'touch_enabled')) {
@@ -520,11 +535,7 @@ function applyConfigPatch(cfg) {
520
535
  agent.config.set('models', 'fallback_on_unavailable', value === true || value === 'on');
521
536
  break;
522
537
  case 'switchTendency':
523
- agent.config.set('models', 'auto_switch_preference', value);
524
- break;
525
- case 'clearLearnedAutoPreferences':
526
- if (value === true)
527
- agent.clearLearnedModelPreferences();
538
+ agent.config.set('models', 'auto_switch_preference', ['conservative', 'balanced', 'aggressive'].includes(String(value)) ? String(value) : 'balanced');
528
539
  break;
529
540
  case 'openAIApiMode':
530
541
  agent.config.set('models', 'openai_api_mode', ['chat_stream', 'chat', 'responses'].includes(String(value)) ? value : 'chat_stream');
@@ -594,7 +605,7 @@ function jsonResponse(res, data, code = 200) {
594
605
  res.end(body);
595
606
  }
596
607
  async function handleApi(req, res, body) {
597
- const url = new URL(req.url || '/', `http://localhost:${PORT}`);
608
+ const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
598
609
  const pathname = url.pathname;
599
610
  if (pathname.startsWith('/api/mobile/')) {
600
611
  if (agent && !agent.config.getBool('remote', 'touch_enabled')) {
@@ -616,6 +627,7 @@ async function handleApi(req, res, body) {
616
627
  jsonResponse(res, {
617
628
  mode: agent.mode, model: agent.modelSelectionValue(), modelLabel: agent.modelLabel(),
618
629
  resolvedDeployment: agent.activeDeployment(), routeDecision: agent.lastRouteDecision,
630
+ jevDecisionModel: agent.jevDecisionModelState(),
619
631
  intelligence: agent.intelligence, status: agent.status, goal: agent.goal,
620
632
  models: agent.allModelNames(), inputMode: agent.inputMode,
621
633
  conversationId: agent.activeConversationId,
@@ -1769,7 +1781,7 @@ function startServer(root, options = {}) {
1769
1781
  res.end();
1770
1782
  return;
1771
1783
  }
1772
- const url = new URL(req.url || '/', `http://localhost:${PORT}`);
1784
+ const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
1773
1785
  if (url.pathname === '/api/mobile/events') {
1774
1786
  handleMobileEvents(req, res);
1775
1787
  return;
@@ -1824,15 +1836,15 @@ function startServer(root, options = {}) {
1824
1836
  if (hostedServer === server)
1825
1837
  hostedServer = null;
1826
1838
  });
1827
- server.listen(PORT, bindHost, () => {
1839
+ server.listen(HOSTED_PORT, bindHost, () => {
1828
1840
  console.log(`\n Newmark Agent v1.0 - Server Mode`);
1829
- console.log(` Bind: ${bindHost}:${PORT}`);
1830
- console.log(` GUI: http://localhost:${PORT}`);
1841
+ console.log(` Bind: ${bindHost}:${HOSTED_PORT}`);
1842
+ console.log(` GUI: http://localhost:${HOSTED_PORT}`);
1831
1843
  console.log(` Tailscale IPv4: ${tailscale || 'not detected'}`);
1832
1844
  console.log(` LAN IPv4: ${lan || 'not detected'}`);
1833
- console.log(` Mobile endpoint: http://${accessHost}:${PORT}/api/mobile/hello?token=<token>`);
1845
+ console.log(` Mobile endpoint: http://${accessHost}:${HOSTED_PORT}/api/mobile/hello?token=<token>`);
1834
1846
  console.log(` Mobile token file: ${tokenPath}`);
1835
- console.log(` Mobile events (SSE): http://${accessHost}:${PORT}/api/mobile/events?token=<token>`);
1847
+ console.log(` Mobile events (SSE): http://${accessHost}:${HOSTED_PORT}/api/mobile/events?token=<token>`);
1836
1848
  console.log(` Press Ctrl+C to stop\n`);
1837
1849
  });
1838
1850
  return server;
@@ -1868,7 +1880,7 @@ async function probeHostedServer(host) {
1868
1880
  return await new Promise(resolve => {
1869
1881
  const request = http.get({
1870
1882
  hostname: host,
1871
- port: PORT,
1883
+ port: HOSTED_PORT,
1872
1884
  path: `/api/mobile/hello?token=${encodeURIComponent(mobileToken)}`,
1873
1885
  timeout: 1800,
1874
1886
  }, response => {
@@ -1935,7 +1947,7 @@ async function hostedServerStatus(enabled = true, probeHost = '') {
1935
1947
  reachable,
1936
1948
  state,
1937
1949
  host,
1938
- port: PORT,
1950
+ port: HOSTED_PORT,
1939
1951
  error: state === 'error' ? (error || 'Mobile server is not reachable') : '',
1940
1952
  checkedAt: new Date().toISOString(),
1941
1953
  startedAt: hostedServerStartedAt,
@@ -30,6 +30,8 @@ export interface ComputerUseOptions {
30
30
  allowEphemeralVisionImage?: boolean;
31
31
  captureMaxWidth?: number;
32
32
  captureMaxHeight?: number;
33
+ sparseWaitMs?: number;
34
+ signal?: AbortSignal;
33
35
  gradientColors?: string[];
34
36
  gradientSpeed?: number;
35
37
  gradientWidth?: number;
@@ -50,10 +52,12 @@ declare function ephemeralScreenshotPath(kind: 'observe' | 'app', directory?: st
50
52
  declare function cleanupStaleScreenshots(options?: ScreenshotCleanupOptions): {
51
53
  removed: number;
52
54
  };
55
+ declare function sparseMeasurementSampleBudget(waitMs: number): number;
53
56
  declare function stringifyComputerUseResult(result: Record<string, unknown>): string;
54
57
  export declare const computerUseInternals: {
55
58
  cleanupStaleScreenshots: typeof cleanupStaleScreenshots;
56
59
  ephemeralScreenshotPath: typeof ephemeralScreenshotPath;
60
+ sparseMeasurementSampleBudget: typeof sparseMeasurementSampleBudget;
57
61
  stringifyComputerUseResult: typeof stringifyComputerUseResult;
58
62
  };
59
63
  export declare function runComputerUse(options: ComputerUseOptions): Promise<string>;
@@ -68,6 +68,7 @@ const DEFAULT_CAPTURE_MAX_HEIGHT = 960;
68
68
  const MIN_CAPTURE_MAX_WIDTH = 320;
69
69
  const MIN_CAPTURE_MAX_HEIGHT = 240;
70
70
  const MAX_CAPTURE_DIMENSION = 2048;
71
+ const MAX_SPARSE_WAIT_MS = 5000;
71
72
  let lastScreenshotCleanupAt = 0;
72
73
  function ephemeralScreenshotPath(kind, directory = tempScreenshotDir(), createdAt = Date.now(), ownerPid = process.pid, suffix = crypto.randomBytes(4).toString('hex')) {
73
74
  const pid = Math.max(1, Math.floor(Number(ownerPid) || process.pid));
@@ -210,6 +211,172 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
210
211
  'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)',
211
212
  ].join('\r\n');
212
213
  }
214
+ function boundedSparseWait(value) {
215
+ const requested = Number(value);
216
+ if (!Number.isFinite(requested) || requested <= 0)
217
+ return 0;
218
+ return Math.min(MAX_SPARSE_WAIT_MS, Math.floor(requested));
219
+ }
220
+ function sparseMeasurementSampleBudget(waitMs) {
221
+ const boundedWait = Math.max(1, Math.floor(Number(waitMs) || 1));
222
+ // A live PowerShell process handshake does not mean its first GDI capture,
223
+ // script execution and IPC response are warm. Reserve completion time on
224
+ // every call so the lane's own wait-elapsed result beats the outer fallback.
225
+ const completionMarginMs = Math.min(Math.max(0, boundedWait - 1), Math.min(1500, Math.max(500, Math.floor(boundedWait * 0.3))));
226
+ return Math.max(1, boundedWait - completionMarginMs);
227
+ }
228
+ function sparseMeasurementScript(waitMs) {
229
+ const boundedWait = Math.min(MAX_SPARSE_WAIT_MS, Math.max(1, Math.floor(waitMs)));
230
+ return [
231
+ '$ErrorActionPreference = "Stop"',
232
+ '$bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen',
233
+ '$x=[int]$bounds.Left; $y=[int]$bounds.Top; $w=[int]$bounds.Width; $h=[int]$bounds.Height',
234
+ 'if ($w -le 0 -or $h -le 0) { throw "Virtual screen geometry is unavailable." }',
235
+ '$gridWidth=32; $gridHeight=18; $pollMs=200; $cellThreshold=24; $strongThreshold=64',
236
+ '$watch = [System.Diagnostics.Stopwatch]::StartNew(); $sampleCount=0; $changedCells=0; $triggered=$false; $geometryChanged=$false',
237
+ '$thumbnail = [NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight)',
238
+ '$baseline = [byte[]]::new($gridWidth * $gridHeight)',
239
+ 'try {',
240
+ ' for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $baseline[$index]=$thumbnail.GetLuma($column,$row) } }',
241
+ ' $sampleCount=1',
242
+ ` while ($watch.ElapsedMilliseconds -lt ${boundedWait}) {`,
243
+ ` $remaining=[Math]::Max(1, ${boundedWait} - [int]$watch.ElapsedMilliseconds); Start-Sleep -Milliseconds ([Math]::Min($pollMs,$remaining))`,
244
+ ' $nextBounds=[System.Windows.Forms.SystemInformation]::VirtualScreen',
245
+ ' if ($nextBounds.Left -ne $x -or $nextBounds.Top -ne $y -or $nextBounds.Width -ne $w -or $nextBounds.Height -ne $h) { $geometryChanged=$true; break }',
246
+ ' $nextThumbnail=[NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight); $thumbnail.Dispose(); $thumbnail=$nextThumbnail',
247
+ ' $changedCells=0; $strongCells=0',
248
+ ' for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $luma=[int]$thumbnail.GetLuma($column,$row); $delta=[Math]::Abs($luma-$baseline[$index]); if ($delta -ge $cellThreshold) { $changedCells++ }; if ($delta -ge $strongThreshold) { $strongCells++ } } }',
249
+ ' $sampleCount++',
250
+ ' if ($changedCells -ge 4 -or $strongCells -ge 2) { $triggered=$true; break }',
251
+ ' }',
252
+ ].join('\r\n') + [
253
+ '} finally { $watch.Stop(); $thumbnail.Dispose() }',
254
+ '$reason = if ($geometryChanged) { "screen_geometry_changed" } elseif ($triggered) { "coarse_change" } else { "wait_elapsed" }',
255
+ 'Write-Output (@{ ok=$true; triggered=$triggered; geometry_changed=$geometryChanged; sample_count=$sampleCount; changed_cells=$changedCells; elapsed_ms=[int]$watch.ElapsedMilliseconds; reason=$reason } | ConvertTo-Json -Compress)',
256
+ ].join('\r\n');
257
+ }
258
+ function sparseAbortError(signal) {
259
+ if (signal.reason instanceof Error)
260
+ return signal.reason;
261
+ return new Error(signal.reason ? String(signal.reason) : 'Computer Use sparse observation was cancelled.');
262
+ }
263
+ async function waitForSparseChange(waitMs, signal) {
264
+ if (signal?.aborted)
265
+ throw sparseAbortError(signal);
266
+ return await new Promise((resolve, reject) => {
267
+ let settled = false;
268
+ let phase = 'preparing';
269
+ const deadlineAt = Date.now() + waitMs;
270
+ let deadlineTimer;
271
+ const cleanup = () => {
272
+ if (deadlineTimer)
273
+ clearTimeout(deadlineTimer);
274
+ signal?.removeEventListener('abort', onAbort);
275
+ };
276
+ const onAbort = () => {
277
+ if (settled)
278
+ return;
279
+ settled = true;
280
+ cleanup();
281
+ (0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
282
+ reject(sparseAbortError(signal));
283
+ };
284
+ deadlineTimer = setTimeout(() => {
285
+ if (settled)
286
+ return;
287
+ settled = true;
288
+ cleanup();
289
+ // A normal wait deadline during helper startup must not kill the shared
290
+ // lane. Its initialization can finish in the background and serve the
291
+ // next observation. Once a sample has actually been submitted, however,
292
+ // stop a worker that failed to return before its bounded sampling window.
293
+ if (phase === 'sampling')
294
+ (0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
295
+ const error = phase === 'preparing'
296
+ ? 'Sparse helper did not become ready before the observation deadline; its startup continues for the next observation.'
297
+ : 'Sparse sample did not return before the observation deadline; continuing with a fresh full observation.';
298
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error });
299
+ }, waitMs);
300
+ signal?.addEventListener('abort', onAbort, { once: true });
301
+ if (signal?.aborted) {
302
+ onAbort();
303
+ return;
304
+ }
305
+ const parseResult = (result) => {
306
+ if (settled)
307
+ return;
308
+ settled = true;
309
+ cleanup();
310
+ if (!result.ok) {
311
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: result.output.slice(0, 500) });
312
+ return;
313
+ }
314
+ try {
315
+ const parsed = JSON.parse(result.output);
316
+ resolve({
317
+ ok: parsed.ok === true,
318
+ triggered: parsed.triggered === true,
319
+ geometryChanged: parsed.geometry_changed === true,
320
+ sampleCount: Math.max(0, Number(parsed.sample_count) || 0),
321
+ changedCells: Math.max(0, Number(parsed.changed_cells) || 0),
322
+ elapsedMs: Math.max(0, Number(parsed.elapsed_ms) || result.elapsedMs),
323
+ reason: String(parsed.reason || 'sampling_failed'),
324
+ ...(parsed.ok === true ? {} : { error: 'Sparse measurement returned an invalid result.' }),
325
+ });
326
+ }
327
+ catch {
328
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: 'Sparse measurement result was not valid JSON.' });
329
+ }
330
+ };
331
+ const prepareAndSample = async () => {
332
+ const preparation = await (0, computerUsePowerShellHost_1.prepareComputerUsePowerShellLane)('sparse', waitMs);
333
+ if (settled)
334
+ return;
335
+ if (!preparation.ready) {
336
+ settled = true;
337
+ cleanup();
338
+ resolve({
339
+ ok: false,
340
+ triggered: false,
341
+ geometryChanged: false,
342
+ sampleCount: null,
343
+ changedCells: null,
344
+ elapsedMs: Math.max(preparation.elapsedMs, Date.now() - (deadlineAt - waitMs)),
345
+ reason: 'wait_elapsed',
346
+ error: 'Sparse helper initialization exceeded the observation deadline; the lane remains available to finish startup.',
347
+ });
348
+ return;
349
+ }
350
+ const remainingMs = Math.max(0, deadlineAt - Date.now());
351
+ if (remainingMs < 100) {
352
+ settled = true;
353
+ cleanup();
354
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error: 'Sparse helper became ready without enough time left to capture a bounded sample.' });
355
+ return;
356
+ }
357
+ const sampleBudgetMs = sparseMeasurementSampleBudget(remainingMs);
358
+ phase = 'sampling';
359
+ try {
360
+ const result = await (0, computerUsePowerShellHost_1.runPersistentPowerShell)(sparseMeasurementScript(sampleBudgetMs), remainingMs, 'sparse');
361
+ parseResult(result);
362
+ }
363
+ catch (error) {
364
+ if (settled)
365
+ return;
366
+ settled = true;
367
+ cleanup();
368
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
369
+ }
370
+ };
371
+ void prepareAndSample().catch(error => {
372
+ if (settled)
373
+ return;
374
+ settled = true;
375
+ cleanup();
376
+ resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
377
+ });
378
+ });
379
+ }
213
380
  function gradientPalette(_input) {
214
381
  return ['#000000', '#ffffff', '#000000', '#ffffff'];
215
382
  }
@@ -901,8 +1068,28 @@ function resolveTarget(targetId, workspacePath, ownerId) {
901
1068
  return { x: 0, y: 0, error: `target_id not found in latest observation: ${id}.` };
902
1069
  return { x: target.center.x, y: target.center.y, target };
903
1070
  }
904
- async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight) {
1071
+ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight, sparseWaitMs, signal) {
905
1072
  const totalStarted = Date.now();
1073
+ const sparseDurationMs = boundedSparseWait(sparseWaitMs);
1074
+ const sparseStarted = Date.now();
1075
+ const sparse = sparseDurationMs > 0 ? await waitForSparseChange(sparseDurationMs, signal) : undefined;
1076
+ const sparseWallMs = sparse ? Date.now() - sparseStarted : 0;
1077
+ if (signal?.aborted)
1078
+ throw sparseAbortError(signal);
1079
+ const sparseDetails = sparse ? {
1080
+ method: 'coarse-grid-luminance-change-wakeup',
1081
+ grid_width: 32,
1082
+ grid_height: 18,
1083
+ sample_count: sparse.sampleCount,
1084
+ changed_cells: sparse.changedCells,
1085
+ triggered: sparse.triggered,
1086
+ reason: sparse.reason,
1087
+ elapsed_ms: sparse.elapsedMs,
1088
+ wall_elapsed_ms: sparseWallMs,
1089
+ fresh_full_observation_follows: true,
1090
+ interpretation: 'Wake signal only. A timeout or no detected change is not evidence that the screen stayed unchanged; small, brief, or color-only changes can be missed.',
1091
+ ...(sparse.error ? { warning: sparse.error } : {}),
1092
+ } : undefined;
906
1093
  const outPath = ephemeralScreenshotPath('observe');
907
1094
  let retainedForVision = false;
908
1095
  try {
@@ -916,14 +1103,16 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
916
1103
  observeUiAutomation(30000, foreground.handle, 750, 'uia_advisory'),
917
1104
  observeAppWindows(30000, 750, 'windows_advisory'),
918
1105
  ]);
1106
+ if (signal?.aborted)
1107
+ throw sparseAbortError(signal);
919
1108
  if (!result.ok)
920
- return captureFailure('observe');
1109
+ return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
921
1110
  let parsed = {};
922
1111
  try {
923
1112
  parsed = JSON.parse(result.output);
924
1113
  }
925
1114
  catch {
926
- return captureFailure('observe');
1115
+ return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
927
1116
  }
928
1117
  const screenWidth = Number(parsed.width || 1);
929
1118
  const screenHeight = Number(parsed.height || 1);
@@ -947,6 +1136,7 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
947
1136
  scene_generation: generation,
948
1137
  scene_summary: buildSceneSummary(objects, screenWidth, screenHeight),
949
1138
  objects: compactSemanticObjects(objects),
1139
+ ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}),
950
1140
  object_usage: 'Prefer target_id from scene_summary.high_priority_objects or objects for click/move/scroll when available; it resolves to the latest observed center coordinate and keeps actions tied to a semantic UI object.',
951
1141
  warning: foreground.error || ui.error || apps.error || undefined,
952
1142
  note: 'Inspired by capture-parse-decide-act and MCP-style Computer Control agents: screenshot capture plus native Windows UI Automation text/control bounding boxes, semantic target objects, stable target IDs, high-priority object summaries, normalized bboxes, allowed actions, and risk hints. It does not copy third-party code.',
@@ -959,6 +1149,13 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
959
1149
  total_ms: Date.now() - totalStarted,
960
1150
  element_count: ui.elements.length,
961
1151
  visited_count: ui.visited,
1152
+ ...(sparse ? {
1153
+ sparse_sampling_ms: sparse.elapsedMs,
1154
+ sparse_wall_ms: sparseWallMs,
1155
+ sparse_configured_wait_ms: sparseDurationMs,
1156
+ sparse_sample_count: sparse.sampleCount,
1157
+ sparse_triggered: sparse.triggered,
1158
+ } : {}),
962
1159
  },
963
1160
  ...parsed,
964
1161
  };
@@ -1207,6 +1404,7 @@ function stringifyComputerUseResult(result) {
1207
1404
  exports.computerUseInternals = {
1208
1405
  cleanupStaleScreenshots,
1209
1406
  ephemeralScreenshotPath,
1407
+ sparseMeasurementSampleBudget,
1210
1408
  stringifyComputerUseResult,
1211
1409
  };
1212
1410
  async function runComputerUse(options) {
@@ -1230,7 +1428,7 @@ async function runComputerUse(options) {
1230
1428
  result = { ok: true, action: 'takeover_stop', takeover: false };
1231
1429
  }
1232
1430
  else if (action === 'observe') {
1233
- result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight);
1431
+ result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight, options.sparseWaitMs, options.signal);
1234
1432
  }
1235
1433
  else if (action === 'app_list') {
1236
1434
  const first = await observeAppWindows(Number(options.maxChars || 60000));
@@ -1,4 +1,4 @@
1
- type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory';
1
+ type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory' | 'sparse';
2
2
  export interface PowerShellResult {
3
3
  ok: boolean;
4
4
  output: string;
@@ -6,6 +6,11 @@ export interface PowerShellResult {
6
6
  }
7
7
  export declare function runPersistentPowerShell(script: string, timeoutMs?: number, lane?: HelperLane): Promise<PowerShellResult>;
8
8
  export declare function computerUsePowerShellLaneReady(lane: HelperLane): boolean;
9
+ export declare function prepareComputerUsePowerShellLane(lane: HelperLane, timeoutMs: number): Promise<{
10
+ ready: boolean;
11
+ elapsedMs: number;
12
+ }>;
13
+ export declare function stopComputerUsePowerShellLane(lane: HelperLane): void;
9
14
  export declare function stopComputerUsePowerShellHost(): void;
10
15
  export {};
11
16
  //# sourceMappingURL=computerUsePowerShellHost.d.ts.map