newmark-agent 0.6.3 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.example.json +13 -3
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +53 -2
- package/dist/cli-discovery.js +3 -0
- package/dist/conversation-utility-host.bundle.cjs +3046 -1038
- package/dist/conversation-utility-host.js +94 -7
- package/dist/core/agent.d.ts +149 -12
- package/dist/core/agent.js +843 -176
- package/dist/core/agentKernelRunner.d.ts +7 -1
- package/dist/core/agentKernelRunner.js +171 -22
- package/dist/core/autoRouter.d.ts +49 -44
- package/dist/core/autoRouter.js +117 -302
- package/dist/core/branchIdentity.d.ts +29 -0
- package/dist/core/branchIdentity.js +54 -0
- package/dist/core/config.js +27 -11
- package/dist/core/continuation/contracts.d.ts +1 -1
- package/dist/core/continuation/store.d.ts +119 -1
- package/dist/core/continuation/store.js +337 -19
- package/dist/core/conversationKernel.d.ts +62 -8
- package/dist/core/conversationKernel.js +361 -47
- package/dist/core/conversationStateDocument.d.ts +18 -0
- package/dist/core/conversationStateDocument.js +88 -0
- package/dist/core/conversationVisualBudget.d.ts +74 -0
- package/dist/core/conversationVisualBudget.js +153 -0
- package/dist/core/electronUtilityAgentClient.d.ts +3 -2
- package/dist/core/electronUtilityAgentClient.js +32 -8
- package/dist/core/electronUtilityRuntimePool.d.ts +28 -3
- package/dist/core/electronUtilityRuntimePool.js +93 -24
- package/dist/core/hostRuntimeHooks.d.ts +23 -0
- package/dist/core/hostRuntimeHooks.js +17 -0
- package/dist/core/jevDecision.d.ts +144 -0
- package/dist/core/jevDecision.js +226 -0
- package/dist/core/memoryProbe.d.ts +24 -0
- package/dist/core/memoryProbe.js +104 -0
- package/dist/core/mobilePairing.d.ts +7 -1
- package/dist/core/mobilePairing.js +9 -1
- package/dist/core/performanceDiagnostics.d.ts +1 -1
- package/dist/core/routeDecisionValidator.d.ts +86 -0
- package/dist/core/routeDecisionValidator.js +249 -0
- package/dist/core/routeEligibility.d.ts +52 -0
- package/dist/core/routeEligibility.js +108 -0
- package/dist/core/runtimeMemoryBudget.d.ts +6 -0
- package/dist/core/runtimeMemoryBudget.js +12 -0
- package/dist/core/utilityAgentProtocol.d.ts +8 -10
- package/dist/core/utilityHostToolRouter.js +2 -0
- package/dist/core/visualDownscale.d.ts +6 -0
- package/dist/core/visualDownscale.js +137 -0
- package/dist/core/wslAgentClient.d.ts +1 -2
- package/dist/core/wslAgentClient.js +0 -8
- package/dist/core/wslAgentProtocol.d.ts +1 -10
- package/dist/core/wslAgentRuntimePool.d.ts +1 -3
- package/dist/core/wslAgentRuntimePool.js +17 -22
- package/dist/llm/provider.d.ts +3 -1
- package/dist/llm/provider.js +53 -9
- package/dist/main.js +108 -31
- package/dist/preload.js +3 -1
- package/dist/server.js +30 -18
- package/dist/tools/computerUse.d.ts +4 -0
- package/dist/tools/computerUse.js +202 -4
- package/dist/tools/computerUsePowerShellHost.d.ts +6 -1
- package/dist/tools/computerUsePowerShellHost.js +105 -31
- package/dist/tools/index.js +4 -1
- package/dist/tui/src/adapters/core-runtime-adapter.js +13 -1
- package/dist/tui/src/i18n.js +12 -1
- package/dist/tui/src/render.js +9 -2
- package/dist/tui/src/settings-schema.js +19 -0
- package/dist/tui/src/state.js +69 -1
- package/dist/ui/index.html +994 -277
- package/dist/ui/lucide-sprite.svg +0 -8
- package/dist/wsl-agent-host.bundle.cjs +2851 -1013
- package/dist/wsl-agent-host.js +0 -3
- package/package.json +38 -9
package/dist/server.js
CHANGED
|
@@ -59,6 +59,17 @@ const workEventCoalescer_1 = require("./core/workEventCoalescer");
|
|
|
59
59
|
const conversationListEvent_1 = require("./core/conversationListEvent");
|
|
60
60
|
const searchMcpPool_1 = require("./core/searchMcpPool");
|
|
61
61
|
const PORT = 47890;
|
|
62
|
+
/**
|
|
63
|
+
* Effective listen/advertised port. Production and every existing harness use
|
|
64
|
+
* `PORT` untouched (the isolated-server gates patch that literal); the
|
|
65
|
+
* `NEWMARK_MOBILE_PORT` override exists so a second, isolated dev/test instance
|
|
66
|
+
* can host a paired client without taking the production port away from the
|
|
67
|
+
* instance the user is already running. It stays in sync with the pairing
|
|
68
|
+
* URL/QR, which resolves the same override through `MOBILE_PORT`.
|
|
69
|
+
*/
|
|
70
|
+
const HOSTED_PORT = Number(process.env.NEWMARK_MOBILE_PORT) > 0
|
|
71
|
+
? Math.trunc(Number(process.env.NEWMARK_MOBILE_PORT))
|
|
72
|
+
: PORT;
|
|
62
73
|
let agent = null;
|
|
63
74
|
let automation = null;
|
|
64
75
|
let workspaceFileRouter = null;
|
|
@@ -104,7 +115,7 @@ function mobileAuthorized(req) {
|
|
|
104
115
|
if (bearer.startsWith('Bearer '))
|
|
105
116
|
return bearer.slice('Bearer '.length).trim() === mobileToken;
|
|
106
117
|
try {
|
|
107
|
-
const url = new URL(req.url || '/', `http://localhost:${
|
|
118
|
+
const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
|
|
108
119
|
const queryToken = url.searchParams.get('token') || '';
|
|
109
120
|
return queryToken === mobileToken;
|
|
110
121
|
}
|
|
@@ -223,7 +234,7 @@ async function handleMobileWorkspaceUpload(req, res) {
|
|
|
223
234
|
req.resume();
|
|
224
235
|
return;
|
|
225
236
|
}
|
|
226
|
-
const url = new URL(req.url || '/', `http://localhost:${
|
|
237
|
+
const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
|
|
227
238
|
const workspaceId = url.searchParams.get('workspaceId') || '';
|
|
228
239
|
const directory = url.searchParams.get('directory') || '';
|
|
229
240
|
const fileName = url.searchParams.get('fileName') || '';
|
|
@@ -392,13 +403,17 @@ function mobileScopedAgent(ws, conversationId) {
|
|
|
392
403
|
}
|
|
393
404
|
function mobileConversationRuntimeBusy(current, ws, conversationId) {
|
|
394
405
|
const scoped = mobileScopedRuntimes.get(mobileRuntimeKey(String(ws.id || ''), conversationId));
|
|
395
|
-
|
|
406
|
+
// Only an in-flight run blocks a mobile mutation. `goal_paused` (a goal the
|
|
407
|
+
// user paused) and `error` (a terminal failure) are not running work, yet
|
|
408
|
+
// treating them as busy refused archiving the conversation the user is
|
|
409
|
+
// actually looking at, which is the common case for a paused goal.
|
|
410
|
+
if (scoped && scoped.agent.status === 'working')
|
|
396
411
|
return true;
|
|
397
412
|
const currentWs = current.workspace.current;
|
|
398
413
|
return !!currentWs
|
|
399
414
|
&& path.resolve(currentWs.path) === path.resolve(ws.path)
|
|
400
415
|
&& current.activeConversationId === conversationId
|
|
401
|
-
&& current.status
|
|
416
|
+
&& current.status === 'working';
|
|
402
417
|
}
|
|
403
418
|
function handleMobileEvents(req, res) {
|
|
404
419
|
if (agent && !agent.config.getBool('remote', 'touch_enabled')) {
|
|
@@ -520,11 +535,7 @@ function applyConfigPatch(cfg) {
|
|
|
520
535
|
agent.config.set('models', 'fallback_on_unavailable', value === true || value === 'on');
|
|
521
536
|
break;
|
|
522
537
|
case 'switchTendency':
|
|
523
|
-
agent.config.set('models', 'auto_switch_preference', value);
|
|
524
|
-
break;
|
|
525
|
-
case 'clearLearnedAutoPreferences':
|
|
526
|
-
if (value === true)
|
|
527
|
-
agent.clearLearnedModelPreferences();
|
|
538
|
+
agent.config.set('models', 'auto_switch_preference', ['conservative', 'balanced', 'aggressive'].includes(String(value)) ? String(value) : 'balanced');
|
|
528
539
|
break;
|
|
529
540
|
case 'openAIApiMode':
|
|
530
541
|
agent.config.set('models', 'openai_api_mode', ['chat_stream', 'chat', 'responses'].includes(String(value)) ? value : 'chat_stream');
|
|
@@ -594,7 +605,7 @@ function jsonResponse(res, data, code = 200) {
|
|
|
594
605
|
res.end(body);
|
|
595
606
|
}
|
|
596
607
|
async function handleApi(req, res, body) {
|
|
597
|
-
const url = new URL(req.url || '/', `http://localhost:${
|
|
608
|
+
const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
|
|
598
609
|
const pathname = url.pathname;
|
|
599
610
|
if (pathname.startsWith('/api/mobile/')) {
|
|
600
611
|
if (agent && !agent.config.getBool('remote', 'touch_enabled')) {
|
|
@@ -616,6 +627,7 @@ async function handleApi(req, res, body) {
|
|
|
616
627
|
jsonResponse(res, {
|
|
617
628
|
mode: agent.mode, model: agent.modelSelectionValue(), modelLabel: agent.modelLabel(),
|
|
618
629
|
resolvedDeployment: agent.activeDeployment(), routeDecision: agent.lastRouteDecision,
|
|
630
|
+
jevDecisionModel: agent.jevDecisionModelState(),
|
|
619
631
|
intelligence: agent.intelligence, status: agent.status, goal: agent.goal,
|
|
620
632
|
models: agent.allModelNames(), inputMode: agent.inputMode,
|
|
621
633
|
conversationId: agent.activeConversationId,
|
|
@@ -1769,7 +1781,7 @@ function startServer(root, options = {}) {
|
|
|
1769
1781
|
res.end();
|
|
1770
1782
|
return;
|
|
1771
1783
|
}
|
|
1772
|
-
const url = new URL(req.url || '/', `http://localhost:${
|
|
1784
|
+
const url = new URL(req.url || '/', `http://localhost:${HOSTED_PORT}`);
|
|
1773
1785
|
if (url.pathname === '/api/mobile/events') {
|
|
1774
1786
|
handleMobileEvents(req, res);
|
|
1775
1787
|
return;
|
|
@@ -1824,15 +1836,15 @@ function startServer(root, options = {}) {
|
|
|
1824
1836
|
if (hostedServer === server)
|
|
1825
1837
|
hostedServer = null;
|
|
1826
1838
|
});
|
|
1827
|
-
server.listen(
|
|
1839
|
+
server.listen(HOSTED_PORT, bindHost, () => {
|
|
1828
1840
|
console.log(`\n Newmark Agent v1.0 - Server Mode`);
|
|
1829
|
-
console.log(` Bind: ${bindHost}:${
|
|
1830
|
-
console.log(` GUI: http://localhost:${
|
|
1841
|
+
console.log(` Bind: ${bindHost}:${HOSTED_PORT}`);
|
|
1842
|
+
console.log(` GUI: http://localhost:${HOSTED_PORT}`);
|
|
1831
1843
|
console.log(` Tailscale IPv4: ${tailscale || 'not detected'}`);
|
|
1832
1844
|
console.log(` LAN IPv4: ${lan || 'not detected'}`);
|
|
1833
|
-
console.log(` Mobile endpoint: http://${accessHost}:${
|
|
1845
|
+
console.log(` Mobile endpoint: http://${accessHost}:${HOSTED_PORT}/api/mobile/hello?token=<token>`);
|
|
1834
1846
|
console.log(` Mobile token file: ${tokenPath}`);
|
|
1835
|
-
console.log(` Mobile events (SSE): http://${accessHost}:${
|
|
1847
|
+
console.log(` Mobile events (SSE): http://${accessHost}:${HOSTED_PORT}/api/mobile/events?token=<token>`);
|
|
1836
1848
|
console.log(` Press Ctrl+C to stop\n`);
|
|
1837
1849
|
});
|
|
1838
1850
|
return server;
|
|
@@ -1868,7 +1880,7 @@ async function probeHostedServer(host) {
|
|
|
1868
1880
|
return await new Promise(resolve => {
|
|
1869
1881
|
const request = http.get({
|
|
1870
1882
|
hostname: host,
|
|
1871
|
-
port:
|
|
1883
|
+
port: HOSTED_PORT,
|
|
1872
1884
|
path: `/api/mobile/hello?token=${encodeURIComponent(mobileToken)}`,
|
|
1873
1885
|
timeout: 1800,
|
|
1874
1886
|
}, response => {
|
|
@@ -1935,7 +1947,7 @@ async function hostedServerStatus(enabled = true, probeHost = '') {
|
|
|
1935
1947
|
reachable,
|
|
1936
1948
|
state,
|
|
1937
1949
|
host,
|
|
1938
|
-
port:
|
|
1950
|
+
port: HOSTED_PORT,
|
|
1939
1951
|
error: state === 'error' ? (error || 'Mobile server is not reachable') : '',
|
|
1940
1952
|
checkedAt: new Date().toISOString(),
|
|
1941
1953
|
startedAt: hostedServerStartedAt,
|
|
@@ -30,6 +30,8 @@ export interface ComputerUseOptions {
|
|
|
30
30
|
allowEphemeralVisionImage?: boolean;
|
|
31
31
|
captureMaxWidth?: number;
|
|
32
32
|
captureMaxHeight?: number;
|
|
33
|
+
sparseWaitMs?: number;
|
|
34
|
+
signal?: AbortSignal;
|
|
33
35
|
gradientColors?: string[];
|
|
34
36
|
gradientSpeed?: number;
|
|
35
37
|
gradientWidth?: number;
|
|
@@ -50,10 +52,12 @@ declare function ephemeralScreenshotPath(kind: 'observe' | 'app', directory?: st
|
|
|
50
52
|
declare function cleanupStaleScreenshots(options?: ScreenshotCleanupOptions): {
|
|
51
53
|
removed: number;
|
|
52
54
|
};
|
|
55
|
+
declare function sparseMeasurementSampleBudget(waitMs: number): number;
|
|
53
56
|
declare function stringifyComputerUseResult(result: Record<string, unknown>): string;
|
|
54
57
|
export declare const computerUseInternals: {
|
|
55
58
|
cleanupStaleScreenshots: typeof cleanupStaleScreenshots;
|
|
56
59
|
ephemeralScreenshotPath: typeof ephemeralScreenshotPath;
|
|
60
|
+
sparseMeasurementSampleBudget: typeof sparseMeasurementSampleBudget;
|
|
57
61
|
stringifyComputerUseResult: typeof stringifyComputerUseResult;
|
|
58
62
|
};
|
|
59
63
|
export declare function runComputerUse(options: ComputerUseOptions): Promise<string>;
|
|
@@ -68,6 +68,7 @@ const DEFAULT_CAPTURE_MAX_HEIGHT = 960;
|
|
|
68
68
|
const MIN_CAPTURE_MAX_WIDTH = 320;
|
|
69
69
|
const MIN_CAPTURE_MAX_HEIGHT = 240;
|
|
70
70
|
const MAX_CAPTURE_DIMENSION = 2048;
|
|
71
|
+
const MAX_SPARSE_WAIT_MS = 5000;
|
|
71
72
|
let lastScreenshotCleanupAt = 0;
|
|
72
73
|
function ephemeralScreenshotPath(kind, directory = tempScreenshotDir(), createdAt = Date.now(), ownerPid = process.pid, suffix = crypto.randomBytes(4).toString('hex')) {
|
|
73
74
|
const pid = Math.max(1, Math.floor(Number(ownerPid) || process.pid));
|
|
@@ -210,6 +211,172 @@ function jpegCaptureScript(outPath, boundsScript, requestedMaxWidth, requestedMa
|
|
|
210
211
|
'Write-Output (@{ ok=$true; left=$x; top=$y; width=$w; height=$h; capture_max_width=$maxWidth; capture_max_height=$maxHeight; image_width=$imageWidth; image_height=$imageHeight; image_bytes=$fileSize; image_quality=$qualityUsed; image_available=$imageAvailable; image_mime="image/jpeg" } | ConvertTo-Json -Compress)',
|
|
211
212
|
].join('\r\n');
|
|
212
213
|
}
|
|
214
|
+
function boundedSparseWait(value) {
|
|
215
|
+
const requested = Number(value);
|
|
216
|
+
if (!Number.isFinite(requested) || requested <= 0)
|
|
217
|
+
return 0;
|
|
218
|
+
return Math.min(MAX_SPARSE_WAIT_MS, Math.floor(requested));
|
|
219
|
+
}
|
|
220
|
+
function sparseMeasurementSampleBudget(waitMs) {
|
|
221
|
+
const boundedWait = Math.max(1, Math.floor(Number(waitMs) || 1));
|
|
222
|
+
// A live PowerShell process handshake does not mean its first GDI capture,
|
|
223
|
+
// script execution and IPC response are warm. Reserve completion time on
|
|
224
|
+
// every call so the lane's own wait-elapsed result beats the outer fallback.
|
|
225
|
+
const completionMarginMs = Math.min(Math.max(0, boundedWait - 1), Math.min(1500, Math.max(500, Math.floor(boundedWait * 0.3))));
|
|
226
|
+
return Math.max(1, boundedWait - completionMarginMs);
|
|
227
|
+
}
|
|
228
|
+
function sparseMeasurementScript(waitMs) {
|
|
229
|
+
const boundedWait = Math.min(MAX_SPARSE_WAIT_MS, Math.max(1, Math.floor(waitMs)));
|
|
230
|
+
return [
|
|
231
|
+
'$ErrorActionPreference = "Stop"',
|
|
232
|
+
'$bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen',
|
|
233
|
+
'$x=[int]$bounds.Left; $y=[int]$bounds.Top; $w=[int]$bounds.Width; $h=[int]$bounds.Height',
|
|
234
|
+
'if ($w -le 0 -or $h -le 0) { throw "Virtual screen geometry is unavailable." }',
|
|
235
|
+
'$gridWidth=32; $gridHeight=18; $pollMs=200; $cellThreshold=24; $strongThreshold=64',
|
|
236
|
+
'$watch = [System.Diagnostics.Stopwatch]::StartNew(); $sampleCount=0; $changedCells=0; $triggered=$false; $geometryChanged=$false',
|
|
237
|
+
'$thumbnail = [NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight)',
|
|
238
|
+
'$baseline = [byte[]]::new($gridWidth * $gridHeight)',
|
|
239
|
+
'try {',
|
|
240
|
+
' for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $baseline[$index]=$thumbnail.GetLuma($column,$row) } }',
|
|
241
|
+
' $sampleCount=1',
|
|
242
|
+
` while ($watch.ElapsedMilliseconds -lt ${boundedWait}) {`,
|
|
243
|
+
` $remaining=[Math]::Max(1, ${boundedWait} - [int]$watch.ElapsedMilliseconds); Start-Sleep -Milliseconds ([Math]::Min($pollMs,$remaining))`,
|
|
244
|
+
' $nextBounds=[System.Windows.Forms.SystemInformation]::VirtualScreen',
|
|
245
|
+
' if ($nextBounds.Left -ne $x -or $nextBounds.Top -ne $y -or $nextBounds.Width -ne $w -or $nextBounds.Height -ne $h) { $geometryChanged=$true; break }',
|
|
246
|
+
' $nextThumbnail=[NewmarkSparseCapture]::CaptureScaled($x,$y,$w,$h,$gridWidth,$gridHeight); $thumbnail.Dispose(); $thumbnail=$nextThumbnail',
|
|
247
|
+
' $changedCells=0; $strongCells=0',
|
|
248
|
+
' for ($row=0; $row -lt $gridHeight; $row++) { for ($column=0; $column -lt $gridWidth; $column++) { $index=$row*$gridWidth+$column; $luma=[int]$thumbnail.GetLuma($column,$row); $delta=[Math]::Abs($luma-$baseline[$index]); if ($delta -ge $cellThreshold) { $changedCells++ }; if ($delta -ge $strongThreshold) { $strongCells++ } } }',
|
|
249
|
+
' $sampleCount++',
|
|
250
|
+
' if ($changedCells -ge 4 -or $strongCells -ge 2) { $triggered=$true; break }',
|
|
251
|
+
' }',
|
|
252
|
+
].join('\r\n') + [
|
|
253
|
+
'} finally { $watch.Stop(); $thumbnail.Dispose() }',
|
|
254
|
+
'$reason = if ($geometryChanged) { "screen_geometry_changed" } elseif ($triggered) { "coarse_change" } else { "wait_elapsed" }',
|
|
255
|
+
'Write-Output (@{ ok=$true; triggered=$triggered; geometry_changed=$geometryChanged; sample_count=$sampleCount; changed_cells=$changedCells; elapsed_ms=[int]$watch.ElapsedMilliseconds; reason=$reason } | ConvertTo-Json -Compress)',
|
|
256
|
+
].join('\r\n');
|
|
257
|
+
}
|
|
258
|
+
function sparseAbortError(signal) {
|
|
259
|
+
if (signal.reason instanceof Error)
|
|
260
|
+
return signal.reason;
|
|
261
|
+
return new Error(signal.reason ? String(signal.reason) : 'Computer Use sparse observation was cancelled.');
|
|
262
|
+
}
|
|
263
|
+
async function waitForSparseChange(waitMs, signal) {
|
|
264
|
+
if (signal?.aborted)
|
|
265
|
+
throw sparseAbortError(signal);
|
|
266
|
+
return await new Promise((resolve, reject) => {
|
|
267
|
+
let settled = false;
|
|
268
|
+
let phase = 'preparing';
|
|
269
|
+
const deadlineAt = Date.now() + waitMs;
|
|
270
|
+
let deadlineTimer;
|
|
271
|
+
const cleanup = () => {
|
|
272
|
+
if (deadlineTimer)
|
|
273
|
+
clearTimeout(deadlineTimer);
|
|
274
|
+
signal?.removeEventListener('abort', onAbort);
|
|
275
|
+
};
|
|
276
|
+
const onAbort = () => {
|
|
277
|
+
if (settled)
|
|
278
|
+
return;
|
|
279
|
+
settled = true;
|
|
280
|
+
cleanup();
|
|
281
|
+
(0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
|
|
282
|
+
reject(sparseAbortError(signal));
|
|
283
|
+
};
|
|
284
|
+
deadlineTimer = setTimeout(() => {
|
|
285
|
+
if (settled)
|
|
286
|
+
return;
|
|
287
|
+
settled = true;
|
|
288
|
+
cleanup();
|
|
289
|
+
// A normal wait deadline during helper startup must not kill the shared
|
|
290
|
+
// lane. Its initialization can finish in the background and serve the
|
|
291
|
+
// next observation. Once a sample has actually been submitted, however,
|
|
292
|
+
// stop a worker that failed to return before its bounded sampling window.
|
|
293
|
+
if (phase === 'sampling')
|
|
294
|
+
(0, computerUsePowerShellHost_1.stopComputerUsePowerShellLane)('sparse');
|
|
295
|
+
const error = phase === 'preparing'
|
|
296
|
+
? 'Sparse helper did not become ready before the observation deadline; its startup continues for the next observation.'
|
|
297
|
+
: 'Sparse sample did not return before the observation deadline; continuing with a fresh full observation.';
|
|
298
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error });
|
|
299
|
+
}, waitMs);
|
|
300
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
301
|
+
if (signal?.aborted) {
|
|
302
|
+
onAbort();
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
const parseResult = (result) => {
|
|
306
|
+
if (settled)
|
|
307
|
+
return;
|
|
308
|
+
settled = true;
|
|
309
|
+
cleanup();
|
|
310
|
+
if (!result.ok) {
|
|
311
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: result.output.slice(0, 500) });
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
try {
|
|
315
|
+
const parsed = JSON.parse(result.output);
|
|
316
|
+
resolve({
|
|
317
|
+
ok: parsed.ok === true,
|
|
318
|
+
triggered: parsed.triggered === true,
|
|
319
|
+
geometryChanged: parsed.geometry_changed === true,
|
|
320
|
+
sampleCount: Math.max(0, Number(parsed.sample_count) || 0),
|
|
321
|
+
changedCells: Math.max(0, Number(parsed.changed_cells) || 0),
|
|
322
|
+
elapsedMs: Math.max(0, Number(parsed.elapsed_ms) || result.elapsedMs),
|
|
323
|
+
reason: String(parsed.reason || 'sampling_failed'),
|
|
324
|
+
...(parsed.ok === true ? {} : { error: 'Sparse measurement returned an invalid result.' }),
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: result.elapsedMs, reason: 'sampling_failed', error: 'Sparse measurement result was not valid JSON.' });
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
const prepareAndSample = async () => {
|
|
332
|
+
const preparation = await (0, computerUsePowerShellHost_1.prepareComputerUsePowerShellLane)('sparse', waitMs);
|
|
333
|
+
if (settled)
|
|
334
|
+
return;
|
|
335
|
+
if (!preparation.ready) {
|
|
336
|
+
settled = true;
|
|
337
|
+
cleanup();
|
|
338
|
+
resolve({
|
|
339
|
+
ok: false,
|
|
340
|
+
triggered: false,
|
|
341
|
+
geometryChanged: false,
|
|
342
|
+
sampleCount: null,
|
|
343
|
+
changedCells: null,
|
|
344
|
+
elapsedMs: Math.max(preparation.elapsedMs, Date.now() - (deadlineAt - waitMs)),
|
|
345
|
+
reason: 'wait_elapsed',
|
|
346
|
+
error: 'Sparse helper initialization exceeded the observation deadline; the lane remains available to finish startup.',
|
|
347
|
+
});
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const remainingMs = Math.max(0, deadlineAt - Date.now());
|
|
351
|
+
if (remainingMs < 100) {
|
|
352
|
+
settled = true;
|
|
353
|
+
cleanup();
|
|
354
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: waitMs, reason: 'wait_elapsed', error: 'Sparse helper became ready without enough time left to capture a bounded sample.' });
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
const sampleBudgetMs = sparseMeasurementSampleBudget(remainingMs);
|
|
358
|
+
phase = 'sampling';
|
|
359
|
+
try {
|
|
360
|
+
const result = await (0, computerUsePowerShellHost_1.runPersistentPowerShell)(sparseMeasurementScript(sampleBudgetMs), remainingMs, 'sparse');
|
|
361
|
+
parseResult(result);
|
|
362
|
+
}
|
|
363
|
+
catch (error) {
|
|
364
|
+
if (settled)
|
|
365
|
+
return;
|
|
366
|
+
settled = true;
|
|
367
|
+
cleanup();
|
|
368
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
void prepareAndSample().catch(error => {
|
|
372
|
+
if (settled)
|
|
373
|
+
return;
|
|
374
|
+
settled = true;
|
|
375
|
+
cleanup();
|
|
376
|
+
resolve({ ok: false, triggered: false, geometryChanged: false, sampleCount: null, changedCells: null, elapsedMs: Date.now() - (deadlineAt - waitMs), reason: 'sampling_failed', error: error instanceof Error ? error.message.slice(0, 500) : String(error).slice(0, 500) });
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
}
|
|
213
380
|
function gradientPalette(_input) {
|
|
214
381
|
return ['#000000', '#ffffff', '#000000', '#ffffff'];
|
|
215
382
|
}
|
|
@@ -901,8 +1068,28 @@ function resolveTarget(targetId, workspacePath, ownerId) {
|
|
|
901
1068
|
return { x: 0, y: 0, error: `target_id not found in latest observation: ${id}.` };
|
|
902
1069
|
return { x: target.center.x, y: target.center.y, target };
|
|
903
1070
|
}
|
|
904
|
-
async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight) {
|
|
1071
|
+
async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, includeRawUi = false, captureMaxWidth, captureMaxHeight, sparseWaitMs, signal) {
|
|
905
1072
|
const totalStarted = Date.now();
|
|
1073
|
+
const sparseDurationMs = boundedSparseWait(sparseWaitMs);
|
|
1074
|
+
const sparseStarted = Date.now();
|
|
1075
|
+
const sparse = sparseDurationMs > 0 ? await waitForSparseChange(sparseDurationMs, signal) : undefined;
|
|
1076
|
+
const sparseWallMs = sparse ? Date.now() - sparseStarted : 0;
|
|
1077
|
+
if (signal?.aborted)
|
|
1078
|
+
throw sparseAbortError(signal);
|
|
1079
|
+
const sparseDetails = sparse ? {
|
|
1080
|
+
method: 'coarse-grid-luminance-change-wakeup',
|
|
1081
|
+
grid_width: 32,
|
|
1082
|
+
grid_height: 18,
|
|
1083
|
+
sample_count: sparse.sampleCount,
|
|
1084
|
+
changed_cells: sparse.changedCells,
|
|
1085
|
+
triggered: sparse.triggered,
|
|
1086
|
+
reason: sparse.reason,
|
|
1087
|
+
elapsed_ms: sparse.elapsedMs,
|
|
1088
|
+
wall_elapsed_ms: sparseWallMs,
|
|
1089
|
+
fresh_full_observation_follows: true,
|
|
1090
|
+
interpretation: 'Wake signal only. A timeout or no detected change is not evidence that the screen stayed unchanged; small, brief, or color-only changes can be missed.',
|
|
1091
|
+
...(sparse.error ? { warning: sparse.error } : {}),
|
|
1092
|
+
} : undefined;
|
|
906
1093
|
const outPath = ephemeralScreenshotPath('observe');
|
|
907
1094
|
let retainedForVision = false;
|
|
908
1095
|
try {
|
|
@@ -916,14 +1103,16 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
|
|
|
916
1103
|
observeUiAutomation(30000, foreground.handle, 750, 'uia_advisory'),
|
|
917
1104
|
observeAppWindows(30000, 750, 'windows_advisory'),
|
|
918
1105
|
]);
|
|
1106
|
+
if (signal?.aborted)
|
|
1107
|
+
throw sparseAbortError(signal);
|
|
919
1108
|
if (!result.ok)
|
|
920
|
-
return captureFailure('observe');
|
|
1109
|
+
return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
|
|
921
1110
|
let parsed = {};
|
|
922
1111
|
try {
|
|
923
1112
|
parsed = JSON.parse(result.output);
|
|
924
1113
|
}
|
|
925
1114
|
catch {
|
|
926
|
-
return captureFailure('observe');
|
|
1115
|
+
return { ...captureFailure('observe'), ...(sparseDetails ? { sparse_measurement: sparseDetails } : {}) };
|
|
927
1116
|
}
|
|
928
1117
|
const screenWidth = Number(parsed.width || 1);
|
|
929
1118
|
const screenHeight = Number(parsed.height || 1);
|
|
@@ -947,6 +1136,7 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
|
|
|
947
1136
|
scene_generation: generation,
|
|
948
1137
|
scene_summary: buildSceneSummary(objects, screenWidth, screenHeight),
|
|
949
1138
|
objects: compactSemanticObjects(objects),
|
|
1139
|
+
...(sparseDetails ? { sparse_measurement: sparseDetails } : {}),
|
|
950
1140
|
object_usage: 'Prefer target_id from scene_summary.high_priority_objects or objects for click/move/scroll when available; it resolves to the latest observed center coordinate and keeps actions tied to a semantic UI object.',
|
|
951
1141
|
warning: foreground.error || ui.error || apps.error || undefined,
|
|
952
1142
|
note: 'Inspired by capture-parse-decide-act and MCP-style Computer Control agents: screenshot capture plus native Windows UI Automation text/control bounding boxes, semantic target objects, stable target IDs, high-priority object summaries, normalized bboxes, allowed actions, and risk hints. It does not copy third-party code.',
|
|
@@ -959,6 +1149,13 @@ async function screenshot(workspacePath, ownerId, allowEphemeralVisionImage, inc
|
|
|
959
1149
|
total_ms: Date.now() - totalStarted,
|
|
960
1150
|
element_count: ui.elements.length,
|
|
961
1151
|
visited_count: ui.visited,
|
|
1152
|
+
...(sparse ? {
|
|
1153
|
+
sparse_sampling_ms: sparse.elapsedMs,
|
|
1154
|
+
sparse_wall_ms: sparseWallMs,
|
|
1155
|
+
sparse_configured_wait_ms: sparseDurationMs,
|
|
1156
|
+
sparse_sample_count: sparse.sampleCount,
|
|
1157
|
+
sparse_triggered: sparse.triggered,
|
|
1158
|
+
} : {}),
|
|
962
1159
|
},
|
|
963
1160
|
...parsed,
|
|
964
1161
|
};
|
|
@@ -1207,6 +1404,7 @@ function stringifyComputerUseResult(result) {
|
|
|
1207
1404
|
exports.computerUseInternals = {
|
|
1208
1405
|
cleanupStaleScreenshots,
|
|
1209
1406
|
ephemeralScreenshotPath,
|
|
1407
|
+
sparseMeasurementSampleBudget,
|
|
1210
1408
|
stringifyComputerUseResult,
|
|
1211
1409
|
};
|
|
1212
1410
|
async function runComputerUse(options) {
|
|
@@ -1230,7 +1428,7 @@ async function runComputerUse(options) {
|
|
|
1230
1428
|
result = { ok: true, action: 'takeover_stop', takeover: false };
|
|
1231
1429
|
}
|
|
1232
1430
|
else if (action === 'observe') {
|
|
1233
|
-
result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight);
|
|
1431
|
+
result = await screenshot(options.workspacePath, ownerId, !!options.allowEphemeralVisionImage, !!options.includeRawUi, options.captureMaxWidth, options.captureMaxHeight, options.sparseWaitMs, options.signal);
|
|
1234
1432
|
}
|
|
1235
1433
|
else if (action === 'app_list') {
|
|
1236
1434
|
const first = await observeAppWindows(Number(options.maxChars || 60000));
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory';
|
|
1
|
+
type HelperLane = 'action' | 'uia' | 'windows' | 'uia_advisory' | 'windows_advisory' | 'sparse';
|
|
2
2
|
export interface PowerShellResult {
|
|
3
3
|
ok: boolean;
|
|
4
4
|
output: string;
|
|
@@ -6,6 +6,11 @@ export interface PowerShellResult {
|
|
|
6
6
|
}
|
|
7
7
|
export declare function runPersistentPowerShell(script: string, timeoutMs?: number, lane?: HelperLane): Promise<PowerShellResult>;
|
|
8
8
|
export declare function computerUsePowerShellLaneReady(lane: HelperLane): boolean;
|
|
9
|
+
export declare function prepareComputerUsePowerShellLane(lane: HelperLane, timeoutMs: number): Promise<{
|
|
10
|
+
ready: boolean;
|
|
11
|
+
elapsedMs: number;
|
|
12
|
+
}>;
|
|
13
|
+
export declare function stopComputerUsePowerShellLane(lane: HelperLane): void;
|
|
9
14
|
export declare function stopComputerUsePowerShellHost(): void;
|
|
10
15
|
export {};
|
|
11
16
|
//# sourceMappingURL=computerUsePowerShellHost.d.ts.map
|