newmark-agent 0.6.3 → 0.6.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/config.example.json +13 -3
  2. package/dist/cli-commands.d.ts +1 -1
  3. package/dist/cli-commands.js +53 -2
  4. package/dist/cli-discovery.js +3 -0
  5. package/dist/conversation-utility-host.bundle.cjs +3046 -1038
  6. package/dist/conversation-utility-host.js +94 -7
  7. package/dist/core/agent.d.ts +149 -12
  8. package/dist/core/agent.js +843 -176
  9. package/dist/core/agentKernelRunner.d.ts +7 -1
  10. package/dist/core/agentKernelRunner.js +171 -22
  11. package/dist/core/autoRouter.d.ts +49 -44
  12. package/dist/core/autoRouter.js +117 -302
  13. package/dist/core/branchIdentity.d.ts +29 -0
  14. package/dist/core/branchIdentity.js +54 -0
  15. package/dist/core/config.js +27 -11
  16. package/dist/core/continuation/contracts.d.ts +1 -1
  17. package/dist/core/continuation/store.d.ts +119 -1
  18. package/dist/core/continuation/store.js +337 -19
  19. package/dist/core/conversationKernel.d.ts +62 -8
  20. package/dist/core/conversationKernel.js +361 -47
  21. package/dist/core/conversationStateDocument.d.ts +18 -0
  22. package/dist/core/conversationStateDocument.js +88 -0
  23. package/dist/core/conversationVisualBudget.d.ts +74 -0
  24. package/dist/core/conversationVisualBudget.js +153 -0
  25. package/dist/core/electronUtilityAgentClient.d.ts +3 -2
  26. package/dist/core/electronUtilityAgentClient.js +32 -8
  27. package/dist/core/electronUtilityRuntimePool.d.ts +28 -3
  28. package/dist/core/electronUtilityRuntimePool.js +93 -24
  29. package/dist/core/hostRuntimeHooks.d.ts +23 -0
  30. package/dist/core/hostRuntimeHooks.js +17 -0
  31. package/dist/core/jevDecision.d.ts +144 -0
  32. package/dist/core/jevDecision.js +226 -0
  33. package/dist/core/memoryProbe.d.ts +24 -0
  34. package/dist/core/memoryProbe.js +104 -0
  35. package/dist/core/mobilePairing.d.ts +7 -1
  36. package/dist/core/mobilePairing.js +9 -1
  37. package/dist/core/performanceDiagnostics.d.ts +1 -1
  38. package/dist/core/routeDecisionValidator.d.ts +86 -0
  39. package/dist/core/routeDecisionValidator.js +249 -0
  40. package/dist/core/routeEligibility.d.ts +52 -0
  41. package/dist/core/routeEligibility.js +108 -0
  42. package/dist/core/runtimeMemoryBudget.d.ts +6 -0
  43. package/dist/core/runtimeMemoryBudget.js +12 -0
  44. package/dist/core/utilityAgentProtocol.d.ts +8 -10
  45. package/dist/core/utilityHostToolRouter.js +2 -0
  46. package/dist/core/visualDownscale.d.ts +6 -0
  47. package/dist/core/visualDownscale.js +137 -0
  48. package/dist/core/wslAgentClient.d.ts +1 -2
  49. package/dist/core/wslAgentClient.js +0 -8
  50. package/dist/core/wslAgentProtocol.d.ts +1 -10
  51. package/dist/core/wslAgentRuntimePool.d.ts +1 -3
  52. package/dist/core/wslAgentRuntimePool.js +17 -22
  53. package/dist/llm/provider.d.ts +3 -1
  54. package/dist/llm/provider.js +53 -9
  55. package/dist/main.js +108 -31
  56. package/dist/preload.js +3 -1
  57. package/dist/server.js +30 -18
  58. package/dist/tools/computerUse.d.ts +4 -0
  59. package/dist/tools/computerUse.js +202 -4
  60. package/dist/tools/computerUsePowerShellHost.d.ts +6 -1
  61. package/dist/tools/computerUsePowerShellHost.js +105 -31
  62. package/dist/tools/index.js +4 -1
  63. package/dist/tui/src/adapters/core-runtime-adapter.js +13 -1
  64. package/dist/tui/src/i18n.js +12 -1
  65. package/dist/tui/src/render.js +9 -2
  66. package/dist/tui/src/settings-schema.js +19 -0
  67. package/dist/tui/src/state.js +69 -1
  68. package/dist/ui/index.html +994 -277
  69. package/dist/ui/lucide-sprite.svg +0 -8
  70. package/dist/wsl-agent-host.bundle.cjs +2851 -1013
  71. package/dist/wsl-agent-host.js +0 -3
  72. package/package.json +38 -9
@@ -35,48 +35,90 @@ var __importStar = (this && this.__importStar) || (function () {
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.runPersistentPowerShell = runPersistentPowerShell;
37
37
  exports.computerUsePowerShellLaneReady = computerUsePowerShellLaneReady;
38
+ exports.prepareComputerUsePowerShellLane = prepareComputerUsePowerShellLane;
39
+ exports.stopComputerUsePowerShellLane = stopComputerUsePowerShellLane;
38
40
  exports.stopComputerUsePowerShellHost = stopComputerUsePowerShellHost;
39
41
  const child_process_1 = require("child_process");
40
42
  const crypto = __importStar(require("crypto"));
41
43
  const readline = __importStar(require("readline"));
42
- const HOST_SCRIPT = [
43
- '[Console]::InputEncoding = [System.Text.UTF8Encoding]::new($false)',
44
- '[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
45
- '$OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
46
- 'Add-Type -AssemblyName System.Windows.Forms',
47
- 'Add-Type -AssemblyName System.Drawing',
48
- 'Add-Type -AssemblyName UIAutomationClient',
49
- 'Add-Type -AssemblyName UIAutomationTypes',
50
- `if (-not ("NewmarkComputerUseNative" -as [type])) { Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class NewmarkComputerUseNative { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, int dwData, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); }' }`,
51
- 'while (($line = [Console]::In.ReadLine()) -ne $null) {',
52
- ' $id = ""',
53
- ' $timer = [System.Diagnostics.Stopwatch]::StartNew()',
54
- ' try {',
55
- ' $request = $line | ConvertFrom-Json',
56
- ' $id = [string]$request.id',
57
- ' $source = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String([string]$request.script))',
58
- ' $previousPreference = $ErrorActionPreference',
59
- ' $ErrorActionPreference = "Stop"',
60
- ' try { $output = (& ([scriptblock]::Create($source)) 2>&1 | Out-String -Width 1048576).Trim() } finally { $ErrorActionPreference = $previousPreference }',
61
- ' $response = @{ id=$id; ok=$true; output=$output; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
62
- ' } catch {',
63
- ' $response = @{ id=$id; ok=$false; output=[string]$_; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
64
- ' }',
65
- ' [Console]::Out.WriteLine(($response | ConvertTo-Json -Compress -Depth 6))',
66
- ' [Console]::Out.Flush()',
67
- '}',
68
- ].join('; ');
44
+ function createHostScript(includeUiAutomation) {
45
+ return [
46
+ '[Console]::InputEncoding = [System.Text.UTF8Encoding]::new($false)',
47
+ '[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
48
+ '$OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
49
+ 'Add-Type -AssemblyName System.Windows.Forms',
50
+ 'Add-Type -AssemblyName System.Drawing',
51
+ ...(includeUiAutomation ? ['Add-Type -AssemblyName UIAutomationClient', 'Add-Type -AssemblyName UIAutomationTypes'] : []),
52
+ ...(includeUiAutomation ? [`if (-not ("NewmarkComputerUseNative" -as [type])) { Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class NewmarkComputerUseNative { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, int dwData, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); }' }`] : []),
53
+ ...(!includeUiAutomation ? [`if (-not ("NewmarkSparseCapture" -as [type])) { Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class NewmarkSparseCapture { [StructLayout(LayoutKind.Sequential)] struct BitmapInfoHeader { public uint Size; public int Width; public int Height; public ushort Planes; public ushort BitCount; public uint Compression; public uint SizeImage; public int XPelsPerMeter; public int YPelsPerMeter; public uint ColorsUsed; public uint ColorsImportant; } [StructLayout(LayoutKind.Sequential)] struct BitmapInfo { public BitmapInfoHeader Header; public uint Colors; } [DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr hWnd); [DllImport("user32.dll")] static extern int ReleaseDC(IntPtr hWnd, IntPtr hdc); [DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr hdc); [DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr hdc); [DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr hdc, ref BitmapInfo info, uint usage, out IntPtr bits, IntPtr section, uint offset); [DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr hdc, IntPtr obj); [DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr obj); [DllImport("gdi32.dll")] static extern int SetStretchBltMode(IntPtr hdc, int mode); [DllImport("gdi32.dll")] static extern bool SetBrushOrgEx(IntPtr hdc, int x, int y, IntPtr oldOrigin); [DllImport("gdi32.dll", SetLastError=true)] static extern bool StretchBlt(IntPtr dst, int dx, int dy, int dw, int dh, IntPtr src, int sx, int sy, int sw, int sh, int operation); [DllImport("gdi32.dll")] static extern bool GdiFlush(); public sealed class Capture : IDisposable { private int width; private byte[] pixels; internal Capture(int w, byte[] p) { width=w; pixels=p; } public byte GetLuma(int x, int y) { int offset=(y*width+x)*4; return (byte)Math.Round((pixels[offset+2]*0.299)+(pixels[offset+1]*0.587)+(pixels[offset]*0.114)); } public void Dispose() { pixels=null; } } public static Capture CaptureScaled(int x, int y, int width, int height, int targetWidth, int targetHeight) { IntPtr screen=GetDC(IntPtr.Zero); if (screen == IntPtr.Zero) throw new InvalidOperationException("Could not read desktop DC."); IntPtr memory=IntPtr.Zero; IntPtr bitmap=IntPtr.Zero; IntPtr previous=IntPtr.Zero; IntPtr bits=IntPtr.Zero; try { memory=CreateCompatibleDC(screen); BitmapInfo info=new BitmapInfo(); info.Header.Size=(uint)System.Runtime.InteropServices.Marshal.SizeOf(typeof(BitmapInfoHeader)); info.Header.Width=targetWidth; info.Header.Height=-targetHeight; info.Header.Planes=1; info.Header.BitCount=32; info.Header.Compression=0; info.Header.SizeImage=(uint)(targetWidth*targetHeight*4); bitmap=CreateDIBSection(screen,ref info,0,out bits,IntPtr.Zero,0); if (memory == IntPtr.Zero || bitmap == IntPtr.Zero || bits == IntPtr.Zero) throw new InvalidOperationException("Could not allocate sparse DIB section."); previous=SelectObject(memory,bitmap); if (previous == IntPtr.Zero) throw new InvalidOperationException("Could not select sparse DIB section."); if (SetStretchBltMode(memory,4)==0 || !SetBrushOrgEx(memory,0,0,IntPtr.Zero)) throw new InvalidOperationException("Could not configure sparse capture scaling."); if (!StretchBlt(memory,0,0,targetWidth,targetHeight,screen,x,y,width,height,0x00CC0020)) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); if (!GdiFlush()) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); byte[] pixels=new byte[targetWidth*targetHeight*4]; Marshal.Copy(bits,pixels,0,pixels.Length); return new Capture(targetWidth,pixels); } finally { if (memory != IntPtr.Zero && previous != IntPtr.Zero) SelectObject(memory,previous); if (bitmap != IntPtr.Zero) DeleteObject(bitmap); if (memory != IntPtr.Zero) DeleteDC(memory); ReleaseDC(IntPtr.Zero,screen); } } }' }`] : []),
54
+ '$hostReady = @{ id="__newmark_ready__"; ready=$true }; [Console]::Out.WriteLine(($hostReady | ConvertTo-Json -Compress)); [Console]::Out.Flush()',
55
+ 'while (($line = [Console]::In.ReadLine()) -ne $null) {',
56
+ ' $id = ""',
57
+ ' $timer = [System.Diagnostics.Stopwatch]::StartNew()',
58
+ ' try {',
59
+ ' $request = $line | ConvertFrom-Json',
60
+ ' $id = [string]$request.id',
61
+ ' $source = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String([string]$request.script))',
62
+ ' $previousPreference = $ErrorActionPreference',
63
+ ' $ErrorActionPreference = "Stop"',
64
+ ' try { $output = (& ([scriptblock]::Create($source)) 2>&1 | Out-String -Width 1048576).Trim() } finally { $ErrorActionPreference = $previousPreference }',
65
+ ' $response = @{ id=$id; ok=$true; output=$output; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
66
+ ' } catch {',
67
+ ' $response = @{ id=$id; ok=$false; output=[string]$_; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
68
+ ' }',
69
+ ' [Console]::Out.WriteLine(($response | ConvertTo-Json -Compress -Depth 6))',
70
+ ' [Console]::Out.Flush()',
71
+ '}',
72
+ ].join('; ');
73
+ }
74
+ const HOST_SCRIPT = createHostScript(true);
75
+ const SPARSE_HOST_SCRIPT = createHostScript(false);
69
76
  class PowerShellWorker {
70
77
  lane;
71
78
  child = null;
79
+ readyChild = null;
72
80
  pending = new Map();
81
+ readinessWaiters = new Set();
73
82
  restarting = false;
74
83
  unavailableUntil = 0;
75
84
  constructor(lane) {
76
85
  this.lane = lane;
77
86
  }
78
87
  ready() {
79
- return Date.now() >= this.unavailableUntil && !!this.child && !this.child.killed && this.child.exitCode === null;
88
+ return Date.now() >= this.unavailableUntil
89
+ && !!this.child
90
+ && !this.child.killed
91
+ && this.child.exitCode === null
92
+ && this.readyChild === this.child;
93
+ }
94
+ async prepare(timeoutMs) {
95
+ const startedAt = Date.now();
96
+ if (Date.now() < this.unavailableUntil)
97
+ return { ready: false, elapsedMs: 0 };
98
+ const child = this.ensureChild();
99
+ if (this.readyChild === child)
100
+ return { ready: true, elapsedMs: Date.now() - startedAt };
101
+ const ready = await new Promise(resolve => {
102
+ let settled = false;
103
+ let waiter;
104
+ const finish = (value) => {
105
+ if (settled)
106
+ return;
107
+ settled = true;
108
+ clearTimeout(waiter.timer);
109
+ this.readinessWaiters.delete(waiter);
110
+ resolve(value);
111
+ };
112
+ waiter = {
113
+ child,
114
+ resolve: finish,
115
+ timer: setTimeout(() => finish(false), Math.max(0, Math.floor(timeoutMs))),
116
+ };
117
+ this.readinessWaiters.add(waiter);
118
+ if (this.readyChild === child)
119
+ finish(true);
120
+ });
121
+ return { ready, elapsedMs: Date.now() - startedAt };
80
122
  }
81
123
  async run(script, timeoutMs) {
82
124
  const startedAt = Date.now();
@@ -114,6 +156,9 @@ class PowerShellWorker {
114
156
  stop() {
115
157
  const child = this.child;
116
158
  this.child = null;
159
+ this.readyChild = null;
160
+ if (child)
161
+ this.resolveReadiness(child, false);
117
162
  if (child && !child.killed)
118
163
  child.kill();
119
164
  this.rejectPending(`Computer Use ${this.lane} helper stopped.`);
@@ -154,14 +199,14 @@ class PowerShellWorker {
154
199
  '-ExecutionPolicy',
155
200
  'Bypass',
156
201
  '-Command',
157
- HOST_SCRIPT,
202
+ this.lane === 'sparse' ? SPARSE_HOST_SCRIPT : HOST_SCRIPT,
158
203
  ], {
159
204
  windowsHide: true,
160
205
  stdio: ['pipe', 'pipe', 'pipe'],
161
206
  });
162
207
  child.stdin.setDefaultEncoding('utf8');
163
208
  const lines = readline.createInterface({ input: child.stdout });
164
- lines.on('line', line => this.handleLine(line));
209
+ lines.on('line', line => this.handleLine(line, child));
165
210
  let stderr = '';
166
211
  child.stderr.on('data', chunk => {
167
212
  stderr = `${stderr}${String(chunk)}`.slice(-8192);
@@ -170,6 +215,8 @@ class PowerShellWorker {
170
215
  if (this.child !== child)
171
216
  return;
172
217
  this.child = null;
218
+ this.readyChild = null;
219
+ this.resolveReadiness(child, false);
173
220
  this.rejectPending(error.message);
174
221
  });
175
222
  child.once('exit', (code, signal) => {
@@ -177,12 +224,14 @@ class PowerShellWorker {
177
224
  if (this.child !== child)
178
225
  return;
179
226
  this.child = null;
227
+ this.readyChild = null;
228
+ this.resolveReadiness(child, false);
180
229
  this.rejectPending(`Computer Use ${this.lane} helper exited (${code ?? signal ?? 'unknown'}).${stderr ? ` ${stderr.trim()}` : ''}`);
181
230
  });
182
231
  this.child = child;
183
232
  return child;
184
233
  }
185
- handleLine(line) {
234
+ handleLine(line, child) {
186
235
  let parsed;
187
236
  try {
188
237
  parsed = JSON.parse(line);
@@ -190,6 +239,13 @@ class PowerShellWorker {
190
239
  catch {
191
240
  return;
192
241
  }
242
+ if (String(parsed.id || '') === '__newmark_ready__' && parsed.ready === true) {
243
+ if (this.child === child) {
244
+ this.readyChild = child;
245
+ this.resolveReadiness(child, true);
246
+ }
247
+ return;
248
+ }
193
249
  const id = String(parsed.id || '');
194
250
  const pending = this.pending.get(id);
195
251
  if (!pending)
@@ -209,6 +265,12 @@ class PowerShellWorker {
209
265
  }
210
266
  this.pending.clear();
211
267
  }
268
+ resolveReadiness(child, ready) {
269
+ for (const waiter of [...this.readinessWaiters]) {
270
+ if (waiter.child === child)
271
+ waiter.resolve(ready);
272
+ }
273
+ }
212
274
  }
213
275
  const workers = {
214
276
  action: new PowerShellWorker('action'),
@@ -216,6 +278,7 @@ const workers = {
216
278
  windows: new PowerShellWorker('windows'),
217
279
  uia_advisory: new PowerShellWorker('uia_advisory'),
218
280
  windows_advisory: new PowerShellWorker('windows_advisory'),
281
+ sparse: new PowerShellWorker('sparse'),
219
282
  };
220
283
  let cleanupRegistered = false;
221
284
  function registerCleanup() {
@@ -228,6 +291,7 @@ function registerCleanup() {
228
291
  workers.windows.stop();
229
292
  workers.uia_advisory.stop();
230
293
  workers.windows_advisory.stop();
294
+ workers.sparse.stop();
231
295
  };
232
296
  process.once('exit', stop);
233
297
  }
@@ -241,11 +305,21 @@ async function runPersistentPowerShell(script, timeoutMs = 30000, lane = 'action
241
305
  function computerUsePowerShellLaneReady(lane) {
242
306
  return process.platform === 'win32' && workers[lane].ready();
243
307
  }
308
+ async function prepareComputerUsePowerShellLane(lane, timeoutMs) {
309
+ if (process.platform !== 'win32')
310
+ return { ready: false, elapsedMs: 0 };
311
+ registerCleanup();
312
+ return await workers[lane].prepare(timeoutMs);
313
+ }
314
+ function stopComputerUsePowerShellLane(lane) {
315
+ workers[lane].stop();
316
+ }
244
317
  function stopComputerUsePowerShellHost() {
245
318
  workers.action.stop();
246
319
  workers.uia.stop();
247
320
  workers.windows.stop();
248
321
  workers.uia_advisory.stop();
249
322
  workers.windows_advisory.stop();
323
+ workers.sparse.stop();
250
324
  }
251
325
  //# sourceMappingURL=computerUsePowerShellHost.js.map
@@ -405,10 +405,11 @@ class ToolExecutor {
405
405
  capture_max_height: { type: 'number', minimum: 240, maximum: 2048, description: 'Maximum ephemeral screenshot height; defaults to 960. Aspect ratio is preserved and the source is never enlarged.' },
406
406
  max_chars: { type: 'number', minimum: 1000, maximum: 100000, description: 'Maximum returned semantic UI text characters.' },
407
407
  }, []),
408
- t('computer_use', 'Native Windows desktop control with persistent observation/action helpers. Use observe/app_observe before acting. sequence may perform up to three stable low-risk move/click/scroll/wait/app_activate steps and stops when focus, window, menu, dialog, scene, or risk changes. Vision screenshots remain one-time inputs and are deleted immediately.', {
408
+ t('computer_use', 'Native Windows desktop control with persistent observation/action helpers. Use observe/app_observe before acting. For passive observe, sparse_wait_ms can wait up to 5 seconds on a coarse local pixel-change signal without a local perception model; after a signal or timeout it always captures a fresh full screenshot and UI Automation tree for the selected vision model. This low-resolution signal can miss small or brief changes and does not prove that no change occurred. sequence may perform up to three stable low-risk move/click/scroll/wait/app_activate steps and stops when focus, window, menu, dialog, scene, or risk changes. Vision screenshots remain one-time inputs and are deleted immediately.', {
409
409
  action: { type: 'string', enum: ['observe', 'app_list', 'app_observe', 'sequence', 'takeover_start', 'takeover_stop', 'move', 'click', 'scroll', 'type', 'key', 'wait', 'app_activate', 'app_click', 'app_scroll', 'app_type', 'app_key'] },
410
410
  capture_max_width: { type: 'number', minimum: 320, maximum: 2048, description: 'For observe/app_observe only. Maximum ephemeral screenshot width; defaults to 1280. Aspect ratio is preserved and the source is never enlarged.' },
411
411
  capture_max_height: { type: 'number', minimum: 240, maximum: 2048, description: 'For observe/app_observe only. Maximum ephemeral screenshot height; defaults to 960. Aspect ratio is preserved and the source is never enlarged.' },
412
+ sparse_wait_ms: { type: 'integer', minimum: 0, maximum: 5000, description: 'Optional for action=observe only; defaults to 0 (immediate capture). Wait this many milliseconds for a coarse 32×18 grayscale change signal with no local neural model, then capture a fresh full screenshot and UI Automation tree whether signaled or timed out. Small/transient changes can be missed.' },
412
413
  x: { type: 'number' },
413
414
  y: { type: 'number' },
414
415
  target_id: { type: 'string', description: 'Stable id from the latest observe perception.objects entry. Preferred over raw coordinates when available.' },
@@ -1063,6 +1064,8 @@ class ToolExecutor {
1063
1064
  allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
1064
1065
  captureMaxWidth: Number(args.capture_max_width),
1065
1066
  captureMaxHeight: Number(args.capture_max_height),
1067
+ sparseWaitMs: Number(args.sparse_wait_ms),
1068
+ signal: context.signal,
1066
1069
  gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors : undefined,
1067
1070
  gradientSpeed: args.gradient_speed !== undefined ? Number(args.gradient_speed) : undefined,
1068
1071
  gradientWidth: args.gradient_width !== undefined ? Number(args.gradient_width) : undefined,
@@ -213,7 +213,10 @@ function createCoreRuntimeAdapter(options = {}) {
213
213
  status: agent.status,
214
214
  connected: true,
215
215
  runtimeRoot: root,
216
- remoteTouchEnabled: agent.config.getBool('remote', 'touch_enabled')
216
+ remoteTouchEnabled: agent.config.getBool('remote', 'touch_enabled'),
217
+ autoSwitchEnabled: agent.config.autoSwitchEnabled(),
218
+ autoSwitchScope: agent.config.autoSwitchScope(),
219
+ jevDecisionModel: agent.jevDecisionModelState()
217
220
  });
218
221
 
219
222
  const snapshotFor = (requested = currentTarget()) => {
@@ -257,6 +260,7 @@ function createCoreRuntimeAdapter(options = {}) {
257
260
  };
258
261
 
259
262
  const applyConfig = (config) => {
263
+ let refused = null;
260
264
  for (const [key, value] of Object.entries(config || {})) {
261
265
  switch (key) {
262
266
  case "theme": agent.config.set("ui", "dark_mode", value); break;
@@ -268,10 +272,18 @@ function createCoreRuntimeAdapter(options = {}) {
268
272
  case "dialogStyle": agent.config.set("ui", "dialog_style", value); break;
269
273
  case "nativeTools": agent.config.set("tools", "enabled", value); break;
270
274
  case "providers": agent.updateProviders(mergeProviderConfig(agent.config.providers(), value)); break;
275
+ case "jevDecisionModel": {
276
+ // The Agent owns the Auto/decision-model contract: while Auto is off
277
+ // it refuses every edit, so the TUI cannot unlock a model change.
278
+ const result = agent.updateJevDecisionModel(value || {});
279
+ if (!result.ok) refused = { ok: false, reason: result.reason, state: result.state };
280
+ break;
281
+ }
271
282
  default: agent.config.set("ui", key, value);
272
283
  }
273
284
  }
274
285
  agent.config.save();
286
+ if (refused) return refused;
275
287
  return { ok: true };
276
288
  };
277
289
 
@@ -73,12 +73,13 @@ const ZH = Object.freeze({
73
73
  "Type a message · Shift+Enter newline · Enter send": "输入消息 · Shift+Enter 换行 · Enter 发送",
74
74
  "Select a conversation · Enter to edit": "选择对话 · Enter 编辑",
75
75
 
76
- // Settings categories
76
+ // Settings categories
77
77
  "General": "通用",
78
78
  "Personalization": "个性化",
79
79
  "Runtime": "运行时",
80
80
  "Providers": "服务商",
81
81
  "Models": "模型",
82
+ "JEV Auto routing": "JEV 自动路由",
82
83
  "Archive": "归档",
83
84
  "Updates": "更新",
84
85
 
@@ -103,6 +104,16 @@ const ZH = Object.freeze({
103
104
  "Retention": "保留期",
104
105
  "Include Memory Lab": "包含记忆实验室",
105
106
  "Export format": "导出格式",
107
+ "Current-model decision": "当前模型决策",
108
+ "Decision source": "决策来源",
109
+ "Current eligible model": "当前符合条件的模型",
110
+ "Thinking mode": "思考模式",
111
+ "Off": "关闭",
112
+ "Output format": "输出格式",
113
+ "Strict JSON Schema": "严格 JSON Schema",
114
+ "Decision timeout": "决策超时",
115
+ "Deterministic fallback": "确定性回退",
116
+ "Auto routing": "自动路由",
106
117
  "Update channel": "更新通道",
107
118
  "Automatic checks": "自动检查",
108
119
  "Automatic download": "自动下载",
@@ -987,7 +987,7 @@ function settingsView(state, width, p) {
987
987
  const style = state.focusRegion === "content" && state.contentColumn === 1 && i === state.selected ? `${p.selected}` : "";
988
988
  if (style) state.contentFocusLine = 4 + i * 2;
989
989
  return [
990
- `${style} ${p.bold}${pad(row.label, 27)}${p.reset} ${p.cyan}${displaySettingValue(row.value)}${p.reset}${row.extension ? ` ${p.amber}TUI EXTENSION${p.reset}` : ""}`,
990
+ `${style} ${p.bold}${pad(row.label, 27)}${p.reset} ${p.cyan}${displaySettingValue(row.value)}${p.reset}${row.locked ? ` ${p.amber}LOCKED${p.reset}` : ""}${row.extension ? ` ${p.amber}TUI EXTENSION${p.reset}` : ""}`,
991
991
  ` ${p.muted}${settingDescription(state.settingsTab, row.key)}${p.reset}`
992
992
  ];
993
993
  }),
@@ -1030,7 +1030,13 @@ function settingDescription(tab, key) {
1030
1030
  channel: "Stable or prerelease update stream.",
1031
1031
  autoCheck: "Check for updates during application startup.",
1032
1032
  autoDownload: "Download only after a successful simulated check.",
1033
- source: "GitHub release or local package source."
1033
+ source: "GitHub release or local package source.",
1034
+ enabled: "Enable or disable model-backed Auto decisions; disabled means Auto keeps the current model.",
1035
+ decisionModel: "Use the current eligible conversation model, or the configured default on the first Auto route.",
1036
+ thinkingMode: "JEV requests the provider's no-thinking mode.",
1037
+ outputFormat: "JEV uses native strict JSON Schema output and validates the returned object before routing.",
1038
+ timeoutMs: "Hard budget for one current-model Auto decision; a timeout stops Auto routing without selecting another model.",
1039
+ autoSwitch: "Auto routing master switch; while it is off current-model decision settings are locked and no model changes."
1034
1040
  };
1035
1041
  return descriptions[key] || `${tab} setting`;
1036
1042
  }
@@ -1077,6 +1083,7 @@ function settingsActionHint(tab) {
1077
1083
  if (tab === "general") return "Enter opens Guide/Next list for Input mode · O open config · R refresh";
1078
1084
  if (tab === "archive") return "L list archive inventory";
1079
1085
  if (tab === "updates") return "U check version and GitHub update";
1086
+ if (tab === "jev") return "Enter changes current-model decision settings · thinking off · strict JSON Schema · failures stop routing";
1080
1087
  return "Enter cycles or toggles the selected setting";
1081
1088
  }
1082
1089
 
@@ -6,6 +6,7 @@ const SETTINGS_CATEGORIES = Object.freeze([
6
6
  { id: "runtime", icon: "[R]", label: "Runtime", source: "agent.* / terminal.*" },
7
7
  { id: "providers", icon: "[V]", label: "Providers", source: "models.providers" },
8
8
  { id: "models", icon: "[M]", label: "Models", source: "models.providers[].models" },
9
+ { id: "jev", icon: "[J]", label: "Auto decision policy", source: "current eligible conversation model + models.jev_* policy" },
9
10
  { id: "tools", icon: "[T]", label: "Tools", source: "tools.enabled" },
10
11
  { id: "archive", icon: "[A]", label: "Archive", source: "archive API / archive.*" },
11
12
  { id: "updates", icon: "[U]", label: "Updates", source: "update API / updates.*" }
@@ -15,6 +16,8 @@ const FONT_CHOICES = Object.freeze(["Terminal default", "Cascadia Mono", "JetBra
15
16
  const FONT_COLOR_CHOICES = Object.freeze(["#E6EAF2", "#B7E4FF", "#C7F9CC", "#FFD6A5", "#F8B4D9"]);
16
17
  const BACKGROUND_COLOR_CHOICES = Object.freeze(["#0A0A1A", "#111827", "#172554", "#1F2937", "#F0F2F8"]);
17
18
 
19
+ const JEV_TIMEOUT_PRESETS = Object.freeze([500, 1000, 2000, 3000, 5000, 10000]);
20
+
18
21
  function settingsRows(state, tab = state.settingsTab) {
19
22
  const s = state.settings;
20
23
  if (tab === "general") return [
@@ -40,6 +43,21 @@ function settingsRows(state, tab = state.settingsTab) {
40
43
  { key: "terminalTimeout", label: "Terminal timeout cap", value: s.runtime.terminalTimeout, choices: [0, 30000, 60000, 120000], save: ["setting", "terminal", "interrupt_timeout_ms"] },
41
44
  { key: "defaultShell", label: "Default shell", value: s.runtime.defaultShell, choices: ["PowerShell", "Command Prompt", "WSL Bash"], save: ["setting", "terminal", "default_shell"] }
42
45
  ];
46
+ if (tab === "jev") {
47
+ const jev = s.jev || {};
48
+ // Auto off locks current-model decision settings: the rows stay visible so the user can see
49
+ // why, but Enter refuses every change (see setSelectedRow in state.js).
50
+ const locked = jev.locked === true || jev.autoEnabled === false;
51
+ const lockedValue = locked ? "Locked (Auto off)" : undefined;
52
+ return [
53
+ { key: "enabled", label: "Current-model decision", value: lockedValue || (jev.enabled === false ? "Off" : "On"), choices: [], action: "jev-toggle-enabled", locked },
54
+ { key: "decisionModel", label: "Decision source", value: "Current eligible model", choices: [], action: "jev-report-current-model", locked: true },
55
+ { key: "thinkingMode", label: "Thinking mode", value: "Off", choices: [], action: "jev-report-current-model", locked: true },
56
+ { key: "outputFormat", label: "Output format", value: "Strict JSON Schema", choices: [], action: "jev-report-current-model", locked: true },
57
+ { key: "timeoutMs", label: "Decision timeout", value: lockedValue || `${Number(jev.timeoutMs) || 3000} ms`, choices: [], action: "jev-cycle-timeout", locked },
58
+ { key: "autoSwitch", label: "Auto routing", value: jev.autoEnabled === false ? "Off" : "On", choices: [], action: "jev-report-auto" }
59
+ ];
60
+ }
43
61
  if (tab === "archive") return [
44
62
  { key: "autoArchive", label: "Automatic archive", value: s.archive.autoArchive, choices: [true, false], save: ["setting", "archive", "automatic"] },
45
63
  { key: "retentionDays", label: "Retention", value: s.archive.retentionDays, choices: [30, 90, 180, 365], save: ["setting", "archive", "retention_days"] },
@@ -67,6 +85,7 @@ module.exports = {
67
85
  BACKGROUND_COLOR_CHOICES,
68
86
  FONT_CHOICES,
69
87
  FONT_COLOR_CHOICES,
88
+ JEV_TIMEOUT_PRESETS,
70
89
  SETTINGS_CATEGORIES,
71
90
  displaySettingValue,
72
91
  settingsRows
@@ -3,7 +3,11 @@
3
3
  const data = require("./data");
4
4
  const { createMockNewmarkAdapter } = require("./adapters/mock-newmark-adapter");
5
5
  const { targetKey, validateSnapshot } = require("./adapters/newmark-contract");
6
- const { SETTINGS_CATEGORIES, settingsRows } = require("./settings-schema");
6
+ const {
7
+ JEV_TIMEOUT_PRESETS,
8
+ SETTINGS_CATEGORIES,
9
+ settingsRows
10
+ } = require("./settings-schema");
7
11
  const INTELLIGENCE_TIERS = Object.freeze(["low", "medium", "high", "xhigh", "max", "ultra"]);
8
12
  const HEX_COLOR_PATTERN = /^#[0-9a-f]{6}$/i;
9
13
  const THEME_APPEARANCE_DEFAULTS = Object.freeze({
@@ -93,6 +97,17 @@ function applySnapshot(state, snapshot) {
93
97
  if (state.runningConversationKeys) {
94
98
  state.busy = state.runningConversationKeys.has(targetKey(valid.target));
95
99
  }
100
+ // Current-model decision settings + Auto master switch drive the locked state of the JEV
101
+ // settings rows, so they must follow every authoritative snapshot.
102
+ if (valid.jevDecisionModel && state.settings) {
103
+ state.jevDecisionModel = valid.jevDecisionModel;
104
+ state.settings.jev = {
105
+ autoEnabled: valid.jevDecisionModel.autoEnabled !== false,
106
+ locked: valid.jevDecisionModel.locked === true || valid.jevDecisionModel.autoEnabled === false,
107
+ enabled: valid.jevDecisionModel.enabled !== false,
108
+ timeoutMs: Number(valid.jevDecisionModel.timeoutMs || 3000)
109
+ };
110
+ }
96
111
  return state;
97
112
  }
98
113
 
@@ -280,6 +295,12 @@ function createState(options = {}) {
280
295
  autoCheck: true,
281
296
  autoDownload: false,
282
297
  source: "GitHub"
298
+ },
299
+ jev: {
300
+ autoEnabled: snapshot.autoSwitchEnabled !== false,
301
+ locked: snapshot.autoSwitchEnabled === false,
302
+ enabled: snapshot.jevDecisionModel ? snapshot.jevDecisionModel.enabled !== false : true,
303
+ timeoutMs: Number(snapshot.jevDecisionModel?.timeoutMs || 3000)
283
304
  }
284
305
  }
285
306
  };
@@ -1072,6 +1093,9 @@ function toggleSelected(state) {
1072
1093
  }
1073
1094
  const row = settingsRows(state)[state.selected];
1074
1095
  if (!row) return;
1096
+ if (typeof row.action === "string" && row.action.startsWith("jev-")) {
1097
+ return applyJevDecisionRow(state, row);
1098
+ }
1075
1099
  if (row.action === "pair-mobile") {
1076
1100
  state.overlay = "pair";
1077
1101
  state.pairingQrLines = ["Loading pairing QR…"];
@@ -1112,6 +1136,50 @@ function toggleSelected(state) {
1112
1136
  }
1113
1137
  }
1114
1138
 
1139
+ /**
1140
+ * Current-model JEV settings rows.
1141
+ *
1142
+ * The Agent is authoritative: while Auto is off it refuses every edit, so the
1143
+ * TUI reports the lock instead of pretending a change happened. The decision
1144
+ * source is always the current eligible conversation model.
1145
+ */
1146
+ function applyJevDecisionRow(state, row) {
1147
+ const jev = state.jevDecisionModel && typeof state.jevDecisionModel === "object" ? state.jevDecisionModel : {};
1148
+ if (row.action === "jev-report-auto") {
1149
+ state.notice = jev.autoEnabled === false
1150
+ ? "Auto routing is off: current-model decision settings are locked and no model may change."
1151
+ : "Auto routing is on: JEV uses the current eligible conversation model with thinking off and strict JSON Schema output.";
1152
+ return true;
1153
+ }
1154
+ if (row.action === "jev-report-current-model") {
1155
+ state.notice = "JEV sends a strict JSON Schema decision request to the current eligible conversation model with thinking disabled.";
1156
+ return true;
1157
+ }
1158
+ if (jev.locked === true || jev.autoEnabled === false) {
1159
+ state.notice = "Auto is off: current-model decision settings are locked. Turn Auto on first — no model may change while Auto is off.";
1160
+ return true;
1161
+ }
1162
+ const patch = {};
1163
+ if (row.action === "jev-toggle-enabled") patch.enabled = jev.enabled === false;
1164
+ if (row.action === "jev-cycle-timeout") {
1165
+ const index = JEV_TIMEOUT_PRESETS.indexOf(Number(jev.timeoutMs));
1166
+ patch.timeoutMs = JEV_TIMEOUT_PRESETS[(index + 1 + JEV_TIMEOUT_PRESETS.length) % JEV_TIMEOUT_PRESETS.length];
1167
+ }
1168
+ if (!Object.keys(patch).length) return true;
1169
+ const result = state.adapter.saveConfig({ jevDecisionModel: patch });
1170
+ if (result && typeof result.then === "function") throw new TypeError("TUI settings adapter must be synchronous");
1171
+ if (!result || result.ok === false) {
1172
+ state.notice = result && result.reason === "auto_disabled"
1173
+ ? "Auto is off: current-model decision settings are locked and no model may change."
1174
+ : "The current-model decision settings were not changed.";
1175
+ if (result && result.state) state.jevDecisionModel = result.state;
1176
+ return true;
1177
+ }
1178
+ state.jevDecisionModel = result.state || jev;
1179
+ state.notice = `JEV current-model settings updated · ${state.adapterKind === "mock" ? "mock" : "persisted"}`;
1180
+ return true;
1181
+ }
1182
+
1115
1183
  function selectedSettingChoiceRow(state) {
1116
1184
  if (state.overlay !== "settings-choice" || !state.settingChoiceKey) return null;
1117
1185
  return settingsRows(state, state.settingChoiceTab)