newmark-agent 0.6.3 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config.example.json +13 -3
- package/dist/cli-commands.d.ts +1 -1
- package/dist/cli-commands.js +53 -2
- package/dist/cli-discovery.js +3 -0
- package/dist/conversation-utility-host.bundle.cjs +3046 -1038
- package/dist/conversation-utility-host.js +94 -7
- package/dist/core/agent.d.ts +149 -12
- package/dist/core/agent.js +843 -176
- package/dist/core/agentKernelRunner.d.ts +7 -1
- package/dist/core/agentKernelRunner.js +171 -22
- package/dist/core/autoRouter.d.ts +49 -44
- package/dist/core/autoRouter.js +117 -302
- package/dist/core/branchIdentity.d.ts +29 -0
- package/dist/core/branchIdentity.js +54 -0
- package/dist/core/config.js +27 -11
- package/dist/core/continuation/contracts.d.ts +1 -1
- package/dist/core/continuation/store.d.ts +119 -1
- package/dist/core/continuation/store.js +337 -19
- package/dist/core/conversationKernel.d.ts +62 -8
- package/dist/core/conversationKernel.js +361 -47
- package/dist/core/conversationStateDocument.d.ts +18 -0
- package/dist/core/conversationStateDocument.js +88 -0
- package/dist/core/conversationVisualBudget.d.ts +74 -0
- package/dist/core/conversationVisualBudget.js +153 -0
- package/dist/core/electronUtilityAgentClient.d.ts +3 -2
- package/dist/core/electronUtilityAgentClient.js +32 -8
- package/dist/core/electronUtilityRuntimePool.d.ts +28 -3
- package/dist/core/electronUtilityRuntimePool.js +93 -24
- package/dist/core/hostRuntimeHooks.d.ts +23 -0
- package/dist/core/hostRuntimeHooks.js +17 -0
- package/dist/core/jevDecision.d.ts +144 -0
- package/dist/core/jevDecision.js +226 -0
- package/dist/core/memoryProbe.d.ts +24 -0
- package/dist/core/memoryProbe.js +104 -0
- package/dist/core/mobilePairing.d.ts +7 -1
- package/dist/core/mobilePairing.js +9 -1
- package/dist/core/performanceDiagnostics.d.ts +1 -1
- package/dist/core/routeDecisionValidator.d.ts +86 -0
- package/dist/core/routeDecisionValidator.js +249 -0
- package/dist/core/routeEligibility.d.ts +52 -0
- package/dist/core/routeEligibility.js +108 -0
- package/dist/core/runtimeMemoryBudget.d.ts +6 -0
- package/dist/core/runtimeMemoryBudget.js +12 -0
- package/dist/core/utilityAgentProtocol.d.ts +8 -10
- package/dist/core/utilityHostToolRouter.js +2 -0
- package/dist/core/visualDownscale.d.ts +6 -0
- package/dist/core/visualDownscale.js +137 -0
- package/dist/core/wslAgentClient.d.ts +1 -2
- package/dist/core/wslAgentClient.js +0 -8
- package/dist/core/wslAgentProtocol.d.ts +1 -10
- package/dist/core/wslAgentRuntimePool.d.ts +1 -3
- package/dist/core/wslAgentRuntimePool.js +17 -22
- package/dist/llm/provider.d.ts +3 -1
- package/dist/llm/provider.js +53 -9
- package/dist/main.js +108 -31
- package/dist/preload.js +3 -1
- package/dist/server.js +30 -18
- package/dist/tools/computerUse.d.ts +4 -0
- package/dist/tools/computerUse.js +202 -4
- package/dist/tools/computerUsePowerShellHost.d.ts +6 -1
- package/dist/tools/computerUsePowerShellHost.js +105 -31
- package/dist/tools/index.js +4 -1
- package/dist/tui/src/adapters/core-runtime-adapter.js +13 -1
- package/dist/tui/src/i18n.js +12 -1
- package/dist/tui/src/render.js +9 -2
- package/dist/tui/src/settings-schema.js +19 -0
- package/dist/tui/src/state.js +69 -1
- package/dist/ui/index.html +994 -277
- package/dist/ui/lucide-sprite.svg +0 -8
- package/dist/wsl-agent-host.bundle.cjs +2851 -1013
- package/dist/wsl-agent-host.js +0 -3
- package/package.json +38 -9
|
@@ -35,48 +35,90 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
36
|
exports.runPersistentPowerShell = runPersistentPowerShell;
|
|
37
37
|
exports.computerUsePowerShellLaneReady = computerUsePowerShellLaneReady;
|
|
38
|
+
exports.prepareComputerUsePowerShellLane = prepareComputerUsePowerShellLane;
|
|
39
|
+
exports.stopComputerUsePowerShellLane = stopComputerUsePowerShellLane;
|
|
38
40
|
exports.stopComputerUsePowerShellHost = stopComputerUsePowerShellHost;
|
|
39
41
|
const child_process_1 = require("child_process");
|
|
40
42
|
const crypto = __importStar(require("crypto"));
|
|
41
43
|
const readline = __importStar(require("readline"));
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
].
|
|
44
|
+
function createHostScript(includeUiAutomation) {
|
|
45
|
+
return [
|
|
46
|
+
'[Console]::InputEncoding = [System.Text.UTF8Encoding]::new($false)',
|
|
47
|
+
'[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
|
|
48
|
+
'$OutputEncoding = [System.Text.UTF8Encoding]::new($false)',
|
|
49
|
+
'Add-Type -AssemblyName System.Windows.Forms',
|
|
50
|
+
'Add-Type -AssemblyName System.Drawing',
|
|
51
|
+
...(includeUiAutomation ? ['Add-Type -AssemblyName UIAutomationClient', 'Add-Type -AssemblyName UIAutomationTypes'] : []),
|
|
52
|
+
...(includeUiAutomation ? [`if (-not ("NewmarkComputerUseNative" -as [type])) { Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class NewmarkComputerUseNative { [DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y); [DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, int dwData, UIntPtr dwExtraInfo); [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); }' }`] : []),
|
|
53
|
+
...(!includeUiAutomation ? [`if (-not ("NewmarkSparseCapture" -as [type])) { Add-Type -TypeDefinition 'using System; using System.Runtime.InteropServices; public static class NewmarkSparseCapture { [StructLayout(LayoutKind.Sequential)] struct BitmapInfoHeader { public uint Size; public int Width; public int Height; public ushort Planes; public ushort BitCount; public uint Compression; public uint SizeImage; public int XPelsPerMeter; public int YPelsPerMeter; public uint ColorsUsed; public uint ColorsImportant; } [StructLayout(LayoutKind.Sequential)] struct BitmapInfo { public BitmapInfoHeader Header; public uint Colors; } [DllImport("user32.dll")] static extern IntPtr GetDC(IntPtr hWnd); [DllImport("user32.dll")] static extern int ReleaseDC(IntPtr hWnd, IntPtr hdc); [DllImport("gdi32.dll")] static extern IntPtr CreateCompatibleDC(IntPtr hdc); [DllImport("gdi32.dll")] static extern bool DeleteDC(IntPtr hdc); [DllImport("gdi32.dll")] static extern IntPtr CreateDIBSection(IntPtr hdc, ref BitmapInfo info, uint usage, out IntPtr bits, IntPtr section, uint offset); [DllImport("gdi32.dll")] static extern IntPtr SelectObject(IntPtr hdc, IntPtr obj); [DllImport("gdi32.dll")] static extern bool DeleteObject(IntPtr obj); [DllImport("gdi32.dll")] static extern int SetStretchBltMode(IntPtr hdc, int mode); [DllImport("gdi32.dll")] static extern bool SetBrushOrgEx(IntPtr hdc, int x, int y, IntPtr oldOrigin); [DllImport("gdi32.dll", SetLastError=true)] static extern bool StretchBlt(IntPtr dst, int dx, int dy, int dw, int dh, IntPtr src, int sx, int sy, int sw, int sh, int operation); [DllImport("gdi32.dll")] static extern bool GdiFlush(); public sealed class Capture : IDisposable { private int width; private byte[] pixels; internal Capture(int w, byte[] p) { width=w; pixels=p; } public byte GetLuma(int x, int y) { int offset=(y*width+x)*4; return (byte)Math.Round((pixels[offset+2]*0.299)+(pixels[offset+1]*0.587)+(pixels[offset]*0.114)); } public void Dispose() { pixels=null; } } public static Capture CaptureScaled(int x, int y, int width, int height, int targetWidth, int targetHeight) { IntPtr screen=GetDC(IntPtr.Zero); if (screen == IntPtr.Zero) throw new InvalidOperationException("Could not read desktop DC."); IntPtr memory=IntPtr.Zero; IntPtr bitmap=IntPtr.Zero; IntPtr previous=IntPtr.Zero; IntPtr bits=IntPtr.Zero; try { memory=CreateCompatibleDC(screen); BitmapInfo info=new BitmapInfo(); info.Header.Size=(uint)System.Runtime.InteropServices.Marshal.SizeOf(typeof(BitmapInfoHeader)); info.Header.Width=targetWidth; info.Header.Height=-targetHeight; info.Header.Planes=1; info.Header.BitCount=32; info.Header.Compression=0; info.Header.SizeImage=(uint)(targetWidth*targetHeight*4); bitmap=CreateDIBSection(screen,ref info,0,out bits,IntPtr.Zero,0); if (memory == IntPtr.Zero || bitmap == IntPtr.Zero || bits == IntPtr.Zero) throw new InvalidOperationException("Could not allocate sparse DIB section."); previous=SelectObject(memory,bitmap); if (previous == IntPtr.Zero) throw new InvalidOperationException("Could not select sparse DIB section."); if (SetStretchBltMode(memory,4)==0 || !SetBrushOrgEx(memory,0,0,IntPtr.Zero)) throw new InvalidOperationException("Could not configure sparse capture scaling."); if (!StretchBlt(memory,0,0,targetWidth,targetHeight,screen,x,y,width,height,0x00CC0020)) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); if (!GdiFlush()) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error()); byte[] pixels=new byte[targetWidth*targetHeight*4]; Marshal.Copy(bits,pixels,0,pixels.Length); return new Capture(targetWidth,pixels); } finally { if (memory != IntPtr.Zero && previous != IntPtr.Zero) SelectObject(memory,previous); if (bitmap != IntPtr.Zero) DeleteObject(bitmap); if (memory != IntPtr.Zero) DeleteDC(memory); ReleaseDC(IntPtr.Zero,screen); } } }' }`] : []),
|
|
54
|
+
'$hostReady = @{ id="__newmark_ready__"; ready=$true }; [Console]::Out.WriteLine(($hostReady | ConvertTo-Json -Compress)); [Console]::Out.Flush()',
|
|
55
|
+
'while (($line = [Console]::In.ReadLine()) -ne $null) {',
|
|
56
|
+
' $id = ""',
|
|
57
|
+
' $timer = [System.Diagnostics.Stopwatch]::StartNew()',
|
|
58
|
+
' try {',
|
|
59
|
+
' $request = $line | ConvertFrom-Json',
|
|
60
|
+
' $id = [string]$request.id',
|
|
61
|
+
' $source = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String([string]$request.script))',
|
|
62
|
+
' $previousPreference = $ErrorActionPreference',
|
|
63
|
+
' $ErrorActionPreference = "Stop"',
|
|
64
|
+
' try { $output = (& ([scriptblock]::Create($source)) 2>&1 | Out-String -Width 1048576).Trim() } finally { $ErrorActionPreference = $previousPreference }',
|
|
65
|
+
' $response = @{ id=$id; ok=$true; output=$output; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
|
|
66
|
+
' } catch {',
|
|
67
|
+
' $response = @{ id=$id; ok=$false; output=[string]$_; elapsed_ms=[int]$timer.ElapsedMilliseconds }',
|
|
68
|
+
' }',
|
|
69
|
+
' [Console]::Out.WriteLine(($response | ConvertTo-Json -Compress -Depth 6))',
|
|
70
|
+
' [Console]::Out.Flush()',
|
|
71
|
+
'}',
|
|
72
|
+
].join('; ');
|
|
73
|
+
}
|
|
74
|
+
const HOST_SCRIPT = createHostScript(true);
|
|
75
|
+
const SPARSE_HOST_SCRIPT = createHostScript(false);
|
|
69
76
|
class PowerShellWorker {
|
|
70
77
|
lane;
|
|
71
78
|
child = null;
|
|
79
|
+
readyChild = null;
|
|
72
80
|
pending = new Map();
|
|
81
|
+
readinessWaiters = new Set();
|
|
73
82
|
restarting = false;
|
|
74
83
|
unavailableUntil = 0;
|
|
75
84
|
constructor(lane) {
|
|
76
85
|
this.lane = lane;
|
|
77
86
|
}
|
|
78
87
|
ready() {
|
|
79
|
-
return Date.now() >= this.unavailableUntil
|
|
88
|
+
return Date.now() >= this.unavailableUntil
|
|
89
|
+
&& !!this.child
|
|
90
|
+
&& !this.child.killed
|
|
91
|
+
&& this.child.exitCode === null
|
|
92
|
+
&& this.readyChild === this.child;
|
|
93
|
+
}
|
|
94
|
+
async prepare(timeoutMs) {
|
|
95
|
+
const startedAt = Date.now();
|
|
96
|
+
if (Date.now() < this.unavailableUntil)
|
|
97
|
+
return { ready: false, elapsedMs: 0 };
|
|
98
|
+
const child = this.ensureChild();
|
|
99
|
+
if (this.readyChild === child)
|
|
100
|
+
return { ready: true, elapsedMs: Date.now() - startedAt };
|
|
101
|
+
const ready = await new Promise(resolve => {
|
|
102
|
+
let settled = false;
|
|
103
|
+
let waiter;
|
|
104
|
+
const finish = (value) => {
|
|
105
|
+
if (settled)
|
|
106
|
+
return;
|
|
107
|
+
settled = true;
|
|
108
|
+
clearTimeout(waiter.timer);
|
|
109
|
+
this.readinessWaiters.delete(waiter);
|
|
110
|
+
resolve(value);
|
|
111
|
+
};
|
|
112
|
+
waiter = {
|
|
113
|
+
child,
|
|
114
|
+
resolve: finish,
|
|
115
|
+
timer: setTimeout(() => finish(false), Math.max(0, Math.floor(timeoutMs))),
|
|
116
|
+
};
|
|
117
|
+
this.readinessWaiters.add(waiter);
|
|
118
|
+
if (this.readyChild === child)
|
|
119
|
+
finish(true);
|
|
120
|
+
});
|
|
121
|
+
return { ready, elapsedMs: Date.now() - startedAt };
|
|
80
122
|
}
|
|
81
123
|
async run(script, timeoutMs) {
|
|
82
124
|
const startedAt = Date.now();
|
|
@@ -114,6 +156,9 @@ class PowerShellWorker {
|
|
|
114
156
|
stop() {
|
|
115
157
|
const child = this.child;
|
|
116
158
|
this.child = null;
|
|
159
|
+
this.readyChild = null;
|
|
160
|
+
if (child)
|
|
161
|
+
this.resolveReadiness(child, false);
|
|
117
162
|
if (child && !child.killed)
|
|
118
163
|
child.kill();
|
|
119
164
|
this.rejectPending(`Computer Use ${this.lane} helper stopped.`);
|
|
@@ -154,14 +199,14 @@ class PowerShellWorker {
|
|
|
154
199
|
'-ExecutionPolicy',
|
|
155
200
|
'Bypass',
|
|
156
201
|
'-Command',
|
|
157
|
-
HOST_SCRIPT,
|
|
202
|
+
this.lane === 'sparse' ? SPARSE_HOST_SCRIPT : HOST_SCRIPT,
|
|
158
203
|
], {
|
|
159
204
|
windowsHide: true,
|
|
160
205
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
161
206
|
});
|
|
162
207
|
child.stdin.setDefaultEncoding('utf8');
|
|
163
208
|
const lines = readline.createInterface({ input: child.stdout });
|
|
164
|
-
lines.on('line', line => this.handleLine(line));
|
|
209
|
+
lines.on('line', line => this.handleLine(line, child));
|
|
165
210
|
let stderr = '';
|
|
166
211
|
child.stderr.on('data', chunk => {
|
|
167
212
|
stderr = `${stderr}${String(chunk)}`.slice(-8192);
|
|
@@ -170,6 +215,8 @@ class PowerShellWorker {
|
|
|
170
215
|
if (this.child !== child)
|
|
171
216
|
return;
|
|
172
217
|
this.child = null;
|
|
218
|
+
this.readyChild = null;
|
|
219
|
+
this.resolveReadiness(child, false);
|
|
173
220
|
this.rejectPending(error.message);
|
|
174
221
|
});
|
|
175
222
|
child.once('exit', (code, signal) => {
|
|
@@ -177,12 +224,14 @@ class PowerShellWorker {
|
|
|
177
224
|
if (this.child !== child)
|
|
178
225
|
return;
|
|
179
226
|
this.child = null;
|
|
227
|
+
this.readyChild = null;
|
|
228
|
+
this.resolveReadiness(child, false);
|
|
180
229
|
this.rejectPending(`Computer Use ${this.lane} helper exited (${code ?? signal ?? 'unknown'}).${stderr ? ` ${stderr.trim()}` : ''}`);
|
|
181
230
|
});
|
|
182
231
|
this.child = child;
|
|
183
232
|
return child;
|
|
184
233
|
}
|
|
185
|
-
handleLine(line) {
|
|
234
|
+
handleLine(line, child) {
|
|
186
235
|
let parsed;
|
|
187
236
|
try {
|
|
188
237
|
parsed = JSON.parse(line);
|
|
@@ -190,6 +239,13 @@ class PowerShellWorker {
|
|
|
190
239
|
catch {
|
|
191
240
|
return;
|
|
192
241
|
}
|
|
242
|
+
if (String(parsed.id || '') === '__newmark_ready__' && parsed.ready === true) {
|
|
243
|
+
if (this.child === child) {
|
|
244
|
+
this.readyChild = child;
|
|
245
|
+
this.resolveReadiness(child, true);
|
|
246
|
+
}
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
193
249
|
const id = String(parsed.id || '');
|
|
194
250
|
const pending = this.pending.get(id);
|
|
195
251
|
if (!pending)
|
|
@@ -209,6 +265,12 @@ class PowerShellWorker {
|
|
|
209
265
|
}
|
|
210
266
|
this.pending.clear();
|
|
211
267
|
}
|
|
268
|
+
resolveReadiness(child, ready) {
|
|
269
|
+
for (const waiter of [...this.readinessWaiters]) {
|
|
270
|
+
if (waiter.child === child)
|
|
271
|
+
waiter.resolve(ready);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
212
274
|
}
|
|
213
275
|
const workers = {
|
|
214
276
|
action: new PowerShellWorker('action'),
|
|
@@ -216,6 +278,7 @@ const workers = {
|
|
|
216
278
|
windows: new PowerShellWorker('windows'),
|
|
217
279
|
uia_advisory: new PowerShellWorker('uia_advisory'),
|
|
218
280
|
windows_advisory: new PowerShellWorker('windows_advisory'),
|
|
281
|
+
sparse: new PowerShellWorker('sparse'),
|
|
219
282
|
};
|
|
220
283
|
let cleanupRegistered = false;
|
|
221
284
|
function registerCleanup() {
|
|
@@ -228,6 +291,7 @@ function registerCleanup() {
|
|
|
228
291
|
workers.windows.stop();
|
|
229
292
|
workers.uia_advisory.stop();
|
|
230
293
|
workers.windows_advisory.stop();
|
|
294
|
+
workers.sparse.stop();
|
|
231
295
|
};
|
|
232
296
|
process.once('exit', stop);
|
|
233
297
|
}
|
|
@@ -241,11 +305,21 @@ async function runPersistentPowerShell(script, timeoutMs = 30000, lane = 'action
|
|
|
241
305
|
function computerUsePowerShellLaneReady(lane) {
|
|
242
306
|
return process.platform === 'win32' && workers[lane].ready();
|
|
243
307
|
}
|
|
308
|
+
async function prepareComputerUsePowerShellLane(lane, timeoutMs) {
|
|
309
|
+
if (process.platform !== 'win32')
|
|
310
|
+
return { ready: false, elapsedMs: 0 };
|
|
311
|
+
registerCleanup();
|
|
312
|
+
return await workers[lane].prepare(timeoutMs);
|
|
313
|
+
}
|
|
314
|
+
function stopComputerUsePowerShellLane(lane) {
|
|
315
|
+
workers[lane].stop();
|
|
316
|
+
}
|
|
244
317
|
function stopComputerUsePowerShellHost() {
|
|
245
318
|
workers.action.stop();
|
|
246
319
|
workers.uia.stop();
|
|
247
320
|
workers.windows.stop();
|
|
248
321
|
workers.uia_advisory.stop();
|
|
249
322
|
workers.windows_advisory.stop();
|
|
323
|
+
workers.sparse.stop();
|
|
250
324
|
}
|
|
251
325
|
//# sourceMappingURL=computerUsePowerShellHost.js.map
|
package/dist/tools/index.js
CHANGED
|
@@ -405,10 +405,11 @@ class ToolExecutor {
|
|
|
405
405
|
capture_max_height: { type: 'number', minimum: 240, maximum: 2048, description: 'Maximum ephemeral screenshot height; defaults to 960. Aspect ratio is preserved and the source is never enlarged.' },
|
|
406
406
|
max_chars: { type: 'number', minimum: 1000, maximum: 100000, description: 'Maximum returned semantic UI text characters.' },
|
|
407
407
|
}, []),
|
|
408
|
-
t('computer_use', 'Native Windows desktop control with persistent observation/action helpers. Use observe/app_observe before acting. sequence may perform up to three stable low-risk move/click/scroll/wait/app_activate steps and stops when focus, window, menu, dialog, scene, or risk changes. Vision screenshots remain one-time inputs and are deleted immediately.', {
|
|
408
|
+
t('computer_use', 'Native Windows desktop control with persistent observation/action helpers. Use observe/app_observe before acting. For passive observe, sparse_wait_ms can wait up to 5 seconds on a coarse local pixel-change signal without a local perception model; after a signal or timeout it always captures a fresh full screenshot and UI Automation tree for the selected vision model. This low-resolution signal can miss small or brief changes and does not prove that no change occurred. sequence may perform up to three stable low-risk move/click/scroll/wait/app_activate steps and stops when focus, window, menu, dialog, scene, or risk changes. Vision screenshots remain one-time inputs and are deleted immediately.', {
|
|
409
409
|
action: { type: 'string', enum: ['observe', 'app_list', 'app_observe', 'sequence', 'takeover_start', 'takeover_stop', 'move', 'click', 'scroll', 'type', 'key', 'wait', 'app_activate', 'app_click', 'app_scroll', 'app_type', 'app_key'] },
|
|
410
410
|
capture_max_width: { type: 'number', minimum: 320, maximum: 2048, description: 'For observe/app_observe only. Maximum ephemeral screenshot width; defaults to 1280. Aspect ratio is preserved and the source is never enlarged.' },
|
|
411
411
|
capture_max_height: { type: 'number', minimum: 240, maximum: 2048, description: 'For observe/app_observe only. Maximum ephemeral screenshot height; defaults to 960. Aspect ratio is preserved and the source is never enlarged.' },
|
|
412
|
+
sparse_wait_ms: { type: 'integer', minimum: 0, maximum: 5000, description: 'Optional for action=observe only; defaults to 0 (immediate capture). Wait this many milliseconds for a coarse 32×18 grayscale change signal with no local neural model, then capture a fresh full screenshot and UI Automation tree whether signaled or timed out. Small/transient changes can be missed.' },
|
|
412
413
|
x: { type: 'number' },
|
|
413
414
|
y: { type: 'number' },
|
|
414
415
|
target_id: { type: 'string', description: 'Stable id from the latest observe perception.objects entry. Preferred over raw coordinates when available.' },
|
|
@@ -1063,6 +1064,8 @@ class ToolExecutor {
|
|
|
1063
1064
|
allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
|
|
1064
1065
|
captureMaxWidth: Number(args.capture_max_width),
|
|
1065
1066
|
captureMaxHeight: Number(args.capture_max_height),
|
|
1067
|
+
sparseWaitMs: Number(args.sparse_wait_ms),
|
|
1068
|
+
signal: context.signal,
|
|
1066
1069
|
gradientColors: Array.isArray(args.gradient_colors) ? args.gradient_colors : undefined,
|
|
1067
1070
|
gradientSpeed: args.gradient_speed !== undefined ? Number(args.gradient_speed) : undefined,
|
|
1068
1071
|
gradientWidth: args.gradient_width !== undefined ? Number(args.gradient_width) : undefined,
|
|
@@ -213,7 +213,10 @@ function createCoreRuntimeAdapter(options = {}) {
|
|
|
213
213
|
status: agent.status,
|
|
214
214
|
connected: true,
|
|
215
215
|
runtimeRoot: root,
|
|
216
|
-
remoteTouchEnabled: agent.config.getBool('remote', 'touch_enabled')
|
|
216
|
+
remoteTouchEnabled: agent.config.getBool('remote', 'touch_enabled'),
|
|
217
|
+
autoSwitchEnabled: agent.config.autoSwitchEnabled(),
|
|
218
|
+
autoSwitchScope: agent.config.autoSwitchScope(),
|
|
219
|
+
jevDecisionModel: agent.jevDecisionModelState()
|
|
217
220
|
});
|
|
218
221
|
|
|
219
222
|
const snapshotFor = (requested = currentTarget()) => {
|
|
@@ -257,6 +260,7 @@ function createCoreRuntimeAdapter(options = {}) {
|
|
|
257
260
|
};
|
|
258
261
|
|
|
259
262
|
const applyConfig = (config) => {
|
|
263
|
+
let refused = null;
|
|
260
264
|
for (const [key, value] of Object.entries(config || {})) {
|
|
261
265
|
switch (key) {
|
|
262
266
|
case "theme": agent.config.set("ui", "dark_mode", value); break;
|
|
@@ -268,10 +272,18 @@ function createCoreRuntimeAdapter(options = {}) {
|
|
|
268
272
|
case "dialogStyle": agent.config.set("ui", "dialog_style", value); break;
|
|
269
273
|
case "nativeTools": agent.config.set("tools", "enabled", value); break;
|
|
270
274
|
case "providers": agent.updateProviders(mergeProviderConfig(agent.config.providers(), value)); break;
|
|
275
|
+
case "jevDecisionModel": {
|
|
276
|
+
// The Agent owns the Auto/decision-model contract: while Auto is off
|
|
277
|
+
// it refuses every edit, so the TUI cannot unlock a model change.
|
|
278
|
+
const result = agent.updateJevDecisionModel(value || {});
|
|
279
|
+
if (!result.ok) refused = { ok: false, reason: result.reason, state: result.state };
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
271
282
|
default: agent.config.set("ui", key, value);
|
|
272
283
|
}
|
|
273
284
|
}
|
|
274
285
|
agent.config.save();
|
|
286
|
+
if (refused) return refused;
|
|
275
287
|
return { ok: true };
|
|
276
288
|
};
|
|
277
289
|
|
package/dist/tui/src/i18n.js
CHANGED
|
@@ -73,12 +73,13 @@ const ZH = Object.freeze({
|
|
|
73
73
|
"Type a message · Shift+Enter newline · Enter send": "输入消息 · Shift+Enter 换行 · Enter 发送",
|
|
74
74
|
"Select a conversation · Enter to edit": "选择对话 · Enter 编辑",
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
// Settings categories
|
|
77
77
|
"General": "通用",
|
|
78
78
|
"Personalization": "个性化",
|
|
79
79
|
"Runtime": "运行时",
|
|
80
80
|
"Providers": "服务商",
|
|
81
81
|
"Models": "模型",
|
|
82
|
+
"JEV Auto routing": "JEV 自动路由",
|
|
82
83
|
"Archive": "归档",
|
|
83
84
|
"Updates": "更新",
|
|
84
85
|
|
|
@@ -103,6 +104,16 @@ const ZH = Object.freeze({
|
|
|
103
104
|
"Retention": "保留期",
|
|
104
105
|
"Include Memory Lab": "包含记忆实验室",
|
|
105
106
|
"Export format": "导出格式",
|
|
107
|
+
"Current-model decision": "当前模型决策",
|
|
108
|
+
"Decision source": "决策来源",
|
|
109
|
+
"Current eligible model": "当前符合条件的模型",
|
|
110
|
+
"Thinking mode": "思考模式",
|
|
111
|
+
"Off": "关闭",
|
|
112
|
+
"Output format": "输出格式",
|
|
113
|
+
"Strict JSON Schema": "严格 JSON Schema",
|
|
114
|
+
"Decision timeout": "决策超时",
|
|
115
|
+
"Deterministic fallback": "确定性回退",
|
|
116
|
+
"Auto routing": "自动路由",
|
|
106
117
|
"Update channel": "更新通道",
|
|
107
118
|
"Automatic checks": "自动检查",
|
|
108
119
|
"Automatic download": "自动下载",
|
package/dist/tui/src/render.js
CHANGED
|
@@ -987,7 +987,7 @@ function settingsView(state, width, p) {
|
|
|
987
987
|
const style = state.focusRegion === "content" && state.contentColumn === 1 && i === state.selected ? `${p.selected}` : "";
|
|
988
988
|
if (style) state.contentFocusLine = 4 + i * 2;
|
|
989
989
|
return [
|
|
990
|
-
`${style} ${p.bold}${pad(row.label, 27)}${p.reset} ${p.cyan}${displaySettingValue(row.value)}${p.reset}${row.extension ? ` ${p.amber}TUI EXTENSION${p.reset}` : ""}`,
|
|
990
|
+
`${style} ${p.bold}${pad(row.label, 27)}${p.reset} ${p.cyan}${displaySettingValue(row.value)}${p.reset}${row.locked ? ` ${p.amber}LOCKED${p.reset}` : ""}${row.extension ? ` ${p.amber}TUI EXTENSION${p.reset}` : ""}`,
|
|
991
991
|
` ${p.muted}${settingDescription(state.settingsTab, row.key)}${p.reset}`
|
|
992
992
|
];
|
|
993
993
|
}),
|
|
@@ -1030,7 +1030,13 @@ function settingDescription(tab, key) {
|
|
|
1030
1030
|
channel: "Stable or prerelease update stream.",
|
|
1031
1031
|
autoCheck: "Check for updates during application startup.",
|
|
1032
1032
|
autoDownload: "Download only after a successful simulated check.",
|
|
1033
|
-
source: "GitHub release or local package source."
|
|
1033
|
+
source: "GitHub release or local package source.",
|
|
1034
|
+
enabled: "Enable or disable model-backed Auto decisions; disabled means Auto keeps the current model.",
|
|
1035
|
+
decisionModel: "Use the current eligible conversation model, or the configured default on the first Auto route.",
|
|
1036
|
+
thinkingMode: "JEV requests the provider's no-thinking mode.",
|
|
1037
|
+
outputFormat: "JEV uses native strict JSON Schema output and validates the returned object before routing.",
|
|
1038
|
+
timeoutMs: "Hard budget for one current-model Auto decision; a timeout stops Auto routing without selecting another model.",
|
|
1039
|
+
autoSwitch: "Auto routing master switch; while it is off current-model decision settings are locked and no model changes."
|
|
1034
1040
|
};
|
|
1035
1041
|
return descriptions[key] || `${tab} setting`;
|
|
1036
1042
|
}
|
|
@@ -1077,6 +1083,7 @@ function settingsActionHint(tab) {
|
|
|
1077
1083
|
if (tab === "general") return "Enter opens Guide/Next list for Input mode · O open config · R refresh";
|
|
1078
1084
|
if (tab === "archive") return "L list archive inventory";
|
|
1079
1085
|
if (tab === "updates") return "U check version and GitHub update";
|
|
1086
|
+
if (tab === "jev") return "Enter changes current-model decision settings · thinking off · strict JSON Schema · failures stop routing";
|
|
1080
1087
|
return "Enter cycles or toggles the selected setting";
|
|
1081
1088
|
}
|
|
1082
1089
|
|
|
@@ -6,6 +6,7 @@ const SETTINGS_CATEGORIES = Object.freeze([
|
|
|
6
6
|
{ id: "runtime", icon: "[R]", label: "Runtime", source: "agent.* / terminal.*" },
|
|
7
7
|
{ id: "providers", icon: "[V]", label: "Providers", source: "models.providers" },
|
|
8
8
|
{ id: "models", icon: "[M]", label: "Models", source: "models.providers[].models" },
|
|
9
|
+
{ id: "jev", icon: "[J]", label: "Auto decision policy", source: "current eligible conversation model + models.jev_* policy" },
|
|
9
10
|
{ id: "tools", icon: "[T]", label: "Tools", source: "tools.enabled" },
|
|
10
11
|
{ id: "archive", icon: "[A]", label: "Archive", source: "archive API / archive.*" },
|
|
11
12
|
{ id: "updates", icon: "[U]", label: "Updates", source: "update API / updates.*" }
|
|
@@ -15,6 +16,8 @@ const FONT_CHOICES = Object.freeze(["Terminal default", "Cascadia Mono", "JetBra
|
|
|
15
16
|
const FONT_COLOR_CHOICES = Object.freeze(["#E6EAF2", "#B7E4FF", "#C7F9CC", "#FFD6A5", "#F8B4D9"]);
|
|
16
17
|
const BACKGROUND_COLOR_CHOICES = Object.freeze(["#0A0A1A", "#111827", "#172554", "#1F2937", "#F0F2F8"]);
|
|
17
18
|
|
|
19
|
+
const JEV_TIMEOUT_PRESETS = Object.freeze([500, 1000, 2000, 3000, 5000, 10000]);
|
|
20
|
+
|
|
18
21
|
function settingsRows(state, tab = state.settingsTab) {
|
|
19
22
|
const s = state.settings;
|
|
20
23
|
if (tab === "general") return [
|
|
@@ -40,6 +43,21 @@ function settingsRows(state, tab = state.settingsTab) {
|
|
|
40
43
|
{ key: "terminalTimeout", label: "Terminal timeout cap", value: s.runtime.terminalTimeout, choices: [0, 30000, 60000, 120000], save: ["setting", "terminal", "interrupt_timeout_ms"] },
|
|
41
44
|
{ key: "defaultShell", label: "Default shell", value: s.runtime.defaultShell, choices: ["PowerShell", "Command Prompt", "WSL Bash"], save: ["setting", "terminal", "default_shell"] }
|
|
42
45
|
];
|
|
46
|
+
if (tab === "jev") {
|
|
47
|
+
const jev = s.jev || {};
|
|
48
|
+
// Auto off locks current-model decision settings: the rows stay visible so the user can see
|
|
49
|
+
// why, but Enter refuses every change (see setSelectedRow in state.js).
|
|
50
|
+
const locked = jev.locked === true || jev.autoEnabled === false;
|
|
51
|
+
const lockedValue = locked ? "Locked (Auto off)" : undefined;
|
|
52
|
+
return [
|
|
53
|
+
{ key: "enabled", label: "Current-model decision", value: lockedValue || (jev.enabled === false ? "Off" : "On"), choices: [], action: "jev-toggle-enabled", locked },
|
|
54
|
+
{ key: "decisionModel", label: "Decision source", value: "Current eligible model", choices: [], action: "jev-report-current-model", locked: true },
|
|
55
|
+
{ key: "thinkingMode", label: "Thinking mode", value: "Off", choices: [], action: "jev-report-current-model", locked: true },
|
|
56
|
+
{ key: "outputFormat", label: "Output format", value: "Strict JSON Schema", choices: [], action: "jev-report-current-model", locked: true },
|
|
57
|
+
{ key: "timeoutMs", label: "Decision timeout", value: lockedValue || `${Number(jev.timeoutMs) || 3000} ms`, choices: [], action: "jev-cycle-timeout", locked },
|
|
58
|
+
{ key: "autoSwitch", label: "Auto routing", value: jev.autoEnabled === false ? "Off" : "On", choices: [], action: "jev-report-auto" }
|
|
59
|
+
];
|
|
60
|
+
}
|
|
43
61
|
if (tab === "archive") return [
|
|
44
62
|
{ key: "autoArchive", label: "Automatic archive", value: s.archive.autoArchive, choices: [true, false], save: ["setting", "archive", "automatic"] },
|
|
45
63
|
{ key: "retentionDays", label: "Retention", value: s.archive.retentionDays, choices: [30, 90, 180, 365], save: ["setting", "archive", "retention_days"] },
|
|
@@ -67,6 +85,7 @@ module.exports = {
|
|
|
67
85
|
BACKGROUND_COLOR_CHOICES,
|
|
68
86
|
FONT_CHOICES,
|
|
69
87
|
FONT_COLOR_CHOICES,
|
|
88
|
+
JEV_TIMEOUT_PRESETS,
|
|
70
89
|
SETTINGS_CATEGORIES,
|
|
71
90
|
displaySettingValue,
|
|
72
91
|
settingsRows
|
package/dist/tui/src/state.js
CHANGED
|
@@ -3,7 +3,11 @@
|
|
|
3
3
|
const data = require("./data");
|
|
4
4
|
const { createMockNewmarkAdapter } = require("./adapters/mock-newmark-adapter");
|
|
5
5
|
const { targetKey, validateSnapshot } = require("./adapters/newmark-contract");
|
|
6
|
-
const {
|
|
6
|
+
const {
|
|
7
|
+
JEV_TIMEOUT_PRESETS,
|
|
8
|
+
SETTINGS_CATEGORIES,
|
|
9
|
+
settingsRows
|
|
10
|
+
} = require("./settings-schema");
|
|
7
11
|
const INTELLIGENCE_TIERS = Object.freeze(["low", "medium", "high", "xhigh", "max", "ultra"]);
|
|
8
12
|
const HEX_COLOR_PATTERN = /^#[0-9a-f]{6}$/i;
|
|
9
13
|
const THEME_APPEARANCE_DEFAULTS = Object.freeze({
|
|
@@ -93,6 +97,17 @@ function applySnapshot(state, snapshot) {
|
|
|
93
97
|
if (state.runningConversationKeys) {
|
|
94
98
|
state.busy = state.runningConversationKeys.has(targetKey(valid.target));
|
|
95
99
|
}
|
|
100
|
+
// Current-model decision settings + Auto master switch drive the locked state of the JEV
|
|
101
|
+
// settings rows, so they must follow every authoritative snapshot.
|
|
102
|
+
if (valid.jevDecisionModel && state.settings) {
|
|
103
|
+
state.jevDecisionModel = valid.jevDecisionModel;
|
|
104
|
+
state.settings.jev = {
|
|
105
|
+
autoEnabled: valid.jevDecisionModel.autoEnabled !== false,
|
|
106
|
+
locked: valid.jevDecisionModel.locked === true || valid.jevDecisionModel.autoEnabled === false,
|
|
107
|
+
enabled: valid.jevDecisionModel.enabled !== false,
|
|
108
|
+
timeoutMs: Number(valid.jevDecisionModel.timeoutMs || 3000)
|
|
109
|
+
};
|
|
110
|
+
}
|
|
96
111
|
return state;
|
|
97
112
|
}
|
|
98
113
|
|
|
@@ -280,6 +295,12 @@ function createState(options = {}) {
|
|
|
280
295
|
autoCheck: true,
|
|
281
296
|
autoDownload: false,
|
|
282
297
|
source: "GitHub"
|
|
298
|
+
},
|
|
299
|
+
jev: {
|
|
300
|
+
autoEnabled: snapshot.autoSwitchEnabled !== false,
|
|
301
|
+
locked: snapshot.autoSwitchEnabled === false,
|
|
302
|
+
enabled: snapshot.jevDecisionModel ? snapshot.jevDecisionModel.enabled !== false : true,
|
|
303
|
+
timeoutMs: Number(snapshot.jevDecisionModel?.timeoutMs || 3000)
|
|
283
304
|
}
|
|
284
305
|
}
|
|
285
306
|
};
|
|
@@ -1072,6 +1093,9 @@ function toggleSelected(state) {
|
|
|
1072
1093
|
}
|
|
1073
1094
|
const row = settingsRows(state)[state.selected];
|
|
1074
1095
|
if (!row) return;
|
|
1096
|
+
if (typeof row.action === "string" && row.action.startsWith("jev-")) {
|
|
1097
|
+
return applyJevDecisionRow(state, row);
|
|
1098
|
+
}
|
|
1075
1099
|
if (row.action === "pair-mobile") {
|
|
1076
1100
|
state.overlay = "pair";
|
|
1077
1101
|
state.pairingQrLines = ["Loading pairing QR…"];
|
|
@@ -1112,6 +1136,50 @@ function toggleSelected(state) {
|
|
|
1112
1136
|
}
|
|
1113
1137
|
}
|
|
1114
1138
|
|
|
1139
|
+
/**
|
|
1140
|
+
* Current-model JEV settings rows.
|
|
1141
|
+
*
|
|
1142
|
+
* The Agent is authoritative: while Auto is off it refuses every edit, so the
|
|
1143
|
+
* TUI reports the lock instead of pretending a change happened. The decision
|
|
1144
|
+
* source is always the current eligible conversation model.
|
|
1145
|
+
*/
|
|
1146
|
+
function applyJevDecisionRow(state, row) {
|
|
1147
|
+
const jev = state.jevDecisionModel && typeof state.jevDecisionModel === "object" ? state.jevDecisionModel : {};
|
|
1148
|
+
if (row.action === "jev-report-auto") {
|
|
1149
|
+
state.notice = jev.autoEnabled === false
|
|
1150
|
+
? "Auto routing is off: current-model decision settings are locked and no model may change."
|
|
1151
|
+
: "Auto routing is on: JEV uses the current eligible conversation model with thinking off and strict JSON Schema output.";
|
|
1152
|
+
return true;
|
|
1153
|
+
}
|
|
1154
|
+
if (row.action === "jev-report-current-model") {
|
|
1155
|
+
state.notice = "JEV sends a strict JSON Schema decision request to the current eligible conversation model with thinking disabled.";
|
|
1156
|
+
return true;
|
|
1157
|
+
}
|
|
1158
|
+
if (jev.locked === true || jev.autoEnabled === false) {
|
|
1159
|
+
state.notice = "Auto is off: current-model decision settings are locked. Turn Auto on first — no model may change while Auto is off.";
|
|
1160
|
+
return true;
|
|
1161
|
+
}
|
|
1162
|
+
const patch = {};
|
|
1163
|
+
if (row.action === "jev-toggle-enabled") patch.enabled = jev.enabled === false;
|
|
1164
|
+
if (row.action === "jev-cycle-timeout") {
|
|
1165
|
+
const index = JEV_TIMEOUT_PRESETS.indexOf(Number(jev.timeoutMs));
|
|
1166
|
+
patch.timeoutMs = JEV_TIMEOUT_PRESETS[(index + 1 + JEV_TIMEOUT_PRESETS.length) % JEV_TIMEOUT_PRESETS.length];
|
|
1167
|
+
}
|
|
1168
|
+
if (!Object.keys(patch).length) return true;
|
|
1169
|
+
const result = state.adapter.saveConfig({ jevDecisionModel: patch });
|
|
1170
|
+
if (result && typeof result.then === "function") throw new TypeError("TUI settings adapter must be synchronous");
|
|
1171
|
+
if (!result || result.ok === false) {
|
|
1172
|
+
state.notice = result && result.reason === "auto_disabled"
|
|
1173
|
+
? "Auto is off: current-model decision settings are locked and no model may change."
|
|
1174
|
+
: "The current-model decision settings were not changed.";
|
|
1175
|
+
if (result && result.state) state.jevDecisionModel = result.state;
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
1178
|
+
state.jevDecisionModel = result.state || jev;
|
|
1179
|
+
state.notice = `JEV current-model settings updated · ${state.adapterKind === "mock" ? "mock" : "persisted"}`;
|
|
1180
|
+
return true;
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1115
1183
|
function selectedSettingChoiceRow(state) {
|
|
1116
1184
|
if (state.overlay !== "settings-choice" || !state.settingChoiceKey) return null;
|
|
1117
1185
|
return settingsRows(state, state.settingChoiceTab)
|