mixdog 1.0.8 → 1.0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/package.json +2 -1
  2. package/scripts/lib/run-node-tests.mjs +13 -0
  3. package/scripts/session-efficiency-diag.mjs +283 -0
  4. package/src/defaults/skills/computer-use/SKILL.md +3 -3
  5. package/src/defaults/skills/local-provider/SKILL.md +2 -1
  6. package/src/defaults/skills/setup/SKILL.md +4 -4
  7. package/src/defaults/skills/setup/references/actions.md +1 -3
  8. package/src/defaults/skills/setup/references/surfaces.md +2 -3
  9. package/src/headless-exec.mjs +1 -0
  10. package/src/rules/shared/70-delivery.md +2 -2
  11. package/src/runtime/agent/orchestrator/config-providers-merge.mjs +8 -0
  12. package/src/runtime/agent/orchestrator/providers/anthropic-fast-mode.mjs +10 -0
  13. package/src/runtime/agent/orchestrator/providers/anthropic-max-tokens.mjs +1 -1
  14. package/src/runtime/agent/orchestrator/providers/anthropic-oauth-client-version.mjs +1 -1
  15. package/src/runtime/agent/orchestrator/providers/anthropic-oauth-request/initial-status.mjs +8 -4
  16. package/src/runtime/agent/orchestrator/providers/anthropic-oauth-request.mjs +10 -1
  17. package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +1 -1
  18. package/src/runtime/agent/orchestrator/providers/antigravity-oauth-tokens.mjs +18 -3
  19. package/src/runtime/agent/orchestrator/providers/client-version-store.mjs +50 -0
  20. package/src/runtime/agent/orchestrator/providers/codex-client-meta.mjs +27 -4
  21. package/src/runtime/agent/orchestrator/providers/cursor-client-version.mjs +9 -5
  22. package/src/runtime/agent/orchestrator/providers/custom-provider.mjs +163 -0
  23. package/src/runtime/agent/orchestrator/providers/effort-configuration.mjs +18 -4
  24. package/src/runtime/agent/orchestrator/providers/grok-client-version.mjs +2 -2
  25. package/src/runtime/agent/orchestrator/providers/lib/anthropic-models.mjs +7 -1
  26. package/src/runtime/agent/orchestrator/providers/model-list-sanitize.mjs +23 -4
  27. package/src/runtime/agent/orchestrator/providers/npm-cli-version.mjs +25 -2
  28. package/src/runtime/agent/orchestrator/providers/oauth-credential-probes.mjs +7 -1
  29. package/src/runtime/agent/orchestrator/providers/openai-codex-model.mjs +4 -0
  30. package/src/runtime/agent/orchestrator/providers/openai-compat-models.mjs +3 -2
  31. package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +8 -0
  32. package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +1 -28
  33. package/src/runtime/agent/orchestrator/providers/openai-direct-request.mjs +17 -3
  34. package/src/runtime/agent/orchestrator/providers/openai-oauth-catalog.mjs +7 -1
  35. package/src/runtime/agent/orchestrator/providers/provider-base-url.mjs +24 -0
  36. package/src/runtime/agent/orchestrator/providers/registry.mjs +14 -4
  37. package/src/runtime/agent/orchestrator/providers/retry-classification.mjs +16 -0
  38. package/src/runtime/agent/orchestrator/runtime-core/builtin-features.mjs +0 -20
  39. package/src/runtime/agent/orchestrator/runtime-core/config-helpers.mjs +3 -2
  40. package/src/runtime/agent/orchestrator/runtime-core/model-capabilities.mjs +12 -10
  41. package/src/runtime/agent/orchestrator/session/compact/runner.mjs +3 -1
  42. package/src/runtime/agent/orchestrator/session/loop/fresh-context.mjs +1 -1
  43. package/src/runtime/agent/orchestrator/session/manager/context-meta.mjs +15 -3
  44. package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +11 -2
  45. package/src/runtime/agent/orchestrator/session/manager/session-prompt-composition.mjs +5 -1
  46. package/src/runtime/agent/orchestrator/session/manager/session-tool-surface.mjs +23 -9
  47. package/src/runtime/agent/orchestrator/tools/builtin/bash-tool/deadline-plan.mjs +15 -6
  48. package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +4 -4
  49. package/src/runtime/agent/orchestrator/tools/patch/native-server.mjs +22 -1
  50. package/src/runtime/channels/lib/voice-transcription.mjs +30 -4
  51. package/src/runtime/computer-bridge/action-schema.mjs +3 -3
  52. package/src/runtime/computer-bridge/actions.mjs +4 -0
  53. package/src/runtime/computer-bridge/bridge-env.test-support.mjs +22 -0
  54. package/src/runtime/computer-bridge/client.mjs +10 -183
  55. package/src/runtime/computer-bridge/core-actions.mjs +2 -1
  56. package/src/runtime/computer-bridge/error-recovery.mjs +46 -34
  57. package/src/runtime/computer-bridge/result-canonical.mjs +181 -0
  58. package/src/runtime/local-provider/asset-installer.mjs +20 -10
  59. package/src/runtime/local-provider/catalog.mjs +4 -7
  60. package/src/runtime/local-provider/data/manifest.json +14 -1
  61. package/src/runtime/local-provider/hardware.mjs +36 -5
  62. package/src/runtime/local-provider/resumable-installations.mjs +2 -2
  63. package/src/runtime/local-provider/server.mjs +1 -1
  64. package/src/runtime/memory/index.mjs +2 -4
  65. package/src/runtime/memory/lib/core-memory-file.mjs +3 -8
  66. package/src/runtime/memory/lib/core-memory-store.mjs +14 -61
  67. package/src/runtime/memory/lib/cycle-scheduler/backlog-probe.mjs +5 -9
  68. package/src/runtime/memory/lib/cycle1/cycle1-plan.mjs +7 -0
  69. package/src/runtime/memory/lib/cycle1/cycle1-rows.mjs +43 -16
  70. package/src/runtime/memory/lib/cycle1/cycle1-window.mjs +11 -2
  71. package/src/runtime/memory/lib/embedding-provider.mjs +2 -5
  72. package/src/runtime/memory/lib/embedding-reindex.mjs +1 -5
  73. package/src/runtime/memory/lib/http-router/lifecycle-routes.mjs +0 -1
  74. package/src/runtime/memory/lib/memory-action-handlers/maintenance-actions.mjs +1 -1
  75. package/src/runtime/memory/lib/memory-schema/core-and-meta.mjs +4 -22
  76. package/src/runtime/memory/lib/memory.mjs +79 -32
  77. package/src/runtime/memory/lib/query-maintenance-handlers.mjs +0 -2
  78. package/src/runtime/office/com/com-adapter.mjs +3 -0
  79. package/src/runtime/office/pdf/pdf-adapter.mjs +50 -14
  80. package/src/runtime/office/pdf/pdf-render.mjs +2 -6
  81. package/src/runtime/office/pdf/pdf-security.mjs +8 -2
  82. package/src/runtime/office/portable/font-provisioner.mjs +28 -0
  83. package/src/runtime/office/portable/portable-docx-edits.mjs +28 -1
  84. package/src/runtime/office/portable/portable-soffice.mjs +44 -7
  85. package/src/runtime/shared/llm/anthropic-betas.mjs +7 -1
  86. package/src/runtime/shared/llm/anthropic-thinking-contract.mjs +1 -1
  87. package/src/runtime/shared/llm/default-context-window.mjs +42 -0
  88. package/src/runtime/shared/llm/model-catalog-projection.mjs +2 -0
  89. package/src/runtime/shared/llm/model-catalog.mjs +2 -0
  90. package/src/runtime/shared/model-alias.mjs +5 -0
  91. package/src/runtime/shared/plugin-paths.mjs +9 -2
  92. package/src/runtime/shared/worker-exec-argv.mjs +25 -0
  93. package/src/runtime/web-search/lib/formatter.mjs +43 -18
  94. package/src/session-runtime/automation-agents.mjs +88 -0
  95. package/src/session-runtime/automation-workflow.mjs +19 -6
  96. package/src/session-runtime/boot/apis.mjs +6 -1
  97. package/src/session-runtime/cwd-plugins/core-memory-context.mjs +14 -12
  98. package/src/session-runtime/internal-tool-executor/bridge-tools.mjs +4 -25
  99. package/src/session-runtime/model-recency.mjs +10 -2
  100. package/src/session-runtime/model-route/set-route.mjs +7 -1
  101. package/src/session-runtime/provider-auth/login.mjs +26 -1
  102. package/src/session-runtime/quick-model-rows/provider-rows.mjs +17 -2
  103. package/src/session-runtime/resource-api.mjs +8 -2
  104. package/src/session-runtime/resource-plugins-api.mjs +14 -2
  105. package/src/session-runtime/route-state.mjs +6 -2
  106. package/src/session-runtime/schedule-session-run.mjs +7 -1
  107. package/src/session-runtime/services/agent-tool/render.mjs +0 -5
  108. package/src/session-runtime/services/provider-admin.mjs +201 -3
  109. package/src/session-runtime/settings-builtin-tools-api.mjs +0 -16
  110. package/src/session-runtime/setup-tool/executor.mjs +7 -13
  111. package/src/session-runtime/setup-tool/tool-defs.mjs +2 -3
  112. package/src/session-runtime/tool-policy-refresh.mjs +0 -2
  113. package/src/session-runtime/webhook-session-run.mjs +10 -11
  114. package/src/session-runtime/workflow-agents-api/agent-route.mjs +5 -0
  115. package/src/standalone/daemon-agent-control.mjs +8 -1
  116. package/src/standalone/daemon.mjs +4 -0
  117. package/src/standalone/session-protocol.mjs +4 -0
  118. package/src/tui/app/model-options.mjs +10 -1
  119. package/src/tui/dist/index.mjs +12 -3
  120. package/src/tui/session/agent-job-feed/notification-router.mjs +7 -0
  121. package/src/tui/session/live-share.mjs +16 -1
  122. package/src/tui/session/notification-plan.mjs +4 -0
  123. package/src/tui/session/session-api/integrations.mjs +12 -0
  124. package/src/session-runtime/bridge-first-use-gate.mjs +0 -72
@@ -13,13 +13,20 @@ import {
13
13
  toComputerHostCommand,
14
14
  validateComputerToolArgs,
15
15
  } from './action-schema.mjs';
16
- import { computerResultRecovery, formatComputerToolError } from './error-recovery.mjs';
16
+ import { formatComputerToolError } from './error-recovery.mjs';
17
+ import {
18
+ canonicalComputerResultIsError,
19
+ canonicalComputerResultText,
20
+ computerImageDimensions,
21
+ } from './result-canonical.mjs';
17
22
  import { computerErrorCode } from './error-code.mjs';
18
23
  import { bridgeDiscoveryChanged, readBridgeDiscovery } from '../bridge-discovery.mjs';
19
24
  import { MAX_COMPUTER_REQUEST_BYTES, readComputerBridgeJson, validateComputerReply } from './limits.mjs';
20
25
  import { computerActionHas } from './actions.mjs';
21
26
  import { continuePendingComputerWork, isPendingComputerWork } from './pending-continuation.mjs';
22
27
 
28
+ export { canonicalComputerResultText };
29
+
23
30
  const DISCOVERY_FILE = 'computer-bridge.json';
24
31
  // Desktop UI Automation queries and input dispatch can be slow; sit above the
25
32
  // bridge's own per-action timeouts so its specific error wins over a bare abort.
@@ -48,186 +55,6 @@ const activeComputerExecutions = new Set();
48
55
  const BRIDGE_UNAVAILABLE_MESSAGE =
49
56
  'computer use is unavailable; open the Mixdog desktop app and enable Computer Use in settings';
50
57
 
51
- const ACT_STEP_STATUSES = new Set(['succeeded', 'failed', 'skipped', 'pending', 'uncertain']);
52
-
53
- function canonicalizeActResult(value, args) {
54
- value.completed_actions = value.completed_steps;
55
- value.total_actions = value.total_steps;
56
- const canonicalStep = (row, index) => {
57
- const normalized = { ...row };
58
- normalized.type = args.input?.actions?.[index]?.type || normalized.action;
59
- if (!ACT_STEP_STATUSES.has(normalized.status)) {
60
- normalized.status = normalized.ok === false ? 'failed' : 'succeeded';
61
- }
62
- delete normalized.action;
63
- delete normalized.ok;
64
- return normalized;
65
- };
66
- value.actions = Array.isArray(value.steps) ? value.steps.map(canonicalStep) : value.steps;
67
- // A single-window act reports the same transition on the act and its step.
68
- const transition = JSON.stringify(value.window_transition);
69
- for (const row of Array.isArray(value.actions) ? value.actions : []) {
70
- if (transition && JSON.stringify(row?.window_transition) === transition) delete row.window_transition;
71
- }
72
- delete value.completed_steps;
73
- delete value.total_steps;
74
- delete value.steps;
75
- }
76
-
77
- // Host diagnostics stay in the host's run record; the model acts on none of them.
78
- const HOST_DIAGNOSTIC_FIELDS = ['timings_ms', 'capture_attempts'];
79
-
80
- function dropHostDiagnostics(value) {
81
- if (!value || typeof value !== 'object') return;
82
- if (Array.isArray(value)) {
83
- for (const item of value) dropHostDiagnostics(item);
84
- return;
85
- }
86
- for (const field of HOST_DIAGNOSTIC_FIELDS) delete value[field];
87
- for (const item of Object.values(value)) dropHostDiagnostics(item);
88
- }
89
-
90
- // Elements and OCR text reach the model one line each, the shape browser
91
- // snapshots use, because a field name costs more than the value it labels:
92
- // `#mark [ref] Role "name" value="…" state=… source=… disabled focused
93
- // @x,y,width,height actions`. Flags appear only when they differ from the
94
- // common case the capture description states (an enabled UIA element), and
95
- // the geometry a som frame repeats as x/y/width/height, center and
96
- // screen_bounds collapses into its one bounds box.
97
- const DERIVED_GEOMETRY_FIELDS = new Set([
98
- 'x',
99
- 'y',
100
- 'width',
101
- 'height',
102
- 'center_x',
103
- 'center_y',
104
- 'center',
105
- 'screen_bounds',
106
- ]);
107
-
108
- const quotedText = (text) => JSON.stringify(String(text));
109
- const bareText = (text) => (/^[^\s"]+$/.test(String(text)) ? String(text) : quotedText(text));
110
- const isUnsetField = (field) =>
111
- field === undefined || field === null || field === '' || field === false || (Array.isArray(field) && !field.length);
112
-
113
- function boundsText(box) {
114
- return box.length === 4 && box.every((edge) => Number.isFinite(edge)) ? `@${box.join(',')}` : null;
115
- }
116
-
117
- function extraFieldTexts(rest) {
118
- return Object.entries(rest)
119
- .filter(([key, field]) => !DERIVED_GEOMETRY_FIELDS.has(key) && !isUnsetField(field))
120
- .map(([key, field]) => `${key}=${typeof field === 'string' ? bareText(field) : JSON.stringify(field)}`);
121
- }
122
-
123
- function elementLine(element) {
124
- if (!element || typeof element !== 'object') return element;
125
- const { mark, ref, role, name, value, state, source, enabled, has_keyboard_focus, bounds, actions, ...rest } =
126
- element;
127
- return [
128
- isUnsetField(mark) ? null : `#${mark}`,
129
- isUnsetField(ref) ? null : `[${ref}]`,
130
- bareText(role || 'Unknown'),
131
- quotedText(name ?? ''),
132
- isUnsetField(value) ? null : `value=${quotedText(value)}`,
133
- isUnsetField(state) || state === source ? null : `state=${bareText(state)}`,
134
- isUnsetField(source) || source === 'uia' ? null : `source=${bareText(source)}`,
135
- enabled === false ? 'disabled' : null,
136
- has_keyboard_focus === true ? 'focused' : null,
137
- boundsText(Array.isArray(bounds) ? bounds : [rest.x, rest.y, rest.width, rest.height]),
138
- Array.isArray(actions) && actions.length ? actions.join(',') : null,
139
- ...extraFieldTexts(rest),
140
- ]
141
- .filter(Boolean)
142
- .join(' ');
143
- }
144
-
145
- function ocrTextLine(row) {
146
- if (!row || typeof row !== 'object') return row;
147
- const { mark, text, line, x, y, width, height, ...rest } = row;
148
- return [
149
- isUnsetField(mark) ? null : `#${mark}`,
150
- quotedText(text ?? ''),
151
- boundsText([x, y, width, height]),
152
- isUnsetField(line) ? null : `line=${line}`,
153
- ...extraFieldTexts(rest),
154
- ]
155
- .filter(Boolean)
156
- .join(' ');
157
- }
158
-
159
- function renderObservationLines(frame) {
160
- if (!frame || typeof frame !== 'object') return;
161
- if (Array.isArray(frame.elements)) frame.elements = frame.elements.map(elementLine);
162
- for (const key of ['lines', 'words']) {
163
- if (Array.isArray(frame.ocr?.[key])) frame.ocr[key] = frame.ocr[key].map(ocrTextLine);
164
- }
165
- }
166
-
167
- export function canonicalComputerResultText(text, args) {
168
- if (args.action === 'clipboard' && args.input?.operation === 'read') return text;
169
- let value;
170
- try {
171
- value = JSON.parse(text);
172
- } catch {
173
- return text;
174
- }
175
- if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.action !== 'string') {
176
- return text;
177
- }
178
- value.action = args.action;
179
- if (args.action === 'list') value.kind = args.input?.kind;
180
- if (args.action === 'capture') value.mode = args.input?.mode || 'state';
181
- if (args.action === 'window') value.operation = args.input?.operation;
182
- if (args.action === 'clipboard') value.operation = args.input?.operation;
183
- if (args.action === 'act') canonicalizeActResult(value, args);
184
- if (value.capture_after && value.observation === undefined) {
185
- value.observation = value.capture_after;
186
- delete value.capture_after;
187
- }
188
- dropHostDiagnostics(value);
189
- renderObservationLines(value);
190
- renderObservationLines(value.observation);
191
- if (value.ok === false && value.recovery === undefined) {
192
- const recovery = computerResultRecovery(value, args);
193
- if (recovery) value.recovery = recovery;
194
- }
195
- return JSON.stringify(value);
196
- }
197
-
198
- // A capture payload reports the frame's real pixel size (`width`/`height`,
199
- // nested under `observation` for action replies). Carrying it on the image
200
- // block lets the context estimator bill a screenshot at its true vision cost
201
- // instead of the flat unknown-image allowance. Provider normalizers rebuild
202
- // the wire block from `type`/`source` alone, so these fields never reach an
203
- // API and never shift the cached prefix.
204
- function computerImageDimensions(text) {
205
- let value;
206
- try {
207
- value = JSON.parse(text);
208
- } catch {
209
- return null;
210
- }
211
- for (const frame of [value, value?.observation]) {
212
- const width = Number(frame?.width);
213
- const height = Number(frame?.height);
214
- if (width > 0 && height > 0 && Number.isFinite(width) && Number.isFinite(height)) return { width, height };
215
- }
216
- return null;
217
- }
218
-
219
- function canonicalComputerResultIsError(text, args) {
220
- if (args.action === 'clipboard' && args.input?.operation === 'read') return false;
221
- try {
222
- const value = JSON.parse(text);
223
- return Boolean(
224
- value && typeof value === 'object' && !Array.isArray(value) && value.action === args.action && value.ok === false
225
- );
226
- } catch {
227
- return false;
228
- }
229
- }
230
-
231
58
  /** Sync gate for the session tool surface (featureDisallowedTools). */
232
59
  export function computerBridgeAvailableSync() {
233
60
  return readBridgeDiscovery(DISCOVERY_FILE) !== null;
@@ -541,8 +368,8 @@ async function abortComputerSession(sessionId) {
541
368
  return aborted;
542
369
  }
543
370
 
544
- /** End the visible Computer Use execution at agent-turn settlement without
545
- * dropping the warm worker and observation refs kept for a possible follow-up. */
371
+ /** End the visible Computer Use execution at agent-turn settlement and hand the
372
+ * desktop back, keeping the warm worker and observation refs for a possible follow-up. */
546
373
  export async function endComputerExecution(sessionId) {
547
374
  const id = String(sessionId || '').trim();
548
375
  if (!id || !activeComputerExecutions.has(id)) return false;
@@ -276,7 +276,8 @@ function actionArgumentsError(action, type, label, frameId) {
276
276
  if (hasOwn(action, 'ref') && hasOwn(action, 'element')) return `${label} accepts only one of ref or element`;
277
277
  return typeof action.value === 'string' ? null : `${label} requires value`;
278
278
  }
279
- if (type === 'key' || type === 'key_down' || type === 'key_up') return typeof action.keys === 'string' ? null : `${label} requires keys`;
279
+ if (type === 'key' || type === 'key_down' || type === 'key_up')
280
+ return typeof action.keys === 'string' ? null : `${label} requires keys`;
280
281
  if (type === 'wait') {
281
282
  const { duration } = action;
282
283
  const valid = typeof duration === 'number' && Number.isFinite(duration) && duration >= 0 && duration <= 5;
@@ -21,11 +21,7 @@ function targetLabel(args) {
21
21
  const LIST_WINDOW_CODES = new Set(['ambiguous_window_target', 'window_stale', 'window_target_not_found']);
22
22
  const STALE_TARGET_CODES = new Set(['stale_frame', 'stale_target', 'target_mismatch']);
23
23
  const USER_YIELD_CODES = new Set(['computer_user_control_active', 'computer_user_takeover', 'user_input_active']);
24
- const DIAGNOSE_CODES = new Set([
25
- 'input_observation_unavailable',
26
- 'input_recovery_unconfirmed',
27
- 'computer_cursor_unavailable',
28
- ]);
24
+ const DIAGNOSE_CODES = new Set(['input_observation_unavailable', 'computer_cursor_unavailable']);
29
25
  const CLEANUP_CODES = new Set([
30
26
  'computer_cleanup_pending',
31
27
  'computer_abort_cleanup_unconfirmed',
@@ -47,6 +43,40 @@ function recaptureLeaseGuidance(code, target) {
47
43
  return `The target lease is now available. Capture ${target} again before issuing any input.`;
48
44
  }
49
45
 
46
+ function timeoutRecovery(code, args, target) {
47
+ if (READ_ONLY_ACTIONS.has(String(args?.action || ''))) {
48
+ return {
49
+ code,
50
+ next: 'capture',
51
+ guidance: `The read exceeded its bounded budget and sent no input, so no cleanup or user-control guard applies. Capture ${target} with mode="state" or "som": pixels and OCR stay usable while that window's accessibility provider is slow.`,
52
+ };
53
+ }
54
+ return {
55
+ code,
56
+ next: 'diagnose',
57
+ guidance: `The command may have executed before timing out. Do not repeat it or switch delivery modes. Diagnose the host first; cleanup and user-control guards must clear through verified recovery. Then capture ${target} and inspect the effect before issuing any new input.`,
58
+ };
59
+ }
60
+
61
+ function foregroundRecovery(code, target) {
62
+ if (code === 'foreground_changed') {
63
+ return {
64
+ code,
65
+ next: 'user',
66
+ guidance:
67
+ 'Focus changed during dispatch. Do not pull it back or retry input automatically. Check user control, then obtain fresh state when control is available.',
68
+ };
69
+ }
70
+ if (code === 'foreground_unavailable') {
71
+ return {
72
+ code,
73
+ next: 'user',
74
+ guidance: `The system did not grant foreground focus (macOS refuses it while another app is active). Ask the user to activate ${target}, or use a foreground click on the window, which activates it; then capture fresh state. Do not substitute background input or repeat the failed gesture.`,
75
+ };
76
+ }
77
+ return undefined;
78
+ }
79
+
50
80
  function recoveryForCode(code, args) {
51
81
  const target = targetLabel(args);
52
82
  if (code === 'computer_background_cleanup_unconfirmed') {
@@ -82,6 +112,14 @@ function recoveryForCode(code, args) {
82
112
  "Computer Use yielded to the user. Call wait_for_user for bounded waiting. The user's Resume on the overlay continues the pause, and ordinary physical input may also resume after the host-configured quiet interval (default 5 seconds); explicit stops and uncertain cleanup/observation require the user. Stop cancels the task. Timeout does not authorize input. After resumed, capture fresh state; never replay interrupted input.",
83
113
  };
84
114
  }
115
+ if (code === 'input_recovery_unconfirmed') {
116
+ return {
117
+ code,
118
+ next: 'wait_for_user',
119
+ guidance:
120
+ 'The desktop state after the action could not be verified, so Computer Use stopped the session and handed the desktop to the user. Do not repeat the mutation. Call wait_for_user: the host resumes by itself once the desktop has been quiet for its interval, or when the user presses Resume. After resumed, capture fresh state and inspect the effect before any new input.',
121
+ };
122
+ }
85
123
  if (DIAGNOSE_CODES.has(code)) {
86
124
  return {
87
125
  code,
@@ -105,20 +143,7 @@ function recoveryForCode(code, args) {
105
143
  guidance: `No input was sent. Foreground delivery needs an observation taken while the user's own input was idle. Let their input settle and capture ${target} again, or send this action through background delivery when it supports one.`,
106
144
  };
107
145
  }
108
- if (code === 'computer_command_timeout') {
109
- if (READ_ONLY_ACTIONS.has(String(args?.action || ''))) {
110
- return {
111
- code,
112
- next: 'capture',
113
- guidance: `The read exceeded its bounded budget and sent no input, so no cleanup or user-control guard applies. Capture ${target} with mode="state" or "som": pixels and OCR stay usable while that window's accessibility provider is slow.`,
114
- };
115
- }
116
- return {
117
- code,
118
- next: 'diagnose',
119
- guidance: `The command may have executed before timing out. Do not repeat it or switch delivery modes. Diagnose the host first; cleanup and user-control guards must clear through verified recovery. Then capture ${target} and inspect the effect before issuing any new input.`,
120
- };
121
- }
146
+ if (code === 'computer_command_timeout') return timeoutRecovery(code, args, target);
122
147
  if (code.startsWith('menu_')) {
123
148
  return {
124
149
  code,
@@ -126,21 +151,8 @@ function recoveryForCode(code, args) {
126
151
  guidance: `Capture ${target} again; empty accessibility automatically uses OCR. Use a fresh OCR mark or frame point and do not retry the same menu path unchanged.`,
127
152
  };
128
153
  }
129
- if (code === 'foreground_changed') {
130
- return {
131
- code,
132
- next: 'user',
133
- guidance:
134
- 'Focus changed during dispatch. Do not pull it back or retry input automatically. Check user control, then obtain fresh state when control is available.',
135
- };
136
- }
137
- if (code === 'foreground_unavailable') {
138
- return {
139
- code,
140
- next: 'user',
141
- guidance: `Windows did not grant foreground focus. Ask the user to activate ${target}, then capture fresh state. Do not substitute background input or repeat the failed gesture.`,
142
- };
143
- }
154
+ const foreground = foregroundRecovery(code, target);
155
+ if (foreground) return foreground;
144
156
  if (code.startsWith('background_')) {
145
157
  return {
146
158
  code,
@@ -0,0 +1,181 @@
1
+ import { computerResultRecovery } from './error-recovery.mjs';
2
+
3
+ const ACT_STEP_STATUSES = new Set(['succeeded', 'failed', 'skipped', 'pending', 'uncertain']);
4
+
5
+ function canonicalizeActResult(value, args) {
6
+ value.completed_actions = value.completed_steps;
7
+ value.total_actions = value.total_steps;
8
+ const canonicalStep = (row, index) => {
9
+ const normalized = { ...row };
10
+ normalized.type = args.input?.actions?.[index]?.type || normalized.action;
11
+ if (!ACT_STEP_STATUSES.has(normalized.status)) {
12
+ normalized.status = normalized.ok === false ? 'failed' : 'succeeded';
13
+ }
14
+ delete normalized.action;
15
+ delete normalized.ok;
16
+ return normalized;
17
+ };
18
+ value.actions = Array.isArray(value.steps) ? value.steps.map(canonicalStep) : value.steps;
19
+ // A single-window act reports the same transition on the act and its step.
20
+ const transition = JSON.stringify(value.window_transition);
21
+ for (const row of Array.isArray(value.actions) ? value.actions : []) {
22
+ if (transition && JSON.stringify(row?.window_transition) === transition) delete row.window_transition;
23
+ }
24
+ delete value.completed_steps;
25
+ delete value.total_steps;
26
+ delete value.steps;
27
+ }
28
+
29
+ // Host diagnostics stay in the host's run record; the model acts on none of them.
30
+ const HOST_DIAGNOSTIC_FIELDS = ['timings_ms', 'capture_attempts'];
31
+
32
+ function dropHostDiagnostics(value) {
33
+ if (!value || typeof value !== 'object') return;
34
+ if (Array.isArray(value)) {
35
+ for (const item of value) dropHostDiagnostics(item);
36
+ return;
37
+ }
38
+ for (const field of HOST_DIAGNOSTIC_FIELDS) delete value[field];
39
+ for (const item of Object.values(value)) dropHostDiagnostics(item);
40
+ }
41
+
42
+ // Elements and OCR text reach the model one line each, the shape browser
43
+ // snapshots use, because a field name costs more than the value it labels:
44
+ // `#mark [ref] Role "name" value="…" state=… source=… disabled focused
45
+ // @x,y,width,height actions`. Flags appear only when they differ from the
46
+ // common case the capture description states (an enabled UIA element), and
47
+ // the geometry a som frame repeats as x/y/width/height, center and
48
+ // screen_bounds collapses into its one bounds box.
49
+ const DERIVED_GEOMETRY_FIELDS = new Set([
50
+ 'x',
51
+ 'y',
52
+ 'width',
53
+ 'height',
54
+ 'center_x',
55
+ 'center_y',
56
+ 'center',
57
+ 'screen_bounds',
58
+ ]);
59
+
60
+ const quotedText = (text) => JSON.stringify(String(text));
61
+ const bareText = (text) => (/^[^\s"]+$/.test(String(text)) ? String(text) : quotedText(text));
62
+ const isUnsetField = (field) =>
63
+ field === undefined || field === null || field === '' || field === false || (Array.isArray(field) && !field.length);
64
+
65
+ function boundsText(box) {
66
+ return box.length === 4 && box.every((edge) => Number.isFinite(edge)) ? `@${box.join(',')}` : null;
67
+ }
68
+
69
+ function extraFieldTexts(rest) {
70
+ return Object.entries(rest)
71
+ .filter(([key, field]) => !DERIVED_GEOMETRY_FIELDS.has(key) && !isUnsetField(field))
72
+ .map(([key, field]) => `${key}=${typeof field === 'string' ? bareText(field) : JSON.stringify(field)}`);
73
+ }
74
+
75
+ function elementLine(element) {
76
+ if (!element || typeof element !== 'object') return element;
77
+ const { mark, ref, role, name, value, state, source, enabled, has_keyboard_focus, bounds, actions, ...rest } =
78
+ element;
79
+ return [
80
+ isUnsetField(mark) ? null : `#${mark}`,
81
+ isUnsetField(ref) ? null : `[${ref}]`,
82
+ bareText(role || 'Unknown'),
83
+ quotedText(name ?? ''),
84
+ isUnsetField(value) ? null : `value=${quotedText(value)}`,
85
+ isUnsetField(state) || state === source ? null : `state=${bareText(state)}`,
86
+ isUnsetField(source) || source === 'uia' ? null : `source=${bareText(source)}`,
87
+ enabled === false ? 'disabled' : null,
88
+ has_keyboard_focus === true ? 'focused' : null,
89
+ boundsText(Array.isArray(bounds) ? bounds : [rest.x, rest.y, rest.width, rest.height]),
90
+ Array.isArray(actions) && actions.length ? actions.join(',') : null,
91
+ ...extraFieldTexts(rest),
92
+ ]
93
+ .filter(Boolean)
94
+ .join(' ');
95
+ }
96
+
97
+ function ocrTextLine(row) {
98
+ if (!row || typeof row !== 'object') return row;
99
+ const { mark, text, line, x, y, width, height, ...rest } = row;
100
+ return [
101
+ isUnsetField(mark) ? null : `#${mark}`,
102
+ quotedText(text ?? ''),
103
+ boundsText([x, y, width, height]),
104
+ isUnsetField(line) ? null : `line=${line}`,
105
+ ...extraFieldTexts(rest),
106
+ ]
107
+ .filter(Boolean)
108
+ .join(' ');
109
+ }
110
+
111
+ function renderObservationLines(frame) {
112
+ if (!frame || typeof frame !== 'object') return;
113
+ if (Array.isArray(frame.elements)) frame.elements = frame.elements.map(elementLine);
114
+ for (const key of ['lines', 'words']) {
115
+ if (Array.isArray(frame.ocr?.[key])) frame.ocr[key] = frame.ocr[key].map(ocrTextLine);
116
+ }
117
+ }
118
+
119
+ export function canonicalComputerResultText(text, args) {
120
+ if (args.action === 'clipboard' && args.input?.operation === 'read') return text;
121
+ let value;
122
+ try {
123
+ value = JSON.parse(text);
124
+ } catch {
125
+ return text;
126
+ }
127
+ if (!value || typeof value !== 'object' || Array.isArray(value) || typeof value.action !== 'string') {
128
+ return text;
129
+ }
130
+ value.action = args.action;
131
+ if (args.action === 'list') value.kind = args.input?.kind;
132
+ if (args.action === 'capture') value.mode = args.input?.mode || 'state';
133
+ if (args.action === 'window') value.operation = args.input?.operation;
134
+ if (args.action === 'clipboard') value.operation = args.input?.operation;
135
+ if (args.action === 'act') canonicalizeActResult(value, args);
136
+ if (value.capture_after && value.observation === undefined) {
137
+ value.observation = value.capture_after;
138
+ delete value.capture_after;
139
+ }
140
+ dropHostDiagnostics(value);
141
+ renderObservationLines(value);
142
+ renderObservationLines(value.observation);
143
+ if (value.ok === false && value.recovery === undefined) {
144
+ const recovery = computerResultRecovery(value, args);
145
+ if (recovery) value.recovery = recovery;
146
+ }
147
+ return JSON.stringify(value);
148
+ }
149
+
150
+ // A capture payload reports the frame's real pixel size (`width`/`height`,
151
+ // nested under `observation` for action replies). Carrying it on the image
152
+ // block lets the context estimator bill a screenshot at its true vision cost
153
+ // instead of the flat unknown-image allowance. Provider normalizers rebuild
154
+ // the wire block from `type`/`source` alone, so these fields never reach an
155
+ // API and never shift the cached prefix.
156
+ export function computerImageDimensions(text) {
157
+ let value;
158
+ try {
159
+ value = JSON.parse(text);
160
+ } catch {
161
+ return null;
162
+ }
163
+ for (const frame of [value, value?.observation]) {
164
+ const width = Number(frame?.width);
165
+ const height = Number(frame?.height);
166
+ if (width > 0 && height > 0 && Number.isFinite(width) && Number.isFinite(height)) return { width, height };
167
+ }
168
+ return null;
169
+ }
170
+
171
+ export function canonicalComputerResultIsError(text, args) {
172
+ if (args.action === 'clipboard' && args.input?.operation === 'read') return false;
173
+ try {
174
+ const value = JSON.parse(text);
175
+ return Boolean(
176
+ value && typeof value === 'object' && !Array.isArray(value) && value.action === args.action && value.ok === false
177
+ );
178
+ } catch {
179
+ return false;
180
+ }
181
+ }
@@ -143,18 +143,26 @@ export async function downloadVerifiedLocalAsset(
143
143
  return destination;
144
144
  }
145
145
 
146
- async function extractZip(zipPath, destination, signal) {
147
- signal?.throwIfAborted();
146
+ // The Windows builds ship as zips (PowerShell's Expand-Archive); the macOS
147
+ // build is a gzipped tarball, which the system tar unpacks.
148
+ function extractCommand(archive, destination) {
149
+ if (!archive.endsWith('.zip')) return ['tar', ['-xzf', archive, '-C', destination], process.env];
148
150
  const command = [
149
151
  "$ErrorActionPreference = 'Stop'",
150
152
  'Expand-Archive -LiteralPath $env:MIXDOG_LOCAL_ARCHIVE -DestinationPath $env:MIXDOG_LOCAL_DESTINATION -Force',
151
153
  ].join('; ');
152
- const child = spawn('powershell', ['-NoProfile', '-NonInteractive', '-Command', command], {
153
- env: {
154
- ...process.env,
155
- MIXDOG_LOCAL_ARCHIVE: zipPath,
156
- MIXDOG_LOCAL_DESTINATION: destination,
157
- },
154
+ return [
155
+ 'powershell',
156
+ ['-NoProfile', '-NonInteractive', '-Command', command],
157
+ { ...process.env, MIXDOG_LOCAL_ARCHIVE: archive, MIXDOG_LOCAL_DESTINATION: destination },
158
+ ];
159
+ }
160
+
161
+ async function extractArchive(archive, destination, signal) {
162
+ signal?.throwIfAborted();
163
+ const [command, args, env] = extractCommand(archive, destination);
164
+ const child = spawn(command, args, {
165
+ env,
158
166
  windowsHide: true,
159
167
  signal,
160
168
  stdio: ['ignore', 'pipe', 'pipe'],
@@ -184,7 +192,9 @@ async function installRuntimeInternal({ dataDir, onProgress, fetchFn, signal })
184
192
  signal?.throwIfAborted();
185
193
  const entry = localProviderRuntimePlatformEntry();
186
194
  if (!entry || !localProviderCatalogStatus({ dataDir, hardware }).available) {
187
- throw new Error('[local-provider] Windows x64 with a compatible NVIDIA RTX GPU is required');
195
+ throw new Error(
196
+ '[local-provider] Windows x64 with a compatible NVIDIA RTX GPU, or an Apple Silicon Mac with enough memory, is required'
197
+ );
188
198
  }
189
199
  const target = localProviderRuntimeDirectory(dataDir);
190
200
  const executable = join(target, entry.executable);
@@ -217,7 +227,7 @@ async function installRuntimeInternal({ dataDir, onProgress, fetchFn, signal })
217
227
  totalBytes: entry.downloadBytes,
218
228
  percent: Math.min(99, Math.round(((completedBytes + asset.size) / entry.downloadBytes) * 100)),
219
229
  });
220
- await extractZip(archive, staging, signal);
230
+ await extractArchive(archive, staging, signal);
221
231
  completedBytes += asset.size;
222
232
  }
223
233
  if (!existsSync(join(staging, entry.executable))) {
@@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url';
4
4
 
5
5
  import { resolvePluginData } from '../shared/plugin-paths.mjs';
6
6
  import { localProviderDiskStatus, partialAssetBytes } from './asset-storage.mjs';
7
- import { localProviderHardwareStatus } from './hardware.mjs';
7
+ import { localProviderHardwareStatus, localProviderPlatformKey } from './hardware.mjs';
8
8
  import { registeredLocalModels } from './registered-models.mjs';
9
9
  import { localModelState } from './model-state.mjs';
10
10
  import { localContextSettings } from './context-settings.mjs';
@@ -24,12 +24,8 @@ export function localProviderModelRoot(dataDir = resolvePluginData()) {
24
24
  return join(dataDir, 'local-provider', 'models');
25
25
  }
26
26
 
27
- function localProviderRuntimePlatformKey(platform = process.platform, arch = process.arch) {
28
- return platform === 'win32' && arch === 'x64' ? 'win32-x64-nvidia' : '';
29
- }
30
-
31
27
  export function localProviderRuntimePlatformEntry(options = {}) {
32
- const key = localProviderRuntimePlatformKey(options.platform, options.arch);
28
+ const key = localProviderPlatformKey(options.platform, options.arch);
33
29
  const entry = key ? LOCAL_PROVIDER_MANIFEST.runtime?.platforms?.[key] : null;
34
30
  return entry ? { key, ...entry } : null;
35
31
  }
@@ -109,7 +105,8 @@ export function localProviderCatalogStatus({
109
105
  const executable = localProviderRuntimeExecutable(dataDir, { platform, arch });
110
106
  const models = localProviderModelEntries(dataDir).map((entry) => publicModel(entry, dataDir, hardware));
111
107
  return {
112
- available: Boolean(platformEntry && hardware?.gpu?.vendor === 'NVIDIA' && models.some((entry) => entry.compatible)),
108
+ platformSupported: Boolean(platformEntry),
109
+ available: Boolean(platformEntry && hardware?.supported && models.some((entry) => entry.compatible)),
113
110
  runtime: {
114
111
  installed: Boolean(executable && existsSync(executable)),
115
112
  version: LOCAL_PROVIDER_MANIFEST.runtime.version,
@@ -23,6 +23,19 @@
23
23
  "sha256": "8c79a9b226de4b3cacfd1f83d24f962d0773be79f1e7b75c6af4ded7e32ae1d6"
24
24
  }
25
25
  ]
26
+ },
27
+ "darwin-arm64-metal": {
28
+ "executable": "llama-b10621/llama-server",
29
+ "backend": "Metal",
30
+ "downloadBytes": 10954823,
31
+ "assets": [
32
+ {
33
+ "name": "llama-b10621-bin-macos-arm64.tar.gz",
34
+ "url": "https://github.com/ggml-org/llama.cpp/releases/download/b10621/llama-b10621-bin-macos-arm64.tar.gz",
35
+ "size": 10954823,
36
+ "sha256": "429c8270608600188035e5e92f7d78dffb7900904fe7dd7e6a84f48068cd13cf"
37
+ }
38
+ ]
26
39
  }
27
40
  }
28
41
  },
@@ -30,7 +43,7 @@
30
43
  {
31
44
  "id": "qwen3.8-27b-q4-k-m",
32
45
  "name": "Qwen3.8 27B Q4_K_M",
33
- "description": "Recommended coding and tool-use model for NVIDIA RTX GPUs with 24 GB VRAM.",
46
+ "description": "Recommended coding and tool-use model for GPUs with 24 GB of memory.",
34
47
  "filename": "Qwen3.8-27B-Q4_K_M.gguf",
35
48
  "url": "https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/0669b98607d47046c7c2b3f801011d54a08cfccf/Qwen3.8-27B-Q4_K_M.gguf?download=true",
36
49
  "revision": "0669b98607d47046c7c2b3f801011d54a08cfccf",