mixdog 1.1.0 → 1.1.2

package/src/runtime/channels/lib/worker-main.mjs

@@ -4,7 +4,6 @@ import { createRequire } from 'node:module';
 import { loadConfig, createProvider, DATA_DIR } from './config.mjs';
 import { resolveVoiceRuntime } from './voice-runtime-fetcher.mjs';
 import { whisperThreadCount } from './voice-transcription.mjs';
-import { TOOL_DEFS } from '../tool-defs.mjs';
 import { ensureReady, stopVoiceWhisperServer } from './whisper-server.mjs';
 import { ensurePrivateRuntimeRoot, resolveRuntimeRoot } from '../../shared/runtime-root.mjs';
 import { Scheduler } from './scheduler.mjs';
@@ -102,7 +101,6 @@ runWorkerBootstrap({
   pruneStalePluginDataLogSiblings,
   DEFAULT_STALE_LOG_SIBLING_MAX,
 });
-const INSTRUCTIONS = '';
 
 // ── Parent notification helper ───────────────────────────────────────
 // This worker has no MCP transport of its own. All notifications flow
@@ -129,10 +127,7 @@ let _reloadDebounce = null;
 const STATUS_FILE = getStatusPath(INSTANCE_ID);
 const statusState = new JsonStateFile(STATUS_FILE, {});
 statusState.ensure();
-const scheduler = new Scheduler(
-  config.nonInteractive ?? [],
-  config.interactive ?? []
-);
+const scheduler = new Scheduler(config.nonInteractive ?? [], config.interactive ?? []);
 // Schedules fire as visible sessions; the session runner belongs to the
 // session-runtime layer, so this worker entry injects it (lazy, like webhooks).
 scheduler.setScheduleRunner(async (schedule, opts) => {
@@ -250,19 +245,12 @@ function wireEventQueueHandlers(eventQueue) {
   // re-read at probe time so a stale-connected flag cannot mask a lost seat.
   eventQueue.setOwnerGetter(() => bridgeRuntimeConnected && currentOwnerState().owned);
 }
-// Inbound messaging, provider interaction handlers, and channel voice-message
-// transcription are DELETED with Discord/Telegram (user decision: PWA replaces
-// channels). The headless provider emits no events, so nothing wires here.
-// Tool dispatch in worker mode goes through the IPC `call` handler at the
-// bottom of this file (parent's `callWorker` → `handleToolCall`). There is no
-// orphan worker-level MCP Server: the parent (server.mjs) owns the single
-// connected transport and routes CallTool through the IPC `call` path.
 // ── Worker/HTTP tool-call dispatch ──────────────────────────────────────────
 // Runtime lifecycle dispatch only (activate_channel_bridge / reload_config):
 // threaded as a lifecycle bag of lazy getters so the module reads live
 // file-level references at call time. Used by the HTTP MCP CallTool path and
 // the worker IPC `call` handler at the bottom of this file.
-const { handleToolCall, handleToolCallWithBridgeRetry } = createToolDispatch({
+const { handleToolCall } = createToolDispatch({
   isChannelsDegraded,
   lifecycle: {
     getChannelBridgeActive: () => channelBridgeActive,
@@ -277,14 +265,6 @@ const { handleToolCall, handleToolCallWithBridgeRetry } = createToolDispatch({
     reloadRuntimeConfig,
   },
 });
-async function init(_sharedMcp) {
-  // _sharedMcp is no longer used. Notifications now flow via IPC to the parent
-  // (sendNotifyToParent above). The parameter is retained for backward
-  // compatibility with any caller that still passes a Server reference.
-  scheduler.setInjectHandler((channelId, name, content, options) => {
-    injectAndRecord(channelId, name, content, options);
-  });
-}
 function ensureConfigWatcher() {
   if (_configWatcher) return;
   try {
@@ -374,19 +354,9 @@ if (_isWorkerMode && process.send && process.env.MIXDOG_DAEMON_HOST !== '1') {
     getProvider: () => provider,
     getConfig: () => config,
     handleMemoryCallResponse,
-    handleToolCallWithBridgeRetry,
+    handleToolCall,
     bootProfile,
   });
 }
 
-export {
-  TOOL_DEFS,
-  handleToolCall,
-  handleToolCallWithBridgeRetry,
-  init,
-  INSTRUCTIONS as instructions,
-  isChannelBridgeActive,
-  isChannelsDegraded,
-  start,
-  stop,
-};
+export { handleToolCall, isChannelBridgeActive, isChannelsDegraded, start, stop };

package/src/runtime/computer-bridge/client.mjs

@@ -23,6 +23,7 @@ import { computerErrorCode } from './error-code.mjs';
 import { bridgeDiscoveryChanged, readBridgeDiscovery } from '../bridge-discovery.mjs';
 import { MAX_COMPUTER_REQUEST_BYTES, readComputerBridgeJson, validateComputerReply } from './limits.mjs';
 import { computerActionHas } from './actions.mjs';
+import { linuxCaptureFallback, linuxComputerPreflight } from './linux-readiness.mjs';
 import { continuePendingComputerWork, isPendingComputerWork } from './pending-continuation.mjs';
 
 export { canonicalComputerResultText };
@@ -245,6 +246,13 @@ function computerToolResult(value, args) {
 /** Execute one `computer` tool call. Returns MCP-shaped content so the
  *  internal-tools normalizer forwards text and screenshot images as-is. */
 export async function executeComputerTool(rawArgs, context = {}) {
+  const note = { text: '' };
+  const result = await executeComputerToolInner(rawArgs, context, note);
+  // The fallback note rides as its own block so the JSON result text stays intact.
+  return note.text && !result.isError ? { ...result, content: [...result.content, { type: 'text', text: note.text }] } : result;
+}
+
+async function executeComputerToolInner(rawArgs, context, note) {
   const discovery = readBridgeDiscovery(DISCOVERY_FILE);
   if (!discovery) return computerErrorResult(`Error: ${BRIDGE_UNAVAILABLE_MESSAGE}`);
   // Resolve the argument shape once so validation, host translation, and the
@@ -252,12 +260,28 @@ export async function executeComputerTool(rawArgs, context = {}) {
   const args = normalizeComputerToolArgs(rawArgs);
   const validationError = validateComputerToolArgs(args);
   if (validationError) return computerErrorResult(`Error: ${validationError}`);
-  const command = toComputerHostCommand(args);
+  let command = toComputerHostCommand(args);
   const sessionId = context?.sessionId ? String(context.sessionId) : '';
+  const fallback = await linuxCaptureFallback({ command });
+  if (fallback) {
+    command = fallback.command;
+    note.text = fallback.note;
+  }
   const encoded = JSON.stringify({ ...command, ...(sessionId ? { session_id: sessionId } : {}) });
   if (Buffer.byteLength(encoded) > MAX_COMPUTER_REQUEST_BYTES) {
     return computerErrorResult('Error: computer request exceeds byte limit; no input was dispatched');
   }
+  // Linux: refuse an action whose own capability (input device, window
+  // control, AT-SPI, ...) is missing, before anything is dispatched, instead of
+  // failing mid-task. Actions that need none of it still run.
+  const notReady = await linuxComputerPreflight({ command });
+  if (notReady) {
+    return {
+      ...computerErrorResult(`Error: ${notReady.message}`),
+      // Structured twin of the text for the transcript card (no text parsing).
+      toolMeta: { code: notReady.code, missing: [...new Set(notReady.missing.map((item) => item.capability))] },
+    };
+  }
   const releaseError = await awaitPendingSessionRelease(sessionId, args, context);
   if (releaseError) return releaseError;
   trackComputerSession(sessionId, command);

package/src/runtime/computer-bridge/linux-action-capabilities.mjs

@@ -0,0 +1,84 @@
+// Which Linux capability each host command really needs, read from
+// native/mixdog-computer (host/*.rs) and the desktop app's observation layer.
+// Only a hard requirement is listed: an action that merely works better with a
+// capability (OCR for capture, AT-SPI for a state capture, which degrades to
+// pixels) does not name it, so a missing optional piece never blocks it.
+//
+//   windows         a window list/handle (X11 EWMH, sway, Hyprland, GNOME Window
+//                   Calls). Every native window target (window_id, app, the
+//                   foreground window) resolves through it.
+//   accessibility   the AT-SPI bus.
+//   semanticActions AT-SPI actions (invoke, toggle, set_value, menu): they act
+//                   on an accessible found through a window handle, so they need
+//                   both and use no input device.
+//   pointer/keyboard the input device (uinput on Wayland, XTest on X11).
+
+const WINDOW_ACTIONS = new Set([
+  'list_windows', 'window_snapshot', 'related_windows', 'window_bounds', 'window_capture',
+  'window_integrity', 'validate_background_input', 'list_apps', 'verify', 'focus_window',
+  'move_window', 'window_state', 'close_window', 'terminate_process',
+]);
+const ACCESSIBILITY_ACTIONS = new Set(['snapshot', 'find', 'window_predicates', 'accessibility_probe']);
+const SEMANTIC_ACTIONS = new Set(['invoke', 'set_value', 'toggle', 'invoke_menu']);
+const POINTER_ACTIONS = new Set([
+  'click', 'double_click', 'right_click', 'middle_click', 'triple_click',
+  'mouse_down', 'mouse_up', 'mouse_move', 'drag', 'scroll',
+]);
+const KEYBOARD_ACTIONS = new Set(['type', 'key', 'key_down', 'key_up']);
+
+const namesWindow = (command) => Boolean(command?.window_id || command?.app || command?.window);
+
+function addStepNeeds(need, step, delivery) {
+  const action = String(step?.action || '');
+  const foreground = delivery === 'foreground';
+  if (step?.ref || step?.element) need.add('accessibility');
+  if (SEMANTIC_ACTIONS.has(action)) {
+    // A foreground invoke is a real click; every other semantic action is AT-SPI.
+    if (action === 'invoke' && foreground) need.add('pointer');
+    else need.add('semanticActions');
+    if (action === 'invoke_menu') need.add('windows');
+  } else if (POINTER_ACTIONS.has(action)) {
+    need.add('pointer');
+  } else if (KEYBOARD_ACTIONS.has(action)) {
+    // Background typing into a settable element goes through its value.
+    if (action === 'type' && step.ref && !foreground) need.add('semanticActions');
+    else need.add('keyboard');
+  }
+}
+
+/** A capture that names no window or screen: it resolves the foreground window. */
+export function isForegroundWindowCapture(command) {
+  return String(command?.action) === 'capture' && !namesWindow(command) && command.screen === undefined;
+}
+
+/** The capabilities a host command (as toComputerHostCommand emits it) requires. */
+export function requiredCapabilities(command) {
+  const need = new Set();
+  const action = String(command?.action || '');
+  if (WINDOW_ACTIONS.has(action)) need.add('windows');
+  else if (ACCESSIBILITY_ACTIONS.has(action)) {
+    need.add('windows');
+    need.add('accessibility');
+  } else if (action === 'capture') {
+    need.add('capture');
+    const mode = String(command.mode || 'state');
+    // Only a screen-index vision capture needs no window: every other mode
+    // resolves a named window or the foreground one.
+    const screenOnly = mode === 'vision' && command.screen !== undefined && !namesWindow(command);
+    if (!screenOnly) need.add('windows');
+    if (mode === 'ax') need.add('accessibility');
+  } else if (action === 'screenshot' || action === 'zoom') {
+    need.add('capture');
+  } else if (action === 'clipboard_read' || action === 'clipboard_write') {
+    need.add('clipboard');
+  } else if (action === 'sequence') {
+    // An act always names its window (the schema requires it).
+    need.add('windows');
+    const delivery = String(command.delivery || 'background');
+    for (const step of Array.isArray(command.steps) ? command.steps : []) addStepNeeds(need, step, delivery);
+  } else {
+    addStepNeeds(need, command, String(command?.delivery || 'background'));
+    if (need.size > 0 && namesWindow(command)) need.add('windows');
+  }
+  return need;
+}

package/src/runtime/computer-bridge/linux-readiness.d.mts

@@ -0,0 +1,52 @@
+export type LinuxComputerCapabilityState = 'ready' | 'needs_setup' | 'unsupported' | 'unknown';
+export interface LinuxComputerCapability {
+  state: LinuxComputerCapabilityState;
+  fix?: 'allow_input' | 'relogin' | 'enable_window_helper' | 'install_package';
+  package?: string;
+  optional?: boolean;
+}
+export type LinuxComputerCapabilityName =
+  | 'capture'
+  | 'windows'
+  | 'accessibility'
+  | 'semanticActions'
+  | 'pointer'
+  | 'keyboard'
+  | 'clipboard'
+  | 'ocr';
+export interface LinuxComputerReadiness {
+  session: 'wayland' | 'x11';
+  desktop: 'gnome' | 'kde' | 'sway' | 'hyprland' | 'other';
+  capabilities: Record<LinuxComputerCapabilityName, LinuxComputerCapability>;
+}
+export interface LinuxComputerMissingCapability {
+  capability: LinuxComputerCapabilityName;
+  state: LinuxComputerCapabilityState;
+  fix?: LinuxComputerCapability['fix'];
+  package?: string;
+  code: string;
+  message: string;
+}
+export interface LinuxComputerRefusal {
+  code: 'computer_capability_unavailable';
+  missing: LinuxComputerMissingCapability[];
+  message: string;
+}
+export const MIN_WINDOW_HELPER_VERSION: number;
+export const CAPABILITY_UNAVAILABLE_CODE: 'computer_capability_unavailable';
+export function detectSession(env: Record<string, string | undefined>): 'wayland' | 'x11';
+export function detectDesktop(env: Record<string, string | undefined>, session: 'wayland' | 'x11'): LinuxComputerReadiness['desktop'];
+export function checkLinuxComputerReadiness(options?: Record<string, unknown>): Promise<LinuxComputerReadiness>;
+export function getLinuxComputerReadiness(options?: Record<string, unknown>): Promise<LinuxComputerReadiness>;
+export function missingCapabilities(
+  readiness: LinuxComputerReadiness,
+  command: Record<string, unknown>
+): LinuxComputerMissingCapability[];
+export function linuxComputerPreflight(options?: Record<string, unknown>): Promise<LinuxComputerRefusal | null>;
+export function resetLinuxComputerPreflightCache(): void;
+export const GNOME_EXTENSION_UUID: 'mixdog-windows@mixdog.io';
+export function gnomeExtensionDirs(env?: Record<string, string | undefined>): string[];
+export const FIX_MESSAGES: Readonly<Record<'allow_input' | 'relogin' | 'enable_window_helper' | 'install_package', string>>;
+export const REFUSAL_MESSAGE: string;
+export const CAPTURE_FALLBACK_NOTE: string;
+export function linuxCaptureFallback(options?: Record<string, unknown>): Promise<{ command: Record<string, unknown>; note: string } | null>;