@vortex-os/computer-use 0.7.0 → 0.7.2

package/package.json

@@ -1,73 +1,74 @@
-{
-  "name": "@vortex-os/computer-use",
-  "version": "0.7.0",
-  "description": "Add-on — read-only screen perception (structured UIA tree + pixel fallback + noise-filtered background watch with an event buffer + sub-second reflex alerts: beep / fixed-phrase / OCR or optional local-VLM description spoken locally, optional higher-quality Supertonic neural TTS with per-app audio ducking, adaptive companion that classifies the on-screen activity and branches its help) exposed as an MCP server, layered on @vortex-os/base. Windows-first. Control (mouse/keyboard) is intentionally out of scope.",
-  "license": "MIT",
-  "author": "vortex-os-project",
-  "homepage": "https://github.com/vortex-os-project/vortex#readme",
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/vortex-os-project/vortex.git",
-    "directory": "modules/computer-use"
-  },
-  "type": "module",
-  "files": [
-    "scripts/mcp-stdio.mjs",
-    "scripts/noise-filter.mjs",
-    "scripts/speech-safety.mjs",
-    "scripts/vlm.mjs",
-    "scripts/ocr.ps1",
-    "scripts/speak.ps1",
-    "scripts/speak-supertonic.mjs",
-    "scripts/audio-duck.ps1",
-    "scripts/fetch-supertonic.mjs",
-    "scripts/worker.ps1",
-    "scripts/lib.ps1",
-    "scripts/probe.ps1",
-    "scripts/read-ui.ps1",
-    "scripts/classify.ps1",
-    "scripts/activity.mjs",
-    "scripts/point-to-ask.ps1",
-    "computer-use.config.example.json",
-    "README.md"
-  ],
-  "bin": {
-    "vortex-mcp-computer-use": "scripts/mcp-stdio.mjs"
-  },
-  "vortex": {
-    "mcpServers": {
-      "vortex-computer-use": {
-        "command": "node",
-        "args": ["node_modules/@vortex-os/computer-use/scripts/mcp-stdio.mjs"]
-      }
-    }
-  },
-  "scripts": {
-    "verify": "node scripts/verify.mjs",
-    "test:filter": "node scripts/test-noise-filter.mjs",
-    "test:speech": "node scripts/test-speech-safety.mjs",
-    "test:vlm": "node scripts/test-vlm.mjs",
-    "test:activity": "node scripts/test-activity.mjs"
-  },
-  "peerDependencies": {
-    "@vortex-os/base": ">=0.3.0 <1.0.0"
-  },
-  "peerDependenciesMeta": {
-    "@vortex-os/base": {
-      "optional": false
-    }
-  },
-  "optionalDependencies": {
-    "@modelcontextprotocol/sdk": "^1.21.0",
-    "onnxruntime-node": "^1.19.2"
-  },
-  "engines": {
-    "node": ">=22"
-  },
-  "os": [
-    "win32"
-  ],
-  "publishConfig": {
-    "access": "public"
-  }
-}
+{
+  "name": "@vortex-os/computer-use",
+  "version": "0.7.2",
+  "description": "Add-on — read-only screen perception (structured UIA tree + pixel fallback + noise-filtered background watch with an event buffer + sub-second reflex alerts: beep / fixed-phrase / OCR or optional local-VLM description spoken locally, optional higher-quality Supertonic neural TTS with per-app audio ducking, adaptive companion that classifies the on-screen activity and branches its help) exposed as an MCP server, layered on @vortex-os/base. Windows-first. Control (mouse/keyboard) is intentionally out of scope.",
+  "license": "MIT",
+  "author": "vortex-os-project",
+  "homepage": "https://github.com/vortex-os-project/vortex#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/vortex-os-project/vortex.git",
+    "directory": "modules/computer-use"
+  },
+  "type": "module",
+  "files": [
+    "scripts/mcp-stdio.mjs",
+    "scripts/noise-filter.mjs",
+    "scripts/speech-safety.mjs",
+    "scripts/vlm.mjs",
+    "scripts/ocr.ps1",
+    "scripts/speak.ps1",
+    "scripts/speak-supertonic.mjs",
+    "scripts/audio-duck.ps1",
+    "scripts/fetch-supertonic.mjs",
+    "scripts/worker.ps1",
+    "scripts/lib.ps1",
+    "scripts/probe.ps1",
+    "scripts/read-ui.ps1",
+    "scripts/classify.ps1",
+    "scripts/activity.mjs",
+    "scripts/point-to-ask.ps1",
+    "computer-use.config.example.json",
+    "README.md"
+  ],
+  "bin": {
+    "vortex-mcp-computer-use": "scripts/mcp-stdio.mjs"
+  },
+  "vortex": {
+    "mcpServers": {
+      "vortex-computer-use": {
+        "command": "node",
+        "args": ["node_modules/@vortex-os/computer-use/scripts/mcp-stdio.mjs"]
+      }
+    }
+  },
+  "scripts": {
+    "verify": "node scripts/verify.mjs",
+    "test:filter": "node scripts/test-noise-filter.mjs",
+    "test:speech": "node scripts/test-speech-safety.mjs",
+    "test:vlm": "node scripts/test-vlm.mjs",
+    "test:activity": "node scripts/test-activity.mjs",
+    "prepublishOnly": "node scripts/require-changelog.mjs && node scripts/require-docs-current.mjs && node scripts/scan-leaks.mjs"
+  },
+  "peerDependencies": {
+    "@vortex-os/base": ">=0.3.0 <1.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@vortex-os/base": {
+      "optional": false
+    }
+  },
+  "optionalDependencies": {
+    "@modelcontextprotocol/sdk": "^1.21.0",
+    "onnxruntime-node": "^1.19.2"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "os": [
+    "win32"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/scripts/activity.mjs

@@ -1,92 +1,92 @@
-// computer-use — activity classifier (pure, testable). Turns the raw signals from classify.ps1
-// (foreground process/title, notification state, UIA count, fullscreen) into an activity CLASS and a help
-// PROFILE the companion uses to pick its cadence/triggers/style. See docs/adaptive-companion.md.
-//
-// Deliberately conservative: process name is the strongest prior; UIA-emptiness marks a GPU canvas
-// (game/video) only when the process isn't a known browser/player; everything is overridable by config and by
-// explicit user requests. All thresholds are starting points to calibrate, surfaced via opts.
-
-// Known-app tables — lowercase process names, no ".exe". Extend via opts.apps.{dev,media,browser,productivity}.
-const DEV = ['code', 'cursor', 'devenv', 'rider64', 'idea64', 'pycharm64', 'webstorm64', 'clion64', 'goland64',
-  'sublime_text', 'notepad++', 'windowsterminal', 'wt', 'powershell', 'pwsh', 'cmd', 'conhost', 'alacritty',
-  'wezterm', 'hx', 'nvim', 'vim', 'emacs'];
-const MEDIA = ['mpv', 'vlc', 'mpc-hc64', 'mpc-be64', 'potplayermini64', 'potplayer', 'wmplayer', 'smplayer',
-  'kodi', 'plex', 'plexmediaplayer', 'netflix'];
-const BROWSER = ['chrome', 'msedge', 'firefox', 'brave', 'opera', 'vivaldi', 'arc', 'librewolf'];
-const PRODUCTIVITY = ['winword', 'excel', 'powerpnt', 'onenote', 'acrobat', 'acrord32', 'sumatrapdf', 'foxitpdfreader',
-  'notepad', 'obsidian', 'notion', 'hwp', 'soffice'];
-
-// Window-title hints that a browser tab is actually video (so a browser → MEDIA, not BROWSING).
-const VIDEO_TITLE = /youtube|netflix|twitch|vimeo|prime\s*video|disney\+?|wavve|tving|watcha|\bwatch\b/i;
-
-// SHQueryUserNotificationState values.
-const NS_NAME = { 1: 'NOT_PRESENT', 2: 'BUSY', 3: 'D3D_FULLSCREEN', 4: 'PRESENTATION', 5: 'ACCEPTS', 6: 'QUIET_TIME', 7: 'APP' };
-
-// Per-class help profile defaults (overridable via config `companion` section).
-export const PROFILES = {
-  GAME: { proactive: true, cadenceSec: 30, mode: 'periodic',
-    note: 'sample change-rate (poll_change) to split fast-action vs strategy; fast-action = break-gated only, announce the limit once' },
-  DEV: { proactive: false, cadenceSec: 0, mode: 'event-error',
-    note: 'silent until an error/build failure/stack trace; scaffolding help on request' },
-  MEDIA: { proactive: false, cadenceSec: 0, mode: 'silent',
-    note: 'on explicit request only; do not talk over it; DRM audio cannot be ducked' },
-  BROWSING: { proactive: false, cadenceSec: 0, mode: 'silent',
-    note: 'on request: summarize / translate / explain' },
-  PRODUCTIVITY: { proactive: false, cadenceSec: 0, mode: 'quiet',
-    note: 'on request; optional gentle risk flags' },
-  UNKNOWN: { proactive: false, cadenceSec: 0, mode: 'low-intrusion',
-    note: 'offer help once, then stay quiet until asked' },
-};
-
-const has = (list, p) => list.includes(p);
-
-/**
- * Derive activity class + profile from raw signals.
- * @param {object} raw  { process, title, notificationState, uiaCount, uiaCapped, fullscreen, ... }
- * @param {object} opts { uiaCanvasMax=5, apps?:{dev,media,browser,productivity}, profiles?:{<CLASS>:{cadenceSec?,proactive?,mode?}} }
- */
-export function classifyActivity(raw = {}, opts = {}) {
-  const uiaCanvasMax = opts.uiaCanvasMax ?? 5;
-  const apps = opts.apps || {};
-  const dev = apps.dev || DEV, media = apps.media || MEDIA, browser = apps.browser || BROWSER, prod = apps.productivity || PRODUCTIVITY;
-
-  const proc = String(raw.process || '').toLowerCase();
-  const title = String(raw.title || '');
-  const ns = Number(raw.notificationState || 0);
-  const nsName = NS_NAME[ns] || 'UNKNOWN';
-  const interruptible = ns === 5; // only ACCEPTS_NOTIFICATIONS is unconditionally clear to speak
-  // uiaCount is null when the UIA walk FAILED (≠ "found 0") — so a UIA error never reads as an empty canvas.
-  const hasUia = typeof raw.uiaCount === 'number';
-  // A capped count means there were MORE elements than the backend cap → never a sparse canvas, even if a large
-  // uiaCanvasMax is configured. So canvas requires a real, uncapped, below-threshold count.
-  const canvas = hasUia && !raw.uiaCapped && raw.uiaCount < uiaCanvasMax;
-
-  let cls;
-  if (has(dev, proc)) cls = 'DEV';
-  else if (has(media, proc)) cls = 'MEDIA';
-  else if (has(browser, proc)) cls = VIDEO_TITLE.test(title) ? 'MEDIA' : 'BROWSING';
-  else if (has(prod, proc)) cls = 'PRODUCTIVITY';
-  else if (canvas && proc) cls = 'GAME';                 // screen-canvas app that isn't a known player/browser
-  else if (hasUia && raw.uiaCount >= uiaCanvasMax) cls = 'PRODUCTIVITY'; // rich tree, unknown app → reading/productivity
-  else cls = 'UNKNOWN';
-
-  // Profile defaults, with optional per-class overrides from the `companion` config (cadence/proactive/mode).
-  const profile = { ...PROFILES[cls], ...((opts.profiles && opts.profiles[cls]) || {}) };
-
-  return {
-    class: cls,
-    process: raw.process || '',
-    title,
-    notificationState: nsName,
-    interruptible,
-    canvas,
-    uiaCount: hasUia ? raw.uiaCount : null,
-    fullscreen: !!raw.fullscreen,
-    profile,
-    // GAME needs a change-rate sample to split fast-action (break-gated) vs strategy (periodic) — the agent
-    // takes a couple of poll_change reads and applies the thresholds in docs/adaptive-companion.md.
-    needsChangeRate: cls === 'GAME',
-  };
-}
-
-export default classifyActivity;
+// computer-use — activity classifier (pure, testable). Turns the raw signals from classify.ps1
+// (foreground process/title, notification state, UIA count, fullscreen) into an activity CLASS and a help
+// PROFILE the companion uses to pick its cadence/triggers/style. See docs/adaptive-companion.md.
+//
+// Deliberately conservative: process name is the strongest prior; UIA-emptiness marks a GPU canvas
+// (game/video) only when the process isn't a known browser/player; everything is overridable by config and by
+// explicit user requests. All thresholds are starting points to calibrate, surfaced via opts.
+
+// Known-app tables — lowercase process names, no ".exe". Extend via opts.apps.{dev,media,browser,productivity}.
+const DEV = ['code', 'cursor', 'devenv', 'rider64', 'idea64', 'pycharm64', 'webstorm64', 'clion64', 'goland64',
+  'sublime_text', 'notepad++', 'windowsterminal', 'wt', 'powershell', 'pwsh', 'cmd', 'conhost', 'alacritty',
+  'wezterm', 'hx', 'nvim', 'vim', 'emacs'];
+const MEDIA = ['mpv', 'vlc', 'mpc-hc64', 'mpc-be64', 'potplayermini64', 'potplayer', 'wmplayer', 'smplayer',
+  'kodi', 'plex', 'plexmediaplayer', 'netflix'];
+const BROWSER = ['chrome', 'msedge', 'firefox', 'brave', 'opera', 'vivaldi', 'arc', 'librewolf'];
+const PRODUCTIVITY = ['winword', 'excel', 'powerpnt', 'onenote', 'acrobat', 'acrord32', 'sumatrapdf', 'foxitpdfreader',
+  'notepad', 'obsidian', 'notion', 'hwp', 'soffice'];
+
+// Window-title hints that a browser tab is actually video (so a browser → MEDIA, not BROWSING).
+const VIDEO_TITLE = /youtube|netflix|twitch|vimeo|prime\s*video|disney\+?|wavve|tving|watcha|\bwatch\b/i;
+
+// SHQueryUserNotificationState values.
+const NS_NAME = { 1: 'NOT_PRESENT', 2: 'BUSY', 3: 'D3D_FULLSCREEN', 4: 'PRESENTATION', 5: 'ACCEPTS', 6: 'QUIET_TIME', 7: 'APP' };
+
+// Per-class help profile defaults (overridable via config `companion` section).
+export const PROFILES = {
+  GAME: { proactive: true, cadenceSec: 30, mode: 'periodic',
+    note: 'sample change-rate (poll_change) to split fast-action vs strategy; fast-action = break-gated only, announce the limit once' },
+  DEV: { proactive: false, cadenceSec: 0, mode: 'event-error',
+    note: 'silent until an error/build failure/stack trace; scaffolding help on request' },
+  MEDIA: { proactive: false, cadenceSec: 0, mode: 'silent',
+    note: 'on explicit request only; do not talk over it; DRM audio cannot be ducked' },
+  BROWSING: { proactive: false, cadenceSec: 0, mode: 'silent',
+    note: 'on request: summarize / translate / explain' },
+  PRODUCTIVITY: { proactive: false, cadenceSec: 0, mode: 'quiet',
+    note: 'on request; optional gentle risk flags' },
+  UNKNOWN: { proactive: false, cadenceSec: 0, mode: 'low-intrusion',
+    note: 'offer help once, then stay quiet until asked' },
+};
+
+const has = (list, p) => list.includes(p);
+
+/**
+ * Derive activity class + profile from raw signals.
+ * @param {object} raw  { process, title, notificationState, uiaCount, uiaCapped, fullscreen, ... }
+ * @param {object} opts { uiaCanvasMax=5, apps?:{dev,media,browser,productivity}, profiles?:{<CLASS>:{cadenceSec?,proactive?,mode?}} }
+ */
+export function classifyActivity(raw = {}, opts = {}) {
+  const uiaCanvasMax = opts.uiaCanvasMax ?? 5;
+  const apps = opts.apps || {};
+  const dev = apps.dev || DEV, media = apps.media || MEDIA, browser = apps.browser || BROWSER, prod = apps.productivity || PRODUCTIVITY;
+
+  const proc = String(raw.process || '').toLowerCase();
+  const title = String(raw.title || '');
+  const ns = Number(raw.notificationState || 0);
+  const nsName = NS_NAME[ns] || 'UNKNOWN';
+  const interruptible = ns === 5; // only ACCEPTS_NOTIFICATIONS is unconditionally clear to speak
+  // uiaCount is null when the UIA walk FAILED (≠ "found 0") — so a UIA error never reads as an empty canvas.
+  const hasUia = typeof raw.uiaCount === 'number';
+  // A capped count means there were MORE elements than the backend cap → never a sparse canvas, even if a large
+  // uiaCanvasMax is configured. So canvas requires a real, uncapped, below-threshold count.
+  const canvas = hasUia && !raw.uiaCapped && raw.uiaCount < uiaCanvasMax;
+
+  let cls;
+  if (has(dev, proc)) cls = 'DEV';
+  else if (has(media, proc)) cls = 'MEDIA';
+  else if (has(browser, proc)) cls = VIDEO_TITLE.test(title) ? 'MEDIA' : 'BROWSING';
+  else if (has(prod, proc)) cls = 'PRODUCTIVITY';
+  else if (canvas && proc) cls = 'GAME';                 // screen-canvas app that isn't a known player/browser
+  else if (hasUia && raw.uiaCount >= uiaCanvasMax) cls = 'PRODUCTIVITY'; // rich tree, unknown app → reading/productivity
+  else cls = 'UNKNOWN';
+
+  // Profile defaults, with optional per-class overrides from the `companion` config (cadence/proactive/mode).
+  const profile = { ...PROFILES[cls], ...((opts.profiles && opts.profiles[cls]) || {}) };
+
+  return {
+    class: cls,
+    process: raw.process || '',
+    title,
+    notificationState: nsName,
+    interruptible,
+    canvas,
+    uiaCount: hasUia ? raw.uiaCount : null,
+    fullscreen: !!raw.fullscreen,
+    profile,
+    // GAME needs a change-rate sample to split fast-action (break-gated) vs strategy (periodic) — the agent
+    // takes a couple of poll_change reads and applies the thresholds in docs/adaptive-companion.md.
+    needsChangeRate: cls === 'GAME',
+  };
+}
+
+export default classifyActivity;