@logictan/dsh-desktop-agent 0.1.1 → 0.1.3

package/README.md

@@ -29,8 +29,14 @@ describing a picture it never received — and answering confidently about it.
 
 ## Requirements
 
-The profile must mount the Cua Driver computer-use provider; the plugin dispatches
-to the `cua_driver_native__*` tools it publishes and has no driver of its own.
+None beyond installing this plugin. It publishes the `cua_driver_native__*` tools
+itself, from the Cua Driver native SDK it depends on, and dispatches to them.
+
+A profile that still mounts the old
+`@deepseek-ai/dsh-experimental-computer-use-cua-driver-native` provider must drop
+that row and its `@deepseek-ai/dsh-computer-use` dependency first: both would
+publish the same tool names, and the second registration is refused with
+`tool "cua_driver_native__click" is already registered`.
 
 ## Settings
 

package/cordis.patch.yml

@@ -1,8 +1,9 @@
 # dsh-desktop-agent bundle patch layer.
 #
 # One bare plugin row. The package ships two halves — the host half
-# (exports ".") registers the `desktop_agent` model tool and exports the
-# `Config` schema the Plugins page derives its form from, the browser half
+# (exports ".") registers the `desktop_agent` model tool, publishes the
+# `cua_driver_native__*` driver tools from `src/driver.js`, and exports the
+# `Config` schema the Plugins page derives its form from; the browser half
 # (exports "./client") registers the Plugins-page configuration card.
 #
 # The row must stay a BARE package name: the web plugin table locates each
@@ -13,6 +14,11 @@
 # Row id `desktop-agent` equals the host half's `export const name` in
 # src/index.js. It is checked by hand against every other row id in this
 # repository; `aggregate.mjs --check` does not validate id uniqueness.
+#
+# This row replaces the two hand-written rows a profile used to need
+# (`computer-use` + `computer-use-cua-driver-native`): the driver runtime now
+# lives in this package, so mounting it twice would collide at tool
+# registration.
 - insert:
     - id: desktop-agent
       name: '@logictan/dsh-desktop-agent'

package/lib/driver.js

@@ -0,0 +1,177 @@
+/**
+ * The Cua Driver native runtime — the driver half.
+ *
+ * Absorbed from `@deepseek-ai/dsh-experimental-computer-use-cua-driver-native`,
+ * which the profile used to mount as its own loader row. The desktop agent drives
+ * the desktop exclusively by dispatching to the driver's `cua_driver_native__*`
+ * tools through the tool registry, so those tools have to be published by
+ * whoever owns the native handle; keeping that ownership in this package makes
+ * the plugin self-contained instead of leaving it inert without a second row.
+ *
+ * What the absorption deliberately drops: the `computerUse` registration slot.
+ * That slot existed to keep two computer-use providers from claiming the same
+ * desktop, and the plugin is now the only publisher of these tools — a profile
+ * that still mounts the old provider collides at `tools.register` with
+ * `tool "cua_driver_native__click" is already registered`, which is the honest
+ * signal that the old row must go.
+ *
+ * @module @logictan/dsh-desktop-agent/driver
+ */
+import { createMcpToolDefinition } from '@deepseek-ai/dsh-mcp-client';
+
+/** Every published tool name carries this prefix; the loop's dispatch names depend on it. */
+export const TOOL_PREFIX = 'cua_driver_native__';
+
+/** DeepSeek's function-name alphabet and maximum length are protocol constants. */
+const TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/u;
+
+/** The prompt section this runtime owns. */
+const SECTION_NAME = 'computer-use:cua-driver-native';
+
+const GUIDANCE = `Cua Driver native computer-use tools operate the host desktop. Discover the exact app and window, then get a fresh window snapshot before acting. Use element_token from that snapshot, or coordinates from its screenshot. A new snapshot of that window invalidates its earlier element tokens. Select either target or the legacy pid/window_id fields; do not combine them.
+
+Prefer background delivery. A refusal does not authorize a foreground retry. Verify the requested outcome from fresh state after an action; a delivered click alone does not prove the outcome. After cancellation, inspect current state before retrying because completed input is not rolled back. Other sessions and applications may change the same desktop.
+
+On macOS, cursor-overlay operations may return facility_unavailable even when screenshots and input work.`;
+
+/**
+ * Load the native SDK module.
+ *
+ * The one external boundary this plugin cannot validate away: the driver ships
+ * as per-platform native binaries, so the module is imported at the point of use
+ * rather than at module load, and a missing binary fails the mount with the
+ * resolver's own message instead of a module-scope import error.
+ *
+ * @returns the SDK module namespace.
+ */
+function loadNativeSdk() {
+  return import('@trycua/cua-driver');
+}
+
+/**
+ * Read the tool catalog out of the driver's JSON.
+ *
+ * The SDK marshals this from a native process, so it is a real untrusted
+ * boundary: a shape change must be reported here, not carried into tool
+ * registration where it would surface as an unexplained schema error.
+ *
+ * @param value - the parsed catalog payload.
+ * @returns the catalog entries.
+ * @throws {Error} when the payload is not a catalog.
+ */
+function readCatalog(value) {
+  const tools = value?.tools;
+  if (!Array.isArray(tools)) throw new Error('Cua Driver returned no tool catalog.');
+  return tools;
+}
+
+/**
+ * Mount the native runtime on `ctx`.
+ *
+ * The effect is registered before the first `await` so that a failure anywhere
+ * after the native handle exists still releases it, and so that unloading the
+ * plugin mid-startup aborts the in-flight discovery call instead of waiting for
+ * it to finish.
+ *
+ * @param ctx - the plugin context, providing `tools` and optionally `systemPrompt`.
+ * @param loadSdk - the SDK loader; overridden by tests, which cannot import the native binaries.
+ * @returns the effect disposer that tears the runtime down.
+ * @throws {Error} when the SDK cannot be loaded, the catalog is unusable, or a registration is refused.
+ */
+export async function mountDriver(ctx, loadSdk = loadNativeSdk) {
+  const lifetime = new AbortController();
+  const pending = new Set();
+  let driver;
+
+  const dispose = ctx.effect(
+    () => async () => {
+      lifetime.abort();
+      // Shutting the native handle down under an in-flight call would release
+      // memory the call still reads, so settlement comes first.
+      await Promise.allSettled(pending);
+      if (driver !== undefined) {
+        await driver.shutdown();
+        driver.uniffiDestroy();
+      }
+    },
+    'desktop-agent: cua driver runtime',
+  );
+
+  try {
+    const { CuaDriver } = await loadSdk();
+    lifetime.signal.throwIfAborted();
+    // The generated constructor returns its class with an owned binding handle,
+    // but declares only CuaDriverLike, which omits uniffiDestroy().
+    driver = CuaDriver.create(undefined);
+    await publishTools(ctx, driver, lifetime, pending);
+  } catch (error) {
+    await dispose();
+    throw error;
+  }
+
+  return dispose;
+}
+
+/**
+ * Publish one model tool per catalog entry and the usage guidance section.
+ *
+ * @param ctx - the plugin context.
+ * @param driver - the live native handle.
+ * @param lifetime - the runtime's abort source.
+ * @param pending - the in-flight native calls the teardown waits for.
+ */
+async function publishTools(ctx, driver, lifetime, pending) {
+  const tools = ctx.get('tools');
+  const catalog = readCatalog(JSON.parse(await driver.listToolsJson({ signal: lifetime.signal })));
+  lifetime.signal.throwIfAborted();
+
+  const names = new Set();
+  for (const tool of catalog) {
+    if (typeof tool?.name !== 'string' || tool.name === '') {
+      throw new Error('Cua Driver listed a tool without a name.');
+    }
+    const publicName = TOOL_PREFIX + tool.name;
+    if (!TOOL_NAME.test(publicName)) {
+      throw new Error(`Cua Driver tool "${tool.name}" exceeds the supported function-name format`);
+    }
+    if (names.has(publicName)) throw new Error(`Cua Driver listed tool "${tool.name}" more than once`);
+    names.add(publicName);
+
+    tools.register(createMcpToolDefinition(ctx, {
+      name: publicName,
+      rawName: tool.name,
+      description: tool.description ?? '',
+      inputSchema: tool.inputSchema,
+      outputSchema: tool.outputSchema,
+      async call(args, execution) {
+        // Two independent cancellations: the caller giving up, and the plugin
+        // being unloaded. The native call must observe both.
+        const combined = AbortSignal.any([execution.signal, lifetime.signal]);
+        combined.throwIfAborted();
+        const operation = driver.callTool(tool.name, JSON.stringify(args), { signal: combined });
+        pending.add(operation);
+        try {
+          const result = await operation;
+          combined.throwIfAborted();
+          return JSON.parse(result.rawJson);
+        } finally {
+          pending.delete(operation);
+        }
+      },
+    }));
+  }
+
+  const systemPrompt = ctx.get('systemPrompt');
+  if (systemPrompt === null || systemPrompt === undefined) {
+    ctx.logger?.warn?.('desktop-agent: the systemPrompt service is unavailable; the Cua Driver guidance was not registered.');
+    return;
+  }
+  ctx.effect(
+    () => systemPrompt.section({
+      name: SECTION_NAME,
+      order: systemPrompt.getSectionOrder('TOOL_COMPUTER_USE'),
+      text: GUIDANCE,
+    }),
+    'desktop-agent: cua driver guidance',
+  );
+}

package/lib/index.js

@@ -6,10 +6,11 @@
  * card reads its model list from.
  *
  * The plugin owns no execution engine of its own. Every action is dispatched to
- * the `cua_driver_native__*` tools the profile already mounts, through the tool
- * registry's own `execute` pipeline — so a call this plugin makes is subject to
- * the same guards, timeouts, and cancellation as one the model made directly, and
- * the driver's own coordinate and delivery contracts are the only ones in play.
+ * the `cua_driver_native__*` tools this package publishes — see `./driver.js` —
+ * through the tool registry's own `execute` pipeline, so a call this plugin makes
+ * is subject to the same guards, timeouts, and cancellation as one the model made
+ * directly, and the driver's own coordinate and delivery contracts are the only
+ * ones in play.
  *
  * Configuration arrives as `apply`'s second argument. As of the 0.1.7-alpha.1
  * settings redesign the plugin's own `Config` schema is the only config surface —
@@ -42,6 +43,7 @@ import { defineTool } from '@deepseek-ai/dsh-tools';
 import { ToolCallId } from '@deepseek-ai/dsh-llm';
 
 import { ENTRY_ID, resolveVisionRoute } from './config.js';
+import { mountDriver } from './driver.js';
 import { run } from './loop.js';
 import { structured } from './observe.js';
 import { makeRoutes } from './routes.js';
@@ -61,12 +63,16 @@ export const name = ENTRY_ID;
 export const inject = [];
 
 /**
- * Register the `desktop_agent` tool and the vision-catalog route.
+ * Register the `desktop_agent` tool, the vision-catalog route, and the Cua
+ * Driver runtime the tool dispatches to.
  *
  * @param ctx - the plugin context.
  * @param config - one live reference per {@link Config} field.
+ * @param loadSdk - the native SDK loader, forwarded to {@link mountDriver}; tests
+ *   pass one because the driver ships native binaries that are not installed in
+ *   this workspace, and activation must not depend on a binary the test cannot load.
  */
-export function apply(ctx, config) {
+export async function apply(ctx, config, loadSdk) {
   const tools = ctx.get('tools');
   if (tools === null || tools === undefined || typeof tools !== 'object') {
     ctx.logger?.warn?.('desktop_agent: the tools service is unavailable; the tool was not registered.');
@@ -146,7 +152,7 @@ export function apply(ctx, config) {
       'Resolves the target application, then runs an observation/decision loop: each step captures the window, ' +
       'asks the configured model what to do, and performs the click, key, or text entry it chose. ' +
       'Works on ordinary applications and on games that expose no usable accessibility tree, because it reads the ' +
-      'picture rather than the element table. Requires the computer-use Cua Driver tools to be mounted. ' +
+      'picture rather than the element table. ' +
       'Note that a covered window cannot be driven, and a game that filters injected input needs foreground ' +
       'delivery — the plugin settings control that. This tool waits for the run to finish and returns a structured ' +
       'trace of every step.',
@@ -215,6 +221,10 @@ export function apply(ctx, config) {
       'desktop-agent: vision-model catalog route',
     );
   });
+
+  // Last, so that a driver that cannot start costs this plugin nothing else: a
+  // rejected mount rolls back the tool and route registrations above with it.
+  await mountDriver(ctx, loadSdk);
 }
 
 /**
@@ -235,7 +245,12 @@ async function resolveTarget(input) {
 
   const value = await dispatch('cua_driver_native__list_windows', {}, signal);
   const payload = structured(value, 'list_windows');
-  const windows = Array.isArray(payload.windows) ? payload.windows : [];
+  // A payload that parses but carries no window array is a driver contract
+  // failure, not an empty desktop. Flattening it to [] would report the two as
+  // the same sentence, and the driver is a third-party boundary whose response
+  // shape this plugin does not control.
+  if (!Array.isArray(payload.windows)) throw new Error('desktop_agent: list_windows returned no window array.');
+  const windows = payload.windows;
   if (windows.length === 0) throw new Error('desktop_agent: the driver listed no windows.');
 
   if (args.windowId !== undefined) {
@@ -262,20 +277,41 @@ async function resolveTarget(input) {
   }
 
   const needle = args.app.toLowerCase();
-  const matches = onScreen.filter(
-    (window) =>
-      String(window.app_name ?? '').toLowerCase().includes(needle) ||
-      String(window.title ?? '').toLowerCase().includes(needle),
-  );
+  const matches = onScreen.filter((window) => matchesApp(window, needle));
   if (matches.length === 0) {
+    // An app that is running but has no window on screen is a different fact
+    // from an app that is not running, and only the second one suggests
+    // launching it. The windowId path already tells them apart.
+    if (windows.some((window) => matchesApp(window, needle))) {
+      throw new Error(
+        'desktop_agent: "' + args.app + '" is open, but no window of it is on screen, so it cannot be captured or acted on.',
+      );
+    }
     const names = [...new Set(onScreen.map((window) => window.app_name).filter(Boolean))];
     throw new Error(
-      'desktop_agent: no on-screen window matches "' + args.app + '". Open windows: ' + (names.join(', ') || '(none)') + '.',
+      'desktop_agent: no on-screen window matches "' + args.app + '". On-screen apps: ' + (names.join(', ') || '(none)') + '.',
     );
   }
   return targetOf(bestMatch(matches));
 }
 
+/**
+ * Whether a window answers to an app argument.
+ *
+ * The needle is matched against the application name and the window title,
+ * because both are things a caller can reasonably name.
+ *
+ * @param window - a window from the driver's list.
+ * @param needle - the app argument, lower-cased.
+ * @returns whether the window matches.
+ */
+function matchesApp(window, needle) {
+  return (
+    String(window.app_name ?? '').toLowerCase().includes(needle) ||
+    String(window.title ?? '').toLowerCase().includes(needle)
+  );
+}
+
 /**
  * Choose the window to drive from the candidates an app argument matched.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@logictan/dsh-desktop-agent",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "DSH desktop agent: a native-vision decision loop that drives the host desktop through the Cua Driver tools. DSH 桌面 Agent 子插件。",
   "type": "module",
   "main": "lib/index.js",
@@ -55,6 +55,7 @@
   "peerDependencies": {
     "@deepseek-ai/cordis": "^4.0.2",
     "@deepseek-ai/dsh-llm": "^0.1.6-alpha.2",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.6-alpha.2",
     "@deepseek-ai/dsh-tools": "^0.1.6-alpha.2",
     "@deepseek-ai/schemastery": "^3.18.3"
   },
@@ -65,6 +66,9 @@
     "@deepseek-ai/dsh-llm": {
       "optional": true
     },
+    "@deepseek-ai/dsh-system-prompt": {
+      "optional": true
+    },
     "@deepseek-ai/dsh-tools": {
       "optional": true
     },
@@ -72,9 +76,15 @@
       "optional": true
     }
   },
+  "dependencies": {
+    "@deepseek-ai/dsh-mcp-client": "^0.1.6-alpha.2",
+    "@trycua/cua-driver": "0.28.0"
+  },
   "devDependencies": {
     "@deepseek-ai/cordis": "^4.0.2",
     "@deepseek-ai/dsh-llm": "^0.1.6-alpha.2",
+    "@deepseek-ai/dsh-mcp-client": "^0.1.6-alpha.2",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.6-alpha.2",
     "@deepseek-ai/dsh-tools": "^0.1.6-alpha.2",
     "@deepseek-ai/schemastery": "^3.18.3"
   }