@abot-ai/runtime 1.4.0 → 1.6.0

package/plugins/system/source/computer/public-action.ts

@@ -0,0 +1,53 @@
+import { computerFields, computerInputError } from "../../../../src/plugin-sdk/computer-access.js";
+
+/** Adapt the SDK's existing flat operation inputs; never parse natural language. */
+export function readPublicComputerAction(
+  params: Record<string, unknown>,
+): Record<string, unknown> {
+  const fields = ["desktop_ref", "observation_ref", "action_kind"];
+  const kind = params.action_kind;
+  switch (kind) {
+    case "click":
+      computerFields(params, [...fields, "x", "y", "button", "count"]);
+      return {
+        kind,
+        point: { x: params.x, y: params.y },
+        button: params.button,
+        count: params.count,
+      };
+    case "move":
+      computerFields(params, [...fields, "x", "y"]);
+      return { kind, point: { x: params.x, y: params.y } };
+    case "drag":
+      computerFields(params, [
+        ...fields,
+        "from_x",
+        "from_y",
+        "to_x",
+        "to_y",
+        "button",
+        "duration_ms",
+      ]);
+      return {
+        kind,
+        from: { x: params.from_x, y: params.from_y },
+        to: { x: params.to_x, y: params.to_y },
+        button: params.button,
+        durationMs: params.duration_ms,
+      };
+    case "scroll":
+      computerFields(params, [...fields, "delta_x", "delta_y"]);
+      return { kind, deltaX: params.delta_x, deltaY: params.delta_y };
+    case "type_text":
+      computerFields(params, [...fields, "text"]);
+      return { kind, text: params.text };
+    case "press_keys":
+      computerFields(params, [...fields, "keys"]);
+      return { kind, keys: params.keys };
+    case "focus_window":
+      computerFields(params, [...fields, "window_ref"]);
+      return { kind, window_ref: params.window_ref };
+    default:
+      return computerInputError("Select a declared computer action operation.");
+  }
+}

package/plugins/system/source/computer/request-computers.ts

@@ -0,0 +1,154 @@
+import { randomUUID } from "node:crypto";
+import type {
+  SystemRequestRoute,
+  SystemTargetSnapshot,
+} from "../request-target-snapshot.js";
+import {
+  routeTargetId,
+  systemRouteMetadata,
+  systemRouteBindingIdentity,
+} from "../request-target-snapshot.js";
+import type { SystemHostConnection } from "../host-dispatch.js";
+import { SystemOperationError } from "../../../../src/plugin-sdk/computer-access.js";
+import type { NativeComputerBackend } from "../../../../src/plugin-sdk/computer-access.js";
+import { createNativeComputerBackend } from "../../../../src/plugin-sdk/computer-access.js";
+import { createCompanionComputerBackend } from "../../../../src/plugin-sdk/computer-access.js";
+import { COMPUTER_CAPABILITY } from "../../../../src/plugin-sdk/computer-access.js";
+import { DesktopBindings } from "./desktop-bindings.js";
+
+export type RequestComputer = {
+  ref: string;
+  route: SystemRequestRoute;
+  bindings: DesktopBindings;
+  backend?: NativeComputerBackend;
+};
+export class RequestComputers {
+  readonly entries: readonly RequestComputer[];
+  private closed = false;
+  constructor(
+    private readonly rootDir: string,
+    routes: SystemTargetSnapshot,
+    private readonly connection: SystemHostConnection,
+    private readonly nativeFactory = createNativeComputerBackend,
+    snapshot?: unknown,
+  ) {
+    this.entries =
+      snapshot === undefined
+        ? routes.map((route) => ({
+            ref: randomUUID(),
+            route,
+            bindings: new DesktopBindings(),
+          }))
+        : restoreComputers(routes, snapshot);
+  }
+  snapshot() {
+    if (this.closed) throw new Error("computer_request_closed");
+    return {
+      kind: "request_computers_v1",
+      entries: this.entries.map((entry) => ({
+        ref: entry.ref,
+        routeIdentity: systemRouteBindingIdentity(entry.route),
+        bindings: entry.bindings.snapshot(),
+      })),
+    };
+  }
+  require(ref: unknown): RequestComputer {
+    if (this.closed)
+      throw new SystemOperationError(
+        "computer_request_closed",
+        "This request's desktop bindings have expired.",
+      );
+    const entry = this.entries.find((candidate) => candidate.ref === ref);
+    if (!entry)
+      throw new SystemOperationError(
+        "computer_desktop_unavailable",
+        "Select a desktop reference offered in this request.",
+      );
+    return entry;
+  }
+  backend(entry: RequestComputer): NativeComputerBackend {
+    if (this.closed)
+      throw new SystemOperationError(
+        "computer_request_closed",
+        "The computer request has closed.",
+      );
+    if (!supportsComputerRoute(entry.route))
+      throw new SystemOperationError(
+        "computer_companion_upgrade_required",
+        "Install the companion from this Runtime build to enable desktop observation and input.",
+      );
+    if (entry.backend) return entry.backend;
+    entry.backend =
+      entry.route.kind === "native"
+        ? this.nativeFactory(entry.route.target)
+        : createCompanionComputerBackend(
+            this.rootDir,
+            entry.route,
+            this.connection,
+          );
+    return entry.backend;
+  }
+  async close(): Promise<void> {
+    this.closed = true;
+    for (const entry of this.entries) entry.bindings.clear();
+    await Promise.allSettled(
+      this.entries.map((entry) => entry.backend?.close()),
+    );
+  }
+}
+export function computerRouteLabel(entry: RequestComputer) {
+  return {
+    desktop_ref: entry.ref,
+    platform: routeTargetId(entry.route),
+    ...systemRouteMetadata(entry.route),
+  };
+}
+export function computerQueueIdentity(entry: RequestComputer): string {
+  // Local transports can alias one physical desktop (for example WSL and companion).
+  // Keep their identities distinct while conservatively serializing the same OS.
+  return `computer-local-platform:${routeTargetId(entry.route)}`;
+}
+function supportsComputerRoute(route: SystemRequestRoute): boolean {
+  if (route.kind === "native") return true;
+  return route.capabilities?.includes(COMPUTER_CAPABILITY) === true;
+}
+
+function restoreComputers(
+  routes: SystemTargetSnapshot,
+  value: unknown,
+): RequestComputer[] {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    !("kind" in value) ||
+    value.kind !== "request_computers_v1" ||
+    !("entries" in value) ||
+    !Array.isArray(value.entries)
+  )
+    throw new Error("request_computers_snapshot_invalid");
+  const refs = new Set<string>(),
+    identities = new Set<string>();
+  return value.entries.flatMap((entry) => {
+    if (
+      !entry ||
+      typeof entry.ref !== "string" ||
+      !entry.ref ||
+      typeof entry.routeIdentity !== "string" ||
+      refs.has(entry.ref) ||
+      identities.has(entry.routeIdentity)
+    )
+      throw new Error("request_computers_snapshot_invalid");
+    refs.add(entry.ref);
+    identities.add(entry.routeIdentity);
+    const bindings = new DesktopBindings();
+    bindings.restore(entry.bindings);
+    const route = routes.find(
+      (route) => systemRouteBindingIdentity(route) === entry.routeIdentity,
+    );
+    // Discovery captured unused routes too. Exact prepared actions still require
+    // their original ref; an unavailable route is never rebound to a new target.
+    const hasSavedRoute = route !== undefined;
+    if (!hasSavedRoute) return [];
+    return [{ ref: entry.ref, route, bindings }];
+  });
+}

package/plugins/system/source/computer/request-modules.ts

@@ -0,0 +1,90 @@
+import {
+  failureResult,
+  type ToolImplementation,
+  type ToolRequestPreparationContext,
+  type ToolModuleDeclaration,
+} from "../../../../src/plugin-sdk/index.js";
+import {
+  systemRouteBindingIdentity,
+  systemRouteMetadata,
+  type SystemTargetSnapshot,
+} from "../request-target-snapshot.js";
+import type { SystemHostConnection } from "../host-dispatch.js";
+import { createComputerHandlers, COMPUTER_TOOLS } from "./handlers.js";
+import { RequestComputers } from "./request-computers.js";
+
+/** Load-time placeholders cannot allocate a desktop outside request ownership. */
+export function unboundComputerHandlers(): Record<string, ToolImplementation> {
+  return Object.fromEntries(
+    COMPUTER_TOOLS.map((name) => [
+      name,
+      async () =>
+        failureResult({
+          errorCode: "computer_request_binding_required",
+          message: "Computer control requires a prepared request.",
+        }),
+    ]),
+  );
+}
+
+export function prepareComputerModules(
+  rootDir: string,
+  modules: readonly ToolModuleDeclaration[],
+  routes: SystemTargetSnapshot,
+  connection: SystemHostConnection,
+  preparation?: ToolRequestPreparationContext,
+): readonly ToolModuleDeclaration[] {
+  if (modules.length === 0) return [];
+  const computers = new RequestComputers(
+    rootDir,
+    routes,
+    connection,
+    undefined,
+    preparation?.requestState?.read("system.computers.v1"),
+  );
+  preparation?.requestState?.register("system.computers.v1", {
+    snapshot: () => computers.snapshot(),
+    dispose: () => computers.close(),
+  });
+  if (computers.entries.length === 0) return [];
+  const handlers = createComputerHandlers(computers);
+  const refs = computers.entries.map((entry) => entry.ref);
+  return modules.map((module) => ({
+    ...module,
+    implementation: handlers[module.definition.name]!,
+    normalInvocation: {
+      ...module.normalInvocation,
+      operations: module.normalInvocation.operations.map((operation) => ({
+        ...operation,
+        input: {
+          ...operation.input,
+          properties: {
+            ...operation.input.properties,
+            ...(Object.hasOwn(operation.input.properties, "desktop_ref")
+              ? { desktop_ref: { type: "string" as const, enum: refs } }
+              : {}),
+          },
+        },
+      })),
+    },
+    adapter: {
+      ...module.adapter,
+      executionBinding: (call) => {
+        if (call.tool === "computer_desktops")
+          return {
+            identity: JSON.stringify(
+              computers.entries.map((entry) =>
+                systemRouteBindingIdentity(entry.route),
+              ),
+            ),
+            metadata: { displayTarget: "available desktops" },
+          };
+        const entry = computers.require(call.params.desktop_ref);
+        return {
+          identity: systemRouteBindingIdentity(entry.route),
+          metadata: systemRouteMetadata(entry.route),
+        };
+      },
+    },
+  }));
+}

package/plugins/system/source/host-dispatch.ts

@@ -1,38 +1,25 @@
+import type { ComputerAccessConnection } from "../../../src/plugin-sdk/computer-access.js";
 import {
   failureResult,
   successResult,
   type ToolImplementation,
-  type ToolImplementationOutput,
 } from "../../../src/plugin-sdk/index.js";
 import {
   executeHostOperation,
   readHostStatus,
-} from "./companion/broker-client.js";
+} from "../../../src/plugin-sdk/computer-access.js";
 import {
   HOST_OPERATIONS,
   type HostOperation,
-  type HostStatus,
-} from "./companion/protocol.js";
-import { SystemOperationError } from "./contracts.js";
+} from "../../../src/plugin-sdk/computer-access.js";
+import { SystemOperationError } from "../../../src/plugin-sdk/computer-access.js";
 import {
   observeCapturedSystemTargets,
   requireSystemRequestRoute,
   type SystemTargetSnapshot,
 } from "./request-target-snapshot.js";
 
-export type SystemHostConnection = Readonly<{
-  readHostStatus(rootDir: string): Promise<HostStatus>;
-  executeHostOperation(
-    rootDir: string,
-    input: Readonly<{
-      hostId: string;
-      connectionId: string;
-      operation: HostOperation;
-      params: Record<string, unknown>;
-      abortSignal?: AbortSignal;
-    }>,
-  ): Promise<ToolImplementationOutput>;
-}>;
+export type SystemHostConnection = ComputerAccessConnection;
 
 export const defaultSystemHostConnection: SystemHostConnection = {
   readHostStatus,

package/plugins/system/source/index.ts

@@ -1,8 +1,10 @@
 import { defineRuntimePlugin } from "../../../src/plugin-sdk/index.js";
-import { createSystemHandlers } from "./handlers.js";
+import { createSystemHandlers } from "../../../src/plugin-sdk/computer-access.js";
 import { prepareSystemRequestModules } from "./request-system-modules.js";
+import { unboundComputerHandlers } from "./computer/request-modules.js";
 
 export default defineRuntimePlugin(({ rootDir }) => ({
-  handlers: createSystemHandlers(),
-  prepareRequest: (modules) => prepareSystemRequestModules(rootDir, modules),
+  handlers: { ...createSystemHandlers(), ...unboundComputerHandlers() },
+  prepareRequest: (modules, preparation) =>
+    prepareSystemRequestModules(rootDir, modules, {}, preparation),
 }));

package/plugins/system/source/request-system-modules.ts

@@ -2,10 +2,16 @@ import type {
   ToolCallAdapter,
   ToolImplementation,
   ToolModuleDeclaration,
+  ToolRequestPreparationContext,
 } from "../../../src/plugin-sdk/index.js";
-import type { SystemTarget, SystemTargetId } from "./contracts.js";
-import { isHostOperation } from "./companion/protocol.js";
-import { createSystemHandlers } from "./handlers.js";
+import type {
+  SystemTarget,
+  SystemTargetId,
+} from "../../../src/plugin-sdk/computer-access.js";
+import { isHostOperation } from "../../../src/plugin-sdk/computer-access.js";
+import { isComputerTool } from "./computer/handlers.js";
+import { prepareComputerModules } from "./computer/request-modules.js";
+import { createSystemHandlers } from "../../../src/plugin-sdk/computer-access.js";
 import {
   createBoundSystemHandlers,
   defaultSystemHostConnection,
@@ -14,7 +20,7 @@ import {
 import {
   observeSystemTargets,
   type SystemTargetObservation,
-} from "./target-observation.js";
+} from "../../../src/plugin-sdk/computer-access.js";
 import {
   captureSystemTargetSnapshot,
   requireSystemRequestRoute,
@@ -37,16 +43,28 @@ export async function prepareSystemRequestModules(
   rootDir: string,
   modules: readonly ToolModuleDeclaration[],
   dependencies: SystemRequestDependencies = {},
+  preparation?: ToolRequestPreparationContext,
 ): Promise<readonly ToolModuleDeclaration[]> {
   if (modules.length === 0) return Object.freeze([]);
-  if (modules.some((module) => !isHostOperation(module.definition.name)))
+  if (modules.some(isUnsupportedSystemModule))
     throw new Error("system_request_module_unknown");
   const connection = dependencies.connection ?? defaultSystemHostConnection;
-  const snapshot = await captureSystemTargetSnapshot({
+  const candidates = await captureSystemTargetSnapshot({
     observeTargets: dependencies.observeTargets ?? observeSystemTargets,
     readHostStatus: () => connection.readHostStatus(rootDir),
+    includeSameOsCompanion: true,
   });
-  if (snapshot.length === 0) return Object.freeze([]);
+  // Existing command targeting retains its one-route-per-OS contract.
+  const snapshot = uniqueSystemCommandRoutes(candidates);
+  const computerModules = prepareComputerModules(
+    rootDir,
+    modules.filter((module) => isComputerTool(module.definition.name)),
+    candidates,
+    connection,
+    preparation,
+  );
+  const hasCommandTargets = snapshot.length > 0;
+  if (!hasCommandTargets) return Object.freeze(computerModules);
   const createNative =
     dependencies.createNativeHandlers ??
     ((resolve) => createSystemHandlers(undefined, resolve));
@@ -59,11 +77,29 @@ export async function prepareSystemRequestModules(
     connection,
     nativeHandlers,
   });
-  return Object.freeze(
-    modules.map((module) =>
-      projectSystemRequestModule(module, snapshot, handlers),
-    ),
-  );
+  return Object.freeze([
+    ...modules
+      .filter((module) => !isComputerTool(module.definition.name))
+      .map((module) => projectSystemRequestModule(module, snapshot, handlers)),
+    ...computerModules,
+  ]);
+}
+
+function isUnsupportedSystemModule(module: ToolModuleDeclaration): boolean {
+  if (isHostOperation(module.definition.name)) return false;
+  return !isComputerTool(module.definition.name);
+}
+
+function uniqueSystemCommandRoutes(
+  candidates: SystemTargetSnapshot,
+): SystemTargetSnapshot {
+  const seen = new Set<SystemTargetId>();
+  return candidates.filter((route) => {
+    const target = routeTargetId(route);
+    if (seen.has(target)) return false;
+    seen.add(target);
+    return true;
+  });
 }
 
 function projectSystemRequestModule(

package/plugins/system/source/request-target-snapshot.ts

@@ -2,14 +2,14 @@ import {
   SystemOperationError,
   type SystemTarget,
   type SystemTargetId,
-} from "./contracts.js";
+} from "../../../src/plugin-sdk/computer-access.js";
 import {
   isHostIdentifier,
   isHostIdentity,
   type HostIdentity,
   type HostStatus,
-} from "./companion/protocol.js";
-import type { SystemTargetObservation } from "./target-observation.js";
+} from "../../../src/plugin-sdk/computer-access.js";
+import type { SystemTargetObservation } from "../../../src/plugin-sdk/computer-access.js";
 
 export type SystemRequestRoute = Readonly<
   | { kind: "native"; target: SystemTarget }
@@ -19,15 +19,17 @@ export type SystemRequestRoute = Readonly<
       hostId: string;
       connectionId: string;
       identity: HostIdentity;
+      capabilities?: readonly string[];
     }
 >;
 export type SystemTargetSnapshot = readonly SystemRequestRoute[];
 
-/** Capture availability once. A companion only supplies an OS missing natively. */
+/** Capture routes once; command callers keep native OS precedence, desktop callers retain distinct routes. */
 export async function captureSystemTargetSnapshot(
   input: Readonly<{
     observeTargets(): Promise<SystemTargetObservation>;
     readHostStatus(): Promise<HostStatus>;
+    includeSameOsCompanion?: boolean;
   }>,
 ): Promise<SystemTargetSnapshot> {
   const [native, companion] = await Promise.allSettled([
@@ -38,7 +40,10 @@ export async function captureSystemTargetSnapshot(
   if (companion.status !== "fulfilled") return Object.freeze(routes);
   const remote = connectedCompanionRoute(companion.value);
   if (!remote) return Object.freeze(routes);
-  if (routes.some((route) => routeTargetId(route) === remote.target))
+  if (
+    !input.includeSameOsCompanion &&
+    routes.some((route) => routeTargetId(route) === remote.target)
+  )
     return Object.freeze(routes);
   return Object.freeze([...routes, remote]);
 }
@@ -76,6 +81,9 @@ function connectedCompanionRoute(
     hostId: status.hostId,
     connectionId: status.connectionId,
     identity: Object.freeze({ ...status.identity }),
+    ...(status.capabilities
+      ? { capabilities: Object.freeze([...status.capabilities]) }
+      : {}),
   });
 }