@abot-ai/runtime 1.4.0 → 1.6.0

package/plugins/system/source/computer/desktop-bindings.ts

@@ -0,0 +1,270 @@
+import { randomUUID } from "node:crypto";
+import { SystemOperationError } from "../../../../src/plugin-sdk/computer-access.js";
+import type {
+  NativeComputerAction,
+  NativeComputerResult,
+  NativeComputerObservation,
+  PhysicalPoint,
+  PhysicalRectangle,
+} from "../../../../src/plugin-sdk/computer-access.js";
+import {
+  computerRecord,
+  computerFields,
+  computerText,
+  readNativeComputerAction,
+  readNativeComputerResult,
+} from "../../../../src/plugin-sdk/computer-access.js";
+
+/** References are owned by a request and never reconstructed from titles or history. */
+export class DesktopBindings {
+  private current?: {
+    id: string;
+    issuedAt: number;
+    result: NativeComputerResult;
+    observation: NativeComputerObservation;
+  };
+  private readonly windows = new Map<string, string>();
+
+  observe(result: NativeComputerResult) {
+    this.windows.clear();
+    const windows = result.windows.map((window) => {
+      const id = randomUUID();
+      this.windows.set(id, window.binding);
+      const { binding: _binding, ...details } = window;
+      return {
+        window_ref: id,
+        ...details,
+        bounds: result.observation
+          ? imageRectangle(window.bounds, result.observation)
+          : window.bounds,
+      };
+    });
+    this.current = undefined;
+    if (!result.observation)
+      return {
+        windows,
+        observation_ref: undefined,
+        coordinateSpace: result.desktop.coordinateSpace ?? "native_desktop",
+      };
+    const id = randomUUID();
+    this.current = {
+      id,
+      issuedAt: Date.now(),
+      result,
+      observation: result.observation,
+    };
+    const {
+      region: _region,
+      accessibility,
+      ...observation
+    } = result.observation;
+    return {
+      windows,
+      observation_ref: id,
+      ...observation,
+      coordinateSpace: "image_pixels",
+      accessibility: accessibility.map((node) => ({
+        ...node,
+        ...(node.bounds
+          ? { bounds: imageRectangle(node.bounds, result.observation!) }
+          : {}),
+      })),
+    };
+  }
+
+  action(observationRef: unknown, value: unknown) {
+    const current = this.current;
+    if (!current || current.id !== observationRef)
+      throw new SystemOperationError(
+        "computer_observation_stale",
+        "Observe this desktop again; the observation reference is not current for this request.",
+      );
+    if (Date.now() - current.issuedAt > 120_000)
+      throw new SystemOperationError(
+        "computer_observation_expired",
+        "The observation is more than two minutes old. Observe again before input.",
+      );
+    const action = this.resolveAction(value);
+    const translated = translateAction(action, current.observation);
+    // Consumed before dispatch: an uncertain result must never replay this observation.
+    this.current = undefined;
+    return {
+      operation: "act" as const,
+      desktopBinding: current.result.desktop.binding,
+      expectedGeometry: current.result.desktop.bounds,
+      ...(current.result.focusedWindow
+        ? { expectedWindow: current.result.focusedWindow }
+        : {}),
+      action: translated,
+      deadlineEpochMs: Date.now() + 15_000,
+    };
+  }
+
+  snapshot() {
+    const current = this.current;
+    const result = current
+      ? (({ image: _image, ...value }) => value)(current.result)
+      : undefined;
+    return {
+      kind: "desktop_bindings_v1",
+      windows: [...this.windows],
+      ...(current
+        ? { current: { id: current.id, issuedAt: current.issuedAt, result } }
+        : {}),
+    };
+  }
+
+  restore(value: unknown): void {
+    const saved = computerRecord(value);
+    computerFields(saved, ["kind", "windows", "current"]);
+    if (
+      saved.kind !== "desktop_bindings_v1" ||
+      !Array.isArray(saved.windows) ||
+      saved.windows.length > 256
+    )
+      throw new Error("desktop_bindings_snapshot_invalid");
+    const windows = new Map<string, string>();
+    for (const item of saved.windows) {
+      if (!Array.isArray(item) || item.length !== 2)
+        throw new Error("desktop_bindings_snapshot_invalid");
+      const ref = computerText(item[0], 128),
+        binding = computerText(item[1]);
+      if (windows.has(ref))
+        throw new Error("desktop_bindings_snapshot_invalid");
+      windows.set(ref, binding);
+    }
+    let current: typeof this.current;
+    if (saved.current !== undefined) {
+      const entry = computerRecord(saved.current);
+      computerFields(entry, ["id", "issuedAt", "result"]);
+      const id = computerText(entry.id, 128);
+      if (
+        typeof entry.issuedAt !== "number" ||
+        !Number.isFinite(entry.issuedAt) ||
+        entry.issuedAt < 0
+      )
+        throw new Error("desktop_bindings_snapshot_invalid");
+      const result = readNativeComputerResult(entry.result);
+      if (!result.observation)
+        throw new Error("desktop_bindings_snapshot_invalid");
+      current = {
+        id,
+        issuedAt: entry.issuedAt,
+        result,
+        observation: result.observation,
+      };
+    }
+    this.clear();
+    this.current = current;
+    for (const [ref, binding] of windows) this.windows.set(ref, binding);
+  }
+
+  clear(): void {
+    this.current = undefined;
+    this.windows.clear();
+  }
+
+  regionalObservation(observationRef: unknown, rectangle: PhysicalRectangle) {
+    const current = this.current;
+    if (!current || current.id !== observationRef)
+      throw new SystemOperationError(
+        "computer_observation_stale",
+        "Use a current observation to select a smaller region.",
+      );
+    const observation = current.observation;
+    if (!isRegionInsideImage(rectangle, observation))
+      throw new SystemOperationError(
+        "computer_region_outside_image",
+        "Select a region inside the returned image.",
+      );
+    const scaleX = observation.region.width / observation.imageWidth;
+    const scaleY = observation.region.height / observation.imageHeight;
+    return {
+      operation: "observe" as const,
+      desktopBinding: current.result.desktop.binding,
+      expectedGeometry: current.result.desktop.bounds,
+      region: {
+        x: Math.round(observation.region.x + rectangle.x * scaleX),
+        y: Math.round(observation.region.y + rectangle.y * scaleY),
+        width: Math.max(1, Math.floor(rectangle.width * scaleX)),
+        height: Math.max(1, Math.floor(rectangle.height * scaleY)),
+      },
+    };
+  }
+
+  private resolveAction(value: unknown): NativeComputerAction {
+    const action = computerRecord(value);
+    if (action.kind !== "focus_window") return readNativeComputerAction(action);
+    const ref = computerText(action.window_ref, 128);
+    if (
+      Object.keys(action).some((key) => !["kind", "window_ref"].includes(key))
+    )
+      throw new SystemOperationError(
+        "computer_input_invalid",
+        "Unexpected focus fields.",
+      );
+    const binding = this.windows.get(ref);
+    if (!binding)
+      throw new SystemOperationError(
+        "computer_window_stale",
+        "Select a window returned by this observation.",
+      );
+    return { kind: "focus_window", windowBinding: binding };
+  }
+}
+
+function imageRectangle(
+  bounds: PhysicalRectangle,
+  observation: NativeComputerObservation,
+): PhysicalRectangle {
+  const scaleX = observation.imageWidth / observation.region.width;
+  const scaleY = observation.imageHeight / observation.region.height;
+  return {
+    x: (bounds.x - observation.region.x) * scaleX,
+    y: (bounds.y - observation.region.y) * scaleY,
+    width: bounds.width * scaleX,
+    height: bounds.height * scaleY,
+  };
+}
+
+function isRegionInsideImage(
+  rectangle: PhysicalRectangle,
+  observation: NativeComputerObservation,
+): boolean {
+  if (rectangle.x < 0 || rectangle.y < 0) return false;
+  if (rectangle.x + rectangle.width > observation.imageWidth) return false;
+  return rectangle.y + rectangle.height <= observation.imageHeight;
+}
+
+function translateAction(
+  action: NativeComputerAction,
+  observation: NativeComputerObservation,
+): NativeComputerAction {
+  const point = (value: PhysicalPoint): PhysicalPoint => {
+    if (
+      value.x < 0 ||
+      value.x >= observation.imageWidth ||
+      value.y < 0 ||
+      value.y >= observation.imageHeight
+    )
+      throw new SystemOperationError(
+        "computer_coordinate_outside_image",
+        "Use coordinates inside the returned image.",
+      );
+    return {
+      x: Math.round(
+        observation.region.x +
+          (value.x * observation.region.width) / observation.imageWidth,
+      ),
+      y: Math.round(
+        observation.region.y +
+          (value.y * observation.region.height) / observation.imageHeight,
+      ),
+    };
+  };
+  if (action.kind === "click" || action.kind === "move")
+    return { ...action, point: point(action.point) };
+  if (action.kind === "drag")
+    return { ...action, from: point(action.from), to: point(action.to) };
+  return action;
+}

package/plugins/system/source/computer/handlers.ts

@@ -0,0 +1,160 @@
+import {
+  failureResult,
+  successResult,
+  type ToolExecutionContext,
+  type ToolImplementation,
+} from "../../../../src/plugin-sdk/index.js";
+import { SystemOperationError } from "../../../../src/plugin-sdk/computer-access.js";
+import { computerFields, readPhysicalRectangle } from "../../../../src/plugin-sdk/computer-access.js";
+import { readPublicComputerAction } from "./public-action.js";
+import { withDesktopQueue } from "../../../../src/plugin-sdk/computer-access.js";
+import { serializeComputerDesktops } from "./observation-output.js";
+import {
+  describeDesktop,
+  projectComputerResult,
+} from "./observation-result.js";
+import {
+  RequestComputers,
+  computerQueueIdentity,
+  computerRouteLabel,
+} from "./request-computers.js";
+
+export const COMPUTER_TOOLS = [
+  "computer_desktops",
+  "computer_observe",
+  "computer_act",
+] as const;
+export function isComputerTool(name: string): boolean {
+  return COMPUTER_TOOLS.some((tool) => tool === name);
+}
+
+export function createComputerHandlers(
+  computers: RequestComputers,
+): Record<string, ToolImplementation> {
+  let registered = false;
+  const guard =
+    (handler: ToolImplementation): ToolImplementation =>
+    async (params, context) => {
+      try {
+        if (!context?.onRequestDispose)
+          throw new SystemOperationError(
+            "computer_lifecycle_unavailable",
+            "Computer control requires request resource cleanup support.",
+          );
+        if (!registered) {
+          context.onRequestDispose(() => computers.close());
+          registered = true;
+        }
+        context.abortSignal?.throwIfAborted();
+        return await handler(params, context);
+      } catch (error) {
+        return failureResult({
+          errorCode:
+            error instanceof SystemOperationError
+              ? error.code
+              : "computer_operation_interrupted",
+          message:
+            error instanceof Error
+              ? error.message
+              : "Computer operation interrupted; effects may be partial. Observe before another action.",
+        });
+      }
+    };
+  return {
+    computer_desktops: guard(async (params, context) => {
+      computerFields(params, []);
+      const desktops = [];
+      for (const entry of computers.entries) {
+        const observation = await withDesktopQueue(
+          computerQueueIdentity(entry),
+          context?.abortSignal,
+          async () => {
+            try {
+              entry.bindings.clear();
+              const result = await computers
+                .backend(entry)
+                .execute({ operation: "inspect" }, context?.abortSignal);
+              return describeDesktop(entry, result);
+            } catch (error) {
+              return {
+                ...computerRouteLabel(entry),
+                available: false,
+                reason:
+                  error instanceof Error
+                    ? error.message
+                    : "Desktop inspection failed.",
+              };
+            }
+          },
+        );
+        desktops.push(observation);
+      }
+      return successResult({
+        output: serializeComputerDesktops(desktops),
+        data: {
+          desktopCount: desktops.length,
+          observationMeta: { kind: "volatile_external", carryPolicy: "never" },
+        },
+      });
+    }),
+    computer_observe: guard(async (params, context) => {
+      computerFields(params, [
+        "desktop_ref",
+        "observation_ref",
+        "x",
+        "y",
+        "width",
+        "height",
+      ]);
+      const entry = computers.require(params.desktop_ref);
+      requireMedia(context);
+      return withDesktopQueue(
+        computerQueueIdentity(entry),
+        context?.abortSignal,
+        async () => {
+          const request =
+            params.observation_ref === undefined
+              ? { operation: "observe" as const }
+              : entry.bindings.regionalObservation(
+                  params.observation_ref,
+                  readPhysicalRectangle({
+                    x: params.x,
+                    y: params.y,
+                    width: params.width,
+                    height: params.height,
+                  }),
+                );
+          entry.bindings.clear();
+          const result = await computers
+            .backend(entry)
+            .execute(request, context?.abortSignal);
+          return projectComputerResult(entry, result, context!);
+        },
+      );
+    }),
+    computer_act: guard(async (params, context) => {
+      const action = readPublicComputerAction(params);
+      const entry = computers.require(params.desktop_ref);
+      requireMedia(context);
+      return withDesktopQueue(
+        computerQueueIdentity(entry),
+        context?.abortSignal,
+        async () => {
+          const request = entry.bindings.action(params.observation_ref, action);
+          const result = await computers
+            .backend(entry)
+            .execute(request, context?.abortSignal);
+          return projectComputerResult(entry, result, context!);
+        },
+      );
+    }),
+  };
+}
+
+function requireMedia(context: ToolExecutionContext | undefined): void {
+  if (!context?.media)
+    throw new SystemOperationError(
+      "computer_media_unavailable",
+      "The runtime does not support temporary tool images.",
+    );
+}

package/plugins/system/source/computer/observation-output.ts

@@ -0,0 +1,145 @@
+/** Leaves room for JSON-string escaping and result metadata in the SDK's 256 KiB envelope. */
+export const COMPUTER_OBSERVATION_OUTPUT_MAX_BYTES = 48 * 1024;
+
+type Observation = Record<string, unknown>;
+type ArrayField = "accessibility" | "windows";
+type Collection = {
+  observation: Observation;
+  field: ArrayField;
+  original: readonly unknown[];
+};
+const ARRAY_FIELDS: readonly ArrayField[] = ["accessibility", "windows"];
+const ESSENTIAL_FIELDS = [
+  "desktop_ref",
+  "observation_ref",
+  "platform",
+  "displayTarget",
+  "transport",
+  "capturedAt",
+  "imageWidth",
+  "imageHeight",
+  "coordinateSpace",
+  "targetingGuarantee",
+  "available",
+  "imageAvailable",
+  "evidenceScope",
+  "requestedEffectVerified",
+  "observationMeta",
+] as const;
+
+export function serializeComputerObservation(
+  description: Observation,
+  data: Observation,
+): string {
+  const observation = { ...description, ...data };
+  return boundedOutput([observation], (items) => items[0]!);
+}
+
+export function serializeComputerDesktops(
+  desktops: readonly Observation[],
+): string {
+  return boundedOutput(desktops, (items) => ({ desktops: items }));
+}
+
+function boundedOutput(
+  observations: readonly Observation[],
+  envelope: (items: Observation[]) => unknown,
+): string {
+  const projected = observations.map((observation) => ({ ...observation }));
+  const serialize = () => JSON.stringify(envelope(projected));
+  let output = serialize();
+  if (fits(output)) return output;
+  const collections: Collection[] = projected.flatMap((observation) =>
+    ARRAY_FIELDS.flatMap((field) =>
+      Array.isArray(observation[field])
+        ? [
+            {
+              observation,
+              field,
+              original: observation[field] as readonly unknown[],
+            },
+          ]
+        : [],
+    ),
+  );
+
+  // Drop only suffixes. The remaining nodes retain their exact original order and values.
+  for (const collection of collections) {
+    setRetainedCount(collection, 0);
+    output = serialize();
+    if (!fits(output)) continue;
+    let low = 0;
+    let high = collection.original.length;
+    while (low < high) {
+      const middle = Math.ceil((low + high) / 2);
+      setRetainedCount(collection, middle);
+      if (fits(serialize())) low = middle;
+      else high = middle - 1;
+    }
+    setRetainedCount(collection, low);
+    return serialize();
+  }
+
+  // A malformed/oversized scalar base must not erase an accepted input receipt.
+  // The explicit projection error replaces optional descriptive metadata, never pixels.
+  const minimal = projected.map(minimalObservation);
+  output = JSON.stringify(envelope(minimal));
+  if (fits(output)) return output;
+  throw new Error("computer_observation_identity_exceeds_output_budget");
+}
+
+function setRetainedCount(collection: Collection, count: number): void {
+  const { observation, field, original } = collection;
+  observation[field] = original.slice(0, count);
+  const omittedCount = original.length - count;
+  if (omittedCount === 0) return;
+  const coverage = record(observation.outputCoverage) ?? {};
+  observation.outputCoverage = {
+    ...coverage,
+    kind: "computer_output_coverage_v1",
+    reason: "output_byte_limit",
+    [field]: { retainedCount: count, omittedCount },
+  };
+  observation[`${field}Truncated`] = true;
+}
+
+function minimalObservation(observation: Observation): Observation {
+  const output: Observation = {};
+  for (const field of ESSENTIAL_FIELDS) {
+    if (observation[field] !== undefined) output[field] = observation[field];
+  }
+  const dispatch = record(observation.dispatch);
+  if (dispatch) {
+    output.dispatch = {
+      status: dispatch.status,
+      requestedInputCount: dispatch.requestedInputCount,
+      ...(dispatch.acceptedInputCount === undefined
+        ? {}
+        : { acceptedInputCount: dispatch.acceptedInputCount }),
+    };
+  }
+  for (const field of ARRAY_FIELDS) {
+    if (Array.isArray(observation[field])) output[field] = [];
+    if (observation[`${field}Truncated`] !== undefined)
+      output[`${field}Truncated`] = observation[`${field}Truncated`];
+  }
+  output.outputCoverage = observation.outputCoverage;
+  output.outputError = {
+    code: "computer_observation_metadata_too_large",
+    message:
+      "Descriptive metadata exceeds the output limit. Identity, image geometry, and input dispatch counts are retained; other metadata is omitted.",
+  };
+  return output;
+}
+
+function record(value: unknown): Observation | undefined {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return undefined;
+  return value as Observation;
+}
+
+function fits(value: string): boolean {
+  return (
+    Buffer.byteLength(value, "utf8") <= COMPUTER_OBSERVATION_OUTPUT_MAX_BYTES
+  );
+}

package/plugins/system/source/computer/observation-result.ts

@@ -0,0 +1,122 @@
+import {
+  failureResult,
+  successResult,
+  type ToolExecutionContext,
+} from "../../../../src/plugin-sdk/index.js";
+import { SystemOperationError } from "../../../../src/plugin-sdk/computer-access.js";
+import type { NativeComputerResult } from "../../../../src/plugin-sdk/computer-access.js";
+import {
+  computerRouteLabel,
+  type RequestComputer,
+} from "./request-computers.js";
+import { serializeComputerObservation } from "./observation-output.js";
+
+export function describeDesktop(
+  entry: RequestComputer,
+  result: NativeComputerResult,
+) {
+  const {
+    binding: _binding,
+    bounds: _bounds,
+    coordinateSpace: _coordinateSpace,
+    ...desktop
+  } = result.desktop;
+  return {
+    ...computerRouteLabel(entry),
+    ...desktop,
+    ...entry.bindings.observe(result),
+  };
+}
+
+export async function projectComputerResult(
+  entry: RequestComputer,
+  result: NativeComputerResult,
+  context: ToolExecutionContext,
+) {
+  const media = [];
+  let imageFailure: string | undefined;
+  if (result.image) {
+    try {
+      const image = await context.media!.writeImage(result.image);
+      if (!hasMatchingImageGeometry(image, result))
+        throw new SystemOperationError(
+          "computer_image_geometry_mismatch",
+          "Native observation dimensions do not match the captured image. Observe again before input.",
+        );
+      media.push(image);
+    } catch (error) {
+      imageFailure =
+        error instanceof Error ? error.message : "Image admission failed.";
+    } finally {
+      result.image.bytes.fill(0);
+    }
+  }
+  const description = describeDesktop(entry, result);
+  if (media.length === 0) {
+    entry.bindings.clear();
+    description.observation_ref = undefined;
+  }
+  const hasPartialInputDispatch = result.dispatch?.status === "partial";
+  const operationError = result.error ?? (hasPartialInputDispatch
+    ? { code: "computer_input_partial", message: "Only part of the input was accepted." }
+    : undefined);
+  const data = {
+    desktop_ref: entry.ref,
+    ...(description.observation_ref
+      ? { observation_ref: description.observation_ref }
+      : {}),
+    ...(result.dispatch ? { dispatch: result.dispatch } : {}),
+    ...inputDispatchEvidence(entry.ref, result.dispatch),
+    currentStateEvidence: Boolean(result.observation),
+    ...(operationError ? { error: operationError } : {}),
+    ...(imageFailure ? { imageError: imageFailure } : {}),
+    imageAvailable: media.length > 0,
+    evidenceScope: "observed_desktop_and_input_receipt",
+    requestedEffectVerified: false,
+    observationMeta: {
+      kind: "volatile_external",
+      carryPolicy: "never",
+    } as const,
+  };
+  const output = serializeComputerObservation(description, data);
+  if (imageFailure || operationError)
+    return failureResult({
+      errorCode: operationError?.code ?? "computer_image_unavailable",
+      message: operationError?.message ?? imageFailure!,
+      output,
+      data,
+      media,
+    });
+  return successResult({ output, data, media });
+}
+
+function inputDispatchEvidence(
+  desktopRef: string,
+  dispatch: NativeComputerResult["dispatch"],
+) {
+  if (!dispatch || !["accepted", "partial"].includes(dispatch.status))
+    return {};
+  const accepted = dispatch.acceptedInputCount;
+  if (!Number.isSafeInteger(accepted) || !accepted || accepted < 0) return {};
+  if (!Number.isSafeInteger(dispatch.requestedInputCount)) return {};
+  if (accepted > dispatch.requestedInputCount) return {};
+  return {
+    mutationEvidence: true,
+    mutationGrounding: JSON.stringify({
+      kind: "computer_input_receipt_v1",
+      desktop_ref: desktopRef,
+      dispatch,
+      evidenceScope: "native_input_dispatch_only",
+      requestedEffectVerified: false,
+    }),
+  };
+}
+
+function hasMatchingImageGeometry(
+  image: { width: number; height: number },
+  result: NativeComputerResult,
+): boolean {
+  if (!result.observation) return false;
+  if (image.width !== result.observation.imageWidth) return false;
+  return image.height === result.observation.imageHeight;
+}