@abot-ai/runtime 1.4.0 → 1.5.0

package/dist/src/computer-access/computer/macos-helper.js

@@ -0,0 +1,117 @@
+import { MACOS_DESKTOP_SOURCE } from "./macos-desktop.js";
+import { MACOS_CAPTURE_SOURCE } from "./macos-capture.js";
+import { MACOS_INPUT_SOURCE } from "./macos-input.js";
+import { MACOS_TOPOLOGY_SOURCE } from "./macos-topology.js";
+const MACOS_COMMAND_LOOP = String.raw `
+func perform(_ id:String,_ request:[String:Any],_ cancel:CancellationFlag) -> [String:Any] {
+  var state=desktopState(),dispatch:[String:Any]?
+  do {
+    let operation=request["operation"] as? String
+    if operation=="inspect" {
+      return state
+    }
+    if operation=="observe" {
+      try requireObservationSource(request,state)
+      return try addCapture(state,request["region"])
+    }
+    guard operation=="act",let action=request["action"] as? [String:Any],let deadline=request["deadlineEpochMs"] as? Double else {
+      throw NativeFailure("computer_request_invalid")
+    }
+    try cancel.check(deadline)
+    guard sessionAvailable() else {
+      throw NativeFailure("macos_session_locked_or_unavailable")
+    }
+    guard AXIsProcessTrusted() else {
+      throw NativeFailure("macos_accessibility_permission_required")
+    }
+    _ = try requireDesktopSourceBinding(request["desktopBinding"] as? String)
+    guard let geometry=readRect(request["expectedGeometry"]),geometry==displayBounds() else {
+      throw NativeFailure("computer_geometry_changed")
+    }
+    if let expected=request["expectedWindow"] as? String,expected != state["focusedWindow"] as? String {
+      throw NativeFailure("computer_focus_changed")
+    }
+    try validateAction(action,state)
+    _ = try requireDesktopSourceBinding(request["desktopBinding"] as? String)
+    dispatch=try executeInput(id,action,cancel,deadline,request["expectedWindow"] as? String)
+    emit(["type":"dispatch","id":id,"dispatch":dispatch!])
+    state=desktopState()
+    state["dispatch"]=dispatch
+    try requireObservationSource(request,state)
+    if let dispatchError=incompleteDispatchError(dispatch!) {
+      state["error"]=dispatchError
+    }
+    if action["kind"] as? String=="focus_window",state["focusedWindow"] as? String != action["windowBinding"] as? String {
+      state["error"]=["code":"computer_focus_not_confirmed","message":"The activation request was dispatched but target focus is not confirmed."]
+    }
+    do {
+      return try addCapture(state,nil)
+    }
+    catch {
+      state["error"]=["code":"computer_post_action_capture_failed","message":"Input dispatch settled but its subsequent capture failed; inspect before repeating input."]
+      return state
+    }
+  }
+  catch {
+    let code=(error as? NativeFailure)?.code ?? "macos_native_operation_failed"
+    state["error"]=["code":code,"message":"The native desktop operation could not complete."]
+    if request["operation"] as? String=="act" {
+      state["dispatch"]=dispatch ?? ["status":"not_dispatched","requestedInputCount":actionCount(request["action"] as? [String:Any] ?? [:]),"acceptedInputCount":0,"reason":code]
+    }
+    return state
+  }
+}
+let operationQueue=DispatchQueue(label:"abot.computer.operations"),pendingLock=NSLock()
+var cancellations=[String:CancellationFlag]()
+func shutdownNativeHelper() {
+  pendingLock.lock()
+  for cancel in cancellations.values { cancel.cancel() }
+  pendingLock.unlock()
+  operationQueue.async { exit(0) }
+}
+signal(SIGTERM,SIG_IGN)
+let terminationSource=DispatchSource.makeSignalSource(signal:SIGTERM,queue:.global())
+terminationSource.setEventHandler { shutdownNativeHelper() }
+terminationSource.resume()
+DispatchQueue.global().async {
+  while let line=readLine() {
+    guard line.utf8.count<=65536,let data=line.data(using:.utf8),let message=(try? JSONSerialization.jsonObject(with:data)) as? [String:Any] else {
+      break
+    }
+    if message["type"] as? String=="close" {
+      break
+    }
+    guard let id=message["id"] as? String else {
+      break
+    }
+    if message["type"] as? String=="cancel" {
+      pendingLock.lock()
+      cancellations[id]?.cancel()
+      pendingLock.unlock()
+      continue
+    }
+    guard message["type"] as? String=="request",let request=message["request"] as? [String:Any] else {
+      break
+    }
+    let cancel=CancellationFlag()
+    pendingLock.lock()
+    cancellations[id]=cancel
+    pendingLock.unlock()
+    operationQueue.async {
+      emit(["type":"result","id":id,"result":perform(id,request,cancel)])
+      pendingLock.lock()
+      cancellations.removeValue(forKey:id)
+      pendingLock.unlock()
+    }
+  }
+  shutdownNativeHelper()
+}
+RunLoop.main.run()
+`;
+export const MACOS_COMPUTER_HELPER = [
+    MACOS_DESKTOP_SOURCE,
+    MACOS_TOPOLOGY_SOURCE,
+    MACOS_CAPTURE_SOURCE,
+    MACOS_INPUT_SOURCE,
+    MACOS_COMMAND_LOOP,
+].join("\n");

package/dist/src/computer-access/computer/macos-input.d.ts

@@ -0,0 +1 @@
+export declare const MACOS_INPUT_SOURCE: string;

package/dist/src/computer-access/computer/macos-input.js

@@ -0,0 +1,267 @@
+export const MACOS_INPUT_SOURCE = String.raw `
+func incompleteDispatchError(_ dispatch:[String:Any]) -> [String:Any]? {
+  if dispatch["status"] as? String == "partial" {
+    return ["code":"computer_input_partial","message":"Native input was only partially dispatched. Inspect the receipt and fresh observation before another action."]
+  }
+  if dispatch["status"] as? String == "unknown" {
+    return ["code":"computer_input_unknown","message":"Native input dispatch is uncertain. Inspect the receipt and fresh observation before another action."]
+  }
+  return nil
+}
+final class CancellationFlag {
+  let lock=NSLock()
+  var cancelled=false
+  func cancel() {
+    lock.lock()
+    cancelled=true
+    lock.unlock()
+  }
+  func check(_ deadline:Double) throws {
+    lock.lock()
+    let stopped=cancelled
+    lock.unlock()
+    if stopped {
+      throw NativeFailure("computer_action_cancelled")
+    }
+    if Date().timeIntervalSince1970*1000>deadline {
+      throw NativeFailure("computer_action_deadline")
+    }
+  }
+}
+let keyCodes:[String:CGKeyCode] = ["a":0,"s":1,"d":2,"f":3,"h":4,"g":5,"z":6,"x":7,"c":8,"v":9,"b":11,"q":12,"w":13,"e":14,"r":15,"y":16,"t":17,"1":18,"2":19,"3":20,"4":21,"6":22,"5":23,"=":24,"9":25,"7":26,"-":27,"8":28,"0":29,"]":30,"o":31,"u":32,"[":33,"i":34,"p":35,"l":37,"j":38,"'":39,"k":40,";":41,"\\":42,",":43,"/":44,"n":45,"m":46,".":47,"\u{60}":50,"Enter":36,"Return":36,"Tab":48,"Space":49,"Backspace":51,"Delete":117,"Esc":53,"Escape":53,"Meta":55,"Super":55,"Command":55,"Shift":56,"Alt":58,"Option":58,"Ctrl":59,"Control":59,"F1":122,"F2":120,"F3":99,"F4":118,"F5":96,"F6":97,"F7":98,"F8":100,"F9":101,"F10":109,"F11":103,"F12":111,"Home":115,"End":119,"PageUp":116,"PageDown":121,"ArrowLeft":123,"ArrowRight":124,"ArrowDown":125,"ArrowUp":126]
+func nativeKey(_ value:String) throws -> CGKeyCode {
+  if let key=keyCodes[value] {
+    return key
+  }
+  if value.count==1,let key=keyCodes[value.lowercased()] {
+    return key
+  }
+  throw NativeFailure("macos_key_unsupported")
+}
+func modifierFlag(_ key:CGKeyCode) -> CGEventFlags {
+  switch key {
+    case 55:return .maskCommand
+    case 56:return .maskShift
+    case 58:return .maskAlternate
+    case 59:return .maskControl
+    default:return []
+  }
+}
+func actionCount(_ action:[String:Any]) -> Int {
+  switch action["kind"] as? String {
+    case "type_text":return ((action["text"] as? String)?.unicodeScalars.count ?? 0)*2
+    case "press_keys":return ((action["keys"] as? [String])?.count ?? 0)*2
+    case "click":return 1+(action["count"] as? Int ?? 1)*2
+    case "drag":return 15
+    default:return 1
+  }
+}
+func readPoint(_ value:Any?) throws -> CGPoint {
+  guard let value=value as? [String:Double],let x=value["x"],let y=value["y"],x.isFinite,y.isFinite else {
+    throw NativeFailure("computer_point_invalid")
+  }
+  return CGPoint(x:x,y:y)
+}
+func validateAction(_ action:[String:Any],_ state:[String:Any]) throws {
+  guard let kind=action["kind"] as? String,["click","move","drag","scroll","type_text","press_keys","focus_window"].contains(kind) else {
+    throw NativeFailure("computer_action_unsupported")
+  }
+  let bounds=displayBounds()
+  let names=kind=="drag" ? ["from","to"] : ["click","move"].contains(kind) ? ["point"] : []
+  for name in names {
+    if !bounds.contains(try readPoint(action[name])) {
+      throw NativeFailure("computer_point_outside_desktop")
+    }
+  }
+  if ["click","drag"].contains(kind),!["left","middle","right"].contains(action["button"] as? String ?? "") {
+    throw NativeFailure("computer_button_invalid")
+  }
+  if kind=="click",![1,2].contains(action["count"] as? Int ?? 0) {
+    throw NativeFailure("computer_click_count_invalid")
+  }
+  if kind=="drag",!(0...5000).contains(action["durationMs"] as? Int ?? -1) {
+    throw NativeFailure("computer_drag_duration_invalid")
+  }
+  if kind=="scroll" {
+    guard let x=action["deltaX"] as? Double,let y=action["deltaY"] as? Double,x.isFinite,y.isFinite,abs(x)<=12000,abs(y)<=12000 else {
+      throw NativeFailure("computer_scroll_invalid")
+    }
+    guard x.truncatingRemainder(dividingBy:120)==0,y.truncatingRemainder(dividingBy:120)==0 else {
+      throw NativeFailure("macos_scroll_fraction_unsupported")
+    }
+  }
+  if kind=="type_text" {
+    guard let text=action["text"] as? String,text.utf16.count<=10000 else {
+      throw NativeFailure("computer_text_limit")
+    }
+  }
+  if kind=="press_keys" {
+    guard let keys=action["keys"] as? [String],!keys.isEmpty,keys.count<=8 else {
+      throw NativeFailure("computer_keys_invalid")
+    }
+    for key in keys {
+      _ = try nativeKey(key)
+    }
+  }
+  if kind=="focus_window",!windowReferences.contains(where:{
+    $0.binding==action["windowBinding"] as? String
+  }
+  ) {
+    throw NativeFailure("computer_window_stale")
+  }
+}
+func requireVerifiedTextFocus(_ action:[String:Any],_ expectedWindow:String?) throws {
+  guard action["kind"] as? String == "type_text" else { return }
+  guard let expectedWindow=expectedWindow else { return }
+  guard let reference=windowReferences.first(where: { $0.binding==expectedWindow }) else {
+    throw NativeFailure("computer_focus_changed")
+  }
+  guard NSWorkspace.shared.frontmostApplication?.processIdentifier==reference.pid else {
+    throw NativeFailure("computer_focus_changed")
+  }
+  let root=AXUIElementCreateApplication(reference.pid)
+  guard let focused=axElement(axAttribute(root,kAXFocusedWindowAttribute)),CFEqual(focused,reference.element) else {
+    throw NativeFailure("computer_focus_changed")
+  }
+}
+func executeInput(_ id:String,_ action:[String:Any],_ cancel:CancellationFlag,_ deadline:Double,_ expectedWindow:String? = nil) throws -> [String:Any] {
+  try cancel.check(deadline)
+  guard let source=CGEventSource(stateID:.privateState) else {
+    throw NativeFailure("macos_input_source_unavailable")
+  }
+  let requested=actionCount(action)
+  var accepted=0,heldKeys=[CGKeyCode](),heldButton:CGMouseButton?,flags=CGEventFlags()
+  var lastPoint=CGEvent(source:nil)?.location ?? .zero
+  func post(_ event:CGEvent?) throws {
+    try cancel.check(deadline)
+    try requireVerifiedTextFocus(action,expectedWindow)
+    guard let event=event else {
+      throw NativeFailure("macos_input_event_unavailable")
+    }
+    event.flags=flags
+    event.post(tap:.cghidEventTap)
+    accepted+=1
+  }
+  func keyboard(_ key:CGKeyCode,_ down:Bool) throws {
+    let flag=modifierFlag(key)
+    if down {
+      flags.formUnion(flag)
+    }
+    else {
+      flags.subtract(flag)
+    }
+    try post(CGEvent(keyboardEventSource:source,virtualKey:key,keyDown:down))
+    if down {
+      heldKeys.append(key)
+    }
+    else {
+      heldKeys.removeAll {
+        $0==key
+      }
+    }
+  }
+  func mouse(_ type:CGEventType,_ point:CGPoint,_ button:CGMouseButton,_ click:Int=1) throws {
+    let event=CGEvent(mouseEventSource:source,mouseType:type,mouseCursorPosition:point,mouseButton:button)
+    event?.setIntegerValueField(.mouseEventClickState,value:Int64(click))
+    try post(event)
+    lastPoint=point
+  }
+  emit(["type":"dispatch","id":id,"dispatch":["status":"unknown","requestedInputCount":requested,"reason":"native_dispatch_started"]])
+  var failure:String?
+  do {
+    let kind=action["kind"] as! String
+    if kind=="move" {
+      try mouse(.mouseMoved,try readPoint(action["point"]),.left)
+    }
+    if kind=="click" || kind=="drag" {
+      let button:CGMouseButton = action["button"] as? String == "right" ? .right : action["button"] as? String == "middle" ? .center : .left
+      let down:CGEventType = button == .left ? .leftMouseDown : button == .right ? .rightMouseDown : .otherMouseDown
+      let up:CGEventType = button == .left ? .leftMouseUp : button == .right ? .rightMouseUp : .otherMouseUp
+      let start=try readPoint(action[kind=="click" ? "point":"from"])
+      try mouse(.mouseMoved,start,button)
+      if kind=="click" {
+        for number in 1...(action["count"] as! Int) {
+          try mouse(down,start,button,number)
+          heldButton=button
+          try mouse(up,start,button,number)
+          heldButton=nil
+          if number==1 && action["count"] as! Int==2 {
+            Thread.sleep(forTimeInterval:0.05)
+          }
+        }
+      }
+      else {
+        let end=try readPoint(action["to"])
+        try mouse(down,start,button)
+        heldButton=button
+        let dragged:CGEventType = button == .left ? .leftMouseDragged : button == .right ? .rightMouseDragged : .otherMouseDragged
+        for step in 1...12 {
+          let ratio=Double(step)/12
+          try mouse(dragged,CGPoint(x:start.x+(end.x-start.x)*ratio,y:start.y+(end.y-start.y)*ratio),button)
+          Thread.sleep(forTimeInterval:Double(action["durationMs"] as! Int)/12000)
+        }
+        try mouse(up,end,button)
+        heldButton=nil
+      }
+    }
+    if kind=="scroll" {
+      let x=Int32((action["deltaX"] as! Double)/120),y=Int32((action["deltaY"] as! Double)/120)
+      try post(CGEvent(scrollWheelEvent2Source:source,units:.line,wheelCount:2,wheel1:-y,wheel2:-x,wheel3:0))
+    }
+    if kind=="press_keys" {
+      let keys=try (action["keys"] as! [String]).map {
+        try nativeKey($0)
+      }
+      for key in keys {
+        try keyboard(key,true)
+      }
+      for key in keys.reversed() {
+        try keyboard(key,false)
+      }
+    }
+    if kind=="type_text" {
+      for scalar in (action["text"] as! String).unicodeScalars {
+        var units=Array(String(scalar).utf16)
+        for down in [true,false] {
+          let event=CGEvent(keyboardEventSource:source,virtualKey:0,keyDown:down)
+          event?.keyboardSetUnicodeString(stringLength:units.count,unicodeString:&units)
+          try post(event)
+          if down { heldKeys.append(0) } else { heldKeys.removeAll { $0 == 0 } }
+        }
+      }
+    }
+    if kind=="focus_window" {
+      try cancel.check(deadline)
+      guard let reference=windowReferences.first(where:{
+        $0.binding==action["windowBinding"] as? String
+      }
+      ),let app=NSRunningApplication(processIdentifier:reference.pid) else {
+        throw NativeFailure("computer_window_stale")
+      }
+      guard app.activate(options:[.activateIgnoringOtherApps]),AXUIElementPerformAction(reference.element,kAXRaiseAction as CFString) == .success else {
+        throw NativeFailure("macos_window_focus_failed")
+      }
+      accepted+=1
+    }
+  }
+  catch {
+    failure=(error as? NativeFailure)?.code ?? "macos_native_input_failed"
+  }
+  // Release only this operation's own held input, even after cooperative cancellation.
+  for key in heldKeys.reversed() {
+    flags.subtract(modifierFlag(key))
+    let event=CGEvent(keyboardEventSource:source,virtualKey:key,keyDown:false)
+    event?.flags=flags
+    event?.post(tap:.cghidEventTap)
+  }
+  if let button=heldButton {
+    let type:CGEventType = button == .left ? .leftMouseUp : button == .right ? .rightMouseUp : .otherMouseUp
+    CGEvent(mouseEventSource:source,mouseType:type,mouseCursorPosition:lastPoint,mouseButton:button)?.post(tap:.cghidEventTap)
+  }
+  var dispatch:[String:Any] = ["status":failure==nil ? "accepted":accepted>0 ? "partial":"unknown","requestedInputCount":requested,"acceptedInputCount":accepted]
+  if let failure=failure {
+    dispatch["reason"]=failure
+  }
+  return dispatch
+}
+`;

package/dist/src/computer-access/computer/macos-prerequisites.d.ts

@@ -0,0 +1,2 @@
+/** Do not launch Apple's developer-tools installer as a side effect of inspection. */
+export declare function missingMacosComputerPrerequisite(): Promise<string | undefined>;

package/dist/src/computer-access/computer/macos-prerequisites.js

@@ -0,0 +1,25 @@
+import { execFile } from "node:child_process";
+import { access } from "node:fs/promises";
+import { promisify } from "node:util";
+const executeFile = promisify(execFile);
+/** Do not launch Apple's developer-tools installer as a side effect of inspection. */
+export async function missingMacosComputerPrerequisite() {
+    try {
+        const version = await executeFile("/usr/bin/sw_vers", ["-productVersion"], {
+            timeout: 1000,
+            maxBuffer: 1024,
+        });
+        const major = Number(version.stdout.trim().split(".")[0]);
+        if (!Number.isInteger(major) || major < 14)
+            return "macos14_computer_backend_required";
+        const directory = await executeFile("/usr/bin/xcode-select", ["-p"], {
+            timeout: 1000,
+            maxBuffer: 4096,
+        });
+        await access(directory.stdout.trim());
+        return undefined;
+    }
+    catch {
+        return "macos_swift_developer_tools_required";
+    }
+}

package/dist/src/computer-access/computer/macos-topology.d.ts

@@ -0,0 +1,2 @@
+/** Quartz capture source identity includes individual display layout and pixel scale. */
+export declare const MACOS_TOPOLOGY_SOURCE: string;

package/dist/src/computer-access/computer/macos-topology.js

@@ -0,0 +1,34 @@
+/** Quartz capture source identity includes individual display layout and pixel scale. */
+export const MACOS_TOPOLOGY_SOURCE = String.raw `
+import CryptoKit
+struct DisplaySourceSnapshot {
+  let binding:String
+  let bounds:CGRect
+  let displays:[CGDirectDisplayID]
+}
+func readDisplaySource() -> DisplaySourceSnapshot? {
+  var count:UInt32=0
+  guard CGGetActiveDisplayList(0,nil,&count) == .success,count>0,count<=128 else { return nil }
+  var displays=[CGDirectDisplayID](repeating:0,count:Int(count))
+  guard CGGetActiveDisplayList(count,&displays,&count) == .success else { return nil }
+  let identifiers=Array(displays.prefix(Int(count))).sorted()
+  var records=[[String:Any]](),bounds=CGRect.null
+  for identifier in identifiers {
+    let frame=CGDisplayBounds(identifier)
+    guard !frame.isEmpty,let mode=CGDisplayCopyDisplayMode(identifier) else { return nil }
+    bounds=bounds.union(frame)
+    records.append(["id":identifier,"rect":rectJSON(frame),"rotation":CGDisplayRotation(identifier),
+      "mode":mode.ioDisplayModeID,"width":mode.width,"height":mode.height,
+      "pixelWidth":mode.pixelWidth,"pixelHeight":mode.pixelHeight,
+      "mirrors":CGDisplayMirrorsDisplay(identifier),"main":CGDisplayIsMain(identifier)])
+  }
+  guard let data=try? JSONSerialization.data(withJSONObject:records,options:[.sortedKeys]) else { return nil }
+  let fingerprint=SHA256.hash(data:data).map { String(format:"%02x",$0) }.joined()
+  return DisplaySourceSnapshot(binding:desktopSessionBinding+":"+fingerprint,bounds:bounds,displays:identifiers)
+}
+func requireDesktopSourceBinding(_ expected:String?) throws -> DisplaySourceSnapshot {
+  guard let source=readDisplaySource() else { throw NativeFailure("macos_topology_unavailable") }
+  guard expected==source.binding else { throw NativeFailure("computer_desktop_stale") }
+  return source
+}
+`;

package/dist/src/computer-access/computer/native-backend.d.ts

@@ -0,0 +1,3 @@
+import type { SystemTarget } from "../contracts.js";
+import type { NativeComputerBackend } from "./native-protocol.js";
+export declare function createNativeComputerBackend(target: SystemTarget): NativeComputerBackend;

package/dist/src/computer-access/computer/native-backend.js

@@ -0,0 +1,10 @@
+import { createWindowsComputerBackend } from "./windows-backend.js";
+import { createMacosComputerBackend } from "./macos-backend.js";
+import { createLinuxComputerBackend } from "./linux-backend.js";
+export function createNativeComputerBackend(target) {
+    if (target.id === "windows")
+        return createWindowsComputerBackend(target);
+    if (target.id === "macos")
+        return createMacosComputerBackend(target);
+    return createLinuxComputerBackend(target);
+}

package/dist/src/computer-access/computer/native-protocol.d.ts

@@ -0,0 +1,129 @@
+/** Private structured OS adapter contract. Never projected as a model tool schema. */
+export type ComputerPlatform = "windows" | "macos" | "linux";
+export type PhysicalPoint = Readonly<{
+    x: number;
+    y: number;
+}>;
+export type PhysicalRectangle = Readonly<PhysicalPoint & {
+    width: number;
+    height: number;
+}>;
+export type ComputerButton = "left" | "middle" | "right";
+/** Plugin maps image coordinates to the backend's native desktop coordinate space. */
+export type NativeComputerAction = {
+    kind: "click";
+    point: PhysicalPoint;
+    button: ComputerButton;
+    count: 1 | 2;
+} | {
+    kind: "move";
+    point: PhysicalPoint;
+} | {
+    kind: "drag";
+    from: PhysicalPoint;
+    to: PhysicalPoint;
+    button: ComputerButton;
+    durationMs: number;
+}
+/** Signed wheel units: 120 is a conventional notch; positive right/down. */
+ | {
+    kind: "scroll";
+    deltaX: number;
+    deltaY: number;
+} | {
+    kind: "type_text";
+    text: string;
+} | {
+    kind: "press_keys";
+    keys: string[];
+} | {
+    kind: "focus_window";
+    windowBinding: string;
+};
+export type ComputerCapability = Readonly<{
+    supported: boolean;
+    available: boolean;
+    reason?: string;
+}>;
+export type NativeComputerWindow = Readonly<{
+    binding: string;
+    title: string;
+    application?: string;
+    bounds: PhysicalRectangle;
+    focused: boolean;
+}>;
+export type ComputerAccessibilityNode = Readonly<{
+    role: string;
+    name?: string;
+    value?: string;
+    bounds?: PhysicalRectangle;
+    focused?: boolean;
+}>;
+export type NativeComputerDesktop = Readonly<{
+    platform: ComputerPlatform;
+    /** Opaque identity of the session and capture-source topology; changes invalidate observations. */
+    binding: string;
+    name: string;
+    available: boolean;
+    reason?: string;
+    bounds: PhysicalRectangle;
+    coordinateSpace?: "physical_pixels" | "logical_points";
+    capabilities: Readonly<{
+        capture: ComputerCapability;
+        accessibility: ComputerCapability;
+        input: ComputerCapability;
+        windows: ComputerCapability;
+    }>;
+    targetingGuarantee: "verified_window" | "observed_surface";
+}>;
+export type NativeComputerObservation = Readonly<{
+    capturedAt: string;
+    region: PhysicalRectangle;
+    imageWidth: number;
+    imageHeight: number;
+    accessibility: readonly ComputerAccessibilityNode[];
+    accessibilityTruncated: boolean;
+}>;
+export type NativeComputerDispatch = Readonly<{
+    status: "not_dispatched" | "accepted" | "partial" | "unknown";
+    requestedInputCount: number;
+    acceptedInputCount?: number;
+    reason?: string;
+}>;
+export type NativeComputerResult = Readonly<{
+    desktop: NativeComputerDesktop;
+    windows: readonly NativeComputerWindow[];
+    focusedWindow?: string;
+    observation?: NativeComputerObservation;
+    image?: Readonly<{
+        mimeType: "image/png";
+        bytes: Uint8Array;
+    }>;
+    dispatch?: NativeComputerDispatch;
+    error?: Readonly<{
+        code: string;
+        message: string;
+    }>;
+}>;
+export type NativeComputerRequest = {
+    operation: "inspect";
+} | {
+    operation: "observe";
+    region?: undefined;
+} | {
+    operation: "observe";
+    region: PhysicalRectangle;
+    desktopBinding: string;
+    expectedGeometry: PhysicalRectangle;
+} | {
+    operation: "act";
+    desktopBinding: string;
+    expectedWindow?: string;
+    expectedGeometry: PhysicalRectangle;
+    action: NativeComputerAction;
+    deadlineEpochMs: number;
+};
+export interface NativeComputerBackend {
+    execute(request: NativeComputerRequest, signal?: AbortSignal): Promise<NativeComputerResult>;
+    close(): Promise<void>;
+}

package/dist/src/computer-access/computer/native-protocol.js

@@ -0,0 +1 @@
+export {};

package/dist/src/computer-access/computer/native-validation.d.ts

@@ -0,0 +1,7 @@
+import type { NativeComputerRequest, NativeComputerResult } from "./native-protocol.js";
+export declare const COMPUTER_IMAGE_MAX_BYTES: number;
+export declare const COMPUTER_FRAME_CHUNK_BYTES: number;
+export declare const COMPUTER_CAPABILITY = "computer_control_v1";
+export declare function readNativeComputerRequest(value: unknown): NativeComputerRequest;
+/** Validate the private transport independently of trusted TypeScript declarations. */
+export declare function readNativeComputerResult(value: unknown): NativeComputerResult;