@anionex/dsh-computer-use 0.1.0

package/src/types.ts

@@ -0,0 +1,299 @@
+/** Public Computer Use types shared by the Service, provider, and Tool consumer. */
+
+import type { Agent } from '@deepseek-ai/dsh-agent'
+import type { Branded } from '@deepseek-ai/dsh-brand'
+import type { CallId } from '@deepseek-ai/dsh-llm'
+
+/** Opaque identifier for one immutable observed UI state. */
+export type ComputerObservationId = Branded<'ComputerObservationId'>
+
+/** Brand a generated observation identifier. */
+export const ComputerObservationId = (value: string): ComputerObservationId => value as ComputerObservationId
+
+/** Opaque reference to one element descriptor captured inside an observation. */
+export type ComputerTargetHandle = Branded<'ComputerTargetHandle'>
+
+/** Brand a generated target handle. */
+export const ComputerTargetHandle = (value: string): ComputerTargetHandle => value as ComputerTargetHandle
+
+/** Opaque one-use grant for one confirmed sensitive action. */
+export type ComputerConfirmationToken = Branded<'ComputerConfirmationToken'>
+
+/** Brand a generated confirmation token. */
+export const ComputerConfirmationToken = (value: string): ComputerConfirmationToken => value as ComputerConfirmationToken
+
+/** Rectangle in screen-global point coordinates unless a containing type states otherwise. */
+export interface ComputerRect {
+  x: number
+  y: number
+  width: number
+  height: number
+}
+
+/** macOS permission state reported without attempting to grant it. */
+export type ComputerPermissionState = 'granted' | 'denied' | 'not-determined' | 'unavailable'
+
+/** Stable identity of one running user-facing application. */
+export interface ComputerAppIdentity {
+  bundleId: string
+  pid: number
+  name: string
+}
+
+/** Application selector accepted from a model or caller. */
+export interface ComputerAppSelector {
+  bundleId?: string
+  pid?: number
+  name?: string
+}
+
+/** Bounded application row returned by discovery. */
+export interface ComputerAppSummary extends ComputerAppIdentity {
+  frontmost: boolean
+  accessibility: ComputerPermissionState
+  screenRecording: ComputerPermissionState
+}
+
+/** One model-addressable element. Its index and opaque handle belong only to the enclosing observation. */
+export interface ComputerElement {
+  index: number
+  targetHandle: ComputerTargetHandle
+  role: string
+  subrole?: string
+  title?: string
+  label?: string
+  value?: string
+  enabled?: boolean
+  focused?: boolean
+  selected?: boolean
+  frame?: ComputerRect
+  actions: string[]
+}
+
+/** Deterministic route used to resolve a target immediately before input. */
+export type ComputerTargetResolutionMode =
+  | 'exact-locator'
+  | 'native-identifier'
+  | 'semantic-rebind'
+
+/** Model-visible evidence describing how an element target was resolved. */
+export interface ComputerTargetResolutionResult {
+  mode: ComputerTargetResolutionMode
+  confidence: number
+  candidateCount: number
+  targetChanged: boolean
+}
+
+/** File artifact emitted by a Computer Use observation. */
+export interface ComputerArtifact {
+  path: string
+  filename: string
+  mimeType: 'image/png'
+  kind: 'image'
+  description: string
+  sourceTool: 'computer_observe' | 'computer_action'
+  previewIntent: 'image'
+  bytes: number
+  width: number
+  height: number
+}
+
+/** Complete model-visible observation. */
+export interface ComputerObservation {
+  observationId: ComputerObservationId
+  app: ComputerAppIdentity
+  createdAt: string
+  expiresAt: string
+  frontmost: boolean
+  window?: {
+    title?: string
+    frame: ComputerRect
+    id?: number
+  }
+  tree: {
+    mode: 'full' | 'diff'
+    text: string
+    truncated: boolean
+  }
+  elements: ComputerElement[]
+  screenshot?: ComputerArtifact
+  permissions: {
+    accessibility: ComputerPermissionState
+    screenRecording: ComputerPermissionState
+  }
+}
+
+/** Whether screenshot capture is omitted, best-effort, or required. */
+export type ComputerScreenshotMode = 'none' | 'optional' | 'required'
+
+/** Request for a fresh observation. */
+export interface ComputerObserveRequest {
+  app: ComputerAppSelector
+  screenshot?: ComputerScreenshotMode
+  full?: boolean
+}
+
+/** Closed set of supported key modifiers. */
+export type ComputerKeyModifier = 'command' | 'control' | 'option' | 'shift'
+
+/** Closed set of supported mouse buttons. */
+export type ComputerMouseButton = 'left' | 'right' | 'middle'
+
+/** Coordinate space accepted by coordinate-based pointer actions. */
+export type ComputerCoordinateSpace = 'window' | 'screen'
+
+/** Shared fields carried by every action against an existing observation. */
+export interface ComputerActionBase {
+  observationId: ComputerObservationId
+  /** True only when the Skill has classified the action as requiring just-in-time confirmation. */
+  sensitive?: boolean
+  /** One-use token returned by {@link ComputerUseService.confirm}. */
+  confirmationToken?: ComputerConfirmationToken
+}
+
+/** Stable target selection fields shared by element-addressed actions. */
+export interface ComputerElementTarget {
+  /** Observation-local compatibility index. It does not authorize rebinding by itself. */
+  elementIndex?: number
+  /** Opaque handle returned on the selected observation element. */
+  targetHandle?: ComputerTargetHandle
+  /** Permit deterministic native-identifier or unique semantic rebinding. */
+  allowRebind?: boolean
+}
+
+/** Click an observed element or an observed-window coordinate. */
+export interface ComputerClickAction extends ComputerActionBase, ComputerElementTarget {
+  kind: 'click'
+  x?: number
+  y?: number
+  /** `window` (default) resolves `x`/`y` inside the observed window frame; `screen` treats them as Quartz screen-global points. */
+  coordinateSpace?: ComputerCoordinateSpace
+  button?: ComputerMouseButton
+  clickCount?: number
+  allowCoordinateFallback?: boolean
+}
+
+/** Set the Accessibility value of an observed editable element. */
+export interface ComputerSetValueAction extends ComputerActionBase, ComputerElementTarget {
+  kind: 'set-value'
+  value: string
+}
+
+/** Insert Unicode into the currently focused control through Accessibility or keyboard fallback, without using the clipboard. */
+export interface ComputerTypeTextAction extends ComputerActionBase {
+  kind: 'type-text'
+  text: string
+}
+
+/** Press one validated key chord. */
+export interface ComputerPressKeyAction extends ComputerActionBase {
+  kind: 'press-key'
+  key: string
+  modifiers?: ComputerKeyModifier[]
+}
+
+/** Scroll at an observed element or coordinate. */
+export interface ComputerScrollAction extends ComputerActionBase, ComputerElementTarget {
+  kind: 'scroll'
+  x?: number
+  y?: number
+  coordinateSpace?: ComputerCoordinateSpace
+  direction: 'up' | 'down' | 'left' | 'right'
+  pages?: number
+}
+
+/** Drag between two points in the observed window or screen coordinate space. */
+export interface ComputerDragAction extends ComputerActionBase {
+  kind: 'drag'
+  fromX: number
+  fromY: number
+  toX: number
+  toY: number
+  coordinateSpace?: ComputerCoordinateSpace
+}
+
+/** Perform one Accessibility action advertised by an observed element. */
+export interface ComputerPerformAction extends ComputerActionBase, ComputerElementTarget {
+  kind: 'perform-action'
+  action: string
+}
+
+/** Wait for a bounded UI condition without mutating state. */
+export interface ComputerWaitAction extends ComputerActionBase {
+  kind: 'wait'
+  condition: {
+    text?: string
+    elementRole?: string
+    elementTitle?: string
+  }
+  timeoutMs?: number
+}
+
+/** Every model-accessible Computer Use action. */
+export type ComputerActionRequest =
+  | ComputerClickAction
+  | ComputerSetValueAction
+  | ComputerTypeTextAction
+  | ComputerPressKeyAction
+  | ComputerScrollAction
+  | ComputerDragAction
+  | ComputerPerformAction
+  | ComputerWaitAction
+
+/** Result of a successful action followed by a fresh observation. */
+export interface ComputerActionResult {
+  action: ComputerActionRequest['kind']
+  channel: 'accessibility' | 'coordinates' | 'keyboard' | 'wait'
+  /** Verifiable target-app foreground state transition requested by the helper for this action. */
+  activation: 'not-requested' | 'already-frontmost' | 'activated'
+  /** Whether the helper emitted mouse, drag, or scroll-wheel input to the target process. */
+  pointerInput: boolean
+  /** Pointer-event route selected by the helper; global HID routing is not supported. */
+  pointerRouting: 'none' | 'target-process'
+  /** Present when the action addressed an observed element. */
+  resolution?: ComputerTargetResolutionResult
+  observation: ComputerObservation
+}
+
+/** Confirmation request binding an exact proposed action to human-readable impact. */
+export interface ComputerConfirmRequest {
+  action: Omit<ComputerActionRequest, 'confirmationToken'>
+  reason: string
+  target: string
+  dataSummary?: string
+}
+
+/** One-use confirmation result. */
+export interface ComputerConfirmation {
+  token: ComputerConfirmationToken
+  observationId: ComputerObservationId
+  app: ComputerAppIdentity
+  expiresAt: string
+}
+
+/** Caller context required for scoping, cancellation, approval audit, and artifact placement. */
+export interface ComputerUseContext {
+  agent: Agent
+  workspace: string
+  callId?: CallId
+  signal: AbortSignal
+}
+
+/** Provider and permission diagnostics exposed to Settings. */
+export interface ComputerUseStatus {
+  platform: NodeJS.Platform
+  provider: 'macos-ax'
+  generation: number
+  ready: boolean
+  helperPath: string
+  helperVersion?: string
+  helperSha256?: string
+  accessibility: ComputerPermissionState
+  screenRecording: ComputerPermissionState
+  lastError?: string
+}
+
+// Type-only circular reference for the link above; the emitted declaration
+// retains the Service name without importing runtime code into this module.
+import type { ComputerUseService } from './service.ts'
+void (0 as unknown as ComputerUseService | undefined)

package/src/web.ts

@@ -0,0 +1,180 @@
+/** Optional Web Settings and provider-health route. */
+
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import type { Context } from '@deepseek-ai/cordis'
+import { SettingsConflictError, type SettingsDescriptor } from '@deepseek-ai/dsh-settings'
+import type {} from '@deepseek-ai/dsh-host-webserver'
+import {
+  COMPUTER_USE_SETTINGS_NAMESPACE,
+  type ComputerUseConfig,
+} from './config.ts'
+import type { ComputerUseStatus } from './types.ts'
+
+/** Exact same-origin Settings endpoint. */
+export const COMPUTER_USE_SETTINGS_ROUTE = '/_dsh/computer-use/settings'
+
+/** Browser-safe Settings snapshot. */
+export interface ComputerUseSettingsSnapshot {
+  schemaVersion: 1
+  writable: boolean
+  settings: {
+    value: ComputerUseConfig
+    user?: unknown
+    base?: unknown
+    revision: number
+    applies: 'live'
+  }
+  provider: ComputerUseStatus
+}
+
+type SettingsRequest =
+  | { action: 'save'; expectedRevision: number; value: ComputerUseConfig }
+  | { action: 'health' }
+  | { action: 'open-settings'; kind: 'accessibility' | 'screen-recording' }
+
+type JsonResponse<T> = { ok: true; value: T } | { ok: false; error: { code: string; message: string } }
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+function descriptorOf(ctx: Context): SettingsDescriptor {
+  const descriptor = ctx.settings.describe().find(row => row.ns === COMPUTER_USE_SETTINGS_NAMESPACE)
+  if (descriptor === undefined) throw new Error('computer-use Settings namespace is not registered')
+  return descriptor
+}
+
+function responseJson<T>(res: ServerResponse, status: number, body: JsonResponse<T>): void {
+  const bytes = Buffer.from(JSON.stringify(body))
+  res.setHeader('Content-Type', 'application/json; charset=utf-8')
+  res.setHeader('Content-Length', String(bytes.length))
+  res.setHeader('Cache-Control', 'no-store')
+  res.setHeader('X-Content-Type-Options', 'nosniff')
+  res.setHeader('Content-Security-Policy', "default-src 'none'; frame-ancestors 'none'")
+  res.writeHead(status)
+  res.end(bytes)
+}
+
+function requestError(res: ServerResponse, status: number, code: string, message: string): void {
+  responseJson(res, status, { ok: false, error: { code, message } })
+}
+
+function sameOriginPost(req: IncomingMessage): boolean {
+  const fetchSite = req.headers['sec-fetch-site']
+  if (fetchSite === 'cross-site') return false
+  const origin = req.headers.origin
+  if (origin === undefined) return fetchSite === 'same-origin' || fetchSite === 'same-site' || fetchSite === 'none'
+  const host = req.headers.host
+  if (host === undefined) return false
+  try {
+    const parsed = new URL(origin)
+    return (parsed.protocol === 'http:' || parsed.protocol === 'https:') && parsed.host === host
+  } catch {
+    return false
+  }
+}
+
+async function readJson(req: IncomingMessage, maxBytes = 128 * 1024): Promise<unknown> {
+  const contentType = req.headers['content-type']?.split(';', 1)[0]?.trim().toLowerCase()
+  if (contentType !== 'application/json') throw new TypeError('Content-Type must be application/json')
+  const chunks: Buffer[] = []
+  let bytes = 0
+  for await (const chunk of req) {
+    const part = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
+    bytes += part.length
+    if (bytes > maxBytes) throw new RangeError(`request body exceeds ${maxBytes} bytes`)
+    chunks.push(part)
+  }
+  return JSON.parse(Buffer.concat(chunks).toString('utf8'))
+}
+
+function parseRequest(value: unknown): SettingsRequest {
+  if (!isRecord(value) || typeof value.action !== 'string') throw new TypeError('request needs an action')
+  if (value.action === 'health') return { action: 'health' }
+  if (value.action === 'open-settings') {
+    if (value.kind !== 'accessibility' && value.kind !== 'screen-recording') throw new TypeError('open-settings needs a valid kind')
+    return { action: 'open-settings', kind: value.kind }
+  }
+  if (value.action === 'save') {
+    if (!Number.isInteger(value.expectedRevision) || !isRecord(value.value)) throw new TypeError('save needs expectedRevision and value')
+    return { action: 'save', expectedRevision: value.expectedRevision as number, value: value.value as ComputerUseConfig }
+  }
+  throw new TypeError(`unknown action: ${value.action}`)
+}
+
+function publicMessage(error: unknown): string {
+  return error instanceof Error ? error.message.slice(0, 1000) : String(error).slice(0, 1000)
+}
+
+/** Same-origin backend used by the optional client Settings section. */
+export class ComputerUseWebBackend {
+  constructor(private readonly ctx: Context) {}
+
+  /** Current browser-safe Settings and health state. */
+  snapshot(): ComputerUseSettingsSnapshot {
+    const descriptor = descriptorOf(this.ctx)
+    return {
+      schemaVersion: 1,
+      writable: this.ctx.settings.writable,
+      settings: {
+        value: descriptor.value as ComputerUseConfig,
+        ...(descriptor.user === undefined ? {} : { user: descriptor.user }),
+        ...(descriptor.base === undefined ? {} : { base: descriptor.base }),
+        revision: descriptor.revision,
+        applies: 'live',
+      },
+      provider: this.ctx.computerUse.status(),
+    }
+  }
+
+  /** Handle one Settings request. */
+  async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    if (req.method === 'GET') {
+      try { responseJson(res, 200, { ok: true, value: this.snapshot() }) }
+      catch (error) { requestError(res, 503, 'settings-unavailable', publicMessage(error)) }
+      return
+    }
+    if (req.method !== 'POST') {
+      res.setHeader('Allow', 'GET, POST')
+      requestError(res, 405, 'method-not-allowed', 'Use GET or POST')
+      return
+    }
+    if (!sameOriginPost(req)) {
+      requestError(res, 403, 'origin-rejected', 'The request must originate from this DSH Web application')
+      return
+    }
+    let parsed: SettingsRequest
+    try { parsed = parseRequest(await readJson(req)) }
+    catch (error) {
+      requestError(res, error instanceof RangeError ? 413 : 400, 'invalid-request', publicMessage(error))
+      return
+    }
+    try {
+      if (parsed.action === 'save') {
+        if (!this.ctx.settings.writable) throw new Error('settings provider is read-only')
+        await this.ctx.settings.replace(COMPUTER_USE_SETTINGS_NAMESPACE, parsed.value as object, parsed.expectedRevision)
+      } else if (parsed.action === 'health') {
+        await this.ctx.computerUse.health(AbortSignal.timeout(30000))
+      } else {
+        await this.ctx.computerUse.openPermissionSettings(parsed.kind, AbortSignal.timeout(10000))
+      }
+      responseJson(res, 200, { ok: true, value: this.snapshot() })
+    } catch (error) {
+      const conflict = error instanceof SettingsConflictError
+      this.ctx.logger.warn('dsh-computer-use Web action=%s failed: %s', parsed.action, publicMessage(error))
+      requestError(res, conflict ? 409 : 400, conflict ? 'settings-conflict' : 'action-failed', publicMessage(error))
+    }
+  }
+}
+
+/** Attach the optional route when a Web host is present. */
+export function installComputerUseWeb(ctx: Context): void {
+  const backend = new ComputerUseWebBackend(ctx)
+  ctx.inject(['webServer'], (webCtx) => {
+    webCtx.effect(() => webCtx.webServer.register({
+      kind: 'exact',
+      path: COMPUTER_USE_SETTINGS_ROUTE,
+      handler: (req, res) => backend.handle(req, res),
+    }), 'dsh-computer-use: Web Settings route')
+  })
+}