@anionex/dsh-computer-use 0.1.0

package/src/artifacts.ts

@@ -0,0 +1,93 @@
+/** Workspace-fenced screenshot artifact allocation and validation. */
+
+import { randomUUID } from 'node:crypto'
+import { lstat, mkdir, realpath, stat } from 'node:fs/promises'
+import { basename, isAbsolute, relative, resolve, sep } from 'node:path'
+import type { SessionId } from '@deepseek-ai/dsh-session'
+import { ComputerUseError } from './errors.ts'
+import type { ComputerArtifact } from './types.ts'
+
+/** Screenshot description that hands visual analysis to the sibling Vision Toolkit. */
+export const COMPUTER_SCREENSHOT_DESCRIPTION = [
+  'Current macOS application window observation.',
+  'For OCR, visual grounding, or pixel inspection, load the vision-tools Skill and pass this exact path to vision_glance, vision_ground, vision_detect, or vision_crop;',
+  'do not recreate OCR with bash, tesseract, or an ad hoc script.',
+].join(' ')
+
+function isWithin(root: string, candidate: string): boolean {
+  const rel = relative(root, candidate)
+  return rel === '' || (!rel.startsWith(`..${sep}`) && rel !== '..' && !isAbsolute(rel))
+}
+
+async function rejectSymlinkComponents(root: string, target: string): Promise<void> {
+  const rel = relative(root, target)
+  let current = root
+  for (const part of rel.split(sep).filter(Boolean)) {
+    current = resolve(current, part)
+    try {
+      const info = await lstat(current)
+      if (info.isSymbolicLink()) {
+        throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', `artifact path component must not be a symbolic link: ${part}`)
+      }
+    } catch (error) {
+      if (error instanceof ComputerUseError) throw error
+      const code = (error as NodeJS.ErrnoException).code
+      if (code !== 'ENOENT') throw error
+    }
+  }
+}
+
+/** Allocate a unique managed PNG path inside the current Session workspace. */
+export async function allocateScreenshotPath(
+  workspace: string,
+  artifactRoot: string,
+  sessionId: SessionId,
+): Promise<string> {
+  const realWorkspace = await realpath(workspace)
+  const directory = resolve(realWorkspace, artifactRoot, String(sessionId))
+  if (!isWithin(realWorkspace, directory)) {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'artifactRoot escapes the Session workspace')
+  }
+  await rejectSymlinkComponents(realWorkspace, directory)
+  await mkdir(directory, { recursive: true, mode: 0o700 })
+  const realDirectory = await realpath(directory)
+  if (!isWithin(realWorkspace, realDirectory)) {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'artifact directory escaped the Session workspace')
+  }
+  return resolve(realDirectory, `observation-${randomUUID()}.png`)
+}
+
+/** Validate a committed screenshot and return its stable model/client descriptor. */
+export async function describeScreenshot(
+  path: string,
+  width: number,
+  height: number,
+  maxBytes: number,
+  sourceTool: ComputerArtifact['sourceTool'],
+): Promise<ComputerArtifact> {
+  const link = await lstat(path).catch((error: unknown) => {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'the screenshot artifact was not created', { cause: error })
+  })
+  if (link.isSymbolicLink() || !link.isFile()) {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'the screenshot artifact must be a regular non-symbolic-link file')
+  }
+  const info = await stat(path)
+  if (info.size > maxBytes) {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', `screenshot exceeds maxScreenshotBytes (${info.size} > ${maxBytes})`)
+  }
+  if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(height) || height <= 0) {
+    throw new ComputerUseError('COMPUTER_PROVIDER_FAILURE', 'provider returned invalid screenshot dimensions')
+  }
+  return {
+    path,
+    filename: basename(path),
+    mimeType: 'image/png',
+    kind: 'image',
+    description: COMPUTER_SCREENSHOT_DESCRIPTION,
+    sourceTool,
+    previewIntent: 'image',
+    bytes: info.size,
+    width,
+    height,
+  }
+}

package/src/backend.ts

@@ -0,0 +1,103 @@
+/** Provider-facing backend protocol kept below the public Computer Use Service. */
+
+import type {
+  ComputerActionRequest,
+  ComputerAppIdentity,
+  ComputerAppSelector,
+  ComputerAppSummary,
+  ComputerElement,
+  ComputerPermissionState,
+  ComputerRect,
+  ComputerScreenshotMode,
+} from './types.ts'
+import type { ResolvedComputerUseConfig } from './config.ts'
+
+/** Internal provider evidence never exposed to the model or persisted in Session logs. */
+export interface BackendElement extends Omit<ComputerElement, 'targetHandle'> {
+  locator: number[]
+  nativeIdentifier?: string
+}
+
+/** Raw full-state observation returned by a provider before Service diff projection. */
+export interface BackendObservation {
+  app: ComputerAppIdentity
+  stateHash: string
+  frontmost: boolean
+  window?: {
+    title?: string
+    frame: ComputerRect
+    id?: number
+  }
+  treeText: string
+  truncated: boolean
+  elements: BackendElement[]
+  screenshot?: {
+    path: string
+    width: number
+    height: number
+  }
+  permissions: {
+    accessibility: ComputerPermissionState
+    screenRecording: ComputerPermissionState
+  }
+}
+
+/** Provider limits resolved by the configuration owner before a call. */
+export interface BackendObserveOptions {
+  screenshot: ComputerScreenshotMode
+  screenshotPath?: string
+  maxNodes: number
+  maxDepth: number
+  maxTextBytes: number
+}
+
+/** Action bound to fresh provider state and one exact internally resolved target. */
+export interface BackendActionRequest {
+  action: Exclude<ComputerActionRequest, { kind: 'wait' }>
+  app: ComputerAppIdentity
+  expectedStateHash: string
+  interaction: ResolvedComputerUseConfig['interaction']
+  element?: BackendElement
+  window?: BackendObservation['window']
+}
+
+/** Provider action outcome before the Service obtains the mandatory post-action observation. */
+export interface BackendActionResult {
+  channel: 'accessibility' | 'coordinates' | 'keyboard'
+  activation: 'not-requested' | 'already-frontmost' | 'activated'
+  pointerInput: boolean
+  pointerRouting: 'none' | 'target-process'
+}
+
+/** One model-selected point or gesture for the non-interactive Agent cursor overlay. */
+export interface BackendCursorAction {
+  kind: 'click' | 'scroll' | 'drag'
+  from?: { x: number; y: number }
+  to: { x: number; y: number }
+  /** Exact target identity used to prevent the overlay from lingering over another window. */
+  targetPid: number
+  targetWindowNumber: number
+  targetWindowFrame: ComputerRect
+}
+
+/** Health facts obtained without changing permissions. */
+export interface BackendHealth {
+  helperVersion: string
+  helperSha256: string
+  accessibility: ComputerPermissionState
+  screenRecording: ComputerPermissionState
+}
+
+/** Platform backend used by the provider-independent Service implementation. */
+export interface ComputerUseBackend {
+  readonly name: 'macos-ax'
+  readonly helperPath: string
+  resolveApp(selector: ComputerAppSelector, signal: AbortSignal): Promise<ComputerAppIdentity>
+  listApps(signal: AbortSignal): Promise<ComputerAppSummary[]>
+  observe(app: ComputerAppIdentity, options: BackendObserveOptions, signal: AbortSignal): Promise<BackendObservation>
+  act(request: BackendActionRequest, signal: AbortSignal): Promise<BackendActionResult>
+  visualizeCursor(action: BackendCursorAction, phase: 'before' | 'after', signal: AbortSignal): Promise<void>
+  dispose(): Promise<void>
+  health(signal: AbortSignal): Promise<BackendHealth>
+  openSettings(kind: 'accessibility' | 'screen-recording', signal: AbortSignal): Promise<void>
+}