@crosshands/contract 0.1.2

package/src/context.ts

@@ -0,0 +1,107 @@
+import { randomBytes } from 'node:crypto'
+
+import { createComputerError } from './errors.js'
+import type { InteractionContext, ReferenceBindings, TargetReference } from './schemas.js'
+
+export const DEFAULT_INTERACTION_CONTEXT_TTL_MS = 120_000
+export const DEFAULT_INTERACTION_CONTEXT_LIMIT = 32
+
+type InteractionContextStoreOptions = {
+  ttlMs?: number
+  limit?: number
+  now?: () => number
+  tokenFactory?: () => string
+}
+
+export class InteractionContextStore {
+  readonly #ttlMs: number
+  readonly #limit: number
+  readonly #now: () => number
+  readonly #tokenFactory: () => string
+  readonly #contexts = new Map<string, InteractionContext>()
+
+  constructor(options: InteractionContextStoreOptions = {}) {
+    this.#ttlMs = options.ttlMs ?? DEFAULT_INTERACTION_CONTEXT_TTL_MS
+    this.#limit = options.limit ?? DEFAULT_INTERACTION_CONTEXT_LIMIT
+    this.#now = options.now ?? Date.now
+    this.#tokenFactory =
+      options.tokenFactory ?? (() => `ctx_${randomBytes(24).toString('base64url')}`)
+    if (this.#ttlMs <= 0 || this.#limit <= 0)
+      throw new RangeError('Context TTL and limit must be positive')
+  }
+
+  issue(bindings: ReferenceBindings): InteractionContext {
+    const issuedAtMs = this.#now()
+    const context: InteractionContext = {
+      token: this.#tokenFactory(),
+      ...bindings,
+      issuedAt: new Date(issuedAtMs).toISOString(),
+      expiresAt: new Date(issuedAtMs + this.#ttlMs).toISOString()
+    }
+    this.#contexts.delete(context.token)
+    this.#contexts.set(context.token, context)
+    while (this.#contexts.size > this.#limit) {
+      const oldest = this.#contexts.keys().next().value as string | undefined
+      if (oldest === undefined) break
+      this.#contexts.delete(oldest)
+    }
+    return context
+  }
+
+  resolve(token: string, now = this.#now()): InteractionContext {
+    const context = this.#contexts.get(token)
+    if (context === undefined) {
+      throw createComputerError(
+        'interaction_context_invalid',
+        'Interaction context is unknown or forged'
+      )
+    }
+    if (Date.parse(context.expiresAt) < now) {
+      this.#contexts.delete(token)
+      throw createComputerError('interaction_context_expired', 'Interaction context has expired')
+    }
+    return context
+  }
+
+  invalidateAll(): void {
+    this.#contexts.clear()
+  }
+}
+
+function sameProcess(a: ReferenceBindings['process'], b: ReferenceBindings['process']): boolean {
+  return a.pid === b.pid && a.startedAt === b.startedAt && a.executableId === b.executableId
+}
+
+function sameWindow(a: ReferenceBindings['window'], b: ReferenceBindings['window']): boolean {
+  return a.id === b.id && a.ownerPid === b.ownerPid
+}
+
+export function assertReferenceFresh(
+  reference: TargetReference,
+  context: InteractionContext,
+  current: ReferenceBindings,
+  now = Date.now()
+): void {
+  const stale =
+    reference.contextToken !== context.token ||
+    reference.brokerGeneration !== context.brokerGeneration ||
+    reference.providerGeneration !== context.providerGeneration ||
+    reference.graphicalSessionId !== context.graphicalSessionId ||
+    reference.brokerGeneration !== current.brokerGeneration ||
+    reference.providerGeneration !== current.providerGeneration ||
+    reference.graphicalSessionId !== current.graphicalSessionId ||
+    !sameProcess(reference.process, context.process) ||
+    !sameProcess(reference.process, current.process) ||
+    reference.appId !== context.appId ||
+    reference.appId !== current.appId ||
+    !sameWindow(reference.window, context.window) ||
+    !sameWindow(reference.window, current.window) ||
+    reference.snapshotId !== context.snapshotId ||
+    reference.snapshotId !== current.snapshotId ||
+    reference.desktopEpoch !== context.desktopEpoch ||
+    reference.desktopEpoch !== current.desktopEpoch ||
+    Date.parse(reference.expiresAt) < now ||
+    Date.parse(context.expiresAt) < now
+
+  if (stale) throw createComputerError('stale_target', 'Target reference is no longer fresh')
+}

package/src/errors.ts

@@ -0,0 +1,82 @@
+import { z } from 'zod'
+
+export const ERROR_CATALOG = {
+  app_not_found: { retry: true, remediation: 'refresh_apps' },
+  app_unavailable: { retry: true, remediation: 'refresh_apps' },
+  app_blocked: { retry: false, remediation: 'choose_non_sensitive_target' },
+  window_not_found: { retry: true, remediation: 'refresh_windows' },
+  window_unavailable: { retry: true, remediation: 'refresh_windows' },
+  window_not_focused: { retry: true, remediation: 'focus_window' },
+  permission_denied: { retry: false, remediation: 'grant_permission' },
+  stale_target: { retry: true, remediation: 'refresh_state' },
+  element_not_found: { retry: true, remediation: 'refresh_state' },
+  element_not_clickable: { retry: true, remediation: 'choose_actionable_target' },
+  action_not_supported: { retry: false, remediation: 'choose_supported_operation' },
+  value_not_settable: { retry: false, remediation: 'choose_settable_target' },
+  invalid_argument: { retry: false, remediation: 'correct_request' },
+  timeout: { retry: false, remediation: 'inspect_state_before_retry' },
+  unsupported_capability: { retry: false, remediation: 'choose_supported_operation' },
+  provider_unavailable: { retry: true, remediation: 'run_doctor' },
+  provider_crashed: { retry: true, remediation: 'refresh_state' },
+  version_incompatible: { retry: false, remediation: 'upgrade_or_downgrade' },
+  interaction_context_invalid: { retry: true, remediation: 'refresh_state' },
+  interaction_context_expired: { retry: true, remediation: 'refresh_state' },
+  session_unavailable: { retry: true, remediation: 'unlock_graphical_session' },
+  screenshot_failed: { retry: true, remediation: 'check_screenshot_permission' },
+  accessibility_error: { retry: true, remediation: 'check_accessibility_permission' }
+} as const
+
+export type ComputerErrorCode = keyof typeof ERROR_CATALOG
+export type ComputerErrorRemediation = (typeof ERROR_CATALOG)[ComputerErrorCode]['remediation']
+
+export const ComputerErrorCodeSchema = z.enum(
+  Object.keys(ERROR_CATALOG) as [ComputerErrorCode, ...ComputerErrorCode[]]
+)
+
+export const SerializedComputerErrorSchema = z
+  .object({
+    code: ComputerErrorCodeSchema,
+    message: z.string().min(1),
+    retry: z.boolean(),
+    remediation: z.string().min(1),
+    details: z.unknown().optional()
+  })
+  .strict()
+
+export type SerializedComputerError = z.infer<typeof SerializedComputerErrorSchema>
+
+export class ComputerError extends Error {
+  readonly code: ComputerErrorCode
+  readonly retry: boolean
+  readonly remediation: ComputerErrorRemediation
+  readonly details: unknown
+
+  constructor(code: ComputerErrorCode, message: string, details?: unknown) {
+    super(message)
+    this.name = 'ComputerError'
+    this.code = code
+    this.retry = ERROR_CATALOG[code].retry
+    this.remediation = ERROR_CATALOG[code].remediation
+    this.details = details
+  }
+
+  toJSON(): SerializedComputerError {
+    return this.details === undefined
+      ? { code: this.code, message: this.message, retry: this.retry, remediation: this.remediation }
+      : {
+          code: this.code,
+          message: this.message,
+          retry: this.retry,
+          remediation: this.remediation,
+          details: this.details
+        }
+  }
+}
+
+export function createComputerError(
+  code: ComputerErrorCode,
+  message: string,
+  details?: unknown
+): ComputerError {
+  return new ComputerError(code, message, details)
+}

package/src/generate-schemas.ts

@@ -0,0 +1,9 @@
+import { writeFile } from 'node:fs/promises'
+import { dirname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { contractJsonSchemas } from './json-schema.js'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const output = resolve(here, '../schemas/contract.json')
+await writeFile(output, `${JSON.stringify(contractJsonSchemas, null, 2)}\n`, 'utf8')

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './context.js'
+export * from './errors.js'
+export * from './json-schema.js'
+export * from './operations.js'
+export * from './provider.js'
+export * from './schemas.js'
+export * from './versions.js'

package/src/json-schema.ts

@@ -0,0 +1,17 @@
+import { COMPUTER_OPERATIONS } from './operations.js'
+import { CONTRACT_VERSIONS } from './versions.js'
+
+export const contractJsonSchemas = {
+  $schema: 'https://json-schema.org/draft/2020-12/schema',
+  versions: CONTRACT_VERSIONS,
+  operations: Object.fromEntries(
+    Object.entries(COMPUTER_OPERATIONS).map(([name, operation]) => [
+      name,
+      {
+        mutation: operation.mutation,
+        input: operation.input.toJSONSchema({ target: 'draft-2020-12' }),
+        output: operation.output.toJSONSchema({ target: 'draft-2020-12' })
+      }
+    ])
+  )
+} as const

package/src/operations.ts

@@ -0,0 +1,224 @@
+import { z } from 'zod'
+
+import {
+  AppInfoSchema,
+  InteractionContextTokenSchema,
+  MutationResultSchema,
+  ProviderCapabilitiesSchema,
+  ScreenshotSchema,
+  SnapshotResultSchema,
+  TargetReferenceSchema,
+  WindowInfoSchema
+} from './schemas.js'
+
+const EmptyInputSchema = z.object({}).strict()
+const AppQuerySchema = z.string().min(1).max(512)
+const WindowSelectorSchema = z.union([
+  z.object({ id: z.string().min(1) }).strict(),
+  z.object({ index: z.number().int().nonnegative() }).strict()
+])
+const CaptureOptionsShape = {
+  captureScreenshot: z.boolean().optional(),
+  restoreWindow: z.boolean().optional()
+}
+const ContextWindowTargetSchema = z.object({ kind: z.literal('context-window') }).strict()
+const ElementIndexTargetSchema = z
+  .object({ kind: z.literal('element'), elementIndex: z.number().int().nonnegative() })
+  .strict()
+const CoordinateTargetSchema = z
+  .object({
+    kind: z.literal('coordinate'),
+    window: TargetReferenceSchema.optional(),
+    x: z.number().finite(),
+    y: z.number().finite()
+  })
+  .strict()
+const ElementTargetSchema = z
+  .object({ kind: z.literal('element'), ref: TargetReferenceSchema })
+  .strict()
+const ElementActionTargetSchema = z.union([ElementTargetSchema, ElementIndexTargetSchema])
+const ContextWindowMutationBaseShape = {
+  contextToken: InteractionContextTokenSchema,
+  app: AppQuerySchema.optional(),
+  target: ContextWindowTargetSchema,
+  ...CaptureOptionsShape
+}
+const ElementMutationBaseShape = {
+  contextToken: InteractionContextTokenSchema,
+  app: AppQuerySchema.optional(),
+  target: ElementActionTargetSchema,
+  ...CaptureOptionsShape
+}
+const ActionTargetSchema = z.union([
+  ElementTargetSchema,
+  ElementIndexTargetSchema,
+  CoordinateTargetSchema,
+  ContextWindowTargetSchema
+])
+
+export const CLICK_MODIFIER_TOKENS = [
+  'Shift',
+  'Ctrl',
+  'Control',
+  'Alt',
+  'Option',
+  'Meta',
+  'Cmd',
+  'Command',
+  'Super',
+  'Win',
+  'CmdOrCtrl',
+  'CommandOrControl',
+  'shift',
+  'ctrl',
+  'control',
+  'alt',
+  'option',
+  'meta',
+  'cmd',
+  'command',
+  'super',
+  'win',
+  'cmdorctrl',
+  'commandorcontrol'
+] as const
+
+const ClickModifierTokenSchema = z.enum(CLICK_MODIFIER_TOKENS)
+
+const PermissionsResultSchema = z
+  .object({
+    permissions: z.record(z.string(), z.enum(['granted', 'denied', 'unknown', 'not_required']))
+  })
+  .strict()
+const AppListResultSchema = z.object({ apps: z.array(AppInfoSchema) }).strict()
+const WindowListResultSchema = z.object({ windows: z.array(WindowInfoSchema) }).strict()
+
+export const COMPUTER_OPERATIONS = {
+  capabilities: {
+    mutation: false,
+    input: EmptyInputSchema,
+    output: ProviderCapabilitiesSchema
+  },
+  permissions: {
+    mutation: false,
+    input: z.object({ id: z.enum(['accessibility', 'screenshots']).optional() }).strict(),
+    output: PermissionsResultSchema
+  },
+  listApps: {
+    mutation: false,
+    input: EmptyInputSchema,
+    output: AppListResultSchema
+  },
+  listWindows: {
+    mutation: false,
+    input: z.object({ app: AppQuerySchema }).strict(),
+    output: WindowListResultSchema
+  },
+  getAppState: {
+    mutation: false,
+    input: z
+      .object({
+        app: AppQuerySchema,
+        window: WindowSelectorSchema.optional(),
+        ...CaptureOptionsShape
+      })
+      .strict(),
+    output: SnapshotResultSchema
+  },
+  click: {
+    mutation: true,
+    input: z
+      .object({
+        contextToken: InteractionContextTokenSchema,
+        app: AppQuerySchema.optional(),
+        target: ActionTargetSchema,
+        clickCount: z.number().int().min(1).max(3).optional(),
+        button: z.enum(['left', 'right', 'middle']).optional(),
+        modifiers: z.array(ClickModifierTokenSchema).min(1).max(4).optional(),
+        ...CaptureOptionsShape
+      })
+      .strict(),
+    output: MutationResultSchema
+  },
+  performSecondaryAction: {
+    mutation: true,
+    input: z.object({ ...ElementMutationBaseShape, action: z.string().min(1).max(256) }).strict(),
+    output: MutationResultSchema
+  },
+  scroll: {
+    mutation: true,
+    input: z
+      .object({
+        contextToken: InteractionContextTokenSchema,
+        app: AppQuerySchema.optional(),
+        target: ActionTargetSchema,
+        direction: z.enum(['up', 'down', 'left', 'right']),
+        pages: z.number().int().min(1).max(100).optional(),
+        ...CaptureOptionsShape
+      })
+      .strict(),
+    output: MutationResultSchema
+  },
+  drag: {
+    mutation: true,
+    input: z
+      .object({
+        contextToken: InteractionContextTokenSchema,
+        app: AppQuerySchema.optional(),
+        from: ActionTargetSchema,
+        to: ActionTargetSchema,
+        durationMs: z.number().int().min(50).max(30_000).optional(),
+        ...CaptureOptionsShape
+      })
+      .strict(),
+    output: MutationResultSchema
+  },
+  typeText: {
+    mutation: true,
+    input: z
+      .object({ ...ContextWindowMutationBaseShape, text: z.string().max(1_000_000) })
+      .strict(),
+    output: MutationResultSchema
+  },
+  pressKey: {
+    mutation: true,
+    input: z
+      .object({ ...ContextWindowMutationBaseShape, key: z.string().min(1).max(128) })
+      .strict(),
+    output: MutationResultSchema
+  },
+  hotkey: {
+    mutation: true,
+    input: z
+      .object({
+        ...ContextWindowMutationBaseShape,
+        keys: z.array(z.string().min(1).max(128)).min(2).max(5)
+      })
+      .strict(),
+    output: MutationResultSchema
+  },
+  pasteText: {
+    mutation: true,
+    input: z
+      .object({ ...ContextWindowMutationBaseShape, text: z.string().max(1_000_000) })
+      .strict(),
+    output: MutationResultSchema
+  },
+  setValue: {
+    mutation: true,
+    input: z.object({ ...ElementMutationBaseShape, value: z.string().max(1_000_000) }).strict(),
+    output: MutationResultSchema
+  }
+} as const
+
+export type ComputerOperationName = keyof typeof COMPUTER_OPERATIONS
+
+export function parseOperationInput(operation: ComputerOperationName, input: unknown): unknown {
+  return COMPUTER_OPERATIONS[operation].input.parse(input)
+}
+
+export function parseOperationOutput(operation: ComputerOperationName, output: unknown): unknown {
+  return COMPUTER_OPERATIONS[operation].output.parse(output)
+}
+
+export const ScreenshotOutputSchema = ScreenshotSchema

package/src/provider.ts

@@ -0,0 +1,82 @@
+import { z } from 'zod'
+
+import type { SerializedComputerError } from './errors.js'
+import type { ComputerOperationName } from './operations.js'
+import { ProviderHandshakeSchema } from './schemas.js'
+
+export const ProviderRequestSchema = z
+  .object({
+    requestId: z.string().min(1),
+    operation: z.enum([
+      'capabilities',
+      'permissions',
+      'listApps',
+      'listWindows',
+      'getAppState',
+      'click',
+      'performSecondaryAction',
+      'scroll',
+      'drag',
+      'typeText',
+      'pressKey',
+      'hotkey',
+      'pasteText',
+      'setValue'
+    ]),
+    input: z.unknown(),
+    deadlineAt: z.number()
+  })
+  .strict()
+
+export const ProviderResponseSchema = z.union([
+  z
+    .object({
+      requestId: z.string().min(1),
+      dispatched: z.boolean(),
+      result: z.unknown()
+    })
+    .strict(),
+  z
+    .object({
+      requestId: z.string().min(1),
+      dispatched: z.boolean(),
+      error: z
+        .object({
+          code: z.string().min(1),
+          message: z.string().min(1),
+          retry: z.boolean(),
+          remediation: z.string().min(1),
+          details: z.unknown().optional()
+        })
+        .strict()
+    })
+    .strict()
+])
+
+export const ProviderProtocolFrameSchema = z.discriminatedUnion('type', [
+  z.object({ type: z.literal('handshake'), payload: ProviderHandshakeSchema }).strict(),
+  z.object({ type: z.literal('request'), payload: ProviderRequestSchema }).strict(),
+  z.object({ type: z.literal('response'), payload: ProviderResponseSchema }).strict(),
+  z.object({ type: z.literal('cancel'), requestId: z.string().min(1) }).strict()
+])
+
+export type ProviderRequest = {
+  requestId: string
+  operation: ComputerOperationName
+  input: unknown
+  deadlineAt: number
+}
+
+export type ProviderResponse =
+  | { requestId: string; dispatched: boolean; result: unknown; error?: never }
+  | { requestId: string; dispatched: boolean; result?: never; error: SerializedComputerError }
+
+export type ProviderHandshake = z.infer<typeof ProviderHandshakeSchema>
+
+export interface ComputerProvider {
+  readonly generation: string
+  start(): Promise<ProviderHandshake>
+  dispatch(request: ProviderRequest): Promise<ProviderResponse>
+  cancel(requestId: string): Promise<void>
+  close(): Promise<void>
+}

package/src/schemas.ts

@@ -0,0 +1,161 @@
+import { z } from 'zod'
+
+import { SerializedComputerErrorSchema } from './errors.js'
+
+const FiniteNumberSchema = z.number().finite()
+const NonNegativeIntegerSchema = z.number().int().nonnegative()
+const PositiveIntegerSchema = z.number().int().positive()
+const IdentifierSchema = z.string().min(1).max(512)
+export const InteractionContextTokenSchema = z.string().regex(/^ctx_[A-Za-z0-9_-]{32,}$/)
+
+export const ProcessIdentitySchema = z
+  .object({
+    pid: PositiveIntegerSchema,
+    startedAt: z.iso.datetime(),
+    executableId: IdentifierSchema
+  })
+  .strict()
+
+export const WindowIdentitySchema = z
+  .object({
+    id: IdentifierSchema,
+    ownerPid: PositiveIntegerSchema
+  })
+  .strict()
+
+export const ReferenceBindingsSchema = z
+  .object({
+    brokerGeneration: IdentifierSchema,
+    providerGeneration: IdentifierSchema,
+    graphicalSessionId: IdentifierSchema,
+    process: ProcessIdentitySchema,
+    appId: IdentifierSchema,
+    window: WindowIdentitySchema,
+    snapshotId: IdentifierSchema,
+    desktopEpoch: NonNegativeIntegerSchema
+  })
+  .strict()
+
+export type ReferenceBindings = z.infer<typeof ReferenceBindingsSchema>
+
+export const InteractionContextSchema = ReferenceBindingsSchema.extend({
+  token: InteractionContextTokenSchema,
+  issuedAt: z.iso.datetime(),
+  expiresAt: z.iso.datetime()
+}).strict()
+
+export type InteractionContext = z.infer<typeof InteractionContextSchema>
+
+export const TargetReferenceSchema = ReferenceBindingsSchema.extend({
+  ref: IdentifierSchema,
+  kind: z.enum(['app', 'window', 'element']),
+  contextToken: InteractionContextTokenSchema,
+  expiresAt: z.iso.datetime()
+}).strict()
+
+export type TargetReference = z.infer<typeof TargetReferenceSchema>
+
+export const AppInfoSchema = z
+  .object({
+    id: IdentifierSchema,
+    name: z.string().min(1),
+    bundleId: z.string().min(1).nullable(),
+    pid: PositiveIntegerSchema,
+    isRunning: z.boolean()
+  })
+  .strict()
+
+export const WindowInfoSchema = z
+  .object({
+    id: IdentifierSchema,
+    appId: IdentifierSchema,
+    title: z.string(),
+    index: NonNegativeIntegerSchema,
+    bounds: z
+      .object({
+        x: FiniteNumberSchema,
+        y: FiniteNumberSchema,
+        width: z.number().finite().positive(),
+        height: z.number().finite().positive()
+      })
+      .strict(),
+    minimized: z.boolean()
+  })
+  .strict()
+
+export const ScreenshotSchema = z
+  .object({
+    format: z.literal('png'),
+    width: PositiveIntegerSchema,
+    height: PositiveIntegerSchema,
+    scale: z.number().finite().positive(),
+    data: z.string().min(1).optional(),
+    path: z.string().min(1).optional(),
+    expiresAt: z.iso.datetime().optional()
+  })
+  .strict()
+  .refine((value) => value.data !== undefined || value.path !== undefined, {
+    message: 'Screenshot must include data or path'
+  })
+
+export const SnapshotSchema = z
+  .object({
+    id: IdentifierSchema,
+    app: AppInfoSchema,
+    window: WindowInfoSchema,
+    treeText: z.string(),
+    elementCount: NonNegativeIntegerSchema,
+    focusedElementRef: z.string().nullable(),
+    desktopEpoch: NonNegativeIntegerSchema
+  })
+  .strict()
+
+export const MutationOutcomeSchema = z.discriminatedUnion('state', [
+  z.object({ state: z.literal('verified'), evidence: z.unknown().optional() }).strict(),
+  z.object({ state: z.literal('indeterminate'), reason: z.string().optional() }).strict(),
+  z
+    .object({ state: z.literal('failed'), error: SerializedComputerErrorSchema.optional() })
+    .strict(),
+  z
+    .object({ state: z.literal('not_attempted'), error: SerializedComputerErrorSchema.optional() })
+    .strict()
+])
+
+export type MutationOutcome = z.infer<typeof MutationOutcomeSchema>
+
+export const SnapshotResultSchema = z
+  .object({
+    context: InteractionContextSchema,
+    snapshot: SnapshotSchema,
+    screenshot: ScreenshotSchema.nullable(),
+    issues: z.array(SerializedComputerErrorSchema).default([])
+  })
+  .strict()
+
+export const MutationResultSchema = z
+  .object({
+    outcome: MutationOutcomeSchema,
+    freshState: SnapshotResultSchema.optional()
+  })
+  .strict()
+
+export const ProviderCapabilitiesSchema = z
+  .object({
+    platform: z.enum(['darwin', 'win32', 'linux']),
+    provider: IdentifierSchema,
+    providerVersion: z.string().min(1),
+    operations: z.record(z.string(), z.boolean()),
+    permissions: z.record(z.string(), z.enum(['granted', 'denied', 'unknown', 'not_required']))
+  })
+  .strict()
+
+export const ProviderHandshakeSchema = z
+  .object({
+    provider: IdentifierSchema,
+    generation: IdentifierSchema,
+    graphicalSessionId: IdentifierSchema,
+    providerProtocol: PositiveIntegerSchema,
+    publicContract: z.string().min(1),
+    capabilities: ProviderCapabilitiesSchema
+  })
+  .strict()