@crosshands/cli 0.1.6 → 0.2.0

package/src/intent/dispatch.ts

@@ -0,0 +1,276 @@
+import {
+  ComputerError,
+  createComputerError,
+  hasIntentTarget,
+  parsePublicInput,
+  parsePublicOutput,
+  PublicMutationResultSchema,
+  PublicSnapshotResultSchema,
+  SnapshotResultSchema,
+  splitPublicInput,
+  toBrokerInput,
+  type ComputerOperationName,
+  type PublicOperationInput,
+  type SnapshotResult,
+  type Suggestion
+} from '@crosshands/contract'
+
+import { unwrapBrokerResult } from '../broker-result.js'
+import { parseJevEnv, type JevEnv } from './env.js'
+import {
+  liveEvaluator,
+  namedTargetAllowed,
+  suggestionFromAnswers,
+  type EvaluateFn
+} from './evaluate.js'
+import type { JevLogger } from './log.js'
+import { clickableMoves, parseTreeMoves } from './tree.js'
+
+export type IntentBroker = {
+  request(operation: ComputerOperationName, input: unknown): Promise<unknown>
+}
+
+export type DispatchOptions = {
+  env?: NodeJS.ProcessEnv
+  evaluate?: EvaluateFn
+  log?: JevLogger
+}
+
+const NAMED_GATE = new Set<ComputerOperationName>(['click', 'setValue'])
+
+function policyUnavailable() {
+  return createComputerError('policy_unavailable', 'Jev could not rank this snapshot').toJSON()
+}
+
+function snapshotResultOf(value: unknown): SnapshotResult {
+  return SnapshotResultSchema.parse(value)
+}
+
+function notAttempted(
+  error: ReturnType<typeof policyUnavailable>,
+  suggestion?: Suggestion
+): unknown {
+  return PublicMutationResultSchema.parse({
+    outcome: { state: 'not_attempted', error },
+    ...(suggestion === undefined ? {} : { suggestion })
+  })
+}
+
+function withSnapshotIssue(
+  look: SnapshotResult,
+  issue: ReturnType<typeof policyUnavailable>
+): unknown {
+  return PublicSnapshotResultSchema.parse({
+    ...look,
+    issues: [...look.issues, issue]
+  })
+}
+
+function contextTokenOf(input: object): string | undefined {
+  if (!('contextToken' in input) || typeof input.contextToken !== 'string') return undefined
+  return input.contextToken
+}
+
+function elementIndexOf(input: object): number | undefined {
+  if (!('target' in input)) return undefined
+  const target = input.target
+  if (target === null || typeof target !== 'object' || !('elementIndex' in target)) return undefined
+  return typeof target.elementIndex === 'number' ? target.elementIndex : undefined
+}
+
+function boundIndex(operation: ComputerOperationName, suggestion: Suggestion): number | undefined {
+  const move = suggestion.move
+  if (operation === 'setValue' && move.kind === 'setValue') return move.elementIndex
+  if (
+    (operation === 'click' || operation === 'scroll' || operation === 'performSecondaryAction') &&
+    move.kind === 'click'
+  ) {
+    return move.elementIndex
+  }
+  return undefined
+}
+
+async function rank(
+  look: SnapshotResult,
+  goal: string,
+  evaluate: EvaluateFn,
+  log: JevLogger | undefined,
+  app?: string
+): Promise<Suggestion> {
+  const moves = clickableMoves(parseTreeMoves(look.snapshot.treeText))
+  if (moves.length === 0) {
+    return {
+      untrusted: true,
+      snapshotId: look.snapshot.id,
+      move: { kind: 'blocked', reason: 'no_candidate' }
+    }
+  }
+  const suggestion = suggestionFromAnswers(
+    look.snapshot.id,
+    moves,
+    await evaluate({ goal, ...(app === undefined ? {} : { app }), moves })
+  )
+  log?.emit({
+    kind: 'jev.decision',
+    snapshotId: suggestion.snapshotId,
+    move: suggestion.move.kind,
+    ...(suggestion.move.kind === 'click' ||
+    suggestion.move.kind === 'setValue' ||
+    suggestion.move.kind === 'secondary' ||
+    suggestion.move.kind === 'scroll'
+      ? { elementIndex: suggestion.move.elementIndex }
+      : {}),
+    confidence: suggestion.confidence,
+    goal
+  })
+  return suggestion
+}
+
+function evaluatorFor(env: JevEnv, override: EvaluateFn | undefined): EvaluateFn | undefined {
+  if (override !== undefined) return override
+  if (env.kind !== 'on') return undefined
+  return liveEvaluator(env.apiKey)
+}
+
+async function lookAgain(request: IntentBroker['request'], token: string): Promise<SnapshotResult> {
+  return snapshotResultOf(
+    await request('getAppState', { contextToken: token, captureScreenshot: false })
+  )
+}
+
+async function suggestObserve(
+  request: IntentBroker['request'],
+  input: PublicOperationInput<'getAppState'>,
+  goal: string | undefined,
+  env: JevEnv,
+  evaluate: EvaluateFn | undefined,
+  log: JevLogger | undefined
+): Promise<unknown> {
+  const result = await request('getAppState', toBrokerInput('getAppState', input))
+  if (goal === undefined || env.kind === 'off') return result
+  const look = snapshotResultOf(result)
+  if (env.kind === 'fail_closed' || evaluate === undefined) {
+    return withSnapshotIssue(look, policyUnavailable())
+  }
+  try {
+    const app = 'app' in input && typeof input.app === 'string' ? input.app : undefined
+    const suggestion = await rank(look, goal, evaluate, log, app)
+    return parsePublicOutput('getAppState', { ...look, suggestion })
+  } catch {
+    return withSnapshotIssue(look, policyUnavailable())
+  }
+}
+
+async function bindIntentTarget(
+  request: IntentBroker['request'],
+  operation: ComputerOperationName,
+  input: object,
+  goal: string | undefined,
+  env: JevEnv,
+  evaluate: EvaluateFn | undefined,
+  log: JevLogger | undefined
+): Promise<unknown> {
+  if (env.kind === 'off') {
+    throw createComputerError('invalid_argument', 'intent targeting requires CROSSHANDS_JEV=1')
+  }
+  if (env.kind === 'fail_closed' || evaluate === undefined || goal === undefined) {
+    throw createComputerError('intent_unavailable', 'Jev is enabled but no TypeSafe key is set')
+  }
+  const token = contextTokenOf(input)
+  if (token === undefined) {
+    throw createComputerError('invalid_argument', 'intent targeting requires a context token')
+  }
+  let look: SnapshotResult
+  let suggestion: Suggestion
+  try {
+    look = await lookAgain(request, token)
+    suggestion = await rank(look, goal, evaluate, log)
+  } catch (cause) {
+    if (cause instanceof ComputerError) throw cause
+    return notAttempted(policyUnavailable())
+  }
+  const index = boundIndex(operation, suggestion)
+  if (index === undefined) return notAttempted(policyUnavailable(), suggestion)
+  const bound = {
+    ...splitPublicInput(input).rest,
+    contextToken: look.context.token,
+    target: { kind: 'element' as const, elementIndex: index }
+  }
+  const result = await request(operation, toBrokerInput(operation, bound))
+  const body = result !== null && typeof result === 'object' ? result : {}
+  return parsePublicOutput(operation, {
+    ...body,
+    resolvedTarget: { kind: 'element', elementIndex: index },
+    suggestion
+  })
+}
+
+async function gateNamedTarget(
+  request: IntentBroker['request'],
+  input: object,
+  goal: string,
+  evaluate: EvaluateFn,
+  log: JevLogger | undefined
+): Promise<unknown | undefined> {
+  const named = elementIndexOf(input)
+  const token = contextTokenOf(input)
+  if (token === undefined || named === undefined) return undefined
+  try {
+    const look = await lookAgain(request, token)
+    const suggestion = await rank(look, goal, evaluate, log)
+    const picked =
+      suggestion.move.kind === 'click' || suggestion.move.kind === 'setValue'
+        ? suggestion.move.elementIndex
+        : undefined
+    if (picked !== named || !namedTargetAllowed(suggestion.confidence)) {
+      return notAttempted(
+        createComputerError(
+          'goal_mismatch',
+          'Named target does not match the per-call goal'
+        ).toJSON(),
+        suggestion
+      )
+    }
+  } catch {
+    return notAttempted(policyUnavailable())
+  }
+  return undefined
+}
+
+export async function dispatchPublicOperation(
+  broker: IntentBroker,
+  operation: ComputerOperationName,
+  rawInput: unknown,
+  options: DispatchOptions = {}
+): Promise<unknown> {
+  const envState = parseJevEnv(options.env)
+  const parsed = parsePublicInput(operation, rawInput)
+  const { goal } = splitPublicInput(parsed)
+  const evaluate = evaluatorFor(envState, options.evaluate)
+  const request: IntentBroker['request'] = async (name, input) =>
+    unwrapBrokerResult(await broker.request(name, input))
+
+  if (operation === 'getAppState') {
+    return suggestObserve(
+      request,
+      parsed as PublicOperationInput<'getAppState'>,
+      goal,
+      envState,
+      evaluate,
+      options.log
+    )
+  }
+  if (hasIntentTarget(parsed)) {
+    return bindIntentTarget(request, operation, parsed, goal, envState, evaluate, options.log)
+  }
+  if (
+    goal !== undefined &&
+    envState.kind === 'on' &&
+    evaluate !== undefined &&
+    NAMED_GATE.has(operation)
+  ) {
+    const gated = await gateNamedTarget(request, parsed, goal, evaluate, options.log)
+    if (gated !== undefined) return gated
+  }
+  return request(operation, toBrokerInput(operation, parsed))
+}

package/src/intent/env.ts

@@ -0,0 +1,13 @@
+export type JevEnv =
+  | { kind: 'off' }
+  | { kind: 'fail_closed'; reason: 'missing_key' }
+  | { kind: 'on'; apiKey: string }
+
+export function parseJevEnv(env: NodeJS.ProcessEnv = process.env): JevEnv {
+  if (env.CROSSHANDS_JEV !== '1') return { kind: 'off' }
+  const apiKey = env.TYPESAFE_API_KEY
+  if (typeof apiKey !== 'string' || apiKey.length === 0) {
+    return { kind: 'fail_closed', reason: 'missing_key' }
+  }
+  return { kind: 'on', apiKey }
+}

package/src/intent/evaluate.ts

@@ -0,0 +1,155 @@
+import type { Suggestion } from '@crosshands/contract'
+
+import type { TreeMove } from './tree.js'
+
+export type JevAnswers = {
+  move: 'click' | 'setValue' | 'wait' | 'done' | 'blocked'
+  clickWhich?: string
+  setValueWhich?: string
+  confidence?: number
+  goalMet?: number
+  blockedReason?: string
+}
+
+export type EvaluateInput = {
+  goal: string
+  app?: string
+  moves: readonly TreeMove[]
+}
+
+export type EvaluateFn = (input: EvaluateInput) => Promise<JevAnswers>
+
+export const GATE_REFUSE_BELOW = 0.35
+
+export function namedTargetAllowed(confidence: number | undefined): boolean {
+  if (confidence === undefined) return true
+  return confidence >= GATE_REFUSE_BELOW
+}
+
+export function suggestionFromAnswers(
+  snapshotId: string,
+  moves: readonly TreeMove[],
+  answers: JevAnswers
+): Suggestion {
+  const byIndex = new Map(moves.map((move) => [String(move.elementIndex), move]))
+  if (answers.move === 'wait') {
+    return { untrusted: true, snapshotId, move: { kind: 'wait' }, confidence: answers.confidence }
+  }
+  if (answers.move === 'done') {
+    return { untrusted: true, snapshotId, move: { kind: 'done' }, confidence: answers.confidence }
+  }
+  if (answers.move === 'blocked') {
+    return {
+      untrusted: true,
+      snapshotId,
+      move: { kind: 'blocked', reason: answers.blockedReason ?? 'blocked' },
+      confidence: answers.confidence
+    }
+  }
+  const raw = answers.move === 'setValue' ? answers.setValueWhich : answers.clickWhich
+  const selected = raw === undefined ? undefined : byIndex.get(raw)
+  if (selected === undefined) {
+    return {
+      untrusted: true,
+      snapshotId,
+      move: { kind: 'blocked', reason: 'no_candidate' },
+      confidence: answers.confidence
+    }
+  }
+  return {
+    untrusted: true,
+    snapshotId,
+    move: { kind: answers.move, elementIndex: selected.elementIndex },
+    confidence: answers.confidence,
+    label: `${selected.role} ${selected.label}`.trim()
+  }
+}
+
+function criteria(moves: readonly TreeMove[]): Record<string, string> {
+  return Object.fromEntries(
+    moves.map((move) => [String(move.elementIndex), `${move.role} ${move.label}`.trim()])
+  )
+}
+
+type SystemOneAnswer = {
+  type?: string
+  choice?: string
+  noul?: number
+  confidence?: number
+}
+
+export function liveEvaluator(apiKey: string): EvaluateFn {
+  return async (input) => {
+    const clickable = criteria(input.moves)
+    const response = await fetch('https://api.typesafe.ai/v1/systemone', {
+      method: 'POST',
+      headers: {
+        authorization: `Bearer ${apiKey}`,
+        'content-type': 'application/json'
+      },
+      body: JSON.stringify({
+        model: 'jev-latest',
+        state: {
+          goal: input.goal,
+          ...(input.app === undefined ? {} : { app: input.app }),
+          clickable
+        },
+        questions: {
+          move: {
+            type: 'choice',
+            instructions: 'What should we do next for the goal?',
+            criteria: {
+              click: 'Press one clickable control',
+              setValue: 'Pick a field for the caller to fill',
+              wait: 'The window is still loading',
+              done: 'The goal is already met',
+              blocked: 'Stop'
+            }
+          },
+          click_which: {
+            type: 'choice',
+            instructions: 'If the move is click, which control?',
+            criteria: clickable
+          },
+          setvalue_which: {
+            type: 'choice',
+            instructions: 'If the move is setValue, which field?',
+            criteria: clickable
+          },
+          goal_met: {
+            type: 'noul',
+            instructions: 'Is the goal already met?'
+          }
+        }
+      })
+    })
+    if (!response.ok) {
+      throw new Error(`TypeSafe HTTP ${response.status}`)
+    }
+    const body = (await response.json()) as {
+      answers?: Record<string, SystemOneAnswer>
+    }
+    const answers = body.answers ?? {}
+    const move = answers.move?.choice
+    if (
+      move !== 'click' &&
+      move !== 'setValue' &&
+      move !== 'wait' &&
+      move !== 'done' &&
+      move !== 'blocked'
+    ) {
+      return { move: 'blocked', blockedReason: 'no_candidate' }
+    }
+    const clickWhich = answers.click_which?.choice
+    const setValueWhich = answers.setvalue_which?.choice
+    const confidence = answers.move?.confidence ?? answers.click_which?.confidence
+    const goalMet = answers.goal_met?.noul
+    return {
+      move,
+      ...(clickWhich === undefined ? {} : { clickWhich }),
+      ...(setValueWhich === undefined ? {} : { setValueWhich }),
+      ...(confidence === undefined ? {} : { confidence }),
+      ...(goalMet === undefined ? {} : { goalMet })
+    }
+  }
+}

package/src/intent/log.ts

@@ -0,0 +1,47 @@
+import { createHash, randomUUID } from 'node:crypto'
+
+import { JsonlFileWriter, diagnosticsDirectory } from '@crosshands/runtime'
+
+import { localClientPaths } from '../local-client.js'
+import { parseJevEnv } from './env.js'
+
+export type JevLogger = {
+  emit(record: Record<string, unknown>): void
+  close(): Promise<void>
+}
+
+export function hashGoal(goal: string): string {
+  return createHash('sha256').update(goal).digest('hex')
+}
+
+const OMIT = new Set(['goal', 'apiKey', 'treeText', 'text', 'value', 'TYPESAFE_API_KEY'])
+
+export function createJevLogger(
+  graphicalSessionId: string,
+  env: NodeJS.ProcessEnv = process.env
+): JevLogger {
+  const writer = new JsonlFileWriter({
+    directory: diagnosticsDirectory(graphicalSessionId, { env }),
+    generation: `jev-${randomUUID()}`
+  })
+  writer.start()
+  return {
+    emit(record) {
+      const kind = record.kind
+      if (typeof kind !== 'string') return
+      const goal = record.goal
+      const rest: Record<string, unknown> = { ...record, kind, v: 1, ts: new Date().toISOString() }
+      for (const key of OMIT) delete rest[key]
+      writer.emit({
+        ...rest,
+        ...(typeof goal === 'string' ? { goalSha256: hashGoal(goal) } : {})
+      })
+    },
+    close: () => writer.close()
+  }
+}
+
+export function createCliJevLogger(env: NodeJS.ProcessEnv = process.env): JevLogger | undefined {
+  if (parseJevEnv(env).kind === 'off') return undefined
+  return createJevLogger(localClientPaths().identity.graphicalSessionId, env)
+}

package/src/intent/tree.ts

@@ -0,0 +1,108 @@
+export type TreeMove = {
+  elementIndex: number
+  role: string
+  label: string
+  line: string
+  disabled: boolean
+  clickable: boolean
+  settable: boolean
+}
+
+const ROW_START = /^(\t*)(\d+) (.+)$/
+const MULTI_WORD_ROLES = [
+  'search text field',
+  'text field',
+  'text area',
+  'menu button',
+  'pop up button',
+  'popup button',
+  'radio button',
+  'check box',
+  'combo box',
+  'disclosure triangle',
+  'standard window',
+  'scroll area',
+  'split group',
+  'outline row',
+  'table row'
+] as const
+const CLICKABLE_ROLE = new Set([
+  'button',
+  'link',
+  'checkbox',
+  'check box',
+  'radio button',
+  'radio',
+  'tab',
+  'menu button',
+  'pop up button',
+  'popup button',
+  'combo box',
+  'disclosure triangle'
+])
+const SETTABLE_ROLE = new Set(['text field', 'search text field', 'text area', 'combo box'])
+
+function isHeaderOrFooter(line: string): boolean {
+  if (line.length === 0) return true
+  if (line.startsWith('App=')) return true
+  if (line.startsWith('Window:')) return true
+  if (line.startsWith('The focused UI element')) return true
+  if (line.startsWith('No UI element')) return true
+  return false
+}
+
+function roleAndLabel(body: string): { role: string; label: string } {
+  const lower = body.toLowerCase()
+  const multi = MULTI_WORD_ROLES.find(
+    (role) =>
+      lower.startsWith(role) && (body.length === role.length || /[\s,(]/.test(body[role.length]!))
+  )
+  if (multi !== undefined) {
+    return {
+      role: body.slice(0, multi.length),
+      label: body.slice(multi.length).replace(/^[\s,]+/, '')
+    }
+  }
+  const comma = body.indexOf(',')
+  const head = (comma === -1 ? body : body.slice(0, comma)).trim()
+  const space = head.indexOf(' ')
+  if (space === -1) return { role: head, label: '' }
+  return { role: head.slice(0, space), label: head.slice(space + 1).trim() }
+}
+
+export function parseTreeMoves(treeText: string): TreeMove[] {
+  const rows: { elementIndex: number; body: string }[] = []
+  for (const line of treeText.split('\n')) {
+    const match = ROW_START.exec(line)
+    if (match !== null && match[2] !== undefined && match[3] !== undefined) {
+      rows.push({ elementIndex: Number(match[2]), body: match[3] })
+      continue
+    }
+    const previous = rows[rows.length - 1]
+    if (previous !== undefined && !isHeaderOrFooter(line)) previous.body += `\n${line}`
+  }
+  return rows.map((row) => {
+    const { role, label } = roleAndLabel(row.body)
+    const disabled = /\(disabled\)/.test(row.body)
+    const roleKey = role.toLowerCase()
+    const settable = /\(settable\)/.test(row.body) || SETTABLE_ROLE.has(roleKey)
+    const clickable = !disabled && (CLICKABLE_ROLE.has(roleKey) || settable)
+    return {
+      elementIndex: row.elementIndex,
+      role,
+      label,
+      line: `${row.elementIndex} ${row.body}`,
+      disabled,
+      clickable,
+      settable
+    }
+  })
+}
+
+export const JEV_CHOICE_CAP = 255
+
+export function clickableMoves(moves: readonly TreeMove[]): TreeMove[] {
+  const selected = moves.filter((move) => move.clickable)
+  if (selected.length <= JEV_CHOICE_CAP) return selected
+  return selected.slice(0, JEV_CHOICE_CAP)
+}

package/src/local-client.ts

@@ -1,12 +1,18 @@
 import { spawn } from 'node:child_process'
-import { createHash } from 'node:crypto'
-import { chmod, lstat, mkdir, readFile } from 'node:fs/promises'
+import { lstat, readFile } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { isAbsolute, join, resolve } from 'node:path'
 
 import { CONTRACT_VERSIONS, createComputerError } from '@crosshands/contract'
-import { LocalControlClient, brokerEndpoint, type LocalControlIdentity } from '@crosshands/runtime'
+import {
+  LocalControlClient,
+  brokerEndpoint,
+  ensurePrivateDirectory,
+  graphicalSessionKey,
+  type LocalControlIdentity
+} from '@crosshands/runtime'
 
+import { unwrapBrokerResult } from './broker-result.js'
 import type { CliBrokerClient } from './index.js'
 
 export type LocalClientPaths = {
@@ -29,7 +35,7 @@ export function localClientPaths(): LocalClientPaths {
     process.env.SECURITYSESSIONID ??
     process.env.SESSIONNAME ??
     `interactive:${osIdentity}`
-  const sessionKey = createHash('sha256').update(graphicalSessionId).digest('hex').slice(0, 12)
+  const sessionKey = graphicalSessionKey(graphicalSessionId)
   const runtimeDirectory = process.env.CROSSHANDS_RUNTIME_DIR ?? defaultRuntimeDirectory(sessionKey)
   const identity = { osIdentity, graphicalSessionId }
   return {
@@ -69,14 +75,15 @@ function defaultRuntimeDirectory(sessionKey: string): string {
   return join(localAppData, 'CrossHands', 'runtime', sessionKey)
 }
 
-async function prepareRuntimeDirectory(path: string): Promise<void> {
-  await mkdir(path, { recursive: true, mode: 0o700 })
-  const info = await lstat(path)
-  if (!info.isDirectory() || info.isSymbolicLink())
-    throw createComputerError('provider_unavailable', 'CrossHands runtime path is unsafe')
-  if (process.getuid !== undefined && info.uid !== process.getuid())
-    throw createComputerError('provider_unavailable', 'CrossHands runtime path has another owner')
-  await chmod(path, 0o700)
+function prepareRuntimeDirectory(path: string): void {
+  try {
+    ensurePrivateDirectory(path)
+  } catch (cause) {
+    throw createComputerError(
+      'provider_unavailable',
+      cause instanceof Error ? cause.message : 'CrossHands directory is unsafe'
+    )
+  }
 }
 
 async function readSecureToken(path: string): Promise<string> {
@@ -105,12 +112,21 @@ export type ProductionClientOptions = {
   paths?: LocalClientPaths
 }
 
+export function brokerSpawnEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  const next: NodeJS.ProcessEnv = { ...env }
+  delete next.CROSSHANDS_JEV
+  for (const key of Object.keys(next)) {
+    if (key === 'TYPESAFE_API_KEY' || key.startsWith('TYPESAFE_')) delete next[key]
+  }
+  return next
+}
+
 function defaultSpawnBroker(entrypoint: string): void {
   const child = spawn(process.execPath, [entrypoint, 'broker'], {
     detached: true,
     stdio: 'ignore',
     windowsHide: true,
-    env: process.env
+    env: brokerSpawnEnv(process.env)
   })
   child.unref()
 }
@@ -125,7 +141,7 @@ export async function createProductionBrokerClient(
   options: ProductionClientOptions = {}
 ): Promise<CliBrokerClient> {
   const paths = options.paths ?? localClientPaths()
-  await prepareRuntimeDirectory(paths.runtimeDirectory)
+  prepareRuntimeDirectory(paths.runtimeDirectory)
   try {
     const control = await connect(paths)
     return controlAdapter(control)
@@ -165,7 +181,7 @@ export async function createProductionBrokerClient(
 function controlAdapter(control: LocalControlClient): CliBrokerClient {
   return {
     request: async (operation, input) =>
-      control.request({ operation, input }, { deadlineMs: 30_000 }),
+      unwrapBrokerResult(await control.request({ operation, input }, { deadlineMs: 30_000 })),
     close: () => control.close()
   }
 }