@crosshands/cli 0.1.6 → 0.2.1

package/src/intent/evaluate.ts

@@ -0,0 +1,211 @@
+import type { Suggestion } from '@crosshands/contract'
+
+import { elapsedMs, type JevHttpStats } from './http.js'
+import type { TreeMove } from './tree.js'
+
+export type { JevHttpStats }
+
+export class JevEvaluateError extends Error {
+  readonly http: JevHttpStats
+  constructor(message: string, http: JevHttpStats) {
+    super(message)
+    this.name = 'JevEvaluateError'
+    this.http = http
+  }
+}
+
+export type JevAnswers = {
+  move: 'click' | 'setValue' | 'wait' | 'done' | 'blocked'
+  clickWhich?: string
+  setValueWhich?: string
+  confidence?: number
+  goalMet?: number
+  blockedReason?: string
+}
+
+export type EvaluateResult = {
+  answers: JevAnswers
+  http?: JevHttpStats
+}
+
+export type EvaluateInput = {
+  goal: string
+  app?: string
+  moves: readonly TreeMove[]
+}
+
+export type EvaluateFn = (input: EvaluateInput) => Promise<EvaluateResult>
+
+export const GATE_REFUSE_BELOW = 0.35
+
+export function namedTargetAllowed(confidence: number | undefined): boolean {
+  if (confidence === undefined) return true
+  return confidence >= GATE_REFUSE_BELOW
+}
+
+export function suggestionFromAnswers(
+  snapshotId: string,
+  moves: readonly TreeMove[],
+  answers: JevAnswers
+): Suggestion {
+  const byIndex = new Map(moves.map((move) => [String(move.elementIndex), move]))
+  if (answers.move === 'wait') {
+    return { untrusted: true, snapshotId, move: { kind: 'wait' }, confidence: answers.confidence }
+  }
+  if (answers.move === 'done') {
+    return { untrusted: true, snapshotId, move: { kind: 'done' }, confidence: answers.confidence }
+  }
+  if (answers.move === 'blocked') {
+    return {
+      untrusted: true,
+      snapshotId,
+      move: { kind: 'blocked', reason: answers.blockedReason ?? 'blocked' },
+      confidence: answers.confidence
+    }
+  }
+  const raw = answers.move === 'setValue' ? answers.setValueWhich : answers.clickWhich
+  const selected = raw === undefined ? undefined : byIndex.get(raw)
+  if (selected === undefined) {
+    return {
+      untrusted: true,
+      snapshotId,
+      move: { kind: 'blocked', reason: 'no_candidate' },
+      confidence: answers.confidence
+    }
+  }
+  return {
+    untrusted: true,
+    snapshotId,
+    move: { kind: answers.move, elementIndex: selected.elementIndex },
+    confidence: answers.confidence,
+    label: `${selected.role} ${selected.label}`.trim()
+  }
+}
+
+function criteria(moves: readonly TreeMove[]): Record<string, string> {
+  return Object.fromEntries(
+    moves.map((move) => [String(move.elementIndex), `${move.role} ${move.label}`.trim()])
+  )
+}
+
+type SystemOneAnswer = {
+  type?: string
+  choice?: string
+  noul?: number
+  confidence?: number
+}
+
+const JEV_MODEL = 'jev-latest'
+
+function httpStats(
+  started: number,
+  requestBytes: number,
+  status: number,
+  responseBytes: number
+): JevHttpStats {
+  return {
+    status,
+    durationMs: elapsedMs(started),
+    requestBytes,
+    responseBytes
+  }
+}
+
+function answersOf(raw: Record<string, SystemOneAnswer>): JevAnswers {
+  const move = raw.move?.choice
+  if (
+    move !== 'click' &&
+    move !== 'setValue' &&
+    move !== 'wait' &&
+    move !== 'done' &&
+    move !== 'blocked'
+  ) {
+    return { move: 'blocked', blockedReason: 'no_candidate' }
+  }
+  const clickWhich = raw.click_which?.choice
+  const setValueWhich = raw.setvalue_which?.choice
+  const confidence = raw.move?.confidence ?? raw.click_which?.confidence
+  const goalMet = raw.goal_met?.noul
+  return {
+    move,
+    ...(clickWhich === undefined ? {} : { clickWhich }),
+    ...(setValueWhich === undefined ? {} : { setValueWhich }),
+    ...(confidence === undefined ? {} : { confidence }),
+    ...(goalMet === undefined ? {} : { goalMet })
+  }
+}
+
+export function liveEvaluator(apiKey: string): EvaluateFn {
+  return async (input) => {
+    const clickable = criteria(input.moves)
+    const payload = JSON.stringify({
+      model: JEV_MODEL,
+      state: {
+        goal: input.goal,
+        ...(input.app === undefined ? {} : { app: input.app }),
+        clickable
+      },
+      questions: {
+        move: {
+          type: 'choice',
+          instructions: 'What should we do next for the goal?',
+          criteria: {
+            click: 'Press one clickable control',
+            setValue: 'Pick a field for the caller to fill',
+            wait: 'The window is still loading',
+            done: 'The goal is already met',
+            blocked: 'Stop'
+          }
+        },
+        click_which: {
+          type: 'choice',
+          instructions: 'If the move is click, which control?',
+          criteria: clickable
+        },
+        setvalue_which: {
+          type: 'choice',
+          instructions: 'If the move is setValue, which field?',
+          criteria: clickable
+        },
+        goal_met: {
+          type: 'noul',
+          instructions: 'Is the goal already met?'
+        }
+      }
+    })
+    const requestBytes = Buffer.byteLength(payload)
+    const started = Date.now()
+    let status = 0
+    let responseBytes = 0
+    let raw = ''
+    try {
+      const response = await fetch('https://api.typesafe.ai/v1/systemone', {
+        method: 'POST',
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          'content-type': 'application/json'
+        },
+        body: payload
+      })
+      status = response.status
+      raw = await response.text()
+      responseBytes = Buffer.byteLength(raw)
+    } catch (cause) {
+      throw new JevEvaluateError(
+        cause instanceof Error ? cause.message : 'TypeSafe HTTP failed',
+        httpStats(started, requestBytes, 0, 0)
+      )
+    }
+    const http = httpStats(started, requestBytes, status, responseBytes)
+    if (status < 200 || status >= 300) {
+      throw new JevEvaluateError(`TypeSafe HTTP ${status}`, http)
+    }
+    let parsed: { answers?: Record<string, SystemOneAnswer> }
+    try {
+      parsed = JSON.parse(raw) as { answers?: Record<string, SystemOneAnswer> }
+    } catch {
+      throw new JevEvaluateError('TypeSafe HTTP response was not JSON', http)
+    }
+    return { answers: answersOf(parsed.answers ?? {}), http }
+  }
+}

package/src/intent/http.ts

@@ -0,0 +1,10 @@
+export type JevHttpStats = {
+  status: number
+  durationMs: number
+  requestBytes: number
+  responseBytes: number
+}
+
+export function elapsedMs(started: number): number {
+  return Math.max(0, Date.now() - started)
+}

package/src/intent/log.ts

@@ -0,0 +1,219 @@
+import { createHash, randomUUID } from 'node:crypto'
+
+import type { Suggestion } from '@crosshands/contract'
+import { JsonlFileWriter, diagnosticsDirectory } from '@crosshands/runtime'
+
+import { localClientPaths } from '../local-client.js'
+import { parseJevEnv } from './env.js'
+import { elapsedMs, type JevHttpStats } from './http.js'
+
+export type JevPhase = 'observe' | 'bind' | 'named'
+
+type JevShared = {
+  callId: string
+  operation: string
+  snapshotId: string
+  phase: JevPhase
+  goal?: string
+}
+
+export type JevCallRecord = {
+  kind: 'jev.call'
+  callId: string
+  operation: string
+  durationMs: number
+  httpMs: number
+  brokerCalls: number
+  ranks: number
+  phase?: JevPhase
+  bound?: boolean
+  error?: string
+  goal?: string
+}
+
+export type JevHttpRecord = JevShared & {
+  kind: 'jev.http'
+  status: number
+  durationMs: number
+  requestBytes: number
+  responseBytes: number
+  candidateCount: number
+}
+
+export type JevDecisionRecord = JevShared & {
+  kind: 'jev.decision'
+  app?: string
+  move: Suggestion['move']['kind']
+  elementIndex?: number
+  reason?: string
+  confidence?: number
+  goalChars: number
+  treeChars: number
+  elementCount: number
+  candidateCount: number
+  parseMs: number
+  evaluateMs?: number
+}
+
+export type JevRecord = JevCallRecord | JevHttpRecord | JevDecisionRecord
+
+export type JevLogger = {
+  emit(record: JevRecord): void
+  close(): Promise<void>
+}
+
+export type JevCall = {
+  callId: string
+  operation: string
+  goal?: string
+  ranks: number
+  httpMs: number
+  brokerCalls: number
+  phase?: JevPhase
+}
+
+export type JevRankStats = {
+  snapshotId: string
+  treeChars: number
+  elementCount: number
+  candidateCount: number
+  parseMs: number
+  goalChars: number
+  evaluateMs?: number
+  app?: string
+  http?: JevHttpStats
+}
+
+export type RankRecorder = (phase: JevPhase, stats: JevRankStats, suggestion?: Suggestion) => void
+
+export function hashGoal(goal: string): string {
+  return createHash('sha256').update(goal).digest('hex')
+}
+
+const OMIT = new Set(['goal', 'apiKey', 'treeText', 'text', 'value', 'TYPESAFE_API_KEY'])
+
+export function createJevCall(operation: string, goal?: string): JevCall {
+  return {
+    callId: randomUUID(),
+    operation,
+    ...(goal === undefined ? {} : { goal }),
+    ranks: 0,
+    httpMs: 0,
+    brokerCalls: 0
+  }
+}
+
+export function recordRank(
+  log: JevLogger | undefined,
+  call: JevCall | undefined,
+  phase: JevPhase,
+  stats: JevRankStats,
+  suggestion?: Suggestion
+): void {
+  if (call !== undefined) {
+    call.phase = phase
+    call.ranks += 1
+    if (stats.http !== undefined) call.httpMs += stats.http.durationMs
+  }
+  if (log === undefined || call === undefined) return
+  const shared: JevShared = {
+    callId: call.callId,
+    operation: call.operation,
+    snapshotId: stats.snapshotId,
+    phase,
+    ...(call.goal === undefined ? {} : { goal: call.goal })
+  }
+  if (stats.http !== undefined) {
+    log.emit({
+      kind: 'jev.http',
+      ...shared,
+      status: stats.http.status,
+      durationMs: stats.http.durationMs,
+      requestBytes: stats.http.requestBytes,
+      responseBytes: stats.http.responseBytes,
+      candidateCount: stats.candidateCount
+    })
+  }
+  if (suggestion === undefined) return
+  const move = suggestion.move
+  log.emit({
+    kind: 'jev.decision',
+    ...shared,
+    ...(stats.app === undefined ? {} : { app: stats.app }),
+    move: move.kind,
+    ...(move.kind === 'click' ||
+    move.kind === 'setValue' ||
+    move.kind === 'secondary' ||
+    move.kind === 'scroll'
+      ? { elementIndex: move.elementIndex }
+      : {}),
+    ...(move.kind === 'blocked' ? { reason: move.reason } : {}),
+    ...(suggestion.confidence === undefined ? {} : { confidence: suggestion.confidence }),
+    goalChars: stats.goalChars,
+    treeChars: stats.treeChars,
+    elementCount: stats.elementCount,
+    candidateCount: stats.candidateCount,
+    parseMs: stats.parseMs,
+    ...(stats.evaluateMs === undefined ? {} : { evaluateMs: stats.evaluateMs })
+  })
+}
+
+export function emitCall(
+  log: JevLogger | undefined,
+  call: JevCall | undefined,
+  started: number,
+  extra: { error?: string; bound?: boolean } = {}
+): void {
+  if (log === undefined || call === undefined) return
+  log.emit({
+    kind: 'jev.call',
+    callId: call.callId,
+    operation: call.operation,
+    durationMs: elapsedMs(started),
+    httpMs: call.httpMs,
+    brokerCalls: call.brokerCalls,
+    ranks: call.ranks,
+    ...(call.phase === undefined ? {} : { phase: call.phase }),
+    ...(extra.bound === undefined ? {} : { bound: extra.bound }),
+    ...(extra.error === undefined ? {} : { error: extra.error }),
+    ...(call.goal === undefined ? {} : { goal: call.goal })
+  })
+}
+
+export function createJevLogger(
+  graphicalSessionId: string,
+  env: NodeJS.ProcessEnv = process.env
+): JevLogger {
+  const writer = new JsonlFileWriter({
+    directory: diagnosticsDirectory(graphicalSessionId, { env }),
+    generation: `jev-${randomUUID()}`
+  })
+  writer.start()
+  return {
+    emit(record) {
+      try {
+        const raw = record as JevRecord & Record<string, unknown>
+        const goal = raw.goal
+        const rest: Record<string, unknown> = {
+          ...raw,
+          kind: record.kind,
+          v: 1,
+          ts: new Date().toISOString()
+        }
+        for (const key of OMIT) delete rest[key]
+        writer.emit({
+          ...rest,
+          ...(typeof goal === 'string' ? { goalSha256: hashGoal(goal) } : {})
+        })
+      } catch {
+        // Jev logs must not fail computer-use.
+      }
+    },
+    close: () => writer.close()
+  }
+}
+
+export function createCliJevLogger(env: NodeJS.ProcessEnv = process.env): JevLogger | undefined {
+  if (parseJevEnv(env).kind === 'off') return undefined
+  return createJevLogger(localClientPaths().identity.graphicalSessionId, env)
+}

package/src/intent/tree.ts

@@ -0,0 +1,108 @@
+export type TreeMove = {
+  elementIndex: number
+  role: string
+  label: string
+  line: string
+  disabled: boolean
+  clickable: boolean
+  settable: boolean
+}
+
+const ROW_START = /^(\t*)(\d+) (.+)$/
+const MULTI_WORD_ROLES = [
+  'search text field',
+  'text field',
+  'text area',
+  'menu button',
+  'pop up button',
+  'popup button',
+  'radio button',
+  'check box',
+  'combo box',
+  'disclosure triangle',
+  'standard window',
+  'scroll area',
+  'split group',
+  'outline row',
+  'table row'
+] as const
+const CLICKABLE_ROLE = new Set([
+  'button',
+  'link',
+  'checkbox',
+  'check box',
+  'radio button',
+  'radio',
+  'tab',
+  'menu button',
+  'pop up button',
+  'popup button',
+  'combo box',
+  'disclosure triangle'
+])
+const SETTABLE_ROLE = new Set(['text field', 'search text field', 'text area', 'combo box'])
+
+function isHeaderOrFooter(line: string): boolean {
+  if (line.length === 0) return true
+  if (line.startsWith('App=')) return true
+  if (line.startsWith('Window:')) return true
+  if (line.startsWith('The focused UI element')) return true
+  if (line.startsWith('No UI element')) return true
+  return false
+}
+
+function roleAndLabel(body: string): { role: string; label: string } {
+  const lower = body.toLowerCase()
+  const multi = MULTI_WORD_ROLES.find(
+    (role) =>
+      lower.startsWith(role) && (body.length === role.length || /[\s,(]/.test(body[role.length]!))
+  )
+  if (multi !== undefined) {
+    return {
+      role: body.slice(0, multi.length),
+      label: body.slice(multi.length).replace(/^[\s,]+/, '')
+    }
+  }
+  const comma = body.indexOf(',')
+  const head = (comma === -1 ? body : body.slice(0, comma)).trim()
+  const space = head.indexOf(' ')
+  if (space === -1) return { role: head, label: '' }
+  return { role: head.slice(0, space), label: head.slice(space + 1).trim() }
+}
+
+export function parseTreeMoves(treeText: string): TreeMove[] {
+  const rows: { elementIndex: number; body: string }[] = []
+  for (const line of treeText.split('\n')) {
+    const match = ROW_START.exec(line)
+    if (match !== null && match[2] !== undefined && match[3] !== undefined) {
+      rows.push({ elementIndex: Number(match[2]), body: match[3] })
+      continue
+    }
+    const previous = rows[rows.length - 1]
+    if (previous !== undefined && !isHeaderOrFooter(line)) previous.body += `\n${line}`
+  }
+  return rows.map((row) => {
+    const { role, label } = roleAndLabel(row.body)
+    const disabled = /\(disabled\)/.test(row.body)
+    const roleKey = role.toLowerCase()
+    const settable = /\(settable\)/.test(row.body) || SETTABLE_ROLE.has(roleKey)
+    const clickable = !disabled && (CLICKABLE_ROLE.has(roleKey) || settable)
+    return {
+      elementIndex: row.elementIndex,
+      role,
+      label,
+      line: `${row.elementIndex} ${row.body}`,
+      disabled,
+      clickable,
+      settable
+    }
+  })
+}
+
+export const JEV_CHOICE_CAP = 255
+
+export function clickableMoves(moves: readonly TreeMove[]): TreeMove[] {
+  const selected = moves.filter((move) => move.clickable)
+  if (selected.length <= JEV_CHOICE_CAP) return selected
+  return selected.slice(0, JEV_CHOICE_CAP)
+}

package/src/local-client.ts

@@ -1,12 +1,18 @@
 import { spawn } from 'node:child_process'
-import { createHash } from 'node:crypto'
-import { chmod, lstat, mkdir, readFile } from 'node:fs/promises'
+import { lstat, readFile } from 'node:fs/promises'
 import { homedir } from 'node:os'
 import { isAbsolute, join, resolve } from 'node:path'
 
 import { CONTRACT_VERSIONS, createComputerError } from '@crosshands/contract'
-import { LocalControlClient, brokerEndpoint, type LocalControlIdentity } from '@crosshands/runtime'
+import {
+  LocalControlClient,
+  brokerEndpoint,
+  ensurePrivateDirectory,
+  graphicalSessionKey,
+  type LocalControlIdentity
+} from '@crosshands/runtime'
 
+import { unwrapBrokerResult } from './broker-result.js'
 import type { CliBrokerClient } from './index.js'
 
 export type LocalClientPaths = {
@@ -29,7 +35,7 @@ export function localClientPaths(): LocalClientPaths {
     process.env.SECURITYSESSIONID ??
     process.env.SESSIONNAME ??
     `interactive:${osIdentity}`
-  const sessionKey = createHash('sha256').update(graphicalSessionId).digest('hex').slice(0, 12)
+  const sessionKey = graphicalSessionKey(graphicalSessionId)
   const runtimeDirectory = process.env.CROSSHANDS_RUNTIME_DIR ?? defaultRuntimeDirectory(sessionKey)
   const identity = { osIdentity, graphicalSessionId }
   return {
@@ -69,14 +75,15 @@ function defaultRuntimeDirectory(sessionKey: string): string {
   return join(localAppData, 'CrossHands', 'runtime', sessionKey)
 }
 
-async function prepareRuntimeDirectory(path: string): Promise<void> {
-  await mkdir(path, { recursive: true, mode: 0o700 })
-  const info = await lstat(path)
-  if (!info.isDirectory() || info.isSymbolicLink())
-    throw createComputerError('provider_unavailable', 'CrossHands runtime path is unsafe')
-  if (process.getuid !== undefined && info.uid !== process.getuid())
-    throw createComputerError('provider_unavailable', 'CrossHands runtime path has another owner')
-  await chmod(path, 0o700)
+function prepareRuntimeDirectory(path: string): void {
+  try {
+    ensurePrivateDirectory(path)
+  } catch (cause) {
+    throw createComputerError(
+      'provider_unavailable',
+      cause instanceof Error ? cause.message : 'CrossHands directory is unsafe'
+    )
+  }
 }
 
 async function readSecureToken(path: string): Promise<string> {
@@ -105,12 +112,21 @@ export type ProductionClientOptions = {
   paths?: LocalClientPaths
 }
 
+export function brokerSpawnEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  const next: NodeJS.ProcessEnv = { ...env }
+  delete next.CROSSHANDS_JEV
+  for (const key of Object.keys(next)) {
+    if (key === 'TYPESAFE_API_KEY' || key.startsWith('TYPESAFE_')) delete next[key]
+  }
+  return next
+}
+
 function defaultSpawnBroker(entrypoint: string): void {
   const child = spawn(process.execPath, [entrypoint, 'broker'], {
     detached: true,
     stdio: 'ignore',
     windowsHide: true,
-    env: process.env
+    env: brokerSpawnEnv(process.env)
   })
   child.unref()
 }
@@ -125,7 +141,7 @@ export async function createProductionBrokerClient(
   options: ProductionClientOptions = {}
 ): Promise<CliBrokerClient> {
   const paths = options.paths ?? localClientPaths()
-  await prepareRuntimeDirectory(paths.runtimeDirectory)
+  prepareRuntimeDirectory(paths.runtimeDirectory)
   try {
     const control = await connect(paths)
     return controlAdapter(control)
@@ -165,7 +181,7 @@ export async function createProductionBrokerClient(
 function controlAdapter(control: LocalControlClient): CliBrokerClient {
   return {
     request: async (operation, input) =>
-      control.request({ operation, input }, { deadlineMs: 30_000 }),
+      unwrapBrokerResult(await control.request({ operation, input }, { deadlineMs: 30_000 })),
     close: () => control.close()
   }
 }