@crosshands/cli 0.1.6 → 0.2.1

package/src/index.ts

@@ -5,6 +5,10 @@ import { dirname, resolve } from 'node:path'
 
 import type { ComputerOperationName } from '@crosshands/contract'
 
+import { unwrapBrokerResult } from './broker-result.js'
+import { dispatchPublicOperation } from './intent/dispatch.js'
+import { createCliJevLogger } from './intent/log.js'
+
 export type CliBrokerClient = {
   request(operation: ComputerOperationName, input: unknown): Promise<unknown>
   close(): Promise<void>
@@ -52,8 +56,10 @@ const ALLOWED: Record<string, readonly string[]> = {
   'get-app-state': [
     'json',
     'app',
+    'context',
     'window-id',
     'window-index',
+    'goal',
     'no-screenshot',
     'restore-window',
     'screenshot-output'
@@ -68,6 +74,7 @@ const ALLOWED: Record<string, readonly string[]> = {
     'click-count',
     'mouse-button',
     'modifiers',
+    'goal',
     'no-screenshot',
     'restore-window',
     'screenshot-output'
@@ -78,6 +85,7 @@ const ALLOWED: Record<string, readonly string[]> = {
     'context',
     'element-index',
     'action',
+    'goal',
     'no-screenshot',
     'restore-window',
     'screenshot-output'
@@ -91,6 +99,7 @@ const ALLOWED: Record<string, readonly string[]> = {
     'y',
     'direction',
     'pages',
+    'goal',
     'no-screenshot',
     'restore-window',
     'screenshot-output'
@@ -146,6 +155,7 @@ const ALLOWED: Record<string, readonly string[]> = {
     'element-index',
     'value',
     'value-stdin',
+    'goal',
     'no-screenshot',
     'restore-window',
     'screenshot-output'
@@ -165,7 +175,10 @@ const EXIT_CODES: Record<string, number> = {
   action_not_supported: 7,
   timeout: 8,
   version_incompatible: 9,
-  session_unavailable: 10
+  session_unavailable: 10,
+  goal_mismatch: 5,
+  policy_unavailable: 5,
+  intent_unavailable: 2
 }
 
 class CliError extends Error {
@@ -252,12 +265,6 @@ function contextToken(flags: Flags): string {
   return stringFlag(flags, 'context', true)!
 }
 
-function elementTarget(flags: Flags, name = 'element-index'): unknown {
-  const index = numberFlag(flags, name, { integer: true, min: 0 })
-  if (index === undefined) throw new CliError('invalid_argument', `Missing required --${name}`)
-  return { kind: 'element', elementIndex: index }
-}
-
 function chordTokens(raw: string): string[] {
   return raw.split('+')
 }
@@ -271,17 +278,32 @@ function parseClickModifiers(flags: Flags): string[] | undefined {
   return modifiers
 }
 
-function pointOrElement(flags: Flags): unknown {
+function optionalGoal(flags: Flags): { goal?: string } {
+  const goal = stringFlag(flags, 'goal')
+  return goal === undefined ? {} : { goal }
+}
+
+function actionTarget(flags: Flags, options: { coordinates?: boolean } = {}): unknown {
   const element = numberFlag(flags, 'element-index', { integer: true, min: 0 })
-  const x = numberFlag(flags, 'x')
-  const y = numberFlag(flags, 'y')
+  const allowCoordinates = options.coordinates === true
+  const x = allowCoordinates ? numberFlag(flags, 'x') : undefined
+  const y = allowCoordinates ? numberFlag(flags, 'y') : undefined
   const hasCoordinates = x !== undefined || y !== undefined
   if (element !== undefined && hasCoordinates)
     throw new CliError('invalid_argument', 'Choose an element index or coordinates, not both')
   if (element !== undefined) return { kind: 'element', elementIndex: element }
-  if (x === undefined || y === undefined)
-    throw new CliError('invalid_argument', 'Coordinates require both --x and --y')
-  return { kind: 'coordinate', x, y }
+  if (hasCoordinates) {
+    if (x === undefined || y === undefined)
+      throw new CliError('invalid_argument', 'Coordinates require both --x and --y')
+    return { kind: 'coordinate', x, y }
+  }
+  if (stringFlag(flags, 'goal') !== undefined) return { kind: 'intent' }
+  throw new CliError(
+    'invalid_argument',
+    allowCoordinates
+      ? 'Choose an element index, coordinates, or --goal'
+      : 'Missing required --element-index'
+  )
 }
 
 async function protectedText(flags: Flags, name: 'text' | 'value', io: CliIo): Promise<string> {
@@ -314,10 +336,19 @@ async function operationInput(command: string, flags: Flags, io: CliIo): Promise
     case 'list-windows':
       return { app: stringFlag(flags, 'app', true) }
     case 'get-app-state': {
+      const context = stringFlag(flags, 'context')
       const window = windowSelector(flags)
+      if (context !== undefined) {
+        return {
+          contextToken: context,
+          ...optionalGoal(flags),
+          ...observationFlags(flags)
+        }
+      }
       return {
         app: stringFlag(flags, 'app', true),
         ...(window === undefined ? {} : { window }),
+        ...optionalGoal(flags),
         ...observationFlags(flags)
       }
     }
@@ -329,7 +360,8 @@ async function operationInput(command: string, flags: Flags, io: CliIo): Promise
       const modifiers = parseClickModifiers(flags)
       return {
         ...common(),
-        target: pointOrElement(flags),
+        target: actionTarget(flags, { coordinates: true }),
+        ...optionalGoal(flags),
         ...(clickCount === undefined ? {} : { clickCount }),
         ...(button === undefined ? {} : { button }),
         ...(modifiers === undefined ? {} : { modifiers })
@@ -338,8 +370,9 @@ async function operationInput(command: string, flags: Flags, io: CliIo): Promise
     case 'perform-secondary-action':
       return {
         ...common(),
-        target: elementTarget(flags),
-        action: stringFlag(flags, 'action', true)
+        target: actionTarget(flags),
+        action: stringFlag(flags, 'action', true),
+        ...optionalGoal(flags)
       }
     case 'scroll': {
       const direction = stringFlag(flags, 'direction', true)!
@@ -348,8 +381,9 @@ async function operationInput(command: string, flags: Flags, io: CliIo): Promise
       const pages = numberFlag(flags, 'pages', { integer: true, min: 1 })
       return {
         ...common(),
-        target: pointOrElement(flags),
+        target: actionTarget(flags, { coordinates: true }),
         direction,
+        ...optionalGoal(flags),
         ...(pages === undefined ? {} : { pages })
       }
     }
@@ -403,8 +437,9 @@ async function operationInput(command: string, flags: Flags, io: CliIo): Promise
     case 'set-value':
       return {
         ...common(),
-        target: elementTarget(flags),
-        value: await protectedText(flags, 'value', io)
+        target: actionTarget(flags),
+        value: await protectedText(flags, 'value', io),
+        ...optionalGoal(flags)
       }
     default:
       throw new CliError('invalid_argument', `Unknown computer command: ${command}`)
@@ -428,19 +463,9 @@ function readiness(capabilities: unknown): string {
 }
 
 async function runDoctor(client: CliBrokerClient): Promise<unknown> {
-  const capabilities = await client.request('capabilities', {})
+  const capabilities = unwrapBrokerResult(await client.request('capabilities', {}))
   const permissions = await client.request('permissions', {})
-  const capabilityResult =
-    capabilities !== null && typeof capabilities === 'object' && 'result' in capabilities
-      ? (capabilities as Record<string, unknown>).result
-      : capabilities
-  return { readiness: readiness(capabilityResult), checks: { capabilities, permissions } }
-}
-
-function publicBrokerResult(value: unknown): unknown {
-  if (value !== null && typeof value === 'object' && 'requestId' in value && 'result' in value)
-    return (value as Record<string, unknown>).result
-  return value
+  return { readiness: readiness(capabilities), checks: { capabilities, permissions } }
 }
 
 function serializedError(cause: unknown): Record<string, unknown> {
@@ -524,6 +549,7 @@ async function exportScreenshot(value: unknown, destination: string): Promise<vo
 
 export async function runCli(argv: string[], io: CliIo, client: CliBrokerClient): Promise<number> {
   let json = false
+  const log = createCliJevLogger()
   try {
     if (argv[0] !== 'computer')
       throw new CliError('invalid_argument', 'Usage: crosshands computer <command> --json')
@@ -536,8 +562,13 @@ export async function runCli(argv: string[], io: CliIo, client: CliBrokerClient)
     const brokerResult =
       operation === 'doctor'
         ? await runDoctor(client)
-        : publicBrokerResult(
-            await client.request(operation, await operationInput(command, flags, io))
+        : unwrapBrokerResult(
+            await dispatchPublicOperation(
+              client,
+              operation,
+              await operationInput(command, flags, io),
+              { env: process.env, ...(log === undefined ? {} : { log }) }
+            )
           )
     const result = structuredClone(brokerResult)
     const screenshotOutput = stringFlag(flags, 'screenshot-output')
@@ -550,9 +581,14 @@ export async function runCli(argv: string[], io: CliIo, client: CliBrokerClient)
     if (!json) io.stderr(`${String(error.message)}\n`)
     return EXIT_CODES[String(error.code)] ?? 1
   } finally {
-    await client.close().catch(() => undefined)
+    await Promise.all([client.close().catch(() => undefined), log?.close().catch(() => undefined)])
   }
 }
 
-export { createProductionBrokerClient, localClientPaths } from './local-client.js'
+export { unwrapBrokerResult } from './broker-result.js'
+export { createProductionBrokerClient, localClientPaths, brokerSpawnEnv } from './local-client.js'
 export type { LocalClientPaths, ProductionClientOptions } from './local-client.js'
+export { dispatchPublicOperation } from './intent/dispatch.js'
+export { parseJevEnv } from './intent/env.js'
+export { createCliJevLogger, createJevLogger, hashGoal } from './intent/log.js'
+export type { JevLogger } from './intent/log.js'

package/src/intent/dispatch.ts

@@ -0,0 +1,373 @@
+import {
+  ComputerError,
+  createComputerError,
+  hasIntentTarget,
+  parsePublicInput,
+  parsePublicOutput,
+  PublicMutationResultSchema,
+  PublicSnapshotResultSchema,
+  SnapshotResultSchema,
+  splitPublicInput,
+  toBrokerInput,
+  type ComputerOperationName,
+  type PublicOperationInput,
+  type SnapshotResult,
+  type Suggestion
+} from '@crosshands/contract'
+
+import { unwrapBrokerResult } from '../broker-result.js'
+import { parseJevEnv, type JevEnv } from './env.js'
+import {
+  JevEvaluateError,
+  liveEvaluator,
+  namedTargetAllowed,
+  suggestionFromAnswers,
+  type EvaluateFn
+} from './evaluate.js'
+import { elapsedMs } from './http.js'
+import {
+  createJevCall,
+  emitCall,
+  recordRank,
+  type JevCall,
+  type JevLogger,
+  type JevRankStats,
+  type RankRecorder
+} from './log.js'
+import { clickableMoves, parseTreeMoves } from './tree.js'
+
+export type IntentBroker = {
+  request(operation: ComputerOperationName, input: unknown): Promise<unknown>
+}
+
+export type DispatchOptions = {
+  env?: NodeJS.ProcessEnv
+  evaluate?: EvaluateFn
+  log?: JevLogger
+}
+
+const NAMED_GATE = new Set<ComputerOperationName>(['click', 'setValue'])
+
+type RankOutcome =
+  | { ok: true; suggestion: Suggestion; stats: JevRankStats }
+  | { ok: false; stats: JevRankStats }
+
+function policyUnavailable() {
+  return createComputerError('policy_unavailable', 'Jev could not rank this snapshot').toJSON()
+}
+
+function snapshotResultOf(value: unknown): SnapshotResult {
+  return SnapshotResultSchema.parse(value)
+}
+
+function notAttempted(
+  error: ReturnType<typeof policyUnavailable>,
+  suggestion?: Suggestion
+): unknown {
+  return PublicMutationResultSchema.parse({
+    outcome: { state: 'not_attempted', error },
+    ...(suggestion === undefined ? {} : { suggestion })
+  })
+}
+
+function withSnapshotIssue(
+  look: SnapshotResult,
+  issue: ReturnType<typeof policyUnavailable>
+): unknown {
+  return PublicSnapshotResultSchema.parse({
+    ...look,
+    issues: [...look.issues, issue]
+  })
+}
+
+function contextTokenOf(input: object): string | undefined {
+  if (!('contextToken' in input) || typeof input.contextToken !== 'string') return undefined
+  return input.contextToken
+}
+
+function elementIndexOf(input: object): number | undefined {
+  if (!('target' in input)) return undefined
+  const target = input.target
+  if (target === null || typeof target !== 'object' || !('elementIndex' in target)) return undefined
+  return typeof target.elementIndex === 'number' ? target.elementIndex : undefined
+}
+
+function boundIndex(operation: ComputerOperationName, suggestion: Suggestion): number | undefined {
+  const move = suggestion.move
+  if (operation === 'setValue' && move.kind === 'setValue') return move.elementIndex
+  if (
+    (operation === 'click' || operation === 'scroll' || operation === 'performSecondaryAction') &&
+    move.kind === 'click'
+  ) {
+    return move.elementIndex
+  }
+  return undefined
+}
+
+function errorFromResult(result: unknown): string | undefined {
+  const mutation = PublicMutationResultSchema.safeParse(result)
+  if (mutation.success) {
+    const outcome = mutation.data.outcome
+    if (outcome.state === 'not_attempted' || outcome.state === 'failed') return outcome.error?.code
+    return undefined
+  }
+  const snapshot = PublicSnapshotResultSchema.safeParse(result)
+  if (!snapshot.success) return undefined
+  return snapshot.data.issues.find((issue) => issue.code === 'policy_unavailable')?.code
+}
+
+function boundFromCall(call: JevCall | undefined, result: unknown): boolean | undefined {
+  if (call?.phase !== 'bind') return undefined
+  const mutation = PublicMutationResultSchema.safeParse(result)
+  return mutation.success && mutation.data.resolvedTarget !== undefined
+}
+
+async function rank(
+  look: SnapshotResult,
+  goal: string,
+  evaluate: EvaluateFn,
+  app?: string
+): Promise<RankOutcome> {
+  const snapshotId = look.snapshot.id
+  const parseStarted = Date.now()
+  const moves = clickableMoves(parseTreeMoves(look.snapshot.treeText))
+  const stats: JevRankStats = {
+    snapshotId,
+    treeChars: look.snapshot.treeText.length,
+    elementCount: look.snapshot.elementCount,
+    candidateCount: moves.length,
+    parseMs: elapsedMs(parseStarted),
+    goalChars: goal.length,
+    app: look.snapshot.app.id
+  }
+  if (moves.length === 0) {
+    return {
+      ok: true,
+      suggestion: {
+        untrusted: true,
+        snapshotId,
+        move: { kind: 'blocked', reason: 'no_candidate' }
+      },
+      stats
+    }
+  }
+  const evalStarted = Date.now()
+  try {
+    const evaluated = await evaluate({
+      goal,
+      ...(app === undefined ? {} : { app }),
+      moves
+    })
+    return {
+      ok: true,
+      suggestion: suggestionFromAnswers(snapshotId, moves, evaluated.answers),
+      stats: {
+        ...stats,
+        evaluateMs: elapsedMs(evalStarted),
+        ...(evaluated.http === undefined ? {} : { http: evaluated.http })
+      }
+    }
+  } catch (cause) {
+    if (cause instanceof ComputerError) throw cause
+    const http = cause instanceof JevEvaluateError ? cause.http : undefined
+    return {
+      ok: false,
+      stats: {
+        ...stats,
+        evaluateMs: elapsedMs(evalStarted),
+        ...(http === undefined ? {} : { http })
+      }
+    }
+  }
+}
+
+function evaluatorFor(env: JevEnv, override: EvaluateFn | undefined): EvaluateFn | undefined {
+  if (override !== undefined) return override
+  if (env.kind !== 'on') return undefined
+  return liveEvaluator(env.apiKey)
+}
+
+async function lookAgain(request: IntentBroker['request'], token: string): Promise<SnapshotResult> {
+  return snapshotResultOf(
+    await request('getAppState', { contextToken: token, captureScreenshot: false })
+  )
+}
+
+async function suggestObserve(
+  request: IntentBroker['request'],
+  input: PublicOperationInput<'getAppState'>,
+  goal: string | undefined,
+  env: JevEnv,
+  evaluate: EvaluateFn | undefined,
+  record: RankRecorder
+): Promise<unknown> {
+  const result = await request('getAppState', toBrokerInput('getAppState', input))
+  if (goal === undefined || env.kind === 'off') return result
+  const look = snapshotResultOf(result)
+  if (env.kind === 'fail_closed' || evaluate === undefined) {
+    return withSnapshotIssue(look, policyUnavailable())
+  }
+  const app = 'app' in input && typeof input.app === 'string' ? input.app : undefined
+  try {
+    const ranked = await rank(look, goal, evaluate, app)
+    if (!ranked.ok) {
+      record('observe', ranked.stats)
+      return withSnapshotIssue(look, policyUnavailable())
+    }
+    record('observe', ranked.stats, ranked.suggestion)
+    return parsePublicOutput('getAppState', { ...look, suggestion: ranked.suggestion })
+  } catch {
+    return withSnapshotIssue(look, policyUnavailable())
+  }
+}
+
+async function bindIntentTarget(
+  request: IntentBroker['request'],
+  operation: ComputerOperationName,
+  input: object,
+  goal: string | undefined,
+  env: JevEnv,
+  evaluate: EvaluateFn | undefined,
+  record: RankRecorder
+): Promise<unknown> {
+  if (env.kind === 'off') {
+    throw createComputerError('invalid_argument', 'intent targeting requires CROSSHANDS_JEV=1')
+  }
+  if (env.kind === 'fail_closed' || evaluate === undefined || goal === undefined) {
+    throw createComputerError('intent_unavailable', 'Jev is enabled but no TypeSafe key is set')
+  }
+  const token = contextTokenOf(input)
+  if (token === undefined) {
+    throw createComputerError('invalid_argument', 'intent targeting requires a context token')
+  }
+  let look: SnapshotResult
+  try {
+    look = await lookAgain(request, token)
+  } catch (cause) {
+    if (cause instanceof ComputerError) throw cause
+    return notAttempted(policyUnavailable())
+  }
+  const ranked = await rank(look, goal, evaluate)
+  if (!ranked.ok) {
+    record('bind', ranked.stats)
+    return notAttempted(policyUnavailable())
+  }
+  record('bind', ranked.stats, ranked.suggestion)
+  const index = boundIndex(operation, ranked.suggestion)
+  if (index === undefined) return notAttempted(policyUnavailable(), ranked.suggestion)
+  const bound = {
+    ...splitPublicInput(input).rest,
+    contextToken: look.context.token,
+    target: { kind: 'element' as const, elementIndex: index }
+  }
+  const result = await request(operation, toBrokerInput(operation, bound))
+  const body = result !== null && typeof result === 'object' ? result : {}
+  return parsePublicOutput(operation, {
+    ...body,
+    resolvedTarget: { kind: 'element', elementIndex: index },
+    suggestion: ranked.suggestion
+  })
+}
+
+async function gateNamedTarget(
+  request: IntentBroker['request'],
+  input: object,
+  goal: string,
+  evaluate: EvaluateFn,
+  record: RankRecorder
+): Promise<unknown | undefined> {
+  const named = elementIndexOf(input)
+  const token = contextTokenOf(input)
+  if (token === undefined || named === undefined) return undefined
+  try {
+    const look = await lookAgain(request, token)
+    const ranked = await rank(look, goal, evaluate)
+    if (!ranked.ok) {
+      record('named', ranked.stats)
+      return notAttempted(policyUnavailable())
+    }
+    record('named', ranked.stats, ranked.suggestion)
+    const picked =
+      ranked.suggestion.move.kind === 'click' || ranked.suggestion.move.kind === 'setValue'
+        ? ranked.suggestion.move.elementIndex
+        : undefined
+    if (picked !== named || !namedTargetAllowed(ranked.suggestion.confidence)) {
+      return notAttempted(
+        createComputerError(
+          'goal_mismatch',
+          'Named target does not match the per-call goal'
+        ).toJSON(),
+        ranked.suggestion
+      )
+    }
+  } catch {
+    return notAttempted(policyUnavailable())
+  }
+  return undefined
+}
+
+export async function dispatchPublicOperation(
+  broker: IntentBroker,
+  operation: ComputerOperationName,
+  rawInput: unknown,
+  options: DispatchOptions = {}
+): Promise<unknown> {
+  const envState = parseJevEnv(options.env)
+  const parsed = parsePublicInput(operation, rawInput)
+  const { goal } = splitPublicInput(parsed)
+  const evaluate = evaluatorFor(envState, options.evaluate)
+  const log = options.log
+  const call = log === undefined ? undefined : createJevCall(operation, goal)
+  const record: RankRecorder = (phase, stats, suggestion) => {
+    recordRank(log, call, phase, stats, suggestion)
+  }
+  const request: IntentBroker['request'] = async (name, input) => {
+    if (call !== undefined) call.brokerCalls += 1
+    return unwrapBrokerResult(await broker.request(name, input))
+  }
+
+  async function dispatch(): Promise<unknown> {
+    if (operation === 'getAppState') {
+      return suggestObserve(
+        request,
+        parsed as PublicOperationInput<'getAppState'>,
+        goal,
+        envState,
+        evaluate,
+        record
+      )
+    }
+    if (hasIntentTarget(parsed)) {
+      return bindIntentTarget(request, operation, parsed, goal, envState, evaluate, record)
+    }
+    if (
+      goal !== undefined &&
+      envState.kind === 'on' &&
+      evaluate !== undefined &&
+      NAMED_GATE.has(operation)
+    ) {
+      const gated = await gateNamedTarget(request, parsed, goal, evaluate, record)
+      if (gated !== undefined) return gated
+    }
+    return request(operation, toBrokerInput(operation, parsed))
+  }
+
+  const started = Date.now()
+  try {
+    const result = await dispatch()
+    if (call !== undefined) {
+      const extra: { error?: string; bound?: boolean } = {}
+      const error = errorFromResult(result)
+      const bound = boundFromCall(call, result)
+      if (error !== undefined) extra.error = error
+      if (bound !== undefined) extra.bound = bound
+      emitCall(log, call, started, extra)
+    }
+    return result
+  } catch (cause) {
+    if (call !== undefined) {
+      emitCall(log, call, started, cause instanceof ComputerError ? { error: cause.code } : {})
+    }
+    throw cause
+  }
+}

package/src/intent/env.ts

@@ -0,0 +1,13 @@
+export type JevEnv =
+  | { kind: 'off' }
+  | { kind: 'fail_closed'; reason: 'missing_key' }
+  | { kind: 'on'; apiKey: string }
+
+export function parseJevEnv(env: NodeJS.ProcessEnv = process.env): JevEnv {
+  if (env.CROSSHANDS_JEV !== '1') return { kind: 'off' }
+  const apiKey = env.TYPESAFE_API_KEY
+  if (typeof apiKey !== 'string' || apiKey.length === 0) {
+    return { kind: 'fail_closed', reason: 'missing_key' }
+  }
+  return { kind: 'on', apiKey }
+}