@crosshands/cli 0.2.0 → 0.2.1

package/src/intent/evaluate.ts

@@ -1,7 +1,19 @@
 import type { Suggestion } from '@crosshands/contract'
 
+import { elapsedMs, type JevHttpStats } from './http.js'
 import type { TreeMove } from './tree.js'
 
+export type { JevHttpStats }
+
+export class JevEvaluateError extends Error {
+  readonly http: JevHttpStats
+  constructor(message: string, http: JevHttpStats) {
+    super(message)
+    this.name = 'JevEvaluateError'
+    this.http = http
+  }
+}
+
 export type JevAnswers = {
   move: 'click' | 'setValue' | 'wait' | 'done' | 'blocked'
   clickWhich?: string
@@ -11,13 +23,18 @@ export type JevAnswers = {
   blockedReason?: string
 }
 
+export type EvaluateResult = {
+  answers: JevAnswers
+  http?: JevHttpStats
+}
+
 export type EvaluateInput = {
   goal: string
   app?: string
   moves: readonly TreeMove[]
 }
 
-export type EvaluateFn = (input: EvaluateInput) => Promise<JevAnswers>
+export type EvaluateFn = (input: EvaluateInput) => Promise<EvaluateResult>
 
 export const GATE_REFUSE_BELOW = 0.35
 
@@ -78,78 +95,117 @@ type SystemOneAnswer = {
   confidence?: number
 }
 
+const JEV_MODEL = 'jev-latest'
+
+function httpStats(
+  started: number,
+  requestBytes: number,
+  status: number,
+  responseBytes: number
+): JevHttpStats {
+  return {
+    status,
+    durationMs: elapsedMs(started),
+    requestBytes,
+    responseBytes
+  }
+}
+
+function answersOf(raw: Record<string, SystemOneAnswer>): JevAnswers {
+  const move = raw.move?.choice
+  if (
+    move !== 'click' &&
+    move !== 'setValue' &&
+    move !== 'wait' &&
+    move !== 'done' &&
+    move !== 'blocked'
+  ) {
+    return { move: 'blocked', blockedReason: 'no_candidate' }
+  }
+  const clickWhich = raw.click_which?.choice
+  const setValueWhich = raw.setvalue_which?.choice
+  const confidence = raw.move?.confidence ?? raw.click_which?.confidence
+  const goalMet = raw.goal_met?.noul
+  return {
+    move,
+    ...(clickWhich === undefined ? {} : { clickWhich }),
+    ...(setValueWhich === undefined ? {} : { setValueWhich }),
+    ...(confidence === undefined ? {} : { confidence }),
+    ...(goalMet === undefined ? {} : { goalMet })
+  }
+}
+
 export function liveEvaluator(apiKey: string): EvaluateFn {
   return async (input) => {
     const clickable = criteria(input.moves)
-    const response = await fetch('https://api.typesafe.ai/v1/systemone', {
-      method: 'POST',
-      headers: {
-        authorization: `Bearer ${apiKey}`,
-        'content-type': 'application/json'
+    const payload = JSON.stringify({
+      model: JEV_MODEL,
+      state: {
+        goal: input.goal,
+        ...(input.app === undefined ? {} : { app: input.app }),
+        clickable
       },
-      body: JSON.stringify({
-        model: 'jev-latest',
-        state: {
-          goal: input.goal,
-          ...(input.app === undefined ? {} : { app: input.app }),
-          clickable
-        },
-        questions: {
-          move: {
-            type: 'choice',
-            instructions: 'What should we do next for the goal?',
-            criteria: {
-              click: 'Press one clickable control',
-              setValue: 'Pick a field for the caller to fill',
-              wait: 'The window is still loading',
-              done: 'The goal is already met',
-              blocked: 'Stop'
-            }
-          },
-          click_which: {
-            type: 'choice',
-            instructions: 'If the move is click, which control?',
-            criteria: clickable
-          },
-          setvalue_which: {
-            type: 'choice',
-            instructions: 'If the move is setValue, which field?',
-            criteria: clickable
-          },
-          goal_met: {
-            type: 'noul',
-            instructions: 'Is the goal already met?'
+      questions: {
+        move: {
+          type: 'choice',
+          instructions: 'What should we do next for the goal?',
+          criteria: {
+            click: 'Press one clickable control',
+            setValue: 'Pick a field for the caller to fill',
+            wait: 'The window is still loading',
+            done: 'The goal is already met',
+            blocked: 'Stop'
           }
+        },
+        click_which: {
+          type: 'choice',
+          instructions: 'If the move is click, which control?',
+          criteria: clickable
+        },
+        setvalue_which: {
+          type: 'choice',
+          instructions: 'If the move is setValue, which field?',
+          criteria: clickable
+        },
+        goal_met: {
+          type: 'noul',
+          instructions: 'Is the goal already met?'
         }
-      })
+      }
     })
-    if (!response.ok) {
-      throw new Error(`TypeSafe HTTP ${response.status}`)
-    }
-    const body = (await response.json()) as {
-      answers?: Record<string, SystemOneAnswer>
+    const requestBytes = Buffer.byteLength(payload)
+    const started = Date.now()
+    let status = 0
+    let responseBytes = 0
+    let raw = ''
+    try {
+      const response = await fetch('https://api.typesafe.ai/v1/systemone', {
+        method: 'POST',
+        headers: {
+          authorization: `Bearer ${apiKey}`,
+          'content-type': 'application/json'
+        },
+        body: payload
+      })
+      status = response.status
+      raw = await response.text()
+      responseBytes = Buffer.byteLength(raw)
+    } catch (cause) {
+      throw new JevEvaluateError(
+        cause instanceof Error ? cause.message : 'TypeSafe HTTP failed',
+        httpStats(started, requestBytes, 0, 0)
+      )
     }
-    const answers = body.answers ?? {}
-    const move = answers.move?.choice
-    if (
-      move !== 'click' &&
-      move !== 'setValue' &&
-      move !== 'wait' &&
-      move !== 'done' &&
-      move !== 'blocked'
-    ) {
-      return { move: 'blocked', blockedReason: 'no_candidate' }
+    const http = httpStats(started, requestBytes, status, responseBytes)
+    if (status < 200 || status >= 300) {
+      throw new JevEvaluateError(`TypeSafe HTTP ${status}`, http)
     }
-    const clickWhich = answers.click_which?.choice
-    const setValueWhich = answers.setvalue_which?.choice
-    const confidence = answers.move?.confidence ?? answers.click_which?.confidence
-    const goalMet = answers.goal_met?.noul
-    return {
-      move,
-      ...(clickWhich === undefined ? {} : { clickWhich }),
-      ...(setValueWhich === undefined ? {} : { setValueWhich }),
-      ...(confidence === undefined ? {} : { confidence }),
-      ...(goalMet === undefined ? {} : { goalMet })
+    let parsed: { answers?: Record<string, SystemOneAnswer> }
+    try {
+      parsed = JSON.parse(raw) as { answers?: Record<string, SystemOneAnswer> }
+    } catch {
+      throw new JevEvaluateError('TypeSafe HTTP response was not JSON', http)
     }
+    return { answers: answersOf(parsed.answers ?? {}), http }
   }
 }

package/src/intent/http.ts

@@ -0,0 +1,10 @@
+export type JevHttpStats = {
+  status: number
+  durationMs: number
+  requestBytes: number
+  responseBytes: number
+}
+
+export function elapsedMs(started: number): number {
+  return Math.max(0, Date.now() - started)
+}

package/src/intent/log.ts

@@ -1,21 +1,185 @@
 import { createHash, randomUUID } from 'node:crypto'
 
+import type { Suggestion } from '@crosshands/contract'
 import { JsonlFileWriter, diagnosticsDirectory } from '@crosshands/runtime'
 
 import { localClientPaths } from '../local-client.js'
 import { parseJevEnv } from './env.js'
+import { elapsedMs, type JevHttpStats } from './http.js'
+
+export type JevPhase = 'observe' | 'bind' | 'named'
+
+type JevShared = {
+  callId: string
+  operation: string
+  snapshotId: string
+  phase: JevPhase
+  goal?: string
+}
+
+export type JevCallRecord = {
+  kind: 'jev.call'
+  callId: string
+  operation: string
+  durationMs: number
+  httpMs: number
+  brokerCalls: number
+  ranks: number
+  phase?: JevPhase
+  bound?: boolean
+  error?: string
+  goal?: string
+}
+
+export type JevHttpRecord = JevShared & {
+  kind: 'jev.http'
+  status: number
+  durationMs: number
+  requestBytes: number
+  responseBytes: number
+  candidateCount: number
+}
+
+export type JevDecisionRecord = JevShared & {
+  kind: 'jev.decision'
+  app?: string
+  move: Suggestion['move']['kind']
+  elementIndex?: number
+  reason?: string
+  confidence?: number
+  goalChars: number
+  treeChars: number
+  elementCount: number
+  candidateCount: number
+  parseMs: number
+  evaluateMs?: number
+}
+
+export type JevRecord = JevCallRecord | JevHttpRecord | JevDecisionRecord
 
 export type JevLogger = {
-  emit(record: Record<string, unknown>): void
+  emit(record: JevRecord): void
   close(): Promise<void>
 }
 
+export type JevCall = {
+  callId: string
+  operation: string
+  goal?: string
+  ranks: number
+  httpMs: number
+  brokerCalls: number
+  phase?: JevPhase
+}
+
+export type JevRankStats = {
+  snapshotId: string
+  treeChars: number
+  elementCount: number
+  candidateCount: number
+  parseMs: number
+  goalChars: number
+  evaluateMs?: number
+  app?: string
+  http?: JevHttpStats
+}
+
+export type RankRecorder = (phase: JevPhase, stats: JevRankStats, suggestion?: Suggestion) => void
+
 export function hashGoal(goal: string): string {
   return createHash('sha256').update(goal).digest('hex')
 }
 
 const OMIT = new Set(['goal', 'apiKey', 'treeText', 'text', 'value', 'TYPESAFE_API_KEY'])
 
+export function createJevCall(operation: string, goal?: string): JevCall {
+  return {
+    callId: randomUUID(),
+    operation,
+    ...(goal === undefined ? {} : { goal }),
+    ranks: 0,
+    httpMs: 0,
+    brokerCalls: 0
+  }
+}
+
+export function recordRank(
+  log: JevLogger | undefined,
+  call: JevCall | undefined,
+  phase: JevPhase,
+  stats: JevRankStats,
+  suggestion?: Suggestion
+): void {
+  if (call !== undefined) {
+    call.phase = phase
+    call.ranks += 1
+    if (stats.http !== undefined) call.httpMs += stats.http.durationMs
+  }
+  if (log === undefined || call === undefined) return
+  const shared: JevShared = {
+    callId: call.callId,
+    operation: call.operation,
+    snapshotId: stats.snapshotId,
+    phase,
+    ...(call.goal === undefined ? {} : { goal: call.goal })
+  }
+  if (stats.http !== undefined) {
+    log.emit({
+      kind: 'jev.http',
+      ...shared,
+      status: stats.http.status,
+      durationMs: stats.http.durationMs,
+      requestBytes: stats.http.requestBytes,
+      responseBytes: stats.http.responseBytes,
+      candidateCount: stats.candidateCount
+    })
+  }
+  if (suggestion === undefined) return
+  const move = suggestion.move
+  log.emit({
+    kind: 'jev.decision',
+    ...shared,
+    ...(stats.app === undefined ? {} : { app: stats.app }),
+    move: move.kind,
+    ...(move.kind === 'click' ||
+    move.kind === 'setValue' ||
+    move.kind === 'secondary' ||
+    move.kind === 'scroll'
+      ? { elementIndex: move.elementIndex }
+      : {}),
+    ...(move.kind === 'blocked' ? { reason: move.reason } : {}),
+    ...(suggestion.confidence === undefined ? {} : { confidence: suggestion.confidence }),
+    goalChars: stats.goalChars,
+    treeChars: stats.treeChars,
+    elementCount: stats.elementCount,
+    candidateCount: stats.candidateCount,
+    parseMs: stats.parseMs,
+    ...(stats.evaluateMs === undefined ? {} : { evaluateMs: stats.evaluateMs })
+  })
+}
+
+export function emitCall(
+  log: JevLogger | undefined,
+  call: JevCall | undefined,
+  started: number,
+  extra: { error?: string; bound?: boolean } = {}
+): void {
+  if (log === undefined || call === undefined) return
+  log.emit({
+    kind: 'jev.call',
+    callId: call.callId,
+    operation: call.operation,
+    durationMs: elapsedMs(started),
+    httpMs: call.httpMs,
+    brokerCalls: call.brokerCalls,
+    ranks: call.ranks,
+    ...(call.phase === undefined ? {} : { phase: call.phase }),
+    ...(extra.bound === undefined ? {} : { bound: extra.bound }),
+    ...(extra.error === undefined ? {} : { error: extra.error }),
+    ...(call.goal === undefined ? {} : { goal: call.goal })
+  })
+}
+
 export function createJevLogger(
   graphicalSessionId: string,
   env: NodeJS.ProcessEnv = process.env
@@ -27,15 +191,23 @@ export function createJevLogger(
   writer.start()
   return {
     emit(record) {
-      const kind = record.kind
-      if (typeof kind !== 'string') return
-      const goal = record.goal
-      const rest: Record<string, unknown> = { ...record, kind, v: 1, ts: new Date().toISOString() }
-      for (const key of OMIT) delete rest[key]
-      writer.emit({
-        ...rest,
-        ...(typeof goal === 'string' ? { goalSha256: hashGoal(goal) } : {})
-      })
+      try {
+        const raw = record as JevRecord & Record<string, unknown>
+        const goal = raw.goal
+        const rest: Record<string, unknown> = {
+          ...raw,
+          kind: record.kind,
+          v: 1,
+          ts: new Date().toISOString()
+        }
+        for (const key of OMIT) delete rest[key]
+        writer.emit({
+          ...rest,
+          ...(typeof goal === 'string' ? { goalSha256: hashGoal(goal) } : {})
+        })
+      } catch {
+        // Jev logs must not fail computer-use.
+      }
     },
     close: () => writer.close()
   }