@frontera-sdk/cli 1.52.38 → 1.52.40

package/src/local/local-test-baseline.ts

@@ -0,0 +1,113 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import type { LocalProject } from './local-project'
+import type { CaseResult } from './local-test-runner'
+
+/**
+ * The outcome of each case at its last local run, so a run can say which
+ * cases its change made fail (newly failing) rather than leave the agent to
+ * compare two lists. Written by `frontera local test` and the MCP `test_run`,
+ * one entry per case, in the runtime's state folder.
+ */
+
+export interface CaseOutcome {
+  passed: boolean
+  at: string
+  reason?: string
+  ms?: number
+  toolCalls?: number
+}
+
+export type LastRun = Record<string, CaseOutcome>
+
+/** A case's key: its Agent, file and name. */
+export function caseKey(r: Pick<CaseResult, 'agent' | 'file' | 'name'>): string {
+  return `${r.agent}\u0000${r.file}\u0000${r.name}`
+}
+
+function lastRunPath(project: LocalProject): string {
+  return join(project.stateDir, 'test-results.json')
+}
+
+export function readLastRun(project: LocalProject): LastRun {
+  const file = lastRunPath(project)
+  if (!existsSync(file)) return {}
+  try {
+    return JSON.parse(readFileSync(file, 'utf8')) as LastRun
+  } catch {
+    return {}
+  }
+}
+
+/**
+ * Keep these results as each case's last run; cases that did not run keep
+ * theirs, as does a case that couldn't run (the provider refused it, the
+ * runtime did not answer): it says nothing about the Agent.
+ */
+export function recordRun(project: LocalProject, results: CaseResult[], at = new Date().toISOString()): void {
+  const last = readLastRun(project)
+  for (const r of results) if (!r.notRun) last[caseKey(r)] = { passed: r.passed, at, ms: r.ms, ...(r.toolCalls !== undefined ? { toolCalls: r.toolCalls } : {}), ...(r.passed || !r.reason ? {} : { reason: r.reason.slice(0, 300) }) }
+  mkdirSync(project.stateDir, { recursive: true })
+  writeFileSync(lastRunPath(project), `${JSON.stringify(last, null, 2)}\n`)
+}
+
+export interface RunComparison {
+  /** Passed at their last run and fail now: what the change broke. */
+  newlyFailing: CaseResult[]
+  /** Failed at their last run and pass now. */
+  fixed: CaseResult[]
+  /** Failed both times. */
+  stillFailing: CaseResult[]
+  /** No earlier run to compare with. */
+  firstRun: CaseResult[]
+  /**
+   * Couldn't run (the model provider refused them, the turn timed out, the
+   * runtime did not answer): neither newly failing nor still failing, whatever
+   * their last run was.
+   */
+  couldNotRun: CaseResult[]
+}
+
+export function compareWithLastRun(results: CaseResult[], last: LastRun): RunComparison {
+  const out: RunComparison = { newlyFailing: [], fixed: [], stillFailing: [], firstRun: [], couldNotRun: [] }
+  for (const r of results) {
+    const before = last[caseKey(r)]
+    if (r.notRun) out.couldNotRun.push(r)
+    else if (!before) out.firstRun.push(r)
+    else if (before.passed && !r.passed) out.newlyFailing.push(r)
+    else if (!before.passed && r.passed) out.fixed.push(r)
+    else if (!before.passed && !r.passed) out.stillFailing.push(r)
+  }
+  return out
+}
+
+/**
+ * Each case's run when its Agent was last applied to the workspace by a
+ * change request: the baseline a request's evals are compared with (design
+ * 2026-10-18 §6). Kept beside the last run; an Agent never applied from here
+ * has none, and its evals compare with the last run instead.
+ */
+function appliedRunPath(project: LocalProject): string {
+  return join(project.stateDir, 'test-results-applied.json')
+}
+
+export function readAppliedRun(project: LocalProject): LastRun {
+  const file = appliedRunPath(project)
+  if (!existsSync(file)) return {}
+  try {
+    return JSON.parse(readFileSync(file, 'utf8')) as LastRun
+  } catch {
+    return {}
+  }
+}
+
+/** These Agents were applied: their cases' last runs become their applied baseline. */
+export function recordApplied(project: LocalProject, agents: string[]): void {
+  if (!agents.length) return
+  const last = readLastRun(project)
+  const applied = readAppliedRun(project)
+  for (const [key, outcome] of Object.entries(last)) if (agents.includes(key.split('\u0000')[0]!)) applied[key] = outcome
+  mkdirSync(project.stateDir, { recursive: true })
+  writeFileSync(appliedRunPath(project), `${JSON.stringify(applied, null, 2)}\n`)
+}

package/src/local/local-test-cases.ts

@@ -31,13 +31,45 @@ function block(text: string, indent: string): string {
   return `|-\n${text.split('\n').map((line) => `${indent}${line}`).join('\n')}`
 }
 
-export function renderTestCase(args: { conversationId: string; messages: ChatMessage[]; savedAt: string }): string {
+/** How a saved case answers the tool approvals its conversation stopped on: as the person did. */
+export type SavedDecision = 'approve' | 'deny'
+
+/** An approval card's status once someone approved it: approved, then run (executing, succeeded, failed, ambiguous). */
+const APPROVED = new Set(['approved', 'executing', 'succeeded', 'failed', 'ambiguous', 'resumed'])
+
+/**
+ * How the person decided the tool approvals a conversation stopped on, from
+ * the runtime's list of its cards (`/v1/chat/approvals?conversationId=`):
+ * `approve` when every decided card was approved, `deny` when every one was
+ * denied; nothing when none was decided (pending, expired) or the cards were
+ * decided both ways, which one case decision can't replay (`mixed`).
+ */
+export function decisionOfApprovals(cards: Array<{ status?: unknown; pauseReason?: unknown }>): { decision?: SavedDecision; mixed?: true } {
+  const decided = cards
+    .filter((c) => !c.pauseReason || c.pauseReason === 'needs_approval')
+    .map((c) => (typeof c.status === 'string' && APPROVED.has(c.status) ? 'approve' : c.status === 'denied' ? 'deny' : null))
+    .filter((d): d is SavedDecision => d !== null)
+  const kinds = new Set(decided)
+  if (kinds.size > 1) return { mixed: true }
+  return kinds.size ? { decision: [...kinds][0]! } : {}
+}
+
+/** What saving a case says when its conversation's cards were decided both ways. */
+export const MIXED_DECISIONS_LINE = 'the conversation approved some tool approvals and denied others; a case answers every one the same way, so it was saved without `decision`: add `decision: approve` or `deny` by hand, or split it into one case per turn'
+
+export function renderTestCase(args: { conversationId: string; messages: ChatMessage[]; savedAt: string; variables?: Record<string, string | number | boolean>; decision?: SavedDecision }): string {
   const turns = args.messages.filter((m) => (m.role === 'user' || m.role === 'assistant') && textOf(m))
   const lastReply = [...turns].reverse().find((m) => m.role === 'assistant')
   const inputs = turns.slice(0, lastReply ? turns.lastIndexOf(lastReply) : turns.length)
   const lines = [
     `- name: conversation-${args.conversationId.slice(0, 8)}`,
     `  savedFrom: studio ${args.savedAt}`,
+    // The tool approvals it stopped on, answered as the person did, so it replays to the end.
+    ...(args.decision ? [`  decision: ${args.decision}`] : []),
+    // The variables the conversation started with, so the case replays as the same customer.
+    ...(args.variables && Object.keys(args.variables).length
+      ? ['  variables:', ...Object.entries(args.variables).map(([k, v]) => `    ${JSON.stringify(k)}: ${JSON.stringify(v)}`)]
+      : []),
     '  conversation:',
     ...inputs.map((m) => `    - ${m.role}: ${block(textOf(m), '        ')}`),
     `  expected: ${block(lastReply ? textOf(lastReply) : '', '    ')}`,
@@ -56,3 +88,46 @@ export function appendTestCase(root: string, agentSlug: string, yaml: string): s
   appendFileSync(file, yaml.replace(/^/gm, '  ').replace(/ +$/gm, ''))
   return relative(root, file)
 }
+
+/**
+ * The variables a conversation's Test setup recorded (`variables` on its
+ * `TRY_SETUP` session event, when the runtime records them), as a case file
+ * holds them: plain values only.
+ */
+export function variablesOfTimeline(timeline: Array<{ type?: string; payload?: Record<string, unknown> | null }>): Record<string, string | number | boolean> | undefined {
+  const setup = timeline.find((e) => e.payload?.event === 'TRY_SETUP')
+  const variables = setup?.payload?.variables
+  return variables && typeof variables === 'object' && !Array.isArray(variables) ? plainVariables(variables as Record<string, unknown>) : undefined
+}
+
+/** A conversation's variables a case can carry (strings, numbers, booleans), or undefined when none are. */
+export function plainVariables(raw: Record<string, unknown> | undefined): Record<string, string | number | boolean> | undefined {
+  if (!raw) return undefined
+  const plain = Object.fromEntries(Object.entries(raw).filter((e): e is [string, string | number | boolean] => ['string', 'number', 'boolean'].includes(typeof e[1])))
+  return Object.keys(plain).length ? plain : undefined
+}
+
+/**
+ * The sensitive variables a conversation's Test setup was given
+ * (`redactedVariables` on its `TRY_SETUP` session event): their names only,
+ * since the runtime never records their values. A case saved from the
+ * conversation lacks them until someone fills them in by hand.
+ */
+export function redactedVariablesOfTimeline(timeline: Array<{ type?: string; payload?: Record<string, unknown> | null }>): string[] {
+  const setup = timeline.find((e) => e.payload?.event === 'TRY_SETUP')
+  const keys = setup?.payload?.redactedVariables
+  return Array.isArray(keys) ? [...new Set(keys.filter((k): k is string => typeof k === 'string' && k !== ''))] : []
+}
+
+/** A case (one list item, as rendered) with a comment under its name naming the variables to fill in by hand. */
+export function noteRedactedVariables(yaml: string, keys: string[]): string {
+  if (!keys.length) return yaml
+  const at = yaml.indexOf('\n')
+  const note = `  # Fill in by hand under variables: ${keys.join(', ')} (sensitive: the conversation's values are never recorded)`
+  return at < 0 ? `${yaml}\n${note}\n` : `${yaml.slice(0, at + 1)}${note}\n${yaml.slice(at + 1)}`
+}
+
+/** What saving a case says of the sensitive variables it could not keep. */
+export function redactedVariablesLine(keys: string[], path: string): string {
+  return `not kept: ${keys.join(', ')} (sensitive: the Test setup records ${keys.length === 1 ? 'its name' : 'their names'}, never ${keys.length === 1 ? 'its value' : 'their values'}). Fill ${keys.length === 1 ? 'it' : 'them'} in by hand under the case's variables: in ${path}, or the case starts without ${keys.length === 1 ? 'it' : 'them'}.`
+}