@frontera-sdk/cli 1.52.28 → 1.52.29

package/src/local/local-synthetic-data.ts

@@ -0,0 +1,374 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+
+import { base, en, Faker } from '@faker-js/faker'
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
+
+/**
+ * Sample rows for a project's Blueprint, so a Blueprint pulled from a
+ * workspace (which brings no records) can be explored, queried and used by
+ * Agents without copying customer data to a laptop.
+ *
+ * Each object type gets a CSV under `datasets/` with the definition beside
+ * it. Values fit each property: its type, its name (an `email` gets an email,
+ * an `amount` money) and the values its description lists ("open, pending or
+ * closed"). Links stay consistent: a property another type's records are
+ * linked by gets a value from those records, so every relation resolves.
+ * The same seed gives the same rows.
+ *
+ * `data/generate.ts` in the project, when present, adjusts it: rows per type
+ * and a function per property, for scenarios ("five tickets breaching SLA").
+ */
+
+interface Property {
+  apiName: string
+  displayName?: string
+  description?: string
+  valueType?: string
+  required?: boolean
+  propertyType?: string
+}
+
+interface ObjectType {
+  apiName: string
+  displayName?: string
+  pluralDisplayName?: string
+  primaryKey?: string
+  properties?: Property[]
+  backing?: { dataset?: string; mapping?: Record<string, string> }
+}
+
+interface LinkType {
+  apiName: string
+  cardinality?: string
+  from?: { objectType?: string; property?: string }
+  to?: { objectType?: string; property?: string }
+}
+
+/** What `data/generate.ts` may return. */
+export interface Scenario {
+  rows?: number
+  types?: Record<string, { rows?: number; fields?: Record<string, (row: Record<string, unknown>, index: number) => unknown> }>
+}
+
+export interface GeneratedDataset {
+  objectType: string
+  dataset: string
+  rows: number
+  csvPath: string
+  definitionPath: string
+}
+
+export interface SyntheticOptions {
+  rows?: number
+  seed?: number
+  /** Dates are spread over the months before this day; fixed by default so a seed gives the same rows. */
+  today?: Date
+  scenario?: Scenario
+}
+
+function readTree<T>(dir: string): Array<{ file: string; doc: T }> {
+  if (!existsSync(dir)) return []
+  return readdirSync(dir)
+    .filter((f) => /\.(ya?ml|json)$/.test(f) && !f.startsWith('.'))
+    .sort()
+    .map((f) => {
+      const text = readFileSync(join(dir, f), 'utf8')
+      return { file: f, doc: (f.endsWith('.json') ? JSON.parse(text) : parseYaml(text)) as T }
+    })
+}
+
+export function snake(name: string): string {
+  return name.replace(/([a-z0-9])([A-Z])/g, '$1_$2').replace(/[^A-Za-z0-9]+/g, '_').toLowerCase().replace(/^_|_$/g, '')
+}
+
+function plural(name: string): string {
+  if (/[^aeiou]y$/.test(name)) return `${name.slice(0, -1)}ies`
+  if (/(s|x|ch|sh)$/.test(name)) return `${name}es`
+  return `${name}s`
+}
+
+/** The dataset an object type reads, and its column per property. */
+export function backingOf(type: ObjectType): { dataset: string; column: (property: string) => string } {
+  const dataset = type.backing?.dataset || plural(snake(type.apiName))
+  const mapping = type.backing?.mapping ?? {}
+  return { dataset, column: (p) => mapping[p] ?? snake(p) }
+}
+
+/** "open, pending, resolved or closed" → the four values, when a description lists them. */
+export function listedValues(description: string | undefined): string[] | null {
+  if (!description) return null
+  const text = description.replace(/^[^:]*:\s*/, '').trim()
+  if (!/,|\bor\b/.test(text) || text.length > 160) return null
+  const parts = text.split(/\s*,\s*|\s+or\s+|\s+and\s+/).map((p) => p.trim().replace(/^["'`]|["'`.]$/g, ''))
+  if (parts.length < 2 || parts.some((p) => !/^[A-Za-z0-9_-]{1,32}$/.test(p))) return null
+  return [...new Set(parts)]
+}
+
+/** A short, readable key prefix from the type's name: Ticket → TCK, Customer → CUS. */
+function keyPrefix(apiName: string): string {
+  const upper = apiName.toUpperCase().replace(/[^A-Z]/g, '')
+  const consonants = upper[0]! + upper.slice(1).replace(/[AEIOU]/g, '')
+  return (consonants.length >= 3 ? consonants : upper).slice(0, 3).padEnd(3, 'X')
+}
+
+interface Reference {
+  /** The type whose records this property points at. */
+  objectType: string
+  property: string
+}
+
+/** Which property of which type points at which other type's key, from the link types. */
+export function referencesFrom(links: LinkType[]): Map<string, Map<string, Reference>> {
+  const out = new Map<string, Map<string, Reference>>()
+  const add = (type: string, property: string, ref: Reference) => {
+    if (!out.has(type)) out.set(type, new Map())
+    out.get(type)!.set(property, ref)
+  }
+  for (const link of links) {
+    const from = link.from
+    const to = link.to
+    if (!from?.objectType || !from.property || !to?.objectType || !to.property) continue
+    if (link.cardinality === 'many_to_one') add(from.objectType, from.property, { objectType: to.objectType, property: to.property })
+    else if (link.cardinality === 'many_to_many') continue
+    else add(to.objectType, to.property, { objectType: from.objectType, property: from.property })
+  }
+  return out
+}
+
+/** Types in an order where every referenced type comes before the types that point at it. */
+function dependencyOrder(types: ObjectType[], refs: Map<string, Map<string, Reference>>): ObjectType[] {
+  const byName = new Map(types.map((t) => [t.apiName, t]))
+  const done = new Set<string>()
+  const out: ObjectType[] = []
+  const visit = (name: string, trail: Set<string>) => {
+    if (done.has(name) || trail.has(name) || !byName.has(name)) return
+    trail.add(name)
+    for (const ref of refs.get(name)?.values() ?? []) if (ref.objectType !== name) visit(ref.objectType, trail)
+    trail.delete(name)
+    done.add(name)
+    out.push(byName.get(name)!)
+  }
+  for (const t of types) visit(t.apiName, new Set())
+  return out
+}
+
+function words(name: string): string {
+  return name.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase()
+}
+
+/** A value for one property, from its type, its name and its description. */
+function valueFor(faker: Faker, type: ObjectType, p: Property, row: Record<string, unknown>, state: { lastTime: number | null; today: Date }): unknown {
+  const name = words(p.apiName)
+  const listed = listedValues(p.description)
+  if (listed && (p.valueType ?? 'string') === 'string') return faker.helpers.arrayElement(listed)
+  switch (p.valueType ?? 'string') {
+    case 'boolean':
+      return faker.datatype.boolean({ probability: /^(is|has) /.test(name) ? 0.35 : 0.5 })
+    case 'number': {
+      if (/csat|rating|stars|score/.test(name)) return faker.number.int({ min: 1, max: 5 })
+      if (/percent|rate|ratio|share/.test(name)) return faker.number.float({ min: 0, max: 100, fractionDigits: 1 })
+      if (/hours?\b|sla/.test(name)) return faker.helpers.arrayElement([4, 8, 24, 48, 72])
+      if (/days?\b/.test(name)) return faker.number.int({ min: 1, max: 30 })
+      if (/quantity|qty|count|units|items/.test(name)) return faker.number.int({ min: 1, max: 6 })
+      if (/age\b/.test(name)) return faker.number.int({ min: 18, max: 80 })
+      if (/price|amount|total|revenue|cost|value|fee|refund|balance|spend|discount/.test(name)) return Number(faker.commerce.price({ min: 5, max: 900 }))
+      if (/weight|kg/.test(name)) return faker.number.float({ min: 0.1, max: 25, fractionDigits: 2 })
+      if (/lat/.test(name)) return faker.location.latitude()
+      if (/lon|lng/.test(name)) return faker.location.longitude()
+      return faker.number.int({ min: 1, max: 1000 })
+    }
+    case 'date':
+      return faker.date.past({ years: 2, refDate: state.today }).toISOString().slice(0, 10)
+    case 'timestamp': {
+      // Times in a row follow one another (opened, then answered, then resolved).
+      const at = state.lastTime === null
+        ? faker.date.recent({ days: 120, refDate: state.today }).getTime()
+        : state.lastTime + faker.number.int({ min: 20, max: 72 * 60 }) * 60_000
+      const clamped = Math.min(at, state.today.getTime())
+      state.lastTime = clamped
+      return new Date(clamped).toISOString().replace(/\.\d{3}Z$/, 'Z')
+    }
+    case 'json':
+      return JSON.stringify({})
+    case 'geopoint':
+      return JSON.stringify({ lat: faker.location.latitude(), lon: faker.location.longitude() })
+    case 'media_reference':
+      return ''
+    default:
+      break
+  }
+  if (/e ?mail/.test(name)) return faker.internet.email({ provider: 'example.com' }).toLowerCase()
+  if (/phone|mobile/.test(name)) return faker.phone.number({ style: 'international' })
+  if (/first name|given name/.test(name)) return faker.person.firstName()
+  if (/last name|surname|family name/.test(name)) return faker.person.lastName()
+  if (/(^|\b)(name|full name|customer name|contact)$/.test(name) && /customer|person|user|contact|employee|member|agent/i.test(type.apiName + name)) return faker.person.fullName()
+  if (/company|organi[sz]ation|account name|merchant|vendor|supplier/.test(name)) return faker.company.name()
+  if (/product|item name/.test(name)) return faker.commerce.productName()
+  if (/sku|code|reference/.test(name)) return faker.string.alphanumeric({ length: 8, casing: 'upper' })
+  if (/city/.test(name)) return faker.location.city()
+  if (/country/.test(name)) return faker.location.countryCode()
+  if (/postal|zip/.test(name)) return faker.location.zipCode()
+  if (/address|street/.test(name)) return faker.location.streetAddress()
+  if (/currency/.test(name)) return 'USD'
+  if (/url|website|link/.test(name)) return faker.internet.url()
+  if (/carrier|courier/.test(name)) return faker.helpers.arrayElement(['DHL', 'UPS', 'FedEx', 'Royal Mail', 'PostNL'])
+  if (/assignee|owner|agent|rep/.test(name)) return `agent-${faker.person.firstName().toLowerCase()}`
+  if (/segment|tier|plan/.test(name)) return faker.helpers.arrayElement(['standard', 'plus', 'enterprise'])
+  // English words, not lorem ipsum: sample rows are read by people and by Agents.
+  if (/subject|title|summary|headline/.test(name)) return phrase(faker)
+  if (/description|notes?|body|comment|message|reason|details?/.test(name)) return `${phrase(faker)}. ${faker.hacker.phrase()}`
+  if (/name/.test(name)) return faker.commerce.productName()
+  if (/status|state|stage/.test(name)) return faker.helpers.arrayElement(['open', 'in_progress', 'done'])
+  if (/category|type|kind/.test(name)) return faker.helpers.arrayElement(['general', 'billing', 'delivery', 'product'])
+  return faker.word.noun()
+}
+
+/** "Replace damaged charger": a short line that reads like a title. */
+function phrase(faker: Faker): string {
+  const verb = faker.word.verb()
+  return `${verb[0]!.toUpperCase()}${verb.slice(1)} ${faker.word.adjective()} ${faker.word.noun()}`
+}
+
+function csvCell(value: unknown): string {
+  if (value === null || value === undefined) return ''
+  const text = typeof value === 'string' ? value : String(value)
+  return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text
+}
+
+/**
+ * Generate a CSV and a dataset definition for every object type in the
+ * project's Blueprint tree. Returns what was written.
+ */
+export function generateSyntheticData(root: string, opts: SyntheticOptions = {}): GeneratedDataset[] {
+  const types = readTree<ObjectType>(join(root, 'blueprint', 'object-types')).map((f) => f.doc).filter((t) => t?.apiName)
+  const links = readTree<LinkType>(join(root, 'blueprint', 'link-types')).map((f) => f.doc).filter((l) => l?.apiName)
+  if (!types.length) return []
+  const faker = new Faker({ locale: [en, base] })
+  faker.seed(opts.seed ?? 7)
+  const today = opts.today ?? new Date('2026-10-01T00:00:00Z')
+  const refs = referencesFrom(links)
+  const keys = new Map<string, unknown[]>()
+  const out: GeneratedDataset[] = []
+  const baseRows = Math.max(1, Math.min(opts.scenario?.rows ?? opts.rows ?? 200, 100_000))
+  mkdirSync(join(root, 'datasets'), { recursive: true })
+
+  for (const type of dependencyOrder(types, refs)) {
+    const custom = opts.scenario?.types?.[type.apiName]
+    // Types others point at are the fewer ones (customers, products); the rest scale with them.
+    const referenced = [...refs.values()].some((m) => [...m.values()].some((r) => r.objectType === type.apiName))
+    const count = custom?.rows ?? (referenced ? Math.max(5, Math.round(baseRows / 2)) : baseRows)
+    const properties = type.properties ?? []
+    const pk = type.primaryKey ?? properties[0]?.apiName ?? 'id'
+    const { dataset, column } = backingOf(type)
+    const prefix = keyPrefix(type.apiName)
+    const ownRefs = refs.get(type.apiName) ?? new Map<string, Reference>()
+    const rows: Array<Record<string, unknown>> = []
+    for (let i = 0; i < count; i++) {
+      const row: Record<string, unknown> = {}
+      const state = { lastTime: null as number | null, today }
+      for (const p of properties) {
+        const fn = custom?.fields?.[p.apiName]
+        if (fn) {
+          row[p.apiName] = fn(row, i)
+          continue
+        }
+        if (p.apiName === pk) {
+          row[p.apiName] = (p.valueType ?? 'string') === 'number' ? i + 1 : `${prefix}-${String(i + 1).padStart(5, '0')}`
+          continue
+        }
+        const ref = ownRefs.get(p.apiName)
+        if (ref) {
+          const pool = keys.get(`${ref.objectType}.${ref.property}`) ?? []
+          row[p.apiName] = pool.length && (p.required !== false || faker.datatype.boolean({ probability: 0.85 })) ? faker.helpers.arrayElement(pool) : ''
+          continue
+        }
+        // Optional values are sometimes empty, as real records are.
+        if (p.required === false && faker.datatype.boolean({ probability: 0.2 })) {
+          row[p.apiName] = ''
+          continue
+        }
+        row[p.apiName] = valueFor(faker, type, p, row, state)
+      }
+      rows.push(row)
+    }
+    for (const p of properties) keys.set(`${type.apiName}.${p.apiName}`, rows.map((r) => r[p.apiName]).filter((v) => v !== '' && v !== undefined))
+
+    const columns = properties.map((p) => column(p.apiName))
+    const csv = [columns.join(','), ...rows.map((r) => properties.map((p) => csvCell(r[p.apiName])).join(','))].join('\n') + '\n'
+    const csvPath = join('datasets', `${dataset}.csv`)
+    writeFileSync(join(root, csvPath), csv)
+    const definitionPath = join('datasets', `${dataset}.json`)
+    const definition = {
+      apiName: dataset,
+      displayName: type.pluralDisplayName ?? plural(type.displayName ?? type.apiName),
+      description: `Sample ${(type.pluralDisplayName ?? plural(type.apiName)).toLowerCase()} made by \`frontera local data --synthetic\`: not real records.`,
+      kind: 'file',
+      path: `./${dataset}.csv`,
+      includedColumns: columns,
+      keyColumns: [column(pk)],
+      deterministicKeyConfirmed: true,
+    }
+    writeFileSync(join(root, definitionPath), `${JSON.stringify(definition, null, 2)}\n`)
+    out.push({ objectType: type.apiName, dataset, rows: rows.length, csvPath, definitionPath })
+  }
+  return out
+}
+
+/**
+ * Give each object type without a dataset the one sample data writes, so the
+ * Blueprint reads its rows. A type pulled with its own backing keeps it.
+ * Returns the files changed.
+ */
+export function addMissingBackings(root: string): string[] {
+  const dir = join(root, 'blueprint', 'object-types')
+  const changed: string[] = []
+  for (const { file, doc } of readTree<ObjectType>(dir)) {
+    if (!doc?.apiName || doc.backing?.dataset) continue
+    const { dataset, column } = backingOf(doc)
+    const next = { ...doc, backing: { dataset, mapping: Object.fromEntries((doc.properties ?? []).map((p) => [p.apiName, column(p.apiName)])) } }
+    const path = join(dir, file)
+    writeFileSync(path, file.endsWith('.json') ? `${JSON.stringify(next, null, 2)}\n` : stringifyYaml(next, { lineWidth: 0 }))
+    changed.push(join('blueprint', 'object-types', file))
+  }
+  return changed
+}
+
+/** The scenario template written the first time, for the FDE or a coding agent to edit. */
+export const SCENARIO_TEMPLATE = `/**
+ * Adjusts the sample data \`frontera local data --synthetic\` writes into
+ * datasets/. Return rows per object type and, per property, a function that
+ * gives its value; anything left out is generated from the property's type,
+ * name and description. \`faker\` is seeded, so the same seed gives the same rows.
+ *
+ *   frontera local data --synthetic --seed 7
+ *
+ * \`faker\` is the CLI's own (https://fakerjs.dev/api/), so the project
+ * installs nothing for it.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export default function scenario({ faker }: { faker: any }) {
+  return {
+    // rows: 200,
+    types: {
+      // Ticket: {
+      //   rows: 300,
+      //   fields: {
+      //     priority: () => faker.helpers.weightedArrayElement([{ value: 'urgent', weight: 1 }, { value: 'normal', weight: 6 }]),
+      //   },
+      // },
+    },
+  }
+}
+`
+
+/** The project's scenario, when it has `data/generate.ts`; null when it has none. */
+export async function loadScenario(root: string, seed: number): Promise<Scenario | null> {
+  const file = join(root, 'data', 'generate.ts')
+  if (!existsSync(file)) return null
+  const mod = (await import(`${file}?t=${Date.now()}`)) as { default?: (ctx: { faker: Faker }) => Scenario }
+  if (typeof mod.default !== 'function') return null
+  const faker = new Faker({ locale: [en, base] })
+  faker.seed(seed + 1)
+  return mod.default({ faker })
+}

package/src/local/local-test-runner.ts

@@ -12,15 +12,45 @@ import type { LocalCredentials } from './local-state'
  * should end with. The runner opens a fresh conversation with the Agent's
  * draft (or the live version), sends the person's turns in order through the
  * same chat endpoint the platform uses, and compares the final reply with the
- * expectation.
+ * expectation, then runs the case's `checks` against the reply and the tools
+ * the Agent called, as the platform's expectation grading does.
+ *
+ * A scenario case (a conversation with a simulated customer) runs on the
+ * local runtime's own evaluation runner: the file's scenarios are imported
+ * into a dataset there and run against the Agent, and each comes back with
+ * the runner's verdict.
  */
 
+/** A check graded in code, as the platform grades an item's `expectations` (`expectation-grading.ts`). */
+export type TestCheck =
+  | { type: 'contains'; value: string; text?: string }
+  | { type: 'regex'; value: string; flags?: string; text?: string }
+  | { type: 'tool_called' | 'tool_not_called'; tool: string; argsSubset?: Record<string, unknown>; mode?: 'exact' | 'ignore' | 'subset' | 'superset'; text?: string }
+
 export interface TestCase {
   name: string
   conversation: Array<Record<string, string>>
   expected?: string
   /** Pass when the reply contains each of these, instead of matching exactly. */
   expectedContains?: string[]
+  /** Checks on the reply and the tools called; every one must pass too. */
+  checks?: TestCheck[]
+  /** Expectations only the platform's judge grades, in words: kept, not graded here. */
+  judged?: string[]
+  /** Human-certified on the platform. */
+  gold?: boolean
+  /** A conversation with a simulated customer, run by the local runtime's evaluation runner. */
+  scenario?: Record<string, unknown>
+  /** What the scenario expects: its outcome, the actions that must (or must not) run. */
+  expect?: Record<string, unknown>
+}
+
+/** A tool the Agent called during the case, from the chat stream. */
+export interface ToolCall {
+  name: string
+  args: unknown
+  /** It ran to completion. */
+  ok: boolean
 }
 
 export interface CaseResult {
@@ -40,6 +70,8 @@ export interface CaseFile {
   agent: string
   file: string
   cases: TestCase[]
+  /** The Agent's regression dataset in the workspace: `local test --regression` runs only these. */
+  regression?: boolean
 }
 
 export function findCaseFiles(root: string, agent?: string): CaseFile[] {
@@ -53,8 +85,8 @@ export function findCaseFiles(root: string, agent?: string): CaseFile[] {
     for (const name of readdirSync(evalsDir)) {
       if (!/\.ya?ml$/.test(name)) continue
       const path = join(evalsDir, name)
-      const doc = parse(readFileSync(path, 'utf8')) as { cases?: TestCase[] } | null
-      out.push({ agent: slug, file: relative(root, path), cases: doc?.cases ?? [] })
+      const doc = parse(readFileSync(path, 'utf8')) as { cases?: TestCase[]; regression?: unknown } | null
+      out.push({ agent: slug, file: relative(root, path), cases: doc?.cases ?? [], ...(doc?.regression === true ? { regression: true } : {}) })
     }
   }
   return out
@@ -64,16 +96,67 @@ function normalise(text: string): string {
   return text.replace(/\s+/g, ' ').trim()
 }
 
-export function judge(c: TestCase, reply: string): { passed: boolean; reason?: string } {
+export function judge(c: TestCase, reply: string, toolCalls: ToolCall[] = []): { passed: boolean; reason?: string } {
+  const answer = judgeAnswer(c, reply)
+  if (!answer.passed) return answer
+  for (const check of c.checks ?? []) {
+    const failed = runCheck(check, reply, toolCalls)
+    if (failed) return { passed: false, reason: failed }
+  }
+  return { passed: true }
+}
+
+function judgeAnswer(c: TestCase, reply: string): { passed: boolean; reason?: string } {
   if (c.expectedContains?.length) {
     const missing = c.expectedContains.filter((s) => !normalise(reply).toLowerCase().includes(normalise(s).toLowerCase()))
     return missing.length ? { passed: false, reason: `reply does not mention: ${missing.join(', ')}` } : { passed: true }
   }
-  if (c.expected === undefined) return { passed: false, reason: 'the case has no `expected` or `expectedContains`' }
+  // Checks alone are an expectation.
+  if (c.expected === undefined) return c.checks?.length ? { passed: true } : { passed: false, reason: 'the case has no `expected`, `expectedContains` or `checks`' }
   return normalise(reply) === normalise(c.expected) ? { passed: true } : { passed: false, reason: 'reply differs from `expected`' }
 }
 
-interface Api {
+/** A quantified group itself quantified: the pattern shape that backtracks without end, refused as the platform refuses it. */
+const NESTED_QUANTIFIER = /\([^)]*[*+][^)]*\)\s*[*+{]/
+
+const isRecord = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v)
+
+const deepEqual = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b) || (isRecord(a) && isRecord(b) && Object.keys(a).length === Object.keys(b).length && Object.keys(a).every((k) => deepEqual(a[k], b[k])))
+
+const containsAll = (haystack: Record<string, unknown>, needles: Record<string, unknown>) => Object.entries(needles).every(([k, v]) => k in haystack && deepEqual(haystack[k], v))
+
+function argsMatch(check: Extract<TestCheck, { tool: string }>, call: ToolCall): boolean {
+  const mode = check.mode ?? 'superset'
+  if (mode === 'ignore' || !check.argsSubset) return true
+  if (!isRecord(call.args)) return false
+  if (mode === 'superset') return containsAll(call.args, check.argsSubset)
+  if (mode === 'subset') return containsAll(check.argsSubset, call.args)
+  return containsAll(call.args, check.argsSubset) && Object.keys(call.args).length === Object.keys(check.argsSubset).length
+}
+
+/** Why a check failed, or null when it passed. */
+export function runCheck(check: TestCheck, reply: string, toolCalls: ToolCall[]): string | null {
+  const label = check.text ? ` (${check.text})` : ''
+  if (check.type === 'contains') return reply.toLowerCase().includes(check.value.toLowerCase()) ? null : `reply does not mention: ${check.value}${label}`
+  if (check.type === 'regex') {
+    if (check.value.length > 500 || NESTED_QUANTIFIER.test(check.value)) return `the pattern ${check.value} can't run safely${label}`
+    let pattern: RegExp
+    try {
+      pattern = new RegExp(check.value, check.flags ?? 'i')
+    } catch {
+      return `the pattern ${check.value} does not compile${label}`
+    }
+    return pattern.test(reply.slice(0, 20_000)) ? null : `nothing in the reply matches /${check.value}/${label}`
+  }
+  // A call that ran proves a tool was called; any attempt counts against one that must not be.
+  const candidates = check.type === 'tool_called' ? toolCalls.filter((c) => c.ok) : toolCalls
+  const matched = candidates.some((c) => c.name === check.tool && argsMatch(check, c))
+  const made = toolCalls.length ? `calls made: ${[...new Set(toolCalls.map((c) => c.name))].join(', ')}` : 'no tools were called'
+  if (check.type === 'tool_called') return matched ? null : `no matching call to ${check.tool}; ${made}${label}`
+  return matched ? `${check.tool} was called, which the case does not allow${label}` : null
+}
+
+export interface Api {
   serviceUrl: string
   creds: LocalCredentials
 }
@@ -98,8 +181,8 @@ async function call<T>(api: Api, path: string, init: { method?: string; body?: u
 
 const TURN_TIMEOUT_MS = 180_000
 
-/** Send one person turn and collect the streamed reply's text. */
-async function sendTurn(api: Api, conversationId: string, agentName: string, text: string): Promise<string> {
+/** Send one person turn and collect the streamed reply's text and the tools it called. */
+async function sendTurn(api: Api, conversationId: string, agentName: string, text: string): Promise<{ reply: string; toolCalls: ToolCall[] }> {
   const res = await fetch(new URL('/v1/chat', api.serviceUrl), {
     method: 'POST',
     headers: {
@@ -119,6 +202,7 @@ async function sendTurn(api: Api, conversationId: string, agentName: string, tex
   if (!res.ok || !res.body) throw new Error(`chat answered ${res.status}: ${(await res.text()).slice(0, 200)}`)
   let reply = ''
   let buffer = ''
+  const calls = new Map<string, ToolCall>()
   const decoder = new TextDecoder()
   for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
     buffer += decoder.decode(chunk, { stream: true })
@@ -130,8 +214,10 @@ async function sendTurn(api: Api, conversationId: string, agentName: string, tex
       const payload = line.slice(5).trim()
       if (payload === '[DONE]') continue
       try {
-        const event = JSON.parse(payload) as { type?: string; delta?: string; errorText?: string }
+        const event = JSON.parse(payload) as { type?: string; delta?: string; errorText?: string; toolCallId?: string; toolName?: string; input?: unknown }
         if (event.type === 'text-delta' && typeof event.delta === 'string') reply += event.delta
+        if (event.type === 'tool-input-available' && event.toolCallId && event.toolName) calls.set(event.toolCallId, { name: event.toolName, args: event.input, ok: false })
+        if (event.type === 'tool-output-available' && event.toolCallId && calls.has(event.toolCallId)) calls.get(event.toolCallId)!.ok = true
         if (event.type === 'error') throw new Error(event.errorText ?? 'the turn failed')
       } catch (err) {
         if (err instanceof SyntaxError) continue
@@ -139,7 +225,7 @@ async function sendTurn(api: Api, conversationId: string, agentName: string, tex
       }
     }
   }
-  return reply
+  return { reply, toolCalls: [...calls.values()] }
 }
 
 export async function runCases(args: {
@@ -151,7 +237,16 @@ export async function runCases(args: {
 }): Promise<CaseResult[]> {
   const api: Api = { serviceUrl: args.serviceUrl, creds: args.creds }
   const results: CaseResult[] = []
-  for (const file of args.files) {
+  for (const all of args.files) {
+    // Scenarios run on the runtime's evaluation runner, the rest as conversations here.
+    const scenarios = all.cases.filter((c) => c.scenario)
+    const file = { ...all, cases: all.cases.filter((c) => !c.scenario) }
+    if (scenarios.length) {
+      for (const r of await runScenarios(api, all, scenarios, args.live)) {
+        results.push(r)
+        args.onResult(r)
+      }
+    }
     let agentName = file.agent
     let agentId: string | undefined
     let setupError: string | undefined
@@ -175,10 +270,14 @@ export async function runCases(args: {
         const session = await call<{ id: string }>(api, '/v1/sessions', { body: { agentName, workspaceId: args.creds.workspaceId } })
         conversationId = session.id
         let reply = ''
-        for (const turn of c.conversation) {
-          if (typeof turn.user === 'string') reply = await sendTurn(api, session.id, agentName, turn.user)
+        const toolCalls: ToolCall[] = []
+        for (const turn of c.conversation ?? []) {
+          if (typeof turn.user !== 'string') continue
+          const answer = await sendTurn(api, session.id, agentName, turn.user)
+          reply = answer.reply
+          toolCalls.push(...answer.toolCalls)
         }
-        const verdict = judge(c, reply)
+        const verdict = judge(c, reply, toolCalls)
         result = { agent: file.agent, file: file.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, agentId, conversationId }
       } catch (err) {
         result = { agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason: (err as Error).message, ms: Date.now() - started, agentId, conversationId }
@@ -189,3 +288,62 @@ export async function runCases(args: {
   }
   return results
 }
+
+const SCENARIO_POLL_MS = 3_000
+const OUTCOME_WORDS: Record<string, string> = { fail: 'failed', error: 'errored', split: 'passed and failed equally often', ungraded: 'was not graded', skipped: 'was skipped', pending: 'did not finish' }
+const SCENARIO_TIMEOUT_MS = 15 * 60_000
+
+/** The dataset a case file's scenarios are imported into on the local runtime. */
+export function scenarioDataset(file: CaseFile): string {
+  const stem = file.file.split(/[\\/]/).pop()!.replace(/\.ya?ml$/, '')
+  return `local-test-${file.agent}-${stem}`.toLowerCase().replace(/[^a-z0-9-]+/g, '-').slice(0, 120)
+}
+
+/**
+ * A file's scenario cases, run by the local runtime's evaluation runner: the
+ * scenarios are imported into the file's dataset there (by name, so running
+ * again updates them), run against the Agent's draft (`live`: its published
+ * version) with no judges (a scenario is graded on what the Agent did), and
+ * each case reads its item's verdict.
+ */
+export async function runScenarios(api: Api, file: CaseFile, cases: TestCase[], live: boolean, wait: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms))): Promise<CaseResult[]> {
+  const started = Date.now()
+  const dataset = scenarioDataset(file)
+  const fail = (reason: string, agentId?: string): CaseResult[] => cases.map((c) => ({ agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason, ms: Date.now() - started, agentId }))
+  let agentId: string | undefined
+  try {
+    const agent = await call<{ id: string }>(api, `/v1/config/agents/${encodeURIComponent(file.agent)}`)
+    agentId = agent.id
+    // Made once; it exists on every later run.
+    await call(api, '/v1/eval/datasets', { body: { name: dataset, description: `Scenarios from ${file.file} (frontera local test)` } }).catch(() => null)
+    await call(api, `/v1/eval/datasets/${encodeURIComponent(dataset)}/scenarios/import`, {
+      body: { scenarios: cases.map((c) => ({ name: c.name, scenario: c.scenario, expect: c.expect ?? {} })) },
+    })
+    const draft = live ? null : await call<{ revision?: number }>(api, `/v1/config/agents/${agent.id}/draft-v2`).catch(() => null)
+    const run = await call<{ runId: string }>(api, `/v1/eval/datasets/${encodeURIComponent(dataset)}/runs`, {
+      body: { agentId: agent.id, judgeSlugs: [], runName: 'frontera local test', ...(draft?.revision ? { draftRevision: draft.revision } : {}) },
+    })
+    type Run = { status?: string; items?: Array<{ datasetItemId: string; attempt: number; input?: { query?: string } | null; output?: unknown; error?: string | null }>; rollups?: Array<{ datasetItemId: string; outcome: string }> }
+    let state: Run = {}
+    while (Date.now() - started < SCENARIO_TIMEOUT_MS) {
+      state = await call<Run>(api, `/v1/eval/datasets/${encodeURIComponent(dataset)}/runs/${run.runId}`)
+      if (state.status && !['queued', 'running'].includes(state.status)) break
+      await wait(SCENARIO_POLL_MS)
+    }
+    if (!state.status || ['queued', 'running'].includes(state.status)) return fail('the scenario run did not finish in time', agentId)
+    const outcomes = new Map((state.rollups ?? []).map((r) => [r.datasetItemId, r.outcome]))
+    const firstAttempts = (state.items ?? []).filter((i) => i.attempt === 0)
+    return cases.map((c) => {
+      // An item is matched by its opening, which the run records as the item's query.
+      const opening = typeof c.scenario?.opening === 'string' ? c.scenario.opening.trim() : ''
+      const at = firstAttempts.find((i) => (i.input?.query ?? '').trim() === opening)
+      const outcome = at ? outcomes.get(at.datasetItemId) : undefined
+      const reply = typeof at?.output === 'string' ? at.output : ''
+      const passed = outcome === 'pass'
+      const reason = passed ? undefined : !at ? 'the run has no result for this scenario' : at.error ? at.error : `the scenario ${OUTCOME_WORDS[outcome ?? ''] ?? 'did not finish'}`
+      return { agent: file.agent, file: file.file, name: c.name, passed, reply, ...(reason ? { reason } : {}), ms: Date.now() - started, agentId }
+    })
+  } catch (err) {
+    return fail((err as Error).message, agentId)
+  }
+}