@frontera-sdk/cli 1.52.39 → 1.52.41

package/src/commands/local/check.ts

@@ -1,64 +1,677 @@
 import { spawn } from 'node:child_process'
+import { existsSync, lstatSync, readdirSync, readFileSync } from 'node:fs'
+import { join, normalize, relative } from 'node:path'
 
-import { CliError } from '../../errors'
-import { localProject } from '../../local/local-project'
+import { CliError, UsageError } from '../../errors'
+import { EXIT } from '../../exit'
+import { appSdkDrift, isVendoredSpec, sdkDriftLines, vendoredShadows } from '../../local/local-app-sdk'
+import { lockedSdkVersions } from '../../app-lock'
+import { projectBuildRows } from '../../local/local-build-summary'
+import { allInherited, inheritedActionWarnings, lockRefOf, typecheckErrorFiles, unchangedSincePull } from '../../local/local-check-inherited'
+import { appPulledAsIs, keepBuildWrittenFiles } from '../../local/local-app-pulled'
+import { setAsideCopyPaths } from '../../local/local-app-sdk-paths'
+import { repullRefFor } from '../../local/local-app-code'
+import { customAppFolders } from '../../local/local-apps'
+import { readBaseline, restoreBaseline, saveBaseline } from '../../local/local-db-snapshot'
+import { localIngestApi, settleKnowledge, type KnowledgeSettle } from '../../local/local-knowledge-ingestion'
+import { localProject, type LocalProject } from '../../local/local-project'
+import { projectSecretLines } from '../../local/local-project-secrets'
+import { localSecretPresence } from '../../local/local-secrets'
+import { waitForDatabaseReady } from '../../local/local-runtime-health'
 import { isAlive, readState } from '../../local/local-state'
-import { flagBool, type Command } from '../types'
+import { recordRun } from '../../local/local-test-baseline'
+import { notRunGroups, notRunWords } from '../../local/local-case-not-run'
+import { evidenceLine, failingCaseEvidence, type CaseEvidence } from '../../local/local-case-evidence'
+import { agentFilesProblem, caseConcurrency, findCaseFiles, runCases, selectCaseFiles, waitedWords } from '../../local/local-test-runner'
+import { findWorkflowTestFiles, recordWorkflowTestRun, runWorkflowTests, workflowFileState } from '../../local/local-workflow-tests'
+import { vendoredSdkLine, vendoredSdkState } from '../../local/local-vendored-sdk'
+import { withFix } from '../../local/fix-words'
+import { ensureAppPackages } from '../../project-bootstrap'
+import { flagBool, flagString, type Command, type CommandContext } from '../types'
 
 /**
- * `frontera local check`: the whole loop as one command with one exit code,
- * for a pre-push hook or CI (like `supabase start` then `supabase test`):
- * start the runtime if it is not running, apply the solution, run the
- * Agents' test cases. `--stop` stops the runtime afterwards.
+ * `frontera local check`: the one gate, with one exit code, for a pre-push
+ * hook, CI, or a coding agent saying it is done (like `supabase start` then
+ * `supabase test`). Every step runs and is reported, in order; only the
+ * runtime is a prerequisite: when start or apply fails, the steps that test
+ * on the runtime (agents, workflows) are skipped and say why. `--fail-fast`
+ * stops at the first failure instead:
+ *
+ *   start      the runtime, if it is not running
+ *   apply      `local dev --once`: every item applied; a governed Action that didn't deploy fails it
+ *   typecheck  the project's `typecheck` script, else `tsc --noEmit` when it has a tsconfig.json
+ *   unit       `bun test` over the Functions' test files (functions/**\/*.test.ts)
+ *   apps       each Custom App's own `bun run check` (apps/<name>/, or any folder with an App's frontera.config.json)
+ *   agents     the Agents' test cases (agents/<slug>/evals/*.yaml)
+ *   workflows  the workflow tests (workflows/tests/<slug>.yaml)
+ *
+ * A step with nothing to check is skipped, with a note, and is not a
+ * failure. `--json` returns each step's outcome on stdout, failed or not
+ * (exit 1 when it failed). `--stop` stops the runtime afterwards.
+ *
+ * The tests write for real (a workflow test always does, as does an Agent
+ * case with `realActions` or a `decision`), so after the apply the database
+ * is kept as a baseline (`local-db-snapshot.ts`, taken again only when what
+ * is applied changed), and after the tests it is restored: each check starts
+ * from the data the project's files make. `--keep-state` keeps what the tests
+ * wrote, to look at it.
+ *
+ * Knowledge documents are read by a background job whose work lands in the
+ * database: before the baseline and the tests, the check waits (bounded)
+ * until no document is still being read, and queues again any left stuck
+ * with no job behind it; after the restore it queues again every document
+ * the baseline holds unread (`local-knowledge-ingestion.ts`), so each
+ * check's tests search the same documents.
+ *
+ * In a pulled project, a failure only in items or files unchanged since the
+ * pull (`frontera.lock`) is inherited from the workspace: the step passes,
+ * and lists each one, unless `--strict`, which fails on them too. A failure
+ * in anything the change touched or added fails the step as always.
  */
 
-/** Run this CLI once more, its output passed straight through. */
-function runSelf(cwd: string, args: string[]): Promise<number> {
-  const compiled = Bun.main.startsWith('/$bunfs') || Bun.main.startsWith('B:\\~BUN')
-  const self = compiled ? [process.execPath] : [process.execPath, Bun.main]
+/**
+ * The gate's rule for inherited failures, in the sentence the passed result
+ * says it in: `12 inherited failures in items unchanged since the pull, which
+ * pass unless --strict (anything changed or added fails the gate)`.
+ */
+export function inheritedWords(count: number): string {
+  return `${count} inherited failure${count === 1 ? '' : 's'} in items unchanged since the pull, which pass unless --strict (anything changed or added fails the gate)`
+}
+
+export const CHECK_STEPS = ['start', 'apply', 'typecheck', 'unit', 'apps', 'agents', 'workflows'] as const
+export type CheckStepName = (typeof CHECK_STEPS)[number]
+/**
+ * The steps each one needs to have passed: the runtime's tests need it started.
+ * They do not need every item applied: an Agent's or workflow's cases run on
+ * what applied, and the cases of one whose own files did not apply fail with
+ * why (`agentFilesProblem`, `workflowFileState`). So one item that no test
+ * touches (a pulled Function missing a file) never hides every test result;
+ * the apply failure still fails the gate.
+ */
+const NEEDS: Partial<Record<CheckStepName, CheckStepName[]>> = { apply: ['start'], agents: ['start'], workflows: ['start'] }
+
+/** Why a step can't run, given the steps that failed so far; null when it can. */
+export function blockedBy(step: CheckStepName, failed: ReadonlySet<CheckStepName>, failFast = false): string | null {
+  if (failFast && failed.size) return 'not run: an earlier step failed (--fail-fast)'
+  const missing = (NEEDS[step] ?? []).filter((n) => failed.has(n))
+  return missing.length ? `not run: ${missing.join(' and ')} failed, and it runs on the applied runtime` : null
+}
+
+/** What `--skip` may leave out: the runtime and the apply are the gate itself. */
+const SKIPPABLE: readonly CheckStepName[] = ['typecheck', 'unit', 'apps', 'agents', 'workflows']
+
+/** What became of the tests' writes: rolled back to the applied baseline, kept, or not rolled back and why. */
+export interface CheckTestData {
+  /** taken: a new baseline after this apply; reused: the stored one is of this build. */
+  baseline?: 'taken' | 'reused'
+  rolledBack: boolean
+  /** --keep-state */
+  kept?: boolean
+  note: string
+  ms?: number
+}
+
+export interface CheckStep {
+  step: CheckStepName
+  status: 'passed' | 'failed' | 'skipped'
+  /**
+   * What failed in items or files unchanged since the pull (`frontera.lock`):
+   * inherited from the workspace, not made by the change. A step whose every
+   * failure is inherited passes unless `--strict`, and lists them here.
+   */
+  inherited?: string[]
+  /**
+   * apply: warnings inherited from the workspace on items that applied (a
+   * governed Action offering a value the semantic type it writes refuses,
+   * deployed as the workspace runs it). Listed; never a failure, even with `--strict`.
+   */
+  inheritedWarnings?: string[]
+  /** What ran, as a person would type it. */
+  command?: string
+  /** Why it was skipped, or what failed, in a line. */
+  note?: string
+  ms: number
+  /**
+   * agents, workflows: each case that failed, with why; an Agent case also with its conversation, the trace of
+   * its last turn (kept after the rollback: traces live outside the platform database) and the start of its reply.
+   */
+  failures?: Array<{ name: string; reason?: string } & CaseEvidence>
+  /**
+   * agents: the cases that couldn't run, by cause (the model provider refused
+   * them, the turn timed out, the runtime did not answer), with the fix. The
+   * step does not pass, but these are not failures of the Agent.
+   */
+  couldNotRun?: Array<{ what: string; fix: string; cases: string[]; notSent?: number }>
+}
+
+/** Run a command in the project, its output on stderr so stdout stays the check's own result. */
+function runIn(cwd: string, command: string[]): Promise<number> {
   return new Promise((resolve) => {
-    const child = spawn(self[0]!, [...self.slice(1), ...args], { cwd, stdio: ['ignore', 'inherit', 'inherit'] })
-    child.on('error', () => resolve(1))
+    const child = spawn(command[0]!, command.slice(1), { cwd, stdio: ['ignore', 2, 2], env: { ...process.env, NO_COLOR: process.env.NO_COLOR ?? '1' } })
+    child.on('error', () => resolve(127))
     child.on('close', (code) => resolve(code ?? 1))
   })
 }
 
+/** As `runIn`, and what it printed, for reading which files a failure is in. */
+function runCapturing(cwd: string, command: string[]): Promise<{ code: number; output: string }> {
+  return new Promise((resolve) => {
+    let output = ''
+    const child = spawn(command[0]!, command.slice(1), { cwd, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, NO_COLOR: process.env.NO_COLOR ?? '1' } })
+    const pass = (chunk: Buffer) => {
+      output += chunk.toString()
+      process.stderr.write(chunk)
+    }
+    child.stdout.on('data', pass)
+    child.stderr.on('data', pass)
+    child.on('error', () => resolve({ code: 127, output }))
+    child.on('close', (code) => resolve({ code: code ?? 1, output }))
+  })
+}
+
+/**
+ * Why an App's own `bun run check` failed, in one line: the first error it
+ * printed (a TypeScript error with its file and line, a missing command, an
+ * `error:` line) and what fixes it. A copy of the SDK vendored under
+ * src/frontera/ while the App's bun.lock installed a published one is named
+ * as the cause, with the command that switches back.
+ */
+export function appCheckFailure(appDir: string, app: string, code: number, output: string): string {
+  const lines = output.replace(/\u001b\[[0-9;]*m/g, '').split('\n').map((l) => l.trim()).filter(Boolean)
+  const tsError = lines.find((l) => /error TS\d+:/.test(l))
+  const missing = lines.find((l) => /command not found|not found: [\w.-]+$|Cannot find module|Could not resolve/i.test(l))
+  const other = lines.find((l) => /^(error|✕|FAIL)\b|\berror:/i.test(l))
+  const first = (tsError ?? missing ?? other ?? '').slice(0, 300)
+  const exited = `exited ${code}`
+  if ((code === 127 || /command not found/i.test(missing ?? '')) && !existsSync(join(appDir, 'node_modules'))) {
+    // A copy shadowing the release its bun.lock installed: a plain install would rewrite the lockfile to the copy.
+    const shadowed = vendoredShadows(appDir)
+    if (shadowed.length) return `${exited}: ${first || 'a command it runs is not installed'}. Its packages aren't installed, and package.json takes the SDK copy under src/frontera/ where its bun.lock installed the published ${shadowed.map((v) => `${v.pkg} ${v.version}`).join(' and ')}: run \`frontera app sdk use-local --npm\` in ${app} (it installs too), then check again`
+    const locked = existsSync(join(appDir, 'bun.lock')) || existsSync(join(appDir, 'bun.lockb'))
+    return `${exited}: ${first || 'a command it runs is not installed'}. Its packages aren't installed: run \`bun install${locked ? ' --frozen-lockfile' : ''}\` in ${app}, then check again`
+  }
+  if (tsError) {
+    const manifest = (() => {
+      try {
+        return JSON.parse(readFileSync(join(appDir, 'package.json'), 'utf8')) as { dependencies?: Record<string, string> }
+      } catch {
+        return null
+      }
+    })()
+    const vendoredSdk = Object.entries(manifest?.dependencies ?? {}).filter(([pkg, spec]) => isVendoredSpec(pkg, spec)).map(([pkg]) => pkg)
+    const locked = lockedSdkVersions(appDir)
+    const published = vendoredSdk.filter((pkg) => locked.has(pkg))
+    if (published.length) {
+      // An earlier pull's folder, still as it left it: the pull that brings the deployed source back comes first.
+      const repull = repullRefFor(appDir)
+      const fix = repull ? `\`frontera local sync pull ${repull}\` brings the deployed source back, or \`frontera app sdk use-local --npm\` in ${app} goes back to that release where it is` : `run \`frontera app sdk use-local --npm\` in ${app}`
+      return `${exited}: ${first}. Its SDK is a copy under src/frontera/, not the published ${published.map((p) => `${p} ${locked.get(p)!.version}`).join(' and ')} its bun.lock installed and its code was written against: ${fix}, then check again`
+    }
+    return `${exited}: ${first}. Fix it in ${app} (\`bun run typecheck\` there shows every error)`
+  }
+  if (first) return `${exited}: ${first}. Run \`bun run check\` in ${app} to see the whole output`
+  return `${exited}. Run \`bun run check\` in ${app} to see why`
+}
+
+/**
+ * An App's packages before its own check, when it has none (a pulled App
+ * comes without node_modules), installed without writing a tracked file: from
+ * its bun.lock as it is (`--frozen-lockfile`), or, when package.json names
+ * the App's own SDK copy under src/frontera/ where bun.lock installed a
+ * published release, as package.json says with `bun install --no-save`
+ * (`ensureAppPackages`). With no lockfile nothing is installed (an install
+ * would write one) and the line says so. The line to say, or null when it
+ * already had them.
+ */
+export async function installForCheck(appDir: string, app: string, install?: Parameters<typeof ensureAppPackages>[1]): Promise<string | null> {
+  if (existsSync(join(appDir, 'node_modules')) || !existsSync(join(appDir, 'package.json'))) return null
+  if (!['bun.lock', 'bun.lockb'].some((f) => existsSync(join(appDir, f)))) return `${app} has no node_modules and no bun.lock: \`bun install\` there first if its check needs its packages (it writes a bun.lock)`
+  const result = await ensureAppPackages(appDir, install)
+  if (!result) return null
+  if ((result.fromCopy || result.fromLock) && result.installed) return `${app} had no node_modules: ${result.summary}`
+  if (result.installed) return `${app} had no node_modules: installed them from its bun.lock as it is (\`bun install --frozen-lockfile\`; the lockfile is unchanged) (${result.summary})`
+  if (result.lockMismatch) return `${app} has no node_modules, and they weren't installed: ${result.summary}`
+  return `${app} has no node_modules, and installing them failed: ${result.summary}`
+}
+
+/** This CLI, again. */
+function self(args: string[]): string[] {
+  const compiled = Bun.main.startsWith('/$bunfs') || Bun.main.startsWith('B:\\~BUN')
+  return compiled ? [process.execPath, ...args] : [process.execPath, Bun.main, ...args]
+}
+
+function readPackage(dir: string): { scripts?: Record<string, string> } | null {
+  try {
+    return JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { scripts?: Record<string, string> }
+  } catch {
+    return null
+  }
+}
+
+/** How the project is typechecked: its own script first, else the compiler over its tsconfig; null when it has neither. */
+export function typecheckCommand(root: string): string[] | null {
+  if (readPackage(root)?.scripts?.typecheck) return ['bun', 'run', 'typecheck']
+  if (existsSync(join(root, 'tsconfig.json'))) return ['bunx', 'tsc', '--noEmit', '-p', 'tsconfig.json']
+  return null
+}
+
+const SKIP_DIRS = new Set(['node_modules', '.git', '.frontera', 'dist', '.next', 'build'])
+
+/** The Functions' unit test files, relative to the project root, `./`-prefixed so `bun test` takes them as paths. */
+export function functionTestFiles(root: string): string[] {
+  const out: string[] = []
+  const walk = (dir: string, depth: number) => {
+    if (depth > 6 || !existsSync(dir)) return
+    for (const name of readdirSync(dir).sort()) {
+      if (SKIP_DIRS.has(name) || name.startsWith('.')) continue
+      const path = join(dir, name)
+      let stat
+      try {
+        stat = lstatSync(path)
+      } catch {
+        continue
+      }
+      if (stat.isDirectory()) walk(path, depth + 1)
+      else if (/\.(test|spec)\.(ts|tsx|js|mjs)$/.test(name)) out.push(`./${relative(root, path)}`)
+    }
+  }
+  walk(join(root, 'functions'), 0)
+  return out
+}
+
+/**
+ * Each Custom App's code folder that has its own `check` script: apps/<name>/,
+ * and any other folder in the project an App was made in, found by its
+ * manifest (`customAppFolders`). The project's own root is not one.
+ */
+export function appsWithCheck(root: string): string[] {
+  return customAppFolders(root).filter((dir) => !!readPackage(join(root, dir))?.scripts?.check)
+}
+
+function parseSkip(raw: string | undefined): Set<CheckStepName> {
+  const names = (raw ?? '').split(',').map((s) => s.trim()).filter(Boolean)
+  for (const n of names) {
+    if (!SKIPPABLE.includes(n as CheckStepName)) throw new UsageError(`\`${n}\` is not a step --skip can leave out`, `use one or more of: ${SKIPPABLE.join(', ')}`)
+  }
+  return new Set(names as CheckStepName[])
+}
+
+function running(project: LocalProject): boolean {
+  const state = readState(project)
+  return state?.phase === 'ready' && !!state.supervisorPid && isAlive(state.supervisorPid)
+}
+
+async function check(ctx: CommandContext) {
+  const project = localProject(ctx.cwd)
+  const root = project.root
+  const skip = parseSkip(flagString(ctx, 'skip'))
+  const steps: CheckStep[] = []
+  const failedSteps = new Set<CheckStepName>()
+  const failFast = flagBool(ctx, 'fail-fast')
+  /** What secrets.json says is missing here, for `--json`. */
+  const secretNotes: string[] = []
+  /** Apps whose installed `@frontera-sdk/*` is older than (or lacks what is in) this CLI's SDK, for `--json`. */
+  const sdkNotes: string[] = []
+  // A pulled project: what is still as the workspace gave it, so a failure only there is inherited (`--strict` fails on it).
+  const strict = flagBool(ctx, 'strict')
+  // How many Agent cases run at once (`--concurrency`), checked before anything starts.
+  let concurrency: number | undefined
+  try {
+    concurrency = caseConcurrency(flagString(ctx, 'concurrency'))
+  } catch (err) {
+    throw new UsageError((err as Error).message, 'frontera local check --concurrency 1')
+  }
+  const unchanged = unchangedSincePull(root)
+  // The tests' writes: a baseline after the apply, restored after the tests unless `--keep-state`.
+  const keepState = flagBool(ctx, 'keep-state')
+  let testData: CheckTestData | null = null
+  let baselineReady = false
+  /** What became of the knowledge bases' documents around the tests: queued again, still being read, for `--json`. */
+  const knowledgeNotes: string[] = []
+  const sayKnowledge = (settled: KnowledgeSettle) => {
+    for (const line of settled.lines) ctx.output.note(`   ${line}`)
+    knowledgeNotes.push(...settled.lines.map((l) => l.replace(/^warning:\s*/, '')))
+  }
+  /** Whether the Agent or workflow tests will run anything, and so may write. */
+  const testsWillRun = () =>
+    (!skip.has('agents') && !blockedBy('agents', failedSteps, failFast) && findCaseFiles(root).some((f) => f.cases.length > 0))
+    || (!skip.has('workflows') && !blockedBy('workflows', failedSteps, failFast) && findWorkflowTestFiles(root).some((f) => f.cases.length > 0))
+
+  const say = (step: CheckStepName, line: string) => ctx.output.note(`── ${step}: ${line}`)
+  const record = (s: CheckStep) => {
+    steps.push(s)
+    if (s.status === 'failed') failedSteps.add(s.step)
+    if (s.status === 'skipped') say(s.step, `skipped — ${s.note}`)
+    else if (s.status === 'failed') say(s.step, `failed${s.note ? ` — ${s.note}` : ''}`)
+    else say(s.step, `passed (${s.ms} ms)${s.inherited?.length ? `, with ${s.inherited.length} inherited failure${s.inherited.length === 1 ? '' : 's'}` : ''}`)
+    // Inherited failures are listed, never hidden: the gate passes over them only without --strict.
+    if (s.inherited?.length) {
+      ctx.output.note(`   inherited from the workspace (unchanged since the pull${strict ? '' : '; `--strict` fails on them'}):`)
+      for (const i of s.inherited) ctx.output.note(`     ${i}`)
+    }
+    if (s.inheritedWarnings?.length) {
+      ctx.output.note('   warnings inherited from the workspace (they run as the workspace runs them, and fail nothing):')
+      for (const w of s.inheritedWarnings) ctx.output.note(`     ${w}`)
+    }
+  }
+  /** One command; the step fails on a non-zero exit. */
+  const command = async (step: CheckStepName, cwd: string, args: string[], shown: string) => {
+    say(step, shown)
+    const started = Date.now()
+    const code = await runIn(cwd, args)
+    record({ step, status: code === 0 ? 'passed' : 'failed', command: shown, ms: Date.now() - started, ...(code === 0 ? {} : { note: `\`${shown}\` exited ${code}` }) })
+  }
+
+  for (const step of CHECK_STEPS) {
+    const blocked = blockedBy(step, failedSteps, failFast)
+    if (blocked) {
+      record({ step, status: 'skipped', note: blocked, ms: 0 })
+      continue
+    }
+    if (skip.has(step)) {
+      record({ step, status: 'skipped', note: 'left out with --skip', ms: 0 })
+      continue
+    }
+    if (step === 'start') {
+      if (running(project)) record({ step, status: 'skipped', note: 'the runtime is already running', ms: 0 })
+      else await command(step, root, self(['local', 'start']), 'frontera local start')
+      continue
+    }
+    if (step === 'apply') {
+      say(step, 'frontera local dev --once')
+      const started = Date.now()
+      const code = await runIn(root, self(['local', 'dev', '--once']))
+      // Which items did not apply, and whether each is still as the pull wrote it. `local dev --once` makes the
+      // same split and exits 0 when every failure is inherited: then the rows it failed on are those, and
+      // `--strict` fails on them here. A row failing from before, which this apply fixed, is not one.
+      const built = projectBuildRows(root) ?? []
+      const failedRows = built.filter((r) => r.state === 'failed' || r.state === 'blocked')
+      const failing = code === 0 ? failedRows.filter((r) => unchanged.refs.has(lockRefOf(r.ref))) : failedRows
+      const warned = inheritedActionWarnings(built)
+      const warnings = warned.length ? { inheritedWarnings: warned } : {}
+      const lines = failing.map((r) => `${r.ref}${r.reason ? `: ${r.reason}` : ''}`)
+      // `local dev` decides (it also sees a file added to a pulled item, and a runtime that stopped answering): any other exit fails the step.
+      const inheritedOnly = !strict && code === 0
+      if (code === 0 && !failing.length) record({ step, status: 'passed', command: 'frontera local dev --once', ms: Date.now() - started, ...warnings })
+      else if (inheritedOnly) record({ step, status: 'passed', command: 'frontera local dev --once', ms: Date.now() - started, inherited: lines, note: `${failing.length} item${failing.length === 1 ? '' : 's'} unchanged since the pull didn’t apply`, ...warnings })
+      else {
+        const made = failing.filter((r) => !unchanged.refs.has(lockRefOf(r.ref)))
+        const note = failing.length
+          ? `${failing.length} item${failing.length === 1 ? '' : 's'} didn’t apply (${(made.length ? made : failing).slice(0, 4).map((r) => r.ref).join(', ')}${(made.length ? made : failing).length > 4 ? ', …' : ''}); the Agent and workflow tests still run on what applied`
+          : `\`frontera local dev --once\` exited ${code}`
+        record({ step, status: 'failed', command: 'frontera local dev --once', ms: Date.now() - started, note, ...warnings, ...(failing.length ? { failures: failing.map((r) => ({ name: r.ref, ...(r.reason ? { reason: r.reason } : {}) })) } : {}), ...(failing.length - made.length ? { inherited: failing.filter((r) => unchanged.refs.has(lockRefOf(r.ref))).map((r) => `${r.ref}${r.reason ? `: ${r.reason}` : ''}`) } : {}) })
+      }
+      // What secrets.json declares and the workflows and Functions name that the runtime has no value for,
+      // unchanged items' included: said, never a failure (a value is each computer's own).
+      const state = readState(project)
+      if (state?.credentials) {
+        const lines = projectSecretLines(root, await localSecretPresence(`http://127.0.0.1:${state.ports.service}`, state.credentials))
+        for (const line of lines) ctx.output.note(`   ${line.trim()}`)
+        if (lines.length) secretNotes.push(...lines.map((l) => l.trim().replace(/^warning:\s*/, '')))
+      }
+      // The knowledge as the tests and the baseline should see it: a document stuck with no job behind it
+      // (a rollback by an earlier release) queued again, then every document read (bounded), so the
+      // baseline never holds a document mid-read and each check's tests search the same documents.
+      const ingest = state?.credentials && running(project) && testsWillRun() ? localIngestApi(`http://127.0.0.1:${state.ports.service}`, state.credentials) : null
+      if (ingest && state?.credentials) sayKnowledge(await settleKnowledge(ingest, state.credentials.workspaceId, { restoredAt: readBaseline(project, state.credentials.organizationId)?.restoredAt }))
+      // The database as applied, before the tests write to it.
+      if (state?.credentials && running(project) && testsWillRun()) {
+        if (keepState) testData = { rolledBack: false, kept: true, note: 'kept what the tests write (--keep-state)' }
+        else {
+          const saved = await saveBaseline(project, state.credentials.organizationId, { serviceAnswers: () => waitForDatabaseReady(`http://127.0.0.1:${state.ports.service}`) })
+          baselineReady = saved.ok
+          if (saved.ok) testData = { baseline: saved.took ? 'taken' : 'reused', rolledBack: false, note: saved.note }
+          else testData = { rolledBack: false, note: `not rolled back: ${saved.note}` }
+          ctx.output.note(`   test data: ${saved.ok ? saved.note : `couldn’t keep a baseline, so what the tests write stays: ${saved.note}`}`)
+        }
+      }
+      continue
+    }
+    if (step === 'typecheck') {
+      const args = typecheckCommand(root)
+      if (!args) {
+        record({ step, status: 'skipped', note: 'no `typecheck` script and no tsconfig.json', ms: 0 })
+        continue
+      }
+      const shown = args.join(' ')
+      say(step, shown)
+      const started = Date.now()
+      const run = await runCapturing(root, args)
+      // Errors only in files unchanged since the pull are the workspace's, not the change's.
+      const inFiles = typecheckErrorFiles(run.output)
+      const sdkLine = run.code !== 0 ? vendoredSdkLine(vendoredSdkState(root)) : null
+      if (sdkLine && !sdkNotes.includes(sdkLine)) {
+        ctx.output.note(`   ${sdkLine}`)
+        sdkNotes.push(sdkLine)
+      }
+      const inherited = run.code !== 0 && !strict && allInherited(inFiles, unchanged.files)
+      if (run.code === 0) record({ step, status: 'passed', command: shown, ms: Date.now() - started })
+      else if (inherited) record({ step, status: 'passed', command: shown, ms: Date.now() - started, inherited: inFiles.map((f) => `${f}: type errors`) })
+      else record({ step, status: 'failed', command: shown, ms: Date.now() - started, note: `\`${shown}\` exited ${run.code}${inFiles.length ? ` (errors in ${inFiles.filter((f) => !unchanged.files.has(f)).slice(0, 4).join(', ') || inFiles.slice(0, 4).join(', ')})` : ''}`, ...(inFiles.some((f) => unchanged.files.has(f)) ? { inherited: inFiles.filter((f) => unchanged.files.has(f)).map((f) => `${f}: type errors`) } : {}) })
+      continue
+    }
+    if (step === 'unit') {
+      const files = functionTestFiles(root)
+      if (!files.length) {
+        record({ step, status: 'skipped', note: 'no Function unit tests (functions/**/*.test.ts)', ms: 0 })
+        continue
+      }
+      // The change's tests, then the pulled ones still as the workspace gave them: those failing is inherited.
+      const pulled = files.filter((f) => unchanged.files.has(normalize(f.replace(/^\.\//, ''))))
+      const own = files.filter((f) => !pulled.includes(f))
+      const started = Date.now()
+      const described = (list: string[]) => `bun test ${list.length === 1 ? list[0] : `(${list.length} files under functions/)`}`
+      let ownCode = 0
+      if (own.length) {
+        say(step, described(own))
+        ownCode = await runIn(root, ['bun', 'test', ...own])
+      }
+      let pulledCode = 0
+      if (pulled.length) {
+        say(step, `${described(pulled)}, unchanged since the pull`)
+        pulledCode = await runIn(root, ['bun', 'test', ...pulled])
+      }
+      const shown = described(files)
+      // A failing test may be failing on the project's own older copy of the SDK: said, with the command that refreshes it.
+      const sdkLine = ownCode !== 0 || pulledCode !== 0 ? vendoredSdkLine(vendoredSdkState(root)) : null
+      if (sdkLine) {
+        ctx.output.note(`   ${sdkLine}`)
+        sdkNotes.push(sdkLine)
+      }
+      if (ownCode === 0 && pulledCode === 0) record({ step, status: 'passed', command: shown, ms: Date.now() - started })
+      else if (ownCode === 0 && !strict) record({ step, status: 'passed', command: shown, ms: Date.now() - started, inherited: [`${described(pulled)} exited ${pulledCode}: a test file still as the workspace gave it fails`] })
+      else record({ step, status: 'failed', command: shown, ms: Date.now() - started, note: ownCode !== 0 ? `\`${described(own)}\` exited ${ownCode}` : `\`${described(pulled)}\` exited ${pulledCode} (test files unchanged since the pull; --strict)`, ...(ownCode !== 0 && pulledCode !== 0 ? { inherited: [`${described(pulled)} exited ${pulledCode}`] } : {}) })
+      continue
+    }
+    if (step === 'apps') {
+      const apps = appsWithCheck(root)
+      if (!apps.length) {
+        record({ step, status: 'skipped', note: 'no Custom App with a `check` script (apps/<name>/, or a folder with frontera.config.json)', ms: 0 })
+        continue
+      }
+      const started = Date.now()
+      const broken: Array<{ name: string; reason: string }> = []
+      for (const app of apps) {
+        say(step, `bun run check in ${app}`)
+        // A pulled App comes without its packages: installed first, writing no tracked file, so its check checks it.
+        const installNote = await installForCheck(join(root, app), app)
+        if (installNote) ctx.output.note(`   ${installNote}`)
+        // Said, never a failure: the App's own check decides whether the code still builds.
+        for (const line of sdkDriftLines(app, appSdkDrift(join(root, app)))) {
+          ctx.output.note(`   ${line}`)
+          // A note (the App builds against a published release on purpose) is said, not listed as something to fix.
+          if (/^warning:/.test(line)) sdkNotes.push(line.replace(/^warning:\s*/, ''))
+        }
+        // Its build info, next-env.d.ts and tsconfig.json go back as they were: a check doesn't change the tree (a pulled App tracks them).
+        const putBack = keepBuildWrittenFiles(join(root, app))
+        // Installed as its bun.lock records while tsconfig.json still sends the SDK to the older copy under
+        // src/frontera/: the check runs with those paths set aside, so it checks the code against the release it was written for.
+        const aside = setAsideCopyPaths(join(root, app))
+        if (aside) ctx.output.note(`   ${app}: ${aside.line}`)
+        let restored: string[] = []
+        const { code, output } = await runCapturing(join(root, app), ['bun', 'run', 'check']).finally(() => {
+          if (aside) ctx.output.note(`   ${app}: ${aside.restore()}`)
+          restored = putBack()
+        })
+        if (restored.length) ctx.output.note(`   ${app}: its check wrote ${restored.join(', ')}; put back as they were before it`)
+        if (code !== 0) {
+          const why = appCheckFailure(join(root, app), app, code, output)
+          broken.push({ name: app, reason: why })
+          ctx.output.note(`   ${app}: ${why}`)
+        }
+      }
+      // A pulled App whose source is still as the pull left it fails through no change here: inherited, as
+      // a pulled Function's test is (`--strict` fails on it). An App pulled before pulls recorded its files is
+      // told by its frontera.lock entry and the one unpack that wrote it (`appPulledAsIs`).
+      const pulledAsIs = broken.filter((b) => appPulledAsIs(root, b.name) === true)
+      const inheritedApps = pulledAsIs.map((b) => `${b.name}: ${b.reason}`)
+      if (broken.length && !strict && pulledAsIs.length === broken.length) record({ step, status: 'passed', command: 'bun run check', ms: Date.now() - started, inherited: inheritedApps })
+      else record({ step, status: broken.length ? 'failed' : 'passed', command: 'bun run check', ms: Date.now() - started, ...(broken.length ? { note: broken.map((b) => `${b.name}: ${b.reason}`).join('; '), failures: broken } : {}), ...(inheritedApps.length ? { inherited: inheritedApps } : {}) })
+      continue
+    }
+    // The test steps run here, against the runtime the steps above started and applied.
+    const state = readState(project)
+    if (!state?.credentials?.session || !running(project)) {
+      record({ step, status: 'failed', note: 'the local runtime is not running', ms: 0 })
+      continue
+    }
+    const serviceUrl = `http://127.0.0.1:${state.ports.service}`
+    if (step === 'agents') {
+      const files = selectCaseFiles(root)
+      if (!files.some((f) => f.cases.length || f.problem)) {
+        record({ step, status: 'skipped', note: 'no Agent test cases (agents/<slug>/evals/*.yaml)', ms: 0 })
+        continue
+      }
+      say(step, 'frontera local test (Agents)')
+      const started = Date.now()
+      const orgId = state.credentials.organizationId
+      const results = await runCases({
+        serviceUrl,
+        creds: state.credentials,
+        files,
+        live: false,
+        concurrency,
+        staleDraft: (slug) => agentFilesProblem(project, orgId, slug),
+        onResult: (r) => {
+          // The cases not sent after the provider refused every one are said once, below.
+          if (r.notRun?.skipped) return
+          ctx.output.note(`   ${r.passed ? '✓' : r.notRun ? '–' : '✕'} ${r.agent} · ${r.name}${r.passed ? '' : r.notRun ? ` — couldn’t run: ${r.notRun.what}` : ` — ${r.reason}`}${r.passed && r.retried ? ' (run twice: the provider refused the first try for its limits)' : ''}${r.waited?.length ? ` (${waitedWords(r.waited)})` : ''}`)
+        },
+      })
+      recordRun(project, results)
+      const bad = results.filter((r) => !r.passed && !r.notRun)
+      // Each failing case's conversation, last turn's trace and reply, read before the rollback takes the conversation.
+      const evidence = await failingCaseEvidence(project, results)
+      const rollsBack = baselineReady && !!testData && !testData.kept
+      for (const [i, e] of evidence) ctx.output.note(`     ${results[i]!.agent} · ${results[i]!.name}: ${evidenceLine(e, { rolledBack: rollsBack })}`)
+      const notRun = notRunGroups(results)
+      for (const g of notRun) {
+        if (g.skipped) ctx.output.note(`   – ${g.skipped} more case${g.skipped === 1 ? '' : 's'} not sent: ${g.what}`)
+        ctx.output.note(`   couldn’t run ${g.cases.length} of ${results.length} cases: ${withFix(g.what, g.fix)}`)
+      }
+      const passedCount = results.filter((r) => r.passed).length
+      const note = [bad.length ? `${bad.length} of ${results.length} cases failed` : '', notRun.length ? `${notRunWords(notRun)} (${passedCount} passed)` : ''].filter(Boolean).join('; ')
+      record({
+        step,
+        status: bad.length || notRun.length ? 'failed' : 'passed',
+        command: 'frontera local test',
+        ms: Date.now() - started,
+        ...(note ? { note } : {}),
+        ...(bad.length ? { failures: bad.map((r) => ({ name: `${r.agent} · ${r.name}`, reason: r.reason, ...(evidence.get(results.indexOf(r)) ?? {}) })) } : {}),
+        ...(notRun.length ? { couldNotRun: notRun.map((g) => ({ what: `couldn’t run: ${g.what}`, fix: g.fix, cases: g.cases, ...(g.skipped ? { notSent: g.skipped } : {}) })) } : {}),
+      })
+      continue
+    }
+    if (step === 'workflows') {
+      const files = findWorkflowTestFiles(root)
+      if (!files.some((f) => f.cases.length || f.problem)) {
+        record({ step, status: 'skipped', note: 'no workflow tests (workflows/tests/<slug>.yaml)', ms: 0 })
+        continue
+      }
+      say(step, 'frontera local test (workflows)')
+      const started = Date.now()
+      const orgId = state.credentials.organizationId
+      const results = await runWorkflowTests({ serviceUrl, creds: state.credentials, files, fileState: (slug) => workflowFileState(project, orgId, slug), onResult: (r) => ctx.output.note(`   ${r.passed ? '✓' : '✕'} workflow ${r.workflow} · ${r.name}${r.passed ? '' : ` — ${r.reason}`}`) })
+      recordWorkflowTestRun(project, results)
+      const bad = results.filter((r) => !r.passed)
+      record({ step, status: bad.length ? 'failed' : 'passed', command: 'frontera local test', ms: Date.now() - started, ...(bad.length ? { note: `${bad.length} of ${results.length} cases failed`, failures: bad.map((r) => ({ name: `${r.workflow} · ${r.name}`, reason: r.reason })) } : {}) })
+    }
+  }
+
+  // What the tests wrote, rolled back: the database as the apply left it.
+  const testsRan = steps.some((st) => (st.step === 'agents' || st.step === 'workflows') && st.status !== 'skipped')
+  if (baselineReady && testData && testsRan) {
+    const state = readState(project)
+    if (state?.credentials && running(project)) {
+      const restored = await restoreBaseline(project, state.credentials.organizationId, { serviceAnswers: () => waitForDatabaseReady(`http://127.0.0.1:${state.ports.service}`) })
+      // Traces live outside the platform database: a failing case's trace outlives the rollback, its conversation does not.
+      const traced = steps.some((st) => st.failures?.some((f) => f.traceId))
+      testData = { ...testData, rolledBack: restored.ok, ms: restored.ms, note: restored.ok ? `the tests’ writes were rolled back: ${restored.note}${traced ? '; the failing cases’ conversations went with them, their traces stay (trace_get, Studio’s Traces)' : ''}` : `the tests’ writes were not rolled back: ${restored.note}` }
+      ctx.output.note(`── test data: ${testData.note}${restored.ok ? ' (`--keep-state` keeps them)' : ''}`)
+      // The documents the baseline holds unread are read again: their reading went with the database the restore replaced.
+      const ingest = restored.ok ? localIngestApi(`http://127.0.0.1:${state.ports.service}`, state.credentials) : null
+      if (ingest) sayKnowledge(await settleKnowledge(ingest, state.credentials.workspaceId, { afterRestore: true }))
+      // The restore can leave a secret listed without its value: what the runtime needs now, said after it.
+      if (restored.ok) {
+        const lost = projectSecretLines(root, await localSecretPresence(`http://127.0.0.1:${state.ports.service}`, state.credentials))
+          .map((l) => l.trim().replace(/^warning:\s*/, ''))
+          .filter((l) => /^Needs a value for /.test(l) && !secretNotes.includes(l))
+        for (const line of lost) ctx.output.note(`   warning: after the rollback: ${line}`)
+        secretNotes.push(...lost)
+      }
+    }
+  } else if (testData?.kept && testsRan) ctx.output.note('── test data: the tests’ writes were kept (--keep-state); the next `frontera local check` rolls them back')
+  const testDataField = testData && testsRan ? { testData } : {}
+
+  if (flagBool(ctx, 'stop')) {
+    ctx.output.note('── frontera local stop')
+    await runIn(root, self(['local', 'stop']))
+  }
+  // Every step that failed, the first (`failedAt`) first.
+  const failures = steps.filter((st) => st.status === 'failed')
+  const failure = failures[0]
+  // Every inherited failure, by step: said in the result whether the gate passed or not.
+  const inherited = steps.flatMap((st) => (st.inherited ?? []).map((what) => ({ step: st.step, what })))
+  // Warnings inherited from the workspace: said in the result, never a failure.
+  const inheritedWarnings = steps.flatMap((st) => (st.inheritedWarnings ?? []).map((what) => ({ step: st.step, what })))
+  const warningsField = inheritedWarnings.length ? { inheritedWarnings } : {}
+  if (failure) {
+    const others = failures.slice(1).map((f) => f.step)
+    const message = `the check failed at ${failure.step}${failure.note ? `: ${failure.note}` : ''}${others.length ? `; also failed: ${others.join(', ')}` : ''}`
+    // Cases that couldn't run name their own fix (the provider account, the runtime), said with the hint.
+    const notRunFixes = [...new Set(failures.flatMap((f) => (f.couldNotRun ?? []).map((g) => g.fix)))]
+    const hint = [failure.step === 'apply' ? 'the output above names each item that did not apply; `frontera local status` lists them' : failure.couldNotRun && !failure.failures ? null : 'the output above names the cause of each', ...notRunFixes].filter(Boolean).join('; ')
+    const details = { passed: false, failedAt: failure.step, failures: failures.map((f) => ({ step: f.step, ...(f.note ? { note: f.note } : {}), ...(f.failures ? { failures: f.failures } : {}), ...(f.couldNotRun ? { couldNotRun: f.couldNotRun } : {}) })), steps, ...(inherited.length ? { inherited } : {}), ...warningsField, ...(secretNotes.length ? { secrets: secretNotes } : {}), ...(sdkNotes.length ? { sdk: sdkNotes } : {}), ...(knowledgeNotes.length ? { knowledge: knowledgeNotes } : {}), ...testDataField }
+    // `--json`: the check's result IS the data, failed or not, so it goes to stdout (exit 1 says it failed).
+    if (flagBool(ctx, 'json')) {
+      ctx.output.note(`error [LOCAL_CHECK_FAILED]: ${message}`)
+      return { data: { ...details, error: { code: 'LOCAL_CHECK_FAILED', message, hint } }, text: '', exitCode: EXIT.FAILURE }
+    }
+    throw new CliError(message, { code: 'LOCAL_CHECK_FAILED', hint, details })
+  }
+  const ran = steps.filter((s) => s.status === 'passed').map((s) => s.step)
+  const skipped = steps.filter((s) => s.status === 'skipped' && s.step !== 'start')
+  return {
+    data: { passed: true, steps, ...(inherited.length ? { inherited } : {}), ...warningsField, ...(secretNotes.length ? { secrets: secretNotes } : {}), ...(sdkNotes.length ? { sdk: sdkNotes } : {}), ...(knowledgeNotes.length ? { knowledge: knowledgeNotes } : {}), ...testDataField },
+    text: [
+      `Check passed: ${ran.join(', ')}${inherited.length ? `, with ${inheritedWords(inherited.length)}` : ''}.`,
+      ...(testData && testsRan ? [`  test data: ${testData.note}`] : []),
+      ...(inherited.length ? [`  inherited (report each with its fix):`, ...inherited.map((i) => `    ${i.step}: ${i.what}`)] : []),
+      ...(inheritedWarnings.length ? [`  warnings inherited from the workspace (report each; its owner fixes it there):`, ...inheritedWarnings.map((i) => `    ${i.step}: ${i.what}`)] : []),
+      ...skipped.map((s) => `  skipped ${s.step}: ${s.note}`),
+    ].join('\n'),
+  }
+}
+
 export const localCheck: Command = {
   meta: {
     noun: 'local',
     verb: 'check',
     args: [],
-    flags: { stop: 'boolean' },
-    summary: 'Start the runtime if needed, apply the project and run its test cases, with one exit code',
-    examples: ['frontera local check', 'frontera local check --stop'],
+    flags: { stop: 'boolean', skip: 'string', 'fail-fast': 'boolean', strict: 'boolean', 'keep-state': 'boolean', concurrency: 'string' },
+    summary: 'The whole gate with one exit code (start, apply, typecheck, unit tests, App checks, Agent and workflow tests): anything changed or added fails it, and failures only in items unchanged since the pull are listed as inherited and pass unless --strict',
+    examples: ['frontera local check', 'frontera local check --json', 'frontera local check --strict', 'frontera local check --skip typecheck,apps', 'frontera local check --fail-fast', 'frontera local check --keep-state', 'frontera local check --concurrency 1', 'frontera local check --stop'],
     offline: true,
-    internal: true,
-  },
-
-  async run(ctx) {
-    const project = localProject(ctx.cwd)
-    const state = readState(project)
-    const running = state?.phase === 'ready' && !!state.supervisorPid && isAlive(state.supervisorPid)
-    const steps: Array<{ label: string; args: string[] }> = [
-      ...(running ? [] : [{ label: 'start', args: ['local', 'start'] }]),
-      { label: 'apply', args: ['local', 'dev', '--once'] },
-      { label: 'test', args: ['local', 'test'] },
-    ]
-    let failed: string | null = null
-    for (const step of steps) {
-      ctx.output.note(`── frontera ${step.args.join(' ')}`)
-      if ((await runSelf(project.root, step.args)) !== 0) {
-        failed = step.label
-        break
-      }
-    }
-    if (flagBool(ctx, 'stop')) {
-      ctx.output.note('── frontera local stop')
-      await runSelf(project.root, ['local', 'stop'])
-    }
-    if (failed) {
-      throw new CliError(`the check failed at ${failed}`, { code: 'LOCAL_CHECK_FAILED', hint: 'the output above names the cause' })
-    }
-    return { data: { passed: true }, text: 'Check passed: the project applies and its test cases pass.' }
   },
+  run: check,
 }