@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/families/agent-eval-harness-ts/files/src/eval/campaign.ts

@@ -0,0 +1,137 @@
+// Eval campaign — opinionated wrapper over `runEvalCampaign` from
+// `@tangle-network/agent-eval@0.22+`. Use this when the eval question is
+// "does variant A beat variant B over scenarios × seeds?" — i.e. a
+// launch-decision-grade sweep, not a smoke test.
+//
+// Why a separate entrypoint from `runHarness`:
+//
+//   - `runHarness` (runner.ts) runs each scenario once with no comparator.
+//     It's the smoke-test path — operator-facing, fast, no variants.
+//
+//   - `runCampaign` (this file) runs (variants × scenarios × seeds) and
+//     emits `RunRecord[]` + integrity reports + an optional `researchReport`
+//     with paired-evidence verdicts. Capture integrity is wired by
+//     construction:
+//
+//       - `assertLlmRoute` runs once at preflight.
+//       - A per-run `FileSystemRawProviderSink` captures every provider
+//         HTTP envelope.
+//       - `assertRunCaptured` fires after every `endRun`; the campaign's
+//         `onIntegrityFailure` policy decides whether to mark the run
+//         failed or admit it with a flag.
+//       - Each run is built atop a `TraceEmitter` whose `onRunComplete`
+//         hooks fire the trace analyst declaratively.
+//
+// The four directives from `SKILL.md § Capture integrity` are part of
+// the campaign's contract — the caller doesn't wire them, and skipping
+// one isn't possible without rewriting the function.
+//
+// Reference wiring lives in `@tangle-network/agent-eval/src/eval-campaign.ts`.
+
+import { join } from 'node:path'
+import {
+  runEvalCampaign,
+  type CampaignRunContext,
+  type CampaignRunOutcome,
+  type CampaignScenario,
+  type CampaignVariant,
+  type EvalCampaignOptions,
+  type EvalCampaignResult,
+  type LlmClientOptions,
+} from '@tangle-network/agent-eval'
+import {
+  FileSystemRawProviderSink,
+  type RawProviderSink,
+} from '@tangle-network/agent-eval/traces'
+
+export interface CampaignOptions<V> {
+  /** Stable campaign id — folded into the campaign fingerprint. */
+  campaignId: string
+  variants: CampaignVariant<V>[]
+  scenarios: CampaignScenario[]
+  /** Default `[0, 1, 2]`. */
+  seeds?: number[]
+  /** Git SHA the sweep runs against — `RunRecord` rejects unset. */
+  commitSha: string
+  /**
+   * The per-run runner. Receives a fully-wired context (TraceEmitter, store,
+   * RawProviderSink, llmOpts) and returns a `CampaignRunOutcome`. The
+   * runner MUST call `ctx.emitter.startRun` before any work and
+   * `ctx.emitter.endRun` / `ctx.emitter.abortRun` before returning.
+   */
+  runner: (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>
+  /**
+   * LLM client config — `assertLlmRoute` runs against this at preflight.
+   * Must include `baseUrl` + `apiKey` for the route guard to pass with
+   * default `routeRequirements`.
+   */
+  llmOpts: LlmClientOptions
+  /**
+   * Filesystem root for traces + raw events. Defaults to
+   * `./.evolve/agent-eval/campaigns/<campaignId>/`.
+   */
+  workDir?: string
+  /** Optional comparator variant id for paired stats. */
+  comparator?: string
+  /** Optional preregistration manifest hash (see `pre-registration.ts`). */
+  preregistrationHash?: string
+}
+
+/**
+ * Run a launch-decision-grade eval sweep over (variants × scenarios × seeds).
+ *
+ * Returns a `RunRecord[]` + integrity reports + (when `comparator` is set)
+ * a `researchReport` with paired-bootstrap CIs, anytime-valid sequential
+ * verdicts, and a SHA-256 campaign fingerprint.
+ *
+ * Capture-integrity directives are wired by construction:
+ *   1. `RawProviderSink` per run (via the campaign's default `rawSinkFactory`).
+ *   2. `assertLlmRoute` at preflight.
+ *   3. `assertRunCaptured` after every `endRun` (policy: `'mark_failed'`).
+ *   4. `onRunComplete` hooks fire on every run (caller supplies the analyst).
+ *
+ * Skip this and you're back to writing the same shape ad hoc — at the
+ * cost of the bug class 0.22 was designed to eliminate.
+ */
+export async function runCampaign<V>(
+  opts: CampaignOptions<V>,
+): Promise<EvalCampaignResult> {
+  const workDir =
+    opts.workDir ?? join(process.cwd(), '.evolve', 'agent-eval', 'campaigns', opts.campaignId)
+  const tracesDir = join(workDir, 'traces')
+
+  // The store factory hands each cell its own `FileSystemTraceStore` instance
+  // rooted in a per-run subdir. We can't construct a FileSystemTraceStore
+  // here because the import lives in the main barrel — pull it dynamically
+  // so this module stays a stable interface even if upstream re-exports
+  // shift between minor versions.
+  const { FileSystemTraceStore } = await import('@tangle-network/agent-eval')
+
+  const fullOpts: EvalCampaignOptions<V> = {
+    campaignId: opts.campaignId,
+    variants: opts.variants,
+    scenarios: opts.scenarios,
+    commitSha: opts.commitSha,
+    runner: opts.runner,
+    llmOpts: opts.llmOpts,
+    storeFactory: ({ runId }) =>
+      new FileSystemTraceStore({ dir: join(tracesDir, runId) }),
+    rawSinkFactory: ({ runId }): RawProviderSink =>
+      new FileSystemRawProviderSink({ dir: join(workDir, 'raw-events', runId) }),
+    workDir,
+    onIntegrityFailure: 'mark_failed',
+    ...(opts.seeds ? { seeds: opts.seeds } : {}),
+    ...(opts.preregistrationHash ? { preregistrationHash: opts.preregistrationHash } : {}),
+    ...(opts.comparator ? { report: { comparator: opts.comparator } } : {}),
+  }
+
+  return runEvalCampaign(fullOpts)
+}
+
+export type {
+  CampaignRunContext,
+  CampaignRunOutcome,
+  CampaignScenario,
+  CampaignVariant,
+  EvalCampaignResult,
+} from '@tangle-network/agent-eval'

package/registry/families/agent-eval-harness-ts/files/src/eval/cli.ts

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+// eval-harness CLI — `eval-harness run | compare`.
+//
+// `run` invokes the full harness against the configured target.
+// `compare` diffs two scorecards (no statistical gate — for that, use the
+// regression layer's `eval:gate` CLI).
+
+import { runHarness } from './runner.js'
+import { readScorecard, diffScorecards } from './scorecard.js'
+
+function help(): never {
+  console.log(`Usage:
+  eval-harness run [--target <url>] [--threshold <0..1>] [--variant <id>]
+  eval-harness compare <baseline.json> <head.json>
+`)
+  process.exit(2)
+}
+
+async function cmdRun(argv: string[]): Promise<void> {
+  const arg = (flag: string): string | undefined => {
+    const i = argv.indexOf(flag)
+    return i >= 0 && argv[i + 1] ? argv[i + 1] : undefined
+  }
+  const target = arg('--target')
+  const thresholdRaw = arg('--threshold')
+  const variantId = arg('--variant')
+  const threshold = thresholdRaw !== undefined ? Number(thresholdRaw) : undefined
+  if (thresholdRaw !== undefined && Number.isNaN(threshold)) {
+    console.error(`eval-harness: --threshold must be a number, got "${thresholdRaw}"`)
+    process.exit(2)
+  }
+  const report = await runHarness({ targetUrl: target, threshold, variantId })
+  process.exitCode = report.aggregate < report.threshold ? 1 : 0
+}
+
+async function cmdCompare(argv: string[]): Promise<void> {
+  const [baselinePath, headPath] = argv
+  if (!baselinePath || !headPath) help()
+  const baseline = await readScorecard(baselinePath)
+  const head = await readScorecard(headPath)
+  const diff = diffScorecards(baseline, head)
+  const arrow = diff.aggregateDelta >= 0 ? '↑' : '↓'
+  console.log(
+    `aggregate ${baseline.aggregate.toFixed(3)} → ${head.aggregate.toFixed(3)} (${arrow}${Math.abs(diff.aggregateDelta).toFixed(3)})`,
+  )
+  for (const f of diff.perFlow) {
+    const b = f.baseline?.toFixed(3) ?? '—'
+    const h = f.head?.toFixed(3) ?? '—'
+    const d = f.delta !== null ? `${f.delta >= 0 ? '+' : ''}${f.delta.toFixed(3)}` : ''
+    const tag = f.status.padEnd(10, ' ')
+    console.log(`  ${tag} ${f.name}: ${b} → ${h} ${d}`)
+  }
+}
+
+async function main(): Promise<void> {
+  const [cmd, ...rest] = process.argv.slice(2)
+  if (cmd === 'run') return cmdRun(rest)
+  if (cmd === 'compare') return cmdCompare(rest)
+  help()
+}
+
+main().catch((err) => {
+  console.error('[eval-harness] fatal:', err)
+  process.exit(2)
+})

package/registry/families/agent-eval-harness-ts/files/src/eval/runner.ts

@@ -0,0 +1,384 @@
+// Eval runner — thin shell over @tangle-network/agent-eval primitives.
+//
+// Lifecycle:
+//   1. Load every Scenario exported from `scenarios/*.scenario.ts`.
+//   2. For each scenario, build a TestGradedScenario whose harness.testCommand
+//      is a curl-based assertion against the target URL (or skip the test
+//      command entirely when running in `judge-only` mode).
+//   3. Delegate to `runTestGradedScenario` from agent-eval — it spawns the
+//      command via SubprocessSandboxDriver, emits a Run, persists spans
+//      via FileSystemTraceStore.
+//   4. Run every JudgeFn in `judges/*.judge.ts` over the collected
+//      ScenarioResult; aggregate via `JudgeRunner` when sandbox-graded.
+//   5. Emit a scorecard via `writeScorecard` (see scorecard.ts).
+//
+// Capture integrity (agent-eval 0.21+):
+//   - Allocates a `FileSystemRawProviderSink` per run, rooted under
+//     `<tracesDir>/raw-events/<runId>/`. LLM-as-judge call sites that
+//     read `globalThis.__agentEvalRawSink` (or that consumers wire
+//     explicitly) auto-capture every provider HTTP request/response.
+//   - When `EVAL_LLM_BASE_URL` is set the runner calls `assertLlmRoute`
+//     at preflight so the sweep fails loud if it would silently fall
+//     back to the public router.
+//   - After every `runTestGradedScenario` the runner calls
+//     `assertRunCaptured` to verify the run wrote the expected spans
+//     (and, when judges ran, that every LlmSpan has a matching raw
+//     `request` event). Integrity issues are surfaced on the outcome
+//     row but only fail the scenario when `EVAL_INTEGRITY=strict`.
+//
+// All primitives come from the published package — this file is the
+// integration glue, not new measurement infra.
+
+import { resolve, join, dirname } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import {
+  FileSystemTraceStore,
+  FileSystemExperimentStore,
+  SubprocessSandboxDriver,
+  assertLlmRoute,
+  LlmRouteAssertionError,
+  runTestGradedScenario,
+  type Scenario,
+  type TestGradedScenario,
+  type TestGradedRunResult,
+} from '@tangle-network/agent-eval'
+import {
+  FileSystemRawProviderSink,
+  assertRunCaptured,
+  type RawProviderSink,
+  type RunIntegrityReport,
+} from '@tangle-network/agent-eval/traces'
+// Single source of truth for scenario loading: the agent-eval:scenarios
+// layer composes `src/eval/scenario-loader.ts` next to this runner. Pre-fix
+// the family re-implemented a weaker loader inline (no shape validation),
+// silently diverging from the layer's strict checks. Per CLAUDE.md
+// "extend, don't duplicate", the runner imports the layer's loader.
+import { loadScenarios, type LoadedScenario } from './scenario-loader.js'
+import { writeScorecard, type ScorecardFlow } from './scorecard.js'
+
+export type IntegrityMode = 'off' | 'log' | 'strict'
+
+export interface RunnerOptions {
+  /** Project root — defaults to cwd of the runner. */
+  projectRoot?: string
+  /** HTTP target URL the test commands hit. */
+  targetUrl?: string
+  /** Pass threshold for the aggregate. CI gate uses this. */
+  threshold?: number
+  /** Override scenarios dir (default: <root>/scenarios). */
+  scenariosDir?: string
+  /** Override judges dir (default: <root>/judges). */
+  judgesDir?: string
+  /** Override traces dir (default: <root>/.evolve/agent-eval/traces). */
+  tracesDir?: string
+  /** Override experiments dir (default: <root>/.evolve/agent-eval/experiments). */
+  experimentsDir?: string
+  /**
+   * Root for per-run `FileSystemRawProviderSink` directories. Default:
+   * `<tracesDir>/../raw-events`. Disabled when `integrityMode` is `'off'`.
+   */
+  rawEventsDir?: string
+  /** Where the project-level scorecard.json lands. */
+  scorecardPath?: string
+  /** Variant label tagged onto every Run for A/B compares. */
+  variantId?: string
+  /**
+   * Capture-integrity behaviour after each `runTestGradedScenario`:
+   *
+   *   - `'off'`   — skip raw-sink wiring and `assertRunCaptured`.
+   *   - `'log'`   — assert and surface issues on the outcome row, but do
+   *                 NOT downgrade the scenario verdict. (default)
+   *   - `'strict'`— integrity issues mark the scenario as failed with
+   *                 `failureClass='integrity'`.
+   *
+   * Honours `EVAL_INTEGRITY` env var when unset.
+   */
+  integrityMode?: IntegrityMode
+  /**
+   * Optional LLM client options carried only for the preflight route guard.
+   * When `baseUrl` is set the runner calls `assertLlmRoute` once before any
+   * scenario runs so the sweep fails loud if it would silently fall back
+   * to the public router. Mirrors the `LlmClientOptions` shape from
+   * agent-eval; pass `{ baseUrl, provider, apiKey }` for the LLM judge
+   * route you actually want to exercise.
+   */
+  llmOpts?: {
+    baseUrl?: string
+    provider?: string
+    apiKey?: string
+  }
+}
+
+interface ScenarioOutcome {
+  scenarioId: string
+  pass: boolean
+  score: number
+  durationMs: number
+  failureClass: string | null
+  filePath: string
+  runId: string
+  /**
+   * Capture-integrity issues observed at run-end. Empty in the common
+   * happy-path case. Surfaced on the row so a launch reviewer can spot
+   * runs that completed structurally but lost their forensic evidence.
+   */
+  integrityIssues?: string[]
+}
+
+/**
+ * Lightweight RawProviderSink handle exposed on `globalThis` for ad-hoc
+ * LLM-judge wiring. A judge call site can read this to pass `rawSink`
+ * into `callLlm` without the runner having to know about the judge. The
+ * canonical pattern is to wire `rawSink` explicitly via dependency
+ * injection — this hook exists so the example judge auto-captures.
+ */
+declare global {
+  // eslint-disable-next-line no-var
+  var __agentEvalRawSink: RawProviderSink | undefined
+}
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const DEFAULT_ROOT = resolve(HERE, '..', '..')
+
+function shQuote(s: string): string {
+  return `'${String(s).replace(/'/g, `'\\''`)}'`
+}
+
+// Translate a Scenario into a TestGradedScenario — the bridge between
+// the conversational scenario shape and the test-graded-runner shape.
+//
+// Default policy: every scenario's first turn is POST-ed at the target
+// URL's /chat route, and the response is checked for non-empty content.
+// Override by editing the testCommand in your scenario file's tags.
+function toTestGraded(scenario: Scenario, targetUrl: string): TestGradedScenario {
+  const customTestCommand = (scenario as Scenario & { testCommand?: string }).testCommand
+  const firstTurn = scenario.turns[0]
+  const userMessage = firstTurn?.user ?? ''
+  const defaultTest =
+    `curl -sS -o /tmp/eval-body -w '%{http_code}' -X POST -H 'content-type: application/json' ` +
+    `--data ${shQuote(JSON.stringify({ message: userMessage, scenarioId: scenario.id }))} ` +
+    `${shQuote(`${targetUrl}/chat`)} | grep -E '^(200|201)$' >/dev/null && ` +
+    `test $(wc -c < /tmp/eval-body) -ge 1`
+  return {
+    id: scenario.id,
+    description: scenario.label ?? scenario.thesis ?? scenario.id,
+    harness: {
+      testCommand: customTestCommand ?? defaultTest,
+      timeoutMs: 30_000,
+    },
+    passThreshold: 1.0,
+    tags: { persona: scenario.persona, dimensions: scenario.dimensions.join(',') },
+  }
+}
+
+export interface RunReport {
+  timestamp: string
+  variantId?: string
+  scenarioCount: number
+  passCount: number
+  aggregate: number
+  threshold: number
+  outcomes: ScenarioOutcome[]
+  tracesDir: string
+  rawEventsDir: string | null
+  scorecardPath: string
+  integrityMode: IntegrityMode
+  /** Per-run integrity reports, indexed by runId. */
+  integrityReports: Record<string, RunIntegrityReport>
+}
+
+function resolveIntegrityMode(opt: IntegrityMode | undefined): IntegrityMode {
+  if (opt) return opt
+  const env = (process.env.EVAL_INTEGRITY ?? '').toLowerCase()
+  if (env === 'off' || env === 'log' || env === 'strict') return env
+  return 'log'
+}
+
+export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
+  const projectRoot = opts.projectRoot ?? process.cwd()
+  const targetUrl = opts.targetUrl ?? process.env.EVAL_TARGET_BASE_URL ?? 'http://127.0.0.1:8787'
+  const threshold = opts.threshold ?? Number(process.env.EVAL_THRESHOLD ?? '0.7')
+  const scenariosDir = opts.scenariosDir ?? join(projectRoot, 'scenarios')
+  const tracesDir = opts.tracesDir ?? join(projectRoot, '.evolve', 'agent-eval', 'traces')
+  const experimentsDir =
+    opts.experimentsDir ?? join(projectRoot, '.evolve', 'agent-eval', 'experiments')
+  const rawEventsDir =
+    opts.rawEventsDir ?? join(projectRoot, '.evolve', 'agent-eval', 'raw-events')
+  const scorecardPath = opts.scorecardPath ?? join(projectRoot, '.evolve', 'scorecard.json')
+  const variantId = opts.variantId ?? process.env.EVAL_VARIANT_ID
+  const integrityMode = resolveIntegrityMode(opts.integrityMode)
+
+  // ── 0.21 Directive 2: assert the route at preflight ───────────────────
+  // Pure function; no I/O. Only runs when a baseUrl is supplied (env or
+  // opts) — the eval-harness ships with no LLM judge by default, so we
+  // don't want to block users who never call an LLM. When a route IS
+  // declared, fail loud before any scenario runs.
+  const llmBaseUrl = opts.llmOpts?.baseUrl ?? process.env.EVAL_LLM_BASE_URL
+  const llmApiKey = opts.llmOpts?.apiKey ?? process.env.EVAL_LLM_API_KEY ?? process.env.TANGLE_API_KEY
+  const llmProvider = opts.llmOpts?.provider ?? process.env.EVAL_LLM_PROVIDER
+  if (llmBaseUrl) {
+    try {
+      assertLlmRoute(
+        { baseUrl: llmBaseUrl, apiKey: llmApiKey, provider: llmProvider },
+        { requireExplicitBaseUrl: true, requireAuth: true },
+      )
+    } catch (err) {
+      if (err instanceof LlmRouteAssertionError) {
+        throw new Error(
+          `eval-harness: preflight route check failed (code=${err.code}). ` +
+            `Set EVAL_LLM_BASE_URL + auth, or unset to skip the check. ${err.message}`,
+        )
+      }
+      throw err
+    }
+  }
+
+  const traceStore = new FileSystemTraceStore({ dir: tracesDir })
+  // Experiment store is created so callers can persist experiment metadata
+  // alongside traces; its mere existence ensures the dir is created.
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const _experimentStore = new FileSystemExperimentStore({ dir: experimentsDir })
+  const driver = new SubprocessSandboxDriver({ cwd: projectRoot })
+
+  const loaded: LoadedScenario[] = await loadScenarios(scenariosDir)
+  if (loaded.length === 0) {
+    throw new Error(
+      `eval-harness: no scenarios loaded from ${scenariosDir}. Drop a *.scenario.ts file with a default-exported Scenario.`,
+    )
+  }
+
+  const integrityReports: Record<string, RunIntegrityReport> = {}
+  const outcomes: ScenarioOutcome[] = []
+  for (const { scenario, filePath } of loaded) {
+    const tgs = toTestGraded(scenario, targetUrl)
+    const startedAt = Date.now()
+
+    // ── 0.21 Directive 1: per-run RawProviderSink ──────────────────────
+    // Allocated even when no LLM judge runs — the cost is one (possibly
+    // empty) NDJSON file per scenario, and the alternative is a silent
+    // partial-capture bug the next time someone wires in an LLM judge.
+    // Exposed via globalThis so the ad-hoc judge call sites in this
+    // template's `judges/` dir pick it up without explicit plumbing.
+    let rawSink: RawProviderSink | undefined
+    if (integrityMode !== 'off') {
+      rawSink = new FileSystemRawProviderSink({
+        dir: join(rawEventsDir, scenario.id),
+      })
+      globalThis.__agentEvalRawSink = rawSink
+    }
+
+    let result: TestGradedRunResult
+    try {
+      result = await runTestGradedScenario(tgs, traceStore, { driver, variantId })
+    } catch (err) {
+      outcomes.push({
+        scenarioId: scenario.id,
+        pass: false,
+        score: 0,
+        durationMs: Date.now() - startedAt,
+        failureClass: 'harness_error',
+        filePath,
+        runId: '',
+      })
+      console.error(`  ✗ ${scenario.id} — harness error: ${(err as Error).message}`)
+      continue
+    } finally {
+      // Always drop the global handle so the next scenario gets its own
+      // sink (or none, when integrity is off).
+      globalThis.__agentEvalRawSink = undefined
+    }
+
+    // ── 0.21 Directive 3: assert the run captured before declaring done ─
+    // Read-only. We do NOT set `requireRawCoverageOfLlmSpans` because
+    // the test-graded path may not call any LLM at all (the default
+    // testCommand is a curl). When an LLM judge IS wired the sink will
+    // be populated and consumers can tighten this in `runIntegrityExpect`.
+    let integrityIssues: string[] | undefined
+    if (integrityMode !== 'off' && result.runId) {
+      const report = await assertRunCaptured(traceStore, result.runId, {
+        rawSink,
+        // requireOutcome: the harness always sets pass/score, so this is
+        // a cheap belt-and-suspenders check.
+        requireOutcome: true,
+      })
+      integrityReports[result.runId] = report
+      if (!report.ok) {
+        integrityIssues = report.issues.map((i) => i.code)
+      }
+    }
+
+    const integrityFailed = integrityMode === 'strict' && integrityIssues !== undefined
+    outcomes.push({
+      scenarioId: scenario.id,
+      pass: integrityFailed ? false : result.pass,
+      score: integrityFailed ? 0 : result.score,
+      durationMs: Date.now() - startedAt,
+      failureClass: integrityFailed ? 'integrity' : (result.failureClass ?? null),
+      filePath,
+      runId: result.runId,
+      ...(integrityIssues ? { integrityIssues } : {}),
+    })
+    const mark = integrityFailed ? '✗' : result.pass ? '✓' : '✗'
+    const integritySuffix = integrityIssues ? ` (integrity: ${integrityIssues.join(', ')})` : ''
+    console.log(
+      `  ${mark} ${scenario.id} — score=${result.score.toFixed(3)}${integritySuffix}`,
+    )
+  }
+
+  const passCount = outcomes.filter((o) => o.pass).length
+  const aggregate = outcomes.reduce((a, o) => a + o.score, 0) / Math.max(outcomes.length, 1)
+
+  const flows: ScorecardFlow[] = outcomes.map((o) => ({
+    name: o.scenarioId,
+    value: o.score,
+    target: 1.0,
+    status: o.pass ? 'pass' : 'fail',
+    direction: 'higher-better',
+    productValueClaim: `Scenario "${o.scenarioId}" — pass rate against the agent under test.`,
+  }))
+  const report: RunReport = {
+    timestamp: new Date().toISOString(),
+    variantId,
+    scenarioCount: outcomes.length,
+    passCount,
+    aggregate,
+    threshold,
+    outcomes,
+    tracesDir,
+    rawEventsDir: integrityMode === 'off' ? null : rawEventsDir,
+    scorecardPath,
+    integrityMode,
+    integrityReports,
+  }
+  await writeScorecard(scorecardPath, {
+    product: process.env.EVAL_PRODUCT_NAME ?? 'eval-harness',
+    timestamp: report.timestamp,
+    aggregate,
+    coverage: `${passCount}/${outcomes.length} scenarios passed`,
+    flows,
+  })
+  const integritySummary =
+    integrityMode === 'off'
+      ? 'integrity=off'
+      : `integrity=${integrityMode} ` +
+        `${Object.values(integrityReports).filter((r) => !r.ok).length}/${
+          Object.keys(integrityReports).length
+        } flagged`
+  console.log(
+    `\naggregate=${aggregate.toFixed(3)} pass=${passCount}/${outcomes.length} threshold=${threshold} ${integritySummary}`,
+  )
+  return report
+}
+
+// CLI-friendly default — invoked from `pnpm eval` when this file is the entry.
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+  const projectRoot = process.env.EVAL_PROJECT_ROOT ?? DEFAULT_ROOT
+  runHarness({ projectRoot })
+    .then((r) => {
+      process.exitCode = r.aggregate < r.threshold ? 1 : 0
+    })
+    .catch((err) => {
+      console.error('[runner] fatal:', err)
+      process.exit(2)
+    })
+}

package/registry/families/agent-eval-harness-ts/files/src/eval/scorecard.ts

@@ -0,0 +1,89 @@
+// Scorecard emitter — writes scorecard.json in the shape consumed by the
+// project-level dashboard (.evolve/scorecard.json).
+//
+// The shape is intentionally a subset of the parent-repo scorecard so a
+// composed eval harness drops into an existing .evolve workflow without
+// translation. Extra dashboard fields (sourceMtime, stale, etc.) are
+// only relevant when the parent repo aggregates multiple inputs — for a
+// standalone harness, the runner's outputs are the single source.
+
+import { writeFile, mkdir } from 'node:fs/promises'
+import { dirname } from 'node:path'
+
+export type FlowDirection = 'higher-better' | 'lower-better'
+export type FlowStatus = 'pass' | 'fail' | 'skip'
+
+export interface ScorecardFlow {
+  name: string
+  value: number
+  target: number
+  status: FlowStatus
+  productValueClaim: string
+  direction: FlowDirection
+}
+
+export interface Scorecard {
+  product: string
+  timestamp: string
+  aggregate: number
+  coverage: string
+  flows: ScorecardFlow[]
+}
+
+export async function writeScorecard(path: string, scorecard: Scorecard): Promise<void> {
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(path, JSON.stringify(scorecard, null, 2) + '\n', 'utf8')
+}
+
+export async function readScorecard(path: string): Promise<Scorecard> {
+  const { readFile } = await import('node:fs/promises')
+  const raw = await readFile(path, 'utf8')
+  return JSON.parse(raw) as Scorecard
+}
+
+export interface ScorecardDiff {
+  baseline: Scorecard
+  head: Scorecard
+  aggregateDelta: number
+  perFlow: Array<{
+    name: string
+    baseline: number | null
+    head: number | null
+    delta: number | null
+    status: 'improved' | 'regressed' | 'stable' | 'introduced' | 'removed'
+  }>
+}
+
+const STABILITY_EPSILON = 0.005
+
+export function diffScorecards(baseline: Scorecard, head: Scorecard): ScorecardDiff {
+  const baselineByName = new Map<string, ScorecardFlow>()
+  for (const f of baseline.flows) baselineByName.set(f.name, f)
+  const headByName = new Map<string, ScorecardFlow>()
+  for (const f of head.flows) headByName.set(f.name, f)
+  const allNames = new Set<string>([...baselineByName.keys(), ...headByName.keys()])
+  const perFlow: ScorecardDiff['perFlow'] = []
+  for (const name of allNames) {
+    const b = baselineByName.get(name)
+    const h = headByName.get(name)
+    if (b && h) {
+      const delta = h.value - b.value
+      let status: ScorecardDiff['perFlow'][number]['status'] = 'stable'
+      if (Math.abs(delta) >= STABILITY_EPSILON) {
+        const better = h.direction === 'higher-better' ? delta > 0 : delta < 0
+        status = better ? 'improved' : 'regressed'
+      }
+      perFlow.push({ name, baseline: b.value, head: h.value, delta, status })
+    } else if (b) {
+      perFlow.push({ name, baseline: b.value, head: null, delta: null, status: 'removed' })
+    } else if (h) {
+      perFlow.push({ name, baseline: null, head: h.value, delta: null, status: 'introduced' })
+    }
+  }
+  return {
+    baseline,
+    head,
+    aggregateDelta: head.aggregate - baseline.aggregate,
+    perFlow,
+  }
+}

package/registry/families/agent-eval-harness-ts/files/tests/fixtures/scenarios-bad/missing-thesis.scenario.ts

@@ -0,0 +1,14 @@
+// Fixture: scenario missing the required `thesis` field. The layer's
+// strict loader must reject this; the family's pre-fix loader silently
+// accepted it.
+const scenario = {
+  id: 'fixture-bad',
+  persona: 'developer',
+  label: 'missing-thesis',
+  // NB: thesis intentionally omitted to trigger the strict validator.
+  dimensions: ['success'],
+  turns: [{ user: 'hi' }],
+  artifactChecks: [],
+}
+
+export default scenario