@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/families/agent-eval-harness-ts/files/src/eval/runner.ts

@@ -0,0 +1,491 @@
+// Eval runner — thin shell over @tangle-network/agent-eval primitives.
+//
+// Lifecycle:
+//   1. Load every Scenario exported from `scenarios/*.scenario.ts`.
+//   2. For each scenario, build a TestGradedScenario whose harness.testCommand
+//      is a curl-based assertion against the target URL (or skip the test
+//      command entirely when running in `judge-only` mode).
+//   3. Delegate to `runTestGradedScenario` from agent-eval — it spawns the
+//      command via SubprocessSandboxDriver, emits a Run, persists spans
+//      via FileSystemTraceStore.
+//   4. Run every JudgeFn in `judges/*.judge.ts` over the collected
+//      ScenarioResult; aggregate via `JudgeRunner` when sandbox-graded.
+//   5. Emit a scorecard via `writeScorecard` (see scorecard.ts).
+//
+// Capture integrity (agent-eval 0.21+):
+//   - Allocates a `FileSystemRawProviderSink` per run, rooted under
+//     `<tracesDir>/raw-events/<runId>/`. LLM-as-judge call sites that
+//     read `globalThis.__agentEvalRawSink` (or that consumers wire
+//     explicitly) auto-capture every provider HTTP request/response.
+//   - When `EVAL_LLM_BASE_URL` is set the runner calls `assertLlmRoute`
+//     at preflight so the sweep fails loud if it would silently fall
+//     back to the public router.
+//   - After every `runTestGradedScenario` the runner calls
+//     `assertRunCaptured` to verify the run wrote the expected spans
+//     (and, when judges ran, that every LlmSpan has a matching raw
+//     `request` event). Integrity issues are surfaced on the outcome
+//     row but only fail the scenario when `EVAL_INTEGRITY=strict`.
+//
+// All primitives come from the published package — this file is the
+// integration glue, not new measurement infra.
+
+import { readdir } from 'node:fs/promises'
+import { resolve, join, dirname, isAbsolute } from 'node:path'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import {
+  FileSystemTraceStore,
+  fileExperimentStore,
+  SubprocessSandboxDriver,
+  assertLlmRoute,
+  LlmRouteAssertionError,
+  runTestGradedScenario,
+  type Scenario,
+  type TestGradedScenario,
+  type TestGradedRunResult,
+  type JudgeFn,
+  type JudgeInput,
+  type JudgeScore,
+  type CollectedArtifacts,
+} from '@tangle-network/agent-eval'
+import {
+  FileSystemRawProviderSink,
+  assertRunCaptured,
+  type RawProviderSink,
+  type RunIntegrityReport,
+} from '@tangle-network/agent-eval/traces'
+import { TCloud } from '@tangle-network/tcloud'
+// Single source of truth for scenario loading: the agent-eval:scenarios
+// layer composes `src/eval/scenario-loader.ts` next to this runner. Pre-fix
+// the family re-implemented a weaker loader inline (no shape validation),
+// silently diverging from the layer's strict checks. Per CLAUDE.md
+// "extend, don't duplicate", the runner imports the layer's loader.
+import { loadScenarios, type LoadedScenario } from './scenario-loader.js'
+import { writeScorecard, type ScorecardFlow } from './scorecard.js'
+
+export type IntegrityMode = 'off' | 'log' | 'strict'
+
+export interface RunnerOptions {
+  /** Project root — defaults to cwd of the runner. */
+  projectRoot?: string
+  /** HTTP target URL the test commands hit. */
+  targetUrl?: string
+  /** Pass threshold for the aggregate. CI gate uses this. */
+  threshold?: number
+  /** Override scenarios dir (default: <root>/scenarios). */
+  scenariosDir?: string
+  /** Override judges dir (default: <root>/judges). */
+  judgesDir?: string
+  /** Override traces dir (default: <root>/.evolve/agent-eval/traces). */
+  tracesDir?: string
+  /** Override experiments dir (default: <root>/.evolve/agent-eval/experiments). */
+  experimentsDir?: string
+  /**
+   * Root for per-run `FileSystemRawProviderSink` directories. Default:
+   * `<tracesDir>/../raw-events`. Disabled when `integrityMode` is `'off'`.
+   */
+  rawEventsDir?: string
+  /** Where the project-level scorecard.json lands. */
+  scorecardPath?: string
+  /** Variant label tagged onto every Run for A/B compares. */
+  variantId?: string
+  /**
+   * Capture-integrity behaviour after each `runTestGradedScenario`:
+   *
+   *   - `'off'`   — skip raw-sink wiring and `assertRunCaptured`.
+   *   - `'log'`   — assert and surface issues on the outcome row, but do
+   *                 NOT downgrade the scenario verdict. (default)
+   *   - `'strict'`— integrity issues mark the scenario as failed with
+   *                 `failureClass='integrity'`.
+   *
+   * Honours `EVAL_INTEGRITY` env var when unset.
+   */
+  integrityMode?: IntegrityMode
+  /**
+   * Optional LLM client options carried only for the preflight route guard.
+   * When `baseUrl` is set the runner calls `assertLlmRoute` once before any
+   * scenario runs so the sweep fails loud if it would silently fall back
+   * to the public router. Mirrors the `LlmClientOptions` shape from
+   * agent-eval; pass `{ baseUrl, provider, apiKey }` for the LLM judge
+   * route you actually want to exercise.
+   */
+  llmOpts?: {
+    baseUrl?: string
+    provider?: string
+    apiKey?: string
+  }
+}
+
+interface ScenarioOutcome {
+  scenarioId: string
+  pass: boolean
+  score: number
+  durationMs: number
+  failureClass: string | null
+  filePath: string
+  runId: string
+  measuredJudgeCount: number
+  unmeasuredJudgeCount: number
+  /**
+   * Capture-integrity issues observed at run-end. Empty in the common
+   * happy-path case. Surfaced on the row so a launch reviewer can spot
+   * runs that completed structurally but lost their forensic evidence.
+   */
+  integrityIssues?: string[]
+}
+
+/**
+ * Lightweight RawProviderSink handle exposed on `globalThis` for ad-hoc
+ * LLM-judge wiring. A judge call site can read this to pass `rawSink`
+ * into `callLlm` without the runner having to know about the judge. The
+ * canonical pattern is to wire `rawSink` explicitly via dependency
+ * injection — this hook exists so the example judge auto-captures.
+ */
+declare global {
+  // eslint-disable-next-line no-var
+  var __agentEvalRawSink: RawProviderSink | undefined
+}
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+const DEFAULT_ROOT = resolve(HERE, '..', '..')
+
+function shQuote(s: string): string {
+  return `'${String(s).replace(/'/g, `'\\''`)}'`
+}
+
+interface LoadedJudge {
+  fn: JudgeFn
+  filePath: string
+}
+
+async function loadJudgesFrom(dir: string): Promise<LoadedJudge[]> {
+  let entries: string[]
+  try {
+    entries = await readdir(dir)
+  } catch {
+    return []
+  }
+  const judges: LoadedJudge[] = []
+  for (const entry of entries.sort()) {
+    if (!entry.endsWith('.judge.ts') && !entry.endsWith('.judge.js')) continue
+    const filePath = join(dir, entry)
+    const url = pathToFileURL(isAbsolute(filePath) ? filePath : resolve(filePath)).href
+    const mod = (await import(url)) as { default?: JudgeFn }
+    if (typeof mod.default === 'function') judges.push({ fn: mod.default, filePath })
+  }
+  return judges
+}
+
+function createJudgeClient(apiKey: string | undefined, baseUrl: string | undefined): TCloud | undefined {
+  if (!apiKey) return undefined
+  const trimmed = baseUrl?.replace(/\/+$/, '')
+  const baseURL = trimmed ? (trimmed.endsWith('/v1') ? trimmed : `${trimmed}/v1`) : undefined
+  return TCloud.create({ apiKey, ...(baseURL ? { baseURL } : {}) })
+}
+
+// Translate a Scenario into a TestGradedScenario — the bridge between
+// the conversational scenario shape and the test-graded-runner shape.
+//
+// Default policy: every scenario's first turn is POST-ed at the target
+// URL's /chat route, and the response is checked for non-empty content.
+// Override by editing the testCommand in your scenario file's tags.
+function toTestGraded(scenario: Scenario, targetUrl: string): TestGradedScenario {
+  const customTestCommand = (scenario as Scenario & { testCommand?: string }).testCommand
+  const firstTurn = scenario.turns[0]
+  const userMessage = firstTurn?.user ?? ''
+  const bodyPath = `/tmp/eval-body-${scenario.id.replace(/[^a-zA-Z0-9_-]/g, '_')}`
+  const defaultTest =
+    `body=${shQuote(bodyPath)}; status=$(curl -sS -o "$body" -w '%{http_code}' -X POST -H 'content-type: application/json' ` +
+    `--data ${shQuote(JSON.stringify({ message: userMessage, scenarioId: scenario.id }))} ` +
+    `${shQuote(`${targetUrl}/chat`)}); printf '%s' "$status" | grep -E '^(200|201)$' >/dev/null && ` +
+    `test $(wc -c < "$body") -ge 1 && cat "$body"`
+  return {
+    id: scenario.id,
+    description: scenario.label ?? scenario.thesis ?? scenario.id,
+    harness: {
+      testCommand: customTestCommand ?? defaultTest,
+      timeoutMs: 30_000,
+    },
+    passThreshold: 1.0,
+    tags: { persona: scenario.persona, dimensions: scenario.dimensions.join(',') },
+  }
+}
+
+export interface RunReport {
+  timestamp: string
+  variantId?: string
+  scenarioCount: number
+  passCount: number
+  aggregate: number
+  measuredScenarioCount: number
+  unmeasuredScenarioCount: number
+  threshold: number
+  outcomes: ScenarioOutcome[]
+  tracesDir: string
+  rawEventsDir: string | null
+  scorecardPath: string
+  integrityMode: IntegrityMode
+  /** Per-run integrity reports, indexed by runId. */
+  integrityReports: Record<string, RunIntegrityReport>
+}
+
+function resolveIntegrityMode(opt: IntegrityMode | undefined): IntegrityMode {
+  if (opt) return opt
+  const env = (process.env.EVAL_INTEGRITY ?? '').toLowerCase()
+  if (env === 'off' || env === 'log' || env === 'strict') return env
+  return 'log'
+}
+
+export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
+  const projectRoot = opts.projectRoot ?? process.cwd()
+  const targetUrl = opts.targetUrl ?? process.env.EVAL_TARGET_BASE_URL ?? 'http://127.0.0.1:8787'
+  const threshold = opts.threshold ?? Number(process.env.EVAL_THRESHOLD ?? '0.7')
+  const scenariosDir = opts.scenariosDir ?? join(projectRoot, 'scenarios')
+  const judgesDir = opts.judgesDir ?? join(projectRoot, 'judges')
+  const tracesDir = opts.tracesDir ?? join(projectRoot, '.evolve', 'agent-eval', 'traces')
+  const experimentsDir =
+    opts.experimentsDir ?? join(projectRoot, '.evolve', 'agent-eval', 'experiments')
+  const rawEventsDir =
+    opts.rawEventsDir ?? join(projectRoot, '.evolve', 'agent-eval', 'raw-events')
+  const scorecardPath = opts.scorecardPath ?? join(projectRoot, '.evolve', 'scorecard.json')
+  const variantId = opts.variantId ?? process.env.EVAL_VARIANT_ID
+  const integrityMode = resolveIntegrityMode(opts.integrityMode)
+
+  // ── 0.21 Directive 2: assert the route at preflight ───────────────────
+  // Pure function; no I/O. Only runs when a baseUrl is supplied (env or
+  // opts) — the eval-harness ships with no LLM judge by default, so we
+  // don't want to block users who never call an LLM. When a route IS
+  // declared, fail loud before any scenario runs.
+  const llmBaseUrl = opts.llmOpts?.baseUrl ?? process.env.EVAL_LLM_BASE_URL
+  const llmApiKey = opts.llmOpts?.apiKey ?? process.env.EVAL_LLM_API_KEY ?? process.env.TANGLE_API_KEY
+  const llmProvider = opts.llmOpts?.provider ?? process.env.EVAL_LLM_PROVIDER
+  if (llmBaseUrl) {
+    try {
+      assertLlmRoute(
+        { baseUrl: llmBaseUrl, apiKey: llmApiKey, provider: llmProvider },
+        { requireExplicitBaseUrl: true, requireAuth: true },
+      )
+    } catch (err) {
+      if (err instanceof LlmRouteAssertionError) {
+        throw new Error(
+          `eval-harness: preflight route check failed (code=${err.code}). ` +
+            `Set EVAL_LLM_BASE_URL + auth, or unset to skip the check. ${err.message}`,
+        )
+      }
+      throw err
+    }
+  }
+
+  const traceStore = new FileSystemTraceStore({ dir: tracesDir })
+  // Experiment store: agent-eval 0.99 exposes this as the `fileExperimentStore`
+  // factory (replacing the old `FileSystemExperimentStore` class). Constructed
+  // so callers extending this runner can persist experiment metadata alongside
+  // traces; `void` marks the intentional construct-for-availability.
+  void fileExperimentStore(experimentsDir)
+  const driver = new SubprocessSandboxDriver({ cwd: projectRoot }) // muffle-ok: agent-eval honors constructor cwd as fallback when HarnessConfig.cwd is unset; runTestGradedScenario does not thread per-call cwd here, so the constructor arg is the active value.
+
+  const loaded: LoadedScenario[] = await loadScenarios(scenariosDir)
+  if (loaded.length === 0) {
+    throw new Error(
+      `eval-harness: no scenarios loaded from ${scenariosDir}. Drop a *.scenario.ts file with a default-exported Scenario.`,
+    )
+  }
+  const judges = await loadJudgesFrom(judgesDir)
+  const judgeClient = createJudgeClient(
+    llmApiKey,
+    llmBaseUrl ?? process.env.LLM_ROUTER_URL,
+  )
+
+  const integrityReports: Record<string, RunIntegrityReport> = {}
+  const outcomes: ScenarioOutcome[] = []
+  for (const { scenario, filePath } of loaded) {
+    const tgs = toTestGraded(scenario, targetUrl)
+    const startedAt = Date.now()
+
+    // ── 0.21 Directive 1: per-run RawProviderSink ──────────────────────
+    // Allocated even when no LLM judge runs — the cost is one (possibly
+    // empty) NDJSON file per scenario, and the alternative is a silent
+    // partial-capture bug the next time someone wires in an LLM judge.
+    // Exposed via globalThis so the ad-hoc judge call sites in this
+    // template's `judges/` dir pick it up without explicit plumbing.
+    let rawSink: RawProviderSink | undefined
+    if (integrityMode !== 'off') {
+      rawSink = new FileSystemRawProviderSink({
+        dir: join(rawEventsDir, scenario.id),
+      })
+      globalThis.__agentEvalRawSink = rawSink
+    }
+
+    let result: TestGradedRunResult
+    try {
+      result = await runTestGradedScenario(tgs, traceStore, { driver, variantId })
+    } catch (err) {
+      outcomes.push({
+        scenarioId: scenario.id,
+        pass: false,
+        score: 0,
+        durationMs: Date.now() - startedAt,
+        failureClass: 'harness_error',
+        filePath,
+        runId: '',
+        measuredJudgeCount: 0,
+        unmeasuredJudgeCount: 0,
+      })
+      console.error(`  ✗ ${scenario.id} — harness error: ${(err as Error).message}`)
+      globalThis.__agentEvalRawSink = undefined
+      continue
+    }
+
+    const judgeScores: JudgeScore[] = []
+    if (judges.length > 0) {
+      const response = result.harness.test?.stdout ?? ''
+      const judgeInput: JudgeInput = {
+        scenario,
+        turns: [
+          {
+            turnIndex: 0,
+            userMessage: scenario.turns[0]?.user ?? '',
+            agentResponse: response,
+            durationMs: result.harness.test?.wallMs ?? 0,
+            blocksExtracted: [],
+            containsCode: response.includes('```'),
+            containsToolCall: false,
+          },
+        ],
+        artifacts: {
+          vaultFiles: [],
+          blocksExtracted: [],
+          codeBlocks: [],
+          toolCalls: [],
+        } as CollectedArtifacts,
+      }
+      for (const judge of judges) {
+        try {
+          const scores = await judge.fn(judgeClient as TCloud, judgeInput)
+          judgeScores.push(...scores)
+        } catch (err) {
+          console.warn(`  ! judge ${judge.filePath} threw: ${(err as Error).message}`)
+        }
+      }
+    }
+    globalThis.__agentEvalRawSink = undefined
+
+    let measuredJudgeCount = 0
+    let unmeasuredJudgeCount = 0
+    let judgeScoreTotal = 0
+    for (const score of judgeScores) {
+      const status = (score as JudgeScore & { status?: string }).status
+      if (status === 'unmeasured' || !Number.isFinite(score.score)) {
+        unmeasuredJudgeCount += 1
+      } else {
+        measuredJudgeCount += 1
+        judgeScoreTotal += score.score
+      }
+    }
+    const score =
+      measuredJudgeCount > 0
+        ? (result.score + judgeScoreTotal) / (measuredJudgeCount + 1)
+        : result.score
+
+    // ── 0.21 Directive 3: assert the run captured before declaring done ─
+    // Read-only. We do NOT set `requireRawCoverageOfLlmSpans` because
+    // the test-graded path may not call any LLM at all (the default
+    // testCommand is a curl). When an LLM judge IS wired the sink will
+    // be populated and consumers can tighten this in `runIntegrityExpect`.
+    let integrityIssues: string[] | undefined
+    if (integrityMode !== 'off' && result.runId) {
+      const report = await assertRunCaptured(traceStore, result.runId, {
+        rawSink,
+        // requireOutcome: the harness always sets pass/score, so this is
+        // a cheap belt-and-suspenders check.
+        requireOutcome: true,
+      })
+      integrityReports[result.runId] = report
+      if (!report.ok) {
+        integrityIssues = report.issues.map((i) => i.code)
+      }
+    }
+
+    const integrityFailed = integrityMode === 'strict' && integrityIssues !== undefined
+    const scenarioPassed = result.pass && score >= threshold
+    outcomes.push({
+      scenarioId: scenario.id,
+      pass: integrityFailed ? false : scenarioPassed,
+      score: integrityFailed ? 0 : score,
+      durationMs: Date.now() - startedAt,
+      failureClass: integrityFailed ? 'integrity' : (result.failureClass ?? null),
+      filePath,
+      runId: result.runId,
+      measuredJudgeCount,
+      unmeasuredJudgeCount,
+      ...(integrityIssues ? { integrityIssues } : {}),
+    })
+    const mark = integrityFailed ? '✗' : scenarioPassed ? '✓' : '✗'
+    const integritySuffix = integrityIssues ? ` (integrity: ${integrityIssues.join(', ')})` : ''
+    const judgeSuffix =
+      judges.length > 0
+        ? ` judges=${measuredJudgeCount} measured/${unmeasuredJudgeCount} unmeasured`
+        : ''
+    console.log(
+      `  ${mark} ${scenario.id} — score=${score.toFixed(3)}${judgeSuffix}${integritySuffix}`,
+    )
+  }
+
+  const passCount = outcomes.filter((o) => o.pass).length
+  const aggregate = outcomes.reduce((a, o) => a + o.score, 0) / Math.max(outcomes.length, 1)
+
+  const flows: ScorecardFlow[] = outcomes.map((o) => ({
+    name: o.scenarioId,
+    value: o.score,
+    target: 1.0,
+    status: o.pass ? 'pass' : 'fail',
+    direction: 'higher-better',
+    productValueClaim: `Scenario "${o.scenarioId}" — pass rate against the agent under test.`,
+  }))
+  const report: RunReport = {
+    timestamp: new Date().toISOString(),
+    variantId,
+    scenarioCount: outcomes.length,
+    passCount,
+    aggregate,
+    measuredScenarioCount: outcomes.length,
+    unmeasuredScenarioCount: 0,
+    threshold,
+    outcomes,
+    tracesDir,
+    rawEventsDir: integrityMode === 'off' ? null : rawEventsDir,
+    scorecardPath,
+    integrityMode,
+    integrityReports,
+  }
+  await writeScorecard(scorecardPath, {
+    product: process.env.EVAL_PRODUCT_NAME ?? 'eval-harness',
+    timestamp: report.timestamp,
+    aggregate,
+    coverage: `${passCount}/${outcomes.length} scenarios passed`,
+    flows,
+  })
+  const integritySummary =
+    integrityMode === 'off'
+      ? 'integrity=off'
+      : `integrity=${integrityMode} ` +
+        `${Object.values(integrityReports).filter((r) => !r.ok).length}/${
+          Object.keys(integrityReports).length
+        } flagged`
+  console.log(
+    `\naggregate=${aggregate.toFixed(3)} pass=${passCount}/${outcomes.length} threshold=${threshold} ${integritySummary}`,
+  )
+  return report
+}
+
+// CLI-friendly default — invoked from `pnpm eval` when this file is the entry.
+if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) {
+  const projectRoot = process.env.EVAL_PROJECT_ROOT ?? DEFAULT_ROOT
+  runHarness({ projectRoot })
+    .then((r) => {
+      process.exitCode = r.aggregate < r.threshold ? 1 : 0
+    })
+    .catch((err) => {
+      console.error('[runner] fatal:', err)
+      process.exit(2)
+    })
+}

package/registry/families/agent-eval-harness-ts/files/src/eval/scorecard.ts

@@ -0,0 +1,89 @@
+// Scorecard emitter — writes scorecard.json in the shape consumed by the
+// project-level dashboard (.evolve/scorecard.json).
+//
+// The shape is intentionally a subset of the parent-repo scorecard so a
+// composed eval harness drops into an existing .evolve workflow without
+// translation. Extra dashboard fields (sourceMtime, stale, etc.) are
+// only relevant when the parent repo aggregates multiple inputs — for a
+// standalone harness, the runner's outputs are the single source.
+
+import { writeFile, mkdir } from 'node:fs/promises'
+import { dirname } from 'node:path'
+
+export type FlowDirection = 'higher-better' | 'lower-better'
+export type FlowStatus = 'pass' | 'fail' | 'skip'
+
+export interface ScorecardFlow {
+  name: string
+  value: number
+  target: number
+  status: FlowStatus
+  productValueClaim: string
+  direction: FlowDirection
+}
+
+export interface Scorecard {
+  product: string
+  timestamp: string
+  aggregate: number
+  coverage: string
+  flows: ScorecardFlow[]
+}
+
+export async function writeScorecard(path: string, scorecard: Scorecard): Promise<void> {
+  await mkdir(dirname(path), { recursive: true })
+  await writeFile(path, JSON.stringify(scorecard, null, 2) + '\n', 'utf8')
+}
+
+export async function readScorecard(path: string): Promise<Scorecard> {
+  const { readFile } = await import('node:fs/promises')
+  const raw = await readFile(path, 'utf8')
+  return JSON.parse(raw) as Scorecard
+}
+
+export interface ScorecardDiff {
+  baseline: Scorecard
+  head: Scorecard
+  aggregateDelta: number
+  perFlow: Array<{
+    name: string
+    baseline: number | null
+    head: number | null
+    delta: number | null
+    status: 'improved' | 'regressed' | 'stable' | 'introduced' | 'removed'
+  }>
+}
+
+const STABILITY_EPSILON = 0.005
+
+export function diffScorecards(baseline: Scorecard, head: Scorecard): ScorecardDiff {
+  const baselineByName = new Map<string, ScorecardFlow>()
+  for (const f of baseline.flows) baselineByName.set(f.name, f)
+  const headByName = new Map<string, ScorecardFlow>()
+  for (const f of head.flows) headByName.set(f.name, f)
+  const allNames = new Set<string>([...baselineByName.keys(), ...headByName.keys()])
+  const perFlow: ScorecardDiff['perFlow'] = []
+  for (const name of allNames) {
+    const b = baselineByName.get(name)
+    const h = headByName.get(name)
+    if (b && h) {
+      const delta = h.value - b.value
+      let status: ScorecardDiff['perFlow'][number]['status'] = 'stable'
+      if (Math.abs(delta) >= STABILITY_EPSILON) {
+        const better = h.direction === 'higher-better' ? delta > 0 : delta < 0
+        status = better ? 'improved' : 'regressed'
+      }
+      perFlow.push({ name, baseline: b.value, head: h.value, delta, status })
+    } else if (b) {
+      perFlow.push({ name, baseline: b.value, head: null, delta: null, status: 'removed' })
+    } else if (h) {
+      perFlow.push({ name, baseline: null, head: h.value, delta: null, status: 'introduced' })
+    }
+  }
+  return {
+    baseline,
+    head,
+    aggregateDelta: head.aggregate - baseline.aggregate,
+    perFlow,
+  }
+}

package/registry/families/agent-eval-harness-ts/files/tests/fixtures/scenarios-bad/missing-thesis.scenario.ts

@@ -0,0 +1,14 @@
+// Fixture: scenario missing the required `thesis` field. The layer's
+// strict loader must reject this; the family's pre-fix loader silently
+// accepted it.
+const scenario = {
+  id: 'fixture-bad',
+  persona: 'developer',
+  label: 'missing-thesis',
+  // NB: thesis intentionally omitted to trigger the strict validator.
+  dimensions: ['success'],
+  turns: [{ user: 'hi' }],
+  artifactChecks: [],
+}
+
+export default scenario

package/registry/families/agent-eval-harness-ts/files/tests/fixtures/scenarios-good/example.scenario.ts

@@ -0,0 +1,15 @@
+// Fixture for runner tests — a fully-shaped Scenario that the layer's
+// strict loader accepts.
+import type { Scenario } from '@tangle-network/agent-eval'
+
+const scenario: Scenario = {
+  id: 'fixture-good',
+  persona: 'developer',
+  label: 'happy-path',
+  thesis: 'agent must respond non-empty',
+  dimensions: ['success'],
+  turns: [{ user: 'hi', expectedBehaviors: ['responds non-empty'] }],
+  artifactChecks: [],
+}
+
+export default scenario

package/registry/families/agent-eval-harness-ts/files/tests/scenario-loader-import.test.ts

@@ -0,0 +1,63 @@
+// Family runner imports the layer's `loadScenarios` (extend, don't duplicate).
+//
+// Regression bait:
+//   - Pre-fix: the family's `runner.ts` re-implemented a weaker scenario
+//     loader inline. Bad scenarios (missing required fields) silently
+//     loaded; the layer's strict validator was bypassed. Test asserts the
+//     family's runner now wires the layer's loader and rejects malformed
+//     scenarios with the layer's error message.
+
+import test, { describe } from 'node:test'
+import assert from 'node:assert/strict'
+import { dirname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { readFileSync } from 'node:fs'
+
+// Compose-time, the layer's loader lands at ../src/eval/scenario-loader.js
+// next to the family runner. At test-time we import from the layer's
+// source directly so the test exercises what the family will actually
+// ship with after composition.
+import { loadScenarios } from '../../../../layers/agent-eval/scenarios/files/src/eval/scenario-loader.js'
+
+const HERE = dirname(fileURLToPath(import.meta.url))
+// loadScenarios reads .scenario.ts/.scenario.js — at test-time the fixtures
+// have been compiled to .scenario.js next to this file in dist-layer-tests.
+const GOOD = resolve(HERE, 'fixtures', 'scenarios-good')
+const BAD = resolve(HERE, 'fixtures', 'scenarios-bad')
+
+describe('agent-eval-harness-ts: family runner uses layer loader', () => {
+  test('runner.ts imports loadScenarios from scenario-loader (the layer)', () => {
+    // HERE = .../dist-layer-tests/registry/families/agent-eval-harness-ts/files/tests
+    // Walk up to the repo root, then back down to the family's source tree.
+    const repoRoot = resolve(HERE, '..', '..', '..', '..', '..', '..')
+    const runnerPath = resolve(
+      repoRoot,
+      'registry/families/agent-eval-harness-ts/files/src/eval/runner.ts',
+    )
+    const text = readFileSync(runnerPath, 'utf8')
+    assert.match(
+      text,
+      /from\s+'\.\/scenario-loader\.js'/,
+      'runner.ts must import from the composed layer (scenario-loader.js), not re-implement',
+    )
+    assert.doesNotMatch(
+      text,
+      /async function loadScenariosFrom\b/,
+      'family runner must not declare a local loadScenariosFrom — that was the duplicate',
+    )
+  })
+
+  test('layer loader accepts a fully-shaped scenario', async () => {
+    const loaded = await loadScenarios(GOOD)
+    assert.equal(loaded.length, 1)
+    assert.equal(loaded[0]!.scenario.id, 'fixture-good')
+  })
+
+  test('layer loader rejects a scenario missing required fields', async () => {
+    await assert.rejects(
+      () => loadScenarios(BAD),
+      /missing required field "thesis"/,
+      'layer loader must surface the layer\'s strict error message, not silently include the bad scenario',
+    )
+  })
+})