@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/layers/agent-eval/regression/files/src/eval/regression/cli.ts

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+// eval-regression-gate <baseline.json> <head.json>
+//
+// Exit codes:
+//   0 = PROMOTE  (head is stable or improved)
+//   1 = REVERT   (head has BH-significant regressions)
+//   2 = HOLD     (regressions present but not significant, or unstable)
+
+import { readScorecard } from '../scorecard.js'
+import { gate, GATE_EXIT } from './gate.js'
+
+function help(): never {
+  console.log(`Usage:
+  eval-regression-gate <baseline.json> <head.json> [--alpha 0.05] [--fdr 0.1]
+                       [--effect 0.5] [--only flow-a,flow-b]
+                       [--report report.json]
+
+Exit codes:
+  0  PROMOTE
+  1  REVERT
+  2  HOLD
+`)
+  process.exit(2)
+}
+
+interface ParsedArgs {
+  baseline: string
+  head: string
+  alpha: number
+  fdr: number
+  effectThreshold: number
+  onlyFlows?: ReadonlySet<string>
+  reportPath?: string
+}
+
+function parseArgs(argv: string[]): ParsedArgs {
+  const positional: string[] = []
+  let alpha = 0.05
+  let fdr = 0.1
+  let effectThreshold = 0.5
+  let onlyFlows: ReadonlySet<string> | undefined
+  let reportPath: string | undefined
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]!
+    if (a === '--alpha') alpha = Number(argv[++i])
+    else if (a === '--fdr') fdr = Number(argv[++i])
+    else if (a === '--effect') effectThreshold = Number(argv[++i])
+    else if (a === '--only') onlyFlows = new Set(argv[++i]!.split(','))
+    else if (a === '--report') reportPath = argv[++i]
+    else if (a === '-h' || a === '--help') help()
+    else positional.push(a)
+  }
+  if (positional.length < 2) help()
+  if (Number.isNaN(alpha) || Number.isNaN(fdr) || Number.isNaN(effectThreshold)) {
+    console.error('eval-regression-gate: numeric arg parse failed')
+    process.exit(2)
+  }
+  return {
+    baseline: positional[0]!,
+    head: positional[1]!,
+    alpha,
+    fdr,
+    effectThreshold,
+    onlyFlows,
+    reportPath,
+  }
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2))
+  const baseline = await readScorecard(args.baseline)
+  const head = await readScorecard(args.head)
+  const report = gate(baseline, head, {
+    alpha: args.alpha,
+    fdr: args.fdr,
+    effectThreshold: args.effectThreshold,
+    onlyFlows: args.onlyFlows,
+  })
+  console.log(`verdict: ${report.verdict}`)
+  console.log(`reason:  ${report.reason}`)
+  console.log(`alpha=${report.alpha} fdr=${report.fdr}`)
+  console.log('')
+  for (const f of report.perFlow) {
+    const sig = f.bhSignificant ? ' (BH-sig)' : ''
+    const ci =
+      f.bootstrap !== null
+        ? ` ci=[${f.bootstrap.ciLower.toFixed(3)},${f.bootstrap.ciUpper.toFixed(3)}]`
+        : ''
+    const p = f.welch !== null ? ` p=${f.welch.p.toFixed(4)}` : ''
+    console.log(
+      `  [${f.verdict.padEnd(12, ' ')}] ${f.flow}: ${f.baseline.toFixed(3)} → ${f.head.toFixed(3)} (Δ${f.delta >= 0 ? '+' : ''}${f.delta.toFixed(3)})${ci}${p}${sig}`,
+    )
+  }
+  if (args.reportPath) {
+    const { writeFile, mkdir } = await import('node:fs/promises')
+    const { dirname } = await import('node:path')
+    await mkdir(dirname(args.reportPath), { recursive: true })
+    await writeFile(args.reportPath, JSON.stringify(report, null, 2) + '\n', 'utf8')
+  }
+  process.exit(GATE_EXIT[report.verdict])
+}
+
+main().catch((err) => {
+  console.error('[eval-regression-gate] fatal:', err)
+  process.exit(2)
+})

package/registry/layers/agent-eval/regression/files/src/eval/regression/gate.ts

@@ -0,0 +1,260 @@
+// Statistical regression gate.
+//
+// Reads two scorecards (baseline + head), treats every flow's value as a
+// single sample (since scorecards are aggregates, not raw runs), and runs:
+//
+//   1. Per-flow `bootstrapCi` on baseline-vs-head — non-parametric, robust.
+//      With single-sample-per-side scorecards we still get a delta point
+//      estimate; the CI degenerates but the verdict is honest about it
+//      (returns INCONCLUSIVE rather than fake-confident).
+//   2. `welchsTTest` and `cohensD` when the caller passes raw per-scenario
+//      sample arrays via `withSamples`.
+//   3. `compareToBaseline` for a wholesale per-metric verdict shape.
+//   4. `benjaminiHochberg` over the p-values to control FDR across flows.
+//
+// Final verdict is the worst of all flows:
+//   - REVERT if any flow's verdict is `regressed` AND BH-significant.
+//   - HOLD if any flow is `unstable` and no flow is regressed.
+//   - PROMOTE otherwise.
+
+import {
+  bootstrapCi,
+  welchsTTest,
+  cohensD,
+  compareToBaseline,
+  benjaminiHochberg,
+  type BootstrapResult,
+  type MetricSamples,
+  type MetricVerdict,
+} from '@tangle-network/agent-eval'
+import type { Scorecard, ScorecardFlow } from '../scorecard.js'
+
+export type GateVerdict = 'PROMOTE' | 'HOLD' | 'REVERT'
+export const GATE_EXIT: Record<GateVerdict, number> = {
+  PROMOTE: 0,
+  REVERT: 1,
+  HOLD: 2,
+}
+
+export interface GateOptions {
+  /** Confidence level alpha for bootstrap. Default 0.05 → 95% CI. */
+  alpha?: number
+  /** False-discovery-rate target for Benjamini–Hochberg. Default 0.1. */
+  fdr?: number
+  /** Effect size threshold for compareToBaseline. Default 0.5 (medium). */
+  effectThreshold?: number
+  /** Stability threshold for compareToBaseline. Default 0.30. */
+  unstableCvThreshold?: number
+  /** Optional raw samples per metric for higher-rigor gating. */
+  withSamples?: Map<string, { baseline: number[]; candidate: number[] }>
+  /** Optional dimension filter — when set, only these flows gate the build. */
+  onlyFlows?: ReadonlySet<string>
+}
+
+export interface PerFlowVerdict {
+  flow: string
+  baseline: number
+  head: number
+  delta: number
+  bootstrap: BootstrapResult | null
+  welch: { t: number; df: number; p: number } | null
+  cohensD: number | null
+  metricVerdict: MetricVerdict | null
+  bhSignificant: boolean
+  bhQValue: number | null
+  verdict: 'improved' | 'regressed' | 'stable' | 'unstable' | 'inconclusive'
+  direction: 'higher-better' | 'lower-better'
+}
+
+export interface GateReport {
+  verdict: GateVerdict
+  reason: string
+  perFlow: PerFlowVerdict[]
+  generatedAt: string
+  baselineProduct: string
+  headProduct: string
+  alpha: number
+  fdr: number
+}
+
+const STABILITY_EPSILON = 0.005
+
+function flowsByName(s: Scorecard): Map<string, ScorecardFlow> {
+  const m = new Map<string, ScorecardFlow>()
+  for (const f of s.flows) m.set(f.name, f)
+  return m
+}
+
+export function gate(baseline: Scorecard, head: Scorecard, opts: GateOptions = {}): GateReport {
+  const alpha = opts.alpha ?? 0.05
+  const fdr = opts.fdr ?? 0.1
+  const effectThreshold = opts.effectThreshold ?? 0.5
+  const unstableCvThreshold = opts.unstableCvThreshold ?? 0.3
+  const baselineFlows = flowsByName(baseline)
+  const headFlows = flowsByName(head)
+  const allNames = new Set<string>([...baselineFlows.keys(), ...headFlows.keys()])
+  const filteredNames = opts.onlyFlows
+    ? Array.from(allNames).filter((n) => opts.onlyFlows!.has(n))
+    : Array.from(allNames)
+
+  // Build MetricSamples for compareToBaseline. When the caller provides
+  // raw per-scenario samples, use them; otherwise treat the scorecard
+  // values as single-sample arrays (limited rigor, but honest verdicts).
+  const metricSamples: MetricSamples[] = []
+  for (const name of filteredNames) {
+    const b = baselineFlows.get(name)
+    const h = headFlows.get(name)
+    if (!b || !h) continue
+    const samples = opts.withSamples?.get(name)
+    metricSamples.push({
+      metric: name,
+      higherIsBetter: (b.direction ?? 'higher-better') === 'higher-better',
+      baseline: samples?.baseline ?? [b.value],
+      candidate: samples?.candidate ?? [h.value],
+    })
+  }
+  const baselineReport = compareToBaseline(metricSamples, {
+    alpha,
+    effectThreshold,
+    unstableCvThreshold,
+  })
+  const verdictByMetric = new Map<string, MetricVerdict>()
+  for (const m of baselineReport.metrics) verdictByMetric.set(m.metric, m)
+
+  // BH-correct the per-flow p-values.
+  const orderedNames: string[] = []
+  const orderedP: number[] = []
+  for (const m of baselineReport.metrics) {
+    orderedNames.push(m.metric)
+    orderedP.push(m.welchP)
+  }
+  const bh =
+    orderedP.length > 0
+      ? benjaminiHochberg(orderedP, fdr)
+      : { qValues: [] as number[], significant: [] as boolean[] }
+  const bhByName = new Map<string, { q: number; sig: boolean }>()
+  for (let i = 0; i < orderedNames.length; i++) {
+    bhByName.set(orderedNames[i]!, {
+      q: bh.qValues[i] ?? 1,
+      sig: bh.significant[i] ?? false,
+    })
+  }
+
+  const perFlow: PerFlowVerdict[] = []
+  for (const name of filteredNames) {
+    const b = baselineFlows.get(name)
+    const h = headFlows.get(name)
+    if (!b && h) {
+      perFlow.push({
+        flow: name,
+        baseline: 0,
+        head: h.value,
+        delta: h.value,
+        bootstrap: null,
+        welch: null,
+        cohensD: null,
+        metricVerdict: null,
+        bhSignificant: false,
+        bhQValue: null,
+        verdict: 'inconclusive',
+        direction: h.direction ?? 'higher-better',
+      })
+      continue
+    }
+    if (b && !h) {
+      perFlow.push({
+        flow: name,
+        baseline: b.value,
+        head: 0,
+        delta: -b.value,
+        bootstrap: null,
+        welch: null,
+        cohensD: null,
+        metricVerdict: null,
+        bhSignificant: false,
+        bhQValue: null,
+        verdict: 'inconclusive',
+        direction: b.direction ?? 'higher-better',
+      })
+      continue
+    }
+    if (!b || !h) continue
+
+    const samples = opts.withSamples?.get(name)
+    const baselineSamples = samples?.baseline ?? [b.value]
+    const candidateSamples = samples?.candidate ?? [h.value]
+
+    let bootstrap: BootstrapResult | null = null
+    let welch: { t: number; df: number; p: number } | null = null
+    let cd: number | null = null
+    if (baselineSamples.length + candidateSamples.length >= 6) {
+      bootstrap = bootstrapCi(baselineSamples, candidateSamples, { alpha })
+      welch = welchsTTest(baselineSamples, candidateSamples)
+      cd = cohensD(baselineSamples, candidateSamples)
+    }
+
+    const metricVerdict = verdictByMetric.get(name) ?? null
+    const bhEntry = bhByName.get(name) ?? null
+    const direction = b.direction ?? 'higher-better'
+
+    let verdict: PerFlowVerdict['verdict'] = 'stable'
+    if (metricVerdict) {
+      verdict = metricVerdict.verdict
+    } else {
+      // Fallback when compareToBaseline didn't yield a verdict (single sample).
+      const delta = h.value - b.value
+      if (Math.abs(delta) < STABILITY_EPSILON) verdict = 'stable'
+      else {
+        const better = direction === 'higher-better' ? delta > 0 : delta < 0
+        verdict = better ? 'improved' : 'regressed'
+      }
+    }
+
+    perFlow.push({
+      flow: name,
+      baseline: b.value,
+      head: h.value,
+      delta: h.value - b.value,
+      bootstrap,
+      welch,
+      cohensD: cd,
+      metricVerdict,
+      bhSignificant: bhEntry?.sig ?? false,
+      bhQValue: bhEntry?.q ?? null,
+      verdict,
+      direction,
+    })
+  }
+
+  // Final verdict: REVERT if any regressed flow is BH-significant
+  // (or, in single-sample mode, regressed at all). HOLD on unstable
+  // without regressions. PROMOTE otherwise.
+  let finalVerdict: GateVerdict = 'PROMOTE'
+  let reason = 'No flow regressed; aggregate stable or improved.'
+  const regressed = perFlow.filter((p) => p.verdict === 'regressed')
+  const significantRegressions = regressed.filter(
+    (p) => p.bhSignificant || (p.metricVerdict === null && Math.abs(p.delta) >= STABILITY_EPSILON),
+  )
+  const unstable = perFlow.filter((p) => p.verdict === 'unstable')
+  if (significantRegressions.length > 0) {
+    finalVerdict = 'REVERT'
+    reason = `Statistically significant regressions on: ${significantRegressions.map((p) => p.flow).join(', ')}`
+  } else if (regressed.length > 0) {
+    finalVerdict = 'HOLD'
+    reason = `Possible regressions (not BH-significant): ${regressed.map((p) => p.flow).join(', ')}`
+  } else if (unstable.length > 0) {
+    finalVerdict = 'HOLD'
+    reason = `Flows too noisy to gate on: ${unstable.map((p) => p.flow).join(', ')}`
+  }
+
+  return {
+    verdict: finalVerdict,
+    reason,
+    perFlow,
+    generatedAt: new Date().toISOString(),
+    baselineProduct: baseline.product,
+    headProduct: head.product,
+    alpha,
+    fdr,
+  }
+}

package/registry/layers/agent-eval/regression/manifest.json

@@ -0,0 +1,60 @@
+{
+  "id": "regression",
+  "group": "agent-eval",
+  "description": "Statistical regression gate for the eval harness. Reads two scorecards (baseline + head), compares per-flow distributions via `bootstrapCi` + `welchsTTest` + `cohensD`, applies Benjamini–Hochberg FDR correction across dimensions, and emits a PROMOTE / HOLD / REVERT verdict. Ships a CLI (`eval-regression-gate`) and a CI workflow that wires the gate against `origin/main` on every PR.",
+  "appliesTo": [
+    "agent-eval-harness-ts",
+    "agent-research-harness-ts"
+  ],
+  "provides": ["eval:regression"],
+  "requires": ["eval:scenarios"],
+  "defaults": {
+    "alpha": "0.05",
+    "fdr": "0.1",
+    "effectThreshold": "0.5",
+    "unstableCvThreshold": "0.30"
+  },
+  "files": [
+    {
+      "source": "files/src/eval/regression/gate.ts",
+      "target": "src/eval/regression/gate.ts"
+    },
+    {
+      "source": "files/src/eval/regression/cli.ts",
+      "target": "src/eval/regression/cli.ts"
+    },
+    {
+      "source": "files/src/eval/regression/README.md",
+      "target": "src/eval/regression/README.md"
+    },
+    {
+      "source": "files/.github/workflows/regression-gate.yml",
+      "target": ".github/workflows/regression-gate.yml"
+    }
+  ],
+  "validationChecks": [
+    { "type": "file-exists", "path": "src/eval/regression/gate.ts" },
+    { "type": "file-exists", "path": "src/eval/regression/cli.ts" },
+    { "type": "file-exists", "path": "src/eval/regression/README.md" },
+    { "type": "file-exists", "path": ".github/workflows/regression-gate.yml" }
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "eval-regression-gate",
+      "agent-eval-regression",
+      "scorecard-regression-gate",
+      "ci-regression-gate-eval"
+    ],
+    "tier2": [
+      "bootstrap-ci-gate",
+      "welch-t-gate",
+      "cohens-d-effect",
+      "fdr-correction",
+      "benjamini-hochberg"
+    ]
+  },
+  "contextHints": {
+    "commands": ["pnpm eval:gate baseline.json head.json"],
+    "extensionPoints": ["src/eval/regression/gate.ts"]
+  }
+}

package/registry/layers/agent-eval/scenarios/files/scenarios/README.md

@@ -0,0 +1,64 @@
+# scenarios/
+
+Drop one TypeScript file per scenario in this directory. The harness
+loads every `*.scenario.ts` and grades each one against the agent under
+test.
+
+## Shape
+
+A scenario is a `Scenario` from `@tangle-network/agent-eval`. The
+ergonomic re-export lives in `src/eval/scenario-types.ts`.
+
+```ts
+import type { Scenario } from '@/eval/scenario-types'
+
+const scenario: Scenario = {
+  id: 'category/specific-id',
+  persona: 'first-time-user',
+  label: 'Short human-facing label',
+  thesis: 'Why this scenario exists — what regression it would catch.',
+  dimensions: ['correctness', 'helpfulness'],
+  turns: [
+    {
+      user: 'The exact message to send to the agent.',
+      expectedBehaviors: [
+        'one-line description of what the agent should do',
+        'as many as you need',
+      ],
+    },
+  ],
+  artifactChecks: [
+    {
+      type: 'generation_produced',
+      target: 'response',
+      description: 'Agent emitted a non-empty response.',
+    },
+  ],
+}
+
+export default scenario
+```
+
+## Conventions
+
+- **Name the regression.** Every `thesis` field must articulate the
+  specific bug the scenario would catch if it broke. If you can't, the
+  scenario isn't earning its CI time — delete or refactor it.
+- **One file per scenario.** Do not pack multiple scenarios into a
+  single export. The loader supports it (it accepts arrays) but the
+  diff signal is cleaner when one file = one scenario.
+- **Stable ids.** Scenario ids are the dashboard's primary key. Renames
+  break the time series — only rename when you mean to.
+- **Adversarial > happy-path.** Half your scenarios should test
+  failure modes: oversized payloads, malformed input, refusal cases,
+  contradictory followups. The happy path alone is not coverage.
+
+## Categories (suggested)
+
+- `smoke/` — health-check scenarios. Fail = pipeline is dead.
+- `behavior/` — does the agent do what it says.
+- `safety/` — refusals, boundary enforcement, jailbreak resistance.
+- `regression/` — pinned scenarios that captured a real prior bug.
+- `perf/` — latency / token / cost expectations.
+
+The runner doesn't enforce these; they're operator-facing.

package/registry/layers/agent-eval/scenarios/files/src/eval/scenario-loader.ts

@@ -0,0 +1,85 @@
+// Composable scenario loader — used by the harness family's runner and by
+// any consumer that needs to enumerate scenarios outside the runner (e.g.
+// `eval-harness list`, the regression-gate dimension extractor, etc.).
+//
+// Validates the loaded module's default export against the Scenario shape;
+// throws with a precise path on shape mismatch so loader bugs do not
+// silently degrade into "the scenario didn't run".
+
+import { readdir } from 'node:fs/promises'
+import { join, isAbsolute, resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import type { Scenario } from './scenario-types.js'
+
+export interface LoadedScenario {
+  scenario: Scenario
+  filePath: string
+}
+
+const REQUIRED_KEYS: ReadonlyArray<keyof Scenario> = [
+  'id',
+  'persona',
+  'label',
+  'thesis',
+  'dimensions',
+  'turns',
+  'artifactChecks',
+]
+
+function assertScenarioShape(value: unknown, filePath: string): asserts value is Scenario {
+  if (!value || typeof value !== 'object') {
+    throw new Error(`scenario-loader: ${filePath} did not export an object`)
+  }
+  const s = value as Record<string, unknown>
+  for (const k of REQUIRED_KEYS) {
+    if (!(k in s)) {
+      throw new Error(`scenario-loader: ${filePath} default export missing required field "${k}"`)
+    }
+  }
+  if (typeof s.id !== 'string' || s.id.length === 0) {
+    throw new Error(`scenario-loader: ${filePath} has empty/non-string id`)
+  }
+  if (!Array.isArray(s.turns) || s.turns.length === 0) {
+    throw new Error(`scenario-loader: ${filePath} must have at least one turn`)
+  }
+  if (!Array.isArray(s.dimensions) || s.dimensions.length === 0) {
+    throw new Error(`scenario-loader: ${filePath} must declare at least one dimension`)
+  }
+  if (!Array.isArray(s.artifactChecks)) {
+    throw new Error(`scenario-loader: ${filePath} artifactChecks must be an array`)
+  }
+}
+
+export async function loadScenarios(scenariosDir: string): Promise<LoadedScenario[]> {
+  let entries: string[]
+  try {
+    entries = await readdir(scenariosDir)
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return []
+    throw err
+  }
+  const out: LoadedScenario[] = []
+  for (const entry of entries) {
+    if (!entry.endsWith('.scenario.ts') && !entry.endsWith('.scenario.js')) continue
+    const filePath = isAbsolute(scenariosDir)
+      ? join(scenariosDir, entry)
+      : resolve(scenariosDir, entry)
+    const url = pathToFileURL(filePath).href
+    const mod = (await import(url)) as { default?: Scenario | Scenario[] }
+    if (!mod.default) {
+      throw new Error(`scenario-loader: ${filePath} has no default export`)
+    }
+    const list = Array.isArray(mod.default) ? mod.default : [mod.default]
+    for (const scenario of list) {
+      assertScenarioShape(scenario, filePath)
+      out.push({ scenario, filePath })
+    }
+  }
+  // Stable order: id-asc — keeps scorecards diff-stable across runs.
+  out.sort((a, b) => a.scenario.id.localeCompare(b.scenario.id))
+  return out
+}
+
+export function uniqueScenarioIds(loaded: LoadedScenario[]): string[] {
+  return Array.from(new Set(loaded.map((l) => l.scenario.id)))
+}

package/registry/layers/agent-eval/scenarios/files/src/eval/scenario-types.ts

@@ -0,0 +1,15 @@
+// Re-exports the scenario types from @tangle-network/agent-eval so callers
+// can `import type { Scenario } from '@/eval/scenario-types'` without
+// reaching into the package surface every time. Keeps every scenario file's
+// import line short + uniform.
+
+export type {
+  Scenario,
+  Turn,
+  ArtifactCheck,
+  Dataset,
+  DatasetScenario,
+  DatasetManifest,
+  DatasetSplit,
+  ScenarioFile,
+} from '@tangle-network/agent-eval'

package/registry/layers/agent-eval/scenarios/manifest.json

@@ -0,0 +1,44 @@
+{
+  "id": "scenarios",
+  "group": "agent-eval",
+  "slot": "eval-runner",
+  "description": "Base layer for the agent-eval-harness-ts family. Provides the scenario loader + re-exports of `Scenario` and `Dataset` types from @tangle-network/agent-eval. Other agent-eval layers (judge-rubric, regression) require this. Empty appliesTo[] — the consolidator wires this into the harness family at compose time.",
+  "appliesTo": [
+    "agent-eval-harness-ts",
+    "agent-research-harness-ts"
+  ],
+  "provides": ["eval:scenarios"],
+  "defaults": {
+    "scenariosDir": "scenarios"
+  },
+  "files": [
+    {
+      "source": "files/scenarios/README.md",
+      "target": "scenarios/README.md"
+    },
+    {
+      "source": "files/src/eval/scenario-loader.ts",
+      "target": "src/eval/scenario-loader.ts"
+    },
+    {
+      "source": "files/src/eval/scenario-types.ts",
+      "target": "src/eval/scenario-types.ts"
+    }
+  ],
+  "validationChecks": [
+    { "type": "file-exists", "path": "src/eval/scenario-loader.ts" },
+    { "type": "file-exists", "path": "src/eval/scenario-types.ts" },
+    { "type": "file-exists", "path": "scenarios/README.md" }
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "eval-scenarios-loader",
+      "agent-eval-scenarios",
+      "scenarios-base-layer"
+    ],
+    "tier2": ["scenario-runner", "scenario-validation", "dataset-loader"]
+  },
+  "contextHints": {
+    "extensionPoints": ["src/eval/scenario-loader.ts", "scenarios/"]
+  }
+}