@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/families/agent-eval-harness-ts/files/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "lib": ["ES2022"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "allowImportingTsExtensions": false,
+    "outDir": "dist",
+    "rootDir": ".",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    }
+  },
+  "include": ["src", "scenarios", "judges"]
+}

package/registry/families/agent-eval-harness-ts/manifest.json

@@ -0,0 +1,124 @@
+{
+  "id": "agent-eval-harness-ts",
+  "description": "Node-only TypeScript eval harness that wraps @tangle-network/agent-eval. Operator-facing `pnpm eval` runner for scenarios + LLM-as-judge rubrics + statistical regression gates against a baseline scorecard. Composes with eval:scenarios / eval:judge-rubric / eval:regression base layers; emits scorecard.json into .evolve/ for the project-level dashboard. This is the family for any standalone eval bundle that needs to grade an agent under test (HTTP target or in-process function) and gate CI on regressions.",
+  "tags": [
+    "agent-eval",
+    "evaluation",
+    "eval-harness",
+    "judge",
+    "rubric",
+    "regression-gate",
+    "typescript",
+    "node"
+  ],
+  "taxonomy": {
+    "language": "typescript",
+    "runtime": "node",
+    "surface": "eval-runner"
+  },
+  "slots": {
+    "eval-runner": {
+      "options": [
+        "agent-eval:scenarios"
+      ],
+      "default": "agent-eval:scenarios"
+    }
+  },
+  "includes": [
+    "agent-base:tangle",
+    "agent-base:secure",
+    "agent-eval:scenarios",
+    "agent-eval:judge-rubric",
+    "agent-eval:regression"
+  ],
+  "defaults": {
+    "projectType": "agent-eval-harness",
+    "appName": "my-eval-harness",
+    "scorecardDir": ".evolve",
+    "tracesDir": ".evolve/agent-eval/traces",
+    "experimentsDir": ".evolve/agent-eval/experiments",
+    "rawEventsDir": ".evolve/agent-eval/raw-events",
+    "publicVars": [
+      "EVAL_TARGET_BASE_URL",
+      "EVAL_THRESHOLD",
+      "EVAL_INTEGRITY",
+      "EVAL_LLM_BASE_URL",
+      "EVAL_LLM_API_KEY",
+      "EVAL_LLM_PROVIDER",
+      "TANGLE_API_KEY"
+    ],
+    "evalThreshold": "0.7"
+  },
+  "files": [
+    { "source": "files/package.json", "target": "package.json" },
+    { "source": "files/tsconfig.json", "target": "tsconfig.json" },
+    { "source": "files/agent.json", "target": "agent.json" },
+    { "source": "files/AGENTS.md", "target": "AGENTS.md" },
+    { "source": "files/README.md", "target": "README.md" },
+    { "source": "files/src/eval/runner.ts", "target": "src/eval/runner.ts" },
+    { "source": "files/src/eval/scorecard.ts", "target": "src/eval/scorecard.ts" },
+    { "source": "files/src/eval/cli.ts", "target": "src/eval/cli.ts" },
+    { "source": "files/src/eval/campaign.ts", "target": "src/eval/campaign.ts" },
+    { "source": "files/scenarios/.gitkeep", "target": "scenarios/.gitkeep" },
+    { "source": "files/scenarios/example.scenario.ts", "target": "scenarios/example.scenario.ts" },
+    { "source": "files/judges/example.judge.ts", "target": "judges/example.judge.ts" },
+    { "source": "files/.github/workflows/eval.yml", "target": ".github/workflows/eval.yml" },
+    { "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
+  ],
+  "validationChecks": [
+    { "type": "file-exists", "path": "package.json" },
+    { "type": "file-exists", "path": "tsconfig.json" },
+    { "type": "file-exists", "path": "AGENTS.md" },
+    { "type": "agents-md-valid", "path": "AGENTS.md" },
+    { "type": "file-exists", "path": "src/eval/runner.ts" },
+    { "type": "file-exists", "path": "src/eval/scorecard.ts" },
+    { "type": "file-exists", "path": "src/eval/cli.ts" },
+    { "type": "file-exists", "path": "src/eval/campaign.ts" },
+    { "type": "file-exists", "path": "scenarios/example.scenario.ts" },
+    { "type": "file-exists", "path": "judges/example.judge.ts" },
+    { "type": "file-exists", "path": ".github/workflows/eval.yml" }
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "eval-harness-ts",
+      "agent-eval-harness",
+      "eval-runner-bundle",
+      "scorecard-harness-ts",
+      "ci-eval-gate-ts"
+    ],
+    "tier2": [
+      "scenarios-runner",
+      "judge-runner",
+      "rubric-runner",
+      "regression-gate",
+      "scorecard-emit",
+      "baseline-compare"
+    ],
+    "tier3": [
+      "eval harness",
+      "ci gate",
+      "regression gate"
+    ],
+    "archetypes": [
+      "evaluation-harness",
+      "ci-quality-gate"
+    ]
+  },
+  "contextHints": {
+    "commands": [
+      "pnpm install",
+      "pnpm eval",
+      "pnpm eval:compare",
+      "pnpm typecheck"
+    ],
+    "entrypoints": [
+      "src/eval/runner.ts",
+      "src/eval/cli.ts"
+    ],
+    "extensionPoints": [
+      "scenarios/",
+      "judges/",
+      "src/eval/scorecard.ts"
+    ]
+  }
+}

package/registry/families/agent-eval-ui-ts/files/.env.example

@@ -0,0 +1,12 @@
+# Absolute or workspace-relative path to the agent-eval output directory.
+# The UI reads JSONL trace logs (traces.jsonl, structural-assertions.jsonl) +
+# per-run JSON reports (three-layer-report.json, cost-summary.json, etc.)
+# from this tree.
+#
+# Example layouts the loader understands:
+#   $AGENT_EVAL_TRACES_DIR/seeds.json
+#   $AGENT_EVAL_TRACES_DIR/<runDate>/traces.jsonl
+#   $AGENT_EVAL_TRACES_DIR/<runDate>/three-layer-report.json
+#   $AGENT_EVAL_TRACES_DIR/<runDate>/cost-summary.json
+#   $AGENT_EVAL_TRACES_DIR/<runDate>/structural-assertions.jsonl
+AGENT_EVAL_TRACES_DIR=.evolve/agent-eval

package/registry/families/agent-eval-ui-ts/files/README.md

@@ -0,0 +1,66 @@
+# {{appName}} — agent-eval-ui-ts
+
+Vite + React + TypeScript dashboard for evaluators. Reads `agent-eval` output
+(JSONL traces + per-run JSON reports) from a directory you point it at, and
+renders four views:
+
+- **Overview** — aggregate score across every loaded trace, plotted on a multi-dimensional radar.
+- **Trace list** — sidebar with date / family / status / search filters.
+- **Trace detail** — turn-by-turn timeline (user / assistant / tool), score table, per-trace radar, phase summary, and any `:::artifact`-style blocks the run emitted (rendered via `ui-adapter:blocks-renderer`).
+- **Compare** — pick two traces, see per-dimension delta + winner with both runs overlaid on one radar.
+
+## Quick start
+
+```bash
+pnpm install
+cp .env.example .env
+# edit AGENT_EVAL_TRACES_DIR to point at your agent-eval output (e.g. .evolve/agent-eval)
+pnpm dev
+```
+
+Open http://localhost:{{port}}.
+
+## Data sources
+
+The loader reads everything under `$AGENT_EVAL_TRACES_DIR`, bucketed by
+top-level subdir (run date). It understands four file shapes:
+
+| File | What it contributes |
+| --- | --- |
+| `<runDate>/traces.jsonl` | per-phase rows; turns when a row carries them |
+| `<runDate>/three-layer-report.json` | per-project build / meta / runtime scores |
+| `<runDate>/structural-assertions.jsonl` | structural pass/fail per seed |
+| `<runDate>/cost-summary.json` | total cost + p95 latency per seed |
+
+Each unique seed in a run becomes one `AgentEvalTrace` in the UI.
+
+## Scoring dimensions
+
+Default radar dimensions (all normalised to `[0..1]`, with lower-is-better
+dims pre-inverted against a budget so outer = good):
+
+- `correctness`, `helpfulness` — populated when the upstream judge writes them.
+- `structural`, `build`, `runtime`, `runtimePassRate`, `meta` — agent-eval scaffold scores.
+- `latencyP95` — inverted against a 30s budget (override per-call in `radarData`).
+- `costUsd` — inverted against a $1 budget.
+
+Add your own dimensions: write them into the trace's `scores` map; they show
+up automatically in the table and on the radar.
+
+## Production deploy
+
+The dev server uses a Vite middleware (`/__traces`) to read the configured
+directory off the host filesystem. For production:
+
+1. Mount your traces dir at a known path on the same origin and rewrite
+   `loadTraces('/your-static-path')`, OR
+2. Build a tiny API route that mirrors the `/__traces` JSON shape (see
+   `vite.config.ts` for the contract).
+
+Run `pnpm build` for a static bundle in `dist/`.
+
+## What's NOT in scope
+
+- Running evals — this is a viewer for artifacts produced by `pnpm run agent-eval` (or your equivalent). Pair with `scripts/agent-eval-scaffold.ts`.
+- LLM-as-judge logic — judges write into `scores`; this UI only visualises.
+- Multi-tenant auth — stand it up behind your own auth proxy.

package/registry/families/agent-eval-ui-ts/files/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang='en'>
+  <head>
+    <meta charset='UTF-8' />
+    <meta name='viewport' content='width=device-width, initial-scale=1.0' />
+    <title>{{appName}} — agent-eval UI</title>
+  </head>
+  <body>
+    <div id='root'></div>
+    <script type='module' src='/src/main.tsx'></script>
+  </body>
+</html>

package/registry/families/agent-eval-ui-ts/files/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "{{projectName}}",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "preview": "vite preview",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@tangle-network/sandbox-ui": "^0.10.3",
+    "react": "^18.3.0",
+    "react-dom": "^18.3.0",
+    "react-router-dom": "^6.26.0",
+    "recharts": "^2.13.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.7.0",
+    "@types/react": "^18.3.0",
+    "@types/react-dom": "^18.3.0",
+    "@vitejs/plugin-react": "^4.7.0",
+    "typescript": "^5.9.0",
+    "vite": "^6.4.2"
+  }
+}

package/registry/families/agent-eval-ui-ts/files/pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+minimumReleaseAge: 4320
+minimumReleaseAgeExclude:
+  - '@tangle-network/*'

package/registry/families/agent-eval-ui-ts/files/src/App.tsx

@@ -0,0 +1,148 @@
+// App — three views (list / detail / compare) wired to react-router. Trace
+// load is one-shot at mount via the dev-server middleware in vite.config.ts;
+// production deployments should swap that for a static JSON endpoint or a
+// purpose-built API (see README).
+
+import { useEffect, useState } from 'react'
+import { Link, Navigate, Route, Routes, useLocation } from 'react-router-dom'
+import { TraceList } from './components/TraceList'
+import { TraceDetail } from './components/TraceDetail'
+import { RunDiff } from './components/RunDiff'
+import { ScoreRadar } from './components/ScoreRadar'
+import { aggregateScores, radarData } from './lib/scoring'
+import type { AgentEvalTrace } from './lib/traces'
+import { loadTraces } from './lib/traces'
+
+const APP_NAME = (import.meta.env.VITE_APP_NAME as string | undefined) ?? '{{appName}}'
+
+interface LoadState {
+  status: 'idle' | 'loading' | 'ready' | 'error'
+  traces: AgentEvalTrace[]
+  error?: string
+}
+
+export function App() {
+  const [state, setState] = useState<LoadState>({ status: 'idle', traces: [] })
+  const location = useLocation()
+
+  useEffect(() => {
+    let cancelled = false
+    setState({ status: 'loading', traces: [] })
+    loadTraces()
+      .then((traces) => {
+        if (!cancelled) setState({ status: 'ready', traces })
+      })
+      .catch((err: unknown) => {
+        if (!cancelled) {
+          setState({
+            status: 'error',
+            traces: [],
+            error: err instanceof Error ? err.message : String(err),
+          })
+        }
+      })
+    return () => {
+      cancelled = true
+    }
+  }, [])
+
+  return (
+    <div className='app-shell'>
+      <aside className='sidebar'>
+        <div style={{ marginBottom: 12 }}>
+          <Link to='/traces' style={{ fontSize: 16, fontWeight: 600, color: '#e6e8ec' }}>
+            {APP_NAME}
+          </Link>
+          <div style={{ fontSize: 11, color: '#8a93a3', marginTop: 2 }}>agent-eval UI</div>
+        </div>
+        <nav style={{ display: 'flex', gap: 8, marginBottom: 12 }}>
+          <Link to='/traces' className={location.pathname.startsWith('/traces') ? 'trace-row active' : 'trace-row'}>
+            Traces
+          </Link>
+          <Link to='/compare' className={location.pathname.startsWith('/compare') ? 'trace-row active' : 'trace-row'}>
+            Compare
+          </Link>
+        </nav>
+        {state.status === 'loading' && <div className='empty-state'>Loading…</div>}
+        {state.status === 'error' && (
+          <div className='empty-state' style={{ color: '#ef6e6e' }}>
+            Failed to load traces: {state.error}
+          </div>
+        )}
+        {state.status === 'ready' && <TraceList traces={state.traces} />}
+      </aside>
+      <main className='main'>
+        <Routes>
+          <Route path='/' element={<Navigate to='/traces' replace />} />
+          <Route path='/traces' element={<Overview state={state} />} />
+          <Route path='/traces/:id' element={<TraceDetail traces={state.traces} />} />
+          <Route path='/compare' element={<RunDiff traces={state.traces} />} />
+          <Route path='*' element={<div className='empty-state'>Not found.</div>} />
+        </Routes>
+      </main>
+    </div>
+  )
+}
+
+function Overview({ state }: { state: LoadState }) {
+  if (state.status === 'loading') return <div className='empty-state'>Loading traces…</div>
+  if (state.status === 'error') {
+    return (
+      <div className='empty-state' style={{ color: '#ef6e6e' }}>
+        {state.error}
+      </div>
+    )
+  }
+  if (state.traces.length === 0) {
+    return (
+      <div>
+        <h1 className='h-title'>No traces yet</h1>
+        <p className='h-sub'>
+          Set <code>AGENT_EVAL_TRACES_DIR</code> in <code>.env</code> to a directory containing agent-eval output (e.g.{' '}
+          <code>.evolve/agent-eval</code>) and reload.
+        </p>
+      </div>
+    )
+  }
+
+  const aggregate = aggregateScores(state.traces)
+  const passing = state.traces.filter((t) => t.status === 'pass').length
+  const failing = state.traces.filter((t) => t.status === 'fail' || t.status === 'error').length
+  const scaffolds = state.traces.filter((t) => t.status === 'scaffold').length
+
+  return (
+    <div>
+      <h1 className='h-title'>Run overview</h1>
+      <p className='h-sub'>
+        {state.traces.length} traces · {passing} passing · {failing} failing · {scaffolds} scaffold-only
+      </p>
+      <div className='score-grid'>
+        <table className='score-table'>
+          <thead>
+            <tr>
+              <th>Dimension (mean)</th>
+              <th>Value</th>
+            </tr>
+          </thead>
+          <tbody>
+            {Object.entries(aggregate).map(([dim, value]) => (
+              <tr key={dim}>
+                <td>{dim}</td>
+                <td>{formatNumber(dim, value)}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+        <ScoreRadar series={[{ name: 'aggregate', color: '#7aa2ff', data: radarData(aggregate) }]} />
+      </div>
+      <p className='h-sub'>Pick a trace from the sidebar to drill in, or open Compare to diff two runs.</p>
+    </div>
+  )
+}
+
+function formatNumber(dim: string, raw: number): string {
+  if (!Number.isFinite(raw)) return '—'
+  if (dim === 'latencyP95') return `${Math.round(raw)} ms`
+  if (dim === 'costUsd') return `$${raw.toFixed(4)}`
+  return `${(raw * 100).toFixed(1)}%`
+}

package/registry/families/agent-eval-ui-ts/files/src/components/RunDiff.tsx

@@ -0,0 +1,91 @@
+// RunDiff — pick two traces, render side-by-side score deltas, with the
+// winner highlighted per dimension. Uses radarData() to overlay both runs on
+// one chart for at-a-glance comparison.
+
+import { useMemo, useState } from 'react'
+import type { AgentEvalTrace } from '../lib/traces'
+import { diffScores, formatScore, radarData } from '../lib/scoring'
+import { ScoreRadar } from './ScoreRadar'
+
+export interface RunDiffProps {
+  traces: AgentEvalTrace[]
+}
+
+export function RunDiff({ traces }: RunDiffProps) {
+  const sorted = useMemo(() => [...traces].sort((a, b) => (a.id < b.id ? 1 : -1)), [traces])
+  const [aId, setAId] = useState<string>(sorted[0]?.id ?? '')
+  const [bId, setBId] = useState<string>(sorted[1]?.id ?? sorted[0]?.id ?? '')
+
+  const a = sorted.find((t) => t.id === aId)
+  const b = sorted.find((t) => t.id === bId)
+
+  const diff = useMemo(() => (a && b ? diffScores(a, b) : null), [a, b])
+
+  const radar = useMemo(() => {
+    const series: { name: string; color: string; data: Array<{ dim: string; value: number; raw: number }> }[] = []
+    if (a) series.push({ name: a.id, color: '#7aa2ff', data: radarData(a.scores) })
+    if (b) series.push({ name: b.id, color: '#5fd28a', data: radarData(b.scores) })
+    return series
+  }, [a, b])
+
+  if (sorted.length < 2) {
+    return (
+      <div>
+        <h1 className='h-title'>Compare runs</h1>
+        <div className='empty-state'>Need at least two traces loaded to diff. Currently {sorted.length}.</div>
+      </div>
+    )
+  }
+
+  return (
+    <div>
+      <h1 className='h-title'>Compare runs</h1>
+      <div className='compare-pickers'>
+        <select value={aId} onChange={(e) => setAId(e.target.value)}>
+          {sorted.map((t) => (
+            <option key={t.id} value={t.id}>
+              A — {t.family} · {t.id}
+            </option>
+          ))}
+        </select>
+        <select value={bId} onChange={(e) => setBId(e.target.value)}>
+          {sorted.map((t) => (
+            <option key={t.id} value={t.id}>
+              B — {t.family} · {t.id}
+            </option>
+          ))}
+        </select>
+      </div>
+
+      <ScoreRadar series={radar} height={360} />
+
+      {diff && (
+        <table className='score-table' style={{ marginTop: 24 }}>
+          <thead>
+            <tr>
+              <th>Dimension</th>
+              <th>A ({a?.id})</th>
+              <th>B ({b?.id})</th>
+              <th>Δ</th>
+              <th>Winner</th>
+            </tr>
+          </thead>
+          <tbody>
+            {Object.entries(diff).map(([dim, row]) => (
+              <tr key={dim}>
+                <td>{dim}</td>
+                <td>{formatScore(dim, row.a)}</td>
+                <td>{formatScore(dim, row.b)}</td>
+                <td className={row.delta === 0 ? '' : row.delta > 0 ? 'delta-good' : 'delta-bad'}>
+                  {row.delta > 0 ? '+' : ''}
+                  {formatScore(dim, row.delta)}
+                </td>
+                <td>{row.winner === 'tie' ? '—' : row.winner.toUpperCase()}</td>
+              </tr>
+            ))}
+          </tbody>
+        </table>
+      )}
+    </div>
+  )
+}

package/registry/families/agent-eval-ui-ts/files/src/components/ScoreRadar.tsx

@@ -0,0 +1,78 @@
+// ScoreRadar — recharts RadarChart wrapper. Receives a normalised
+// dimension array (every value in [0..1], lower-is-better dims pre-inverted
+// by radarData) and renders a single polygon. Pass two `series` to overlay
+// two runs in the diff view.
+
+import {
+  Radar,
+  RadarChart,
+  PolarAngleAxis,
+  PolarGrid,
+  PolarRadiusAxis,
+  Legend,
+  ResponsiveContainer,
+  Tooltip,
+} from 'recharts'
+
+export interface RadarSeries {
+  name: string
+  /** Stroke + fill base colour. */
+  color: string
+  data: Array<{ dim: string; value: number; raw: number }>
+}
+
+export interface ScoreRadarProps {
+  series: RadarSeries[]
+  height?: number
+}
+
+/**
+ * Recharts requires a single dataset keyed by dimension. We merge the per-series
+ * arrays here so each row looks like { dim, [seriesName]: value, ... }.
+ */
+function mergeSeries(series: RadarSeries[]): Array<Record<string, string | number>> {
+  const dims = new Set<string>()
+  for (const s of series) for (const row of s.data) dims.add(row.dim)
+  const merged: Array<Record<string, string | number>> = []
+  for (const dim of dims) {
+    const row: Record<string, string | number> = { dim }
+    for (const s of series) {
+      const found = s.data.find((d) => d.dim === dim)
+      row[s.name] = found?.value ?? 0
+    }
+    merged.push(row)
+  }
+  return merged
+}
+
+export function ScoreRadar({ series, height = 320 }: ScoreRadarProps) {
+  if (series.length === 0 || series.every((s) => s.data.length === 0)) {
+    return <div className='empty-state'>No scoring dimensions to plot.</div>
+  }
+  const merged = mergeSeries(series)
+  return (
+    <ResponsiveContainer width='100%' height={height}>
+      <RadarChart data={merged} outerRadius='75%'>
+        <PolarGrid stroke='#2a3140' />
+        <PolarAngleAxis dataKey='dim' tick={{ fill: '#8a93a3', fontSize: 11 }} />
+        <PolarRadiusAxis domain={[0, 1]} tick={{ fill: '#8a93a3', fontSize: 10 }} angle={30} />
+        <Tooltip
+          contentStyle={{ background: '#161b25', border: '1px solid #232a36', color: '#e6e8ec', fontSize: 12 }}
+          formatter={(value: number) => `${(Number(value) * 100).toFixed(1)}%`}
+        />
+        <Legend wrapperStyle={{ fontSize: 12, color: '#8a93a3' }} />
+        {series.map((s) => (
+          <Radar
+            key={s.name}
+            name={s.name}
+            dataKey={s.name}
+            stroke={s.color}
+            fill={s.color}
+            fillOpacity={0.25}
+            isAnimationActive={false}
+          />
+        ))}
+      </RadarChart>
+    </ResponsiveContainer>
+  )
+}