@tangle-network/starter-foundry 0.7.2 → 0.13.0

package/registry/families/agent-eval-harness-py/manifest.json

@@ -0,0 +1,209 @@
+{
+  "id": "agent-eval-harness-py",
+  "description": "Python eval harness for agent-under-test scoring. Loads scenarios, runs them against an agent (URL or local fn) inside an isolated tangle-sandbox, scores with multi-dim LLM judges via router.tangle.tools, emits a scorecard.json that matches the TS sibling 1:1 (cross-language compare works), and gates regression with bootstrap CI + Cohen's d + Welch's t-test (scipy). Operator entrypoint: `python -m eval run`.",
+  "tags": [
+    "agent-eval",
+    "python",
+    "eval-harness",
+    "llm-judge",
+    "tangle-sandbox",
+    "regression-gate"
+  ],
+  "taxonomy": {
+    "language": "python",
+    "runtime": "python",
+    "surface": "eval-runner"
+  },
+  "defaults": {
+    "projectType": "agent-eval-harness-py",
+    "entrypoint": "python -m eval",
+    "routerBaseUrl": "https://router.tangle.tools/v1",
+    "judgeModel": "anthropic/claude-sonnet-4.5",
+    "bootstrapResamples": 1000,
+    "bootstrapAlpha": 0.05,
+    "secrets": [
+      "TANGLE_API_KEY",
+      "TANGLE_SANDBOX_API_KEY"
+    ]
+  },
+  "files": [
+    {
+      "source": "files/pyproject.toml",
+      "target": "pyproject.toml"
+    },
+    {
+      "source": "files/requirements.txt",
+      "target": "requirements.txt"
+    },
+    {
+      "source": "files/README.md",
+      "target": "README.md"
+    },
+    {
+      "source": "files/AGENTS.md",
+      "target": "AGENTS.md"
+    },
+    {
+      "source": "files/src/eval/__init__.py",
+      "target": "src/eval/__init__.py"
+    },
+    {
+      "source": "files/src/eval/__main__.py",
+      "target": "src/eval/__main__.py"
+    },
+    {
+      "source": "files/src/eval/cli.py",
+      "target": "src/eval/cli.py"
+    },
+    {
+      "source": "files/src/eval/runner.py",
+      "target": "src/eval/runner.py"
+    },
+    {
+      "source": "files/src/eval/scorecard.py",
+      "target": "src/eval/scorecard.py"
+    },
+    {
+      "source": "files/src/eval/sandbox_runner.py",
+      "target": "src/eval/sandbox_runner.py"
+    },
+    {
+      "source": "files/src/eval/regression.py",
+      "target": "src/eval/regression.py"
+    },
+    {
+      "source": "files/src/eval/judges/__init__.py",
+      "target": "src/eval/judges/__init__.py"
+    },
+    {
+      "source": "files/src/eval/judges/rubric.py",
+      "target": "src/eval/judges/rubric.py"
+    },
+    {
+      "source": "files/src/eval/judges/example_judge.py",
+      "target": "src/eval/judges/example_judge.py"
+    },
+    {
+      "source": "files/scenarios/__init__.py",
+      "target": "scenarios/__init__.py"
+    },
+    {
+      "source": "files/scenarios/example.py",
+      "target": "scenarios/example.py"
+    },
+    {
+      "source": "files/tests/__init__.py",
+      "target": "tests/__init__.py"
+    },
+    {
+      "source": "files/tests/test_runner.py",
+      "target": "tests/test_runner.py"
+    },
+    {
+      "source": "files/tests/test_regression.py",
+      "target": "tests/test_regression.py"
+    },
+    {
+      "source": "files/tests/test_sandbox.py",
+      "target": "tests/test_sandbox.py"
+    },
+    {
+      "source": "files/.github/workflows/eval.yml",
+      "target": ".github/workflows/eval.yml"
+    }
+  ],
+  "validationChecks": [
+    {
+      "type": "file-exists",
+      "path": "pyproject.toml"
+    },
+    {
+      "type": "file-exists",
+      "path": "AGENTS.md"
+    },
+    {
+      "type": "agents-md-valid",
+      "path": "AGENTS.md"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/eval/runner.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/eval/scorecard.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/eval/regression.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/eval/sandbox_runner.py"
+    },
+    {
+      "type": "python-compile",
+      "path": "src/eval/cli.py"
+    }
+  ],
+  "contextHints": {
+    "commands": [
+      "python -m eval run --scenarios scenarios/ --out scorecard.json",
+      "python -m eval compare baseline.json head.json",
+      "pytest -q"
+    ],
+    "entrypoints": [
+      "src/eval/__main__.py",
+      "AGENTS.md"
+    ],
+    "extensionPoints": [
+      "scenarios/",
+      "src/eval/judges/"
+    ]
+  },
+  "keywords": [
+    "agent-eval",
+    "python eval",
+    "llm-as-judge",
+    "scorecard",
+    "regression-gate",
+    "bootstrap-ci",
+    "tangle-sandbox",
+    "cohen-d",
+    "welch-t-test"
+  ],
+  "tieredKeywords": {
+    "tier1": [
+      "agent-eval-python",
+      "agent eval python",
+      "python-eval-runner",
+      "python eval runner",
+      "eval-harness-py",
+      "eval harness python",
+      "python eval harness",
+      "agent eval harness py"
+    ],
+    "tier2": [
+      "llm-as-judge",
+      "llm as judge",
+      "scorecard",
+      "regression-gate",
+      "regression gate",
+      "bootstrap ci",
+      "cohen d",
+      "welch t-test",
+      "scipy stats",
+      "agent under test"
+    ],
+    "tier3": [
+      "eval",
+      "judge",
+      "rubric",
+      "bench"
+    ],
+    "archetypes": [
+      "eval-runner",
+      "agent-evaluator"
+    ]
+  }
+}

package/registry/families/agent-eval-harness-ts/files/.github/workflows/eval.yml

@@ -0,0 +1,55 @@
+name: Eval harness
+
+# Runs the eval harness on every PR + push. Emits .evolve/scorecard.json
+# and uploads it as a build artifact. The regression gate is enforced by
+# the agent-eval:regression layer's workflow — this one is the runner.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: eval-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  eval:
+    name: eval-harness
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'pnpm'
+
+      - name: install
+        run: pnpm install --frozen-lockfile
+
+      - name: typecheck
+        run: pnpm typecheck
+
+      - name: run eval harness
+        env:
+          EVAL_TARGET_BASE_URL: ${{ vars.EVAL_TARGET_BASE_URL }}
+          EVAL_THRESHOLD: ${{ vars.EVAL_THRESHOLD || '0.7' }}
+          TANGLE_API_KEY: ${{ secrets.TANGLE_API_KEY }}
+        run: pnpm eval
+
+      - name: upload scorecard
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-scorecard-${{ github.sha }}
+          path: |
+            .evolve/scorecard.json
+            .evolve/agent-eval/
+          retention-days: 30

package/registry/families/agent-eval-harness-ts/files/AGENTS.md

@@ -0,0 +1,114 @@
+---
+name: eval-harness
+role: Operator-facing eval harness — runs scenarios, judges, and regression gates against an agent under test, emits scorecard.json
+domain: agent-evaluation
+allowedDomains:
+  - api.tangle.tools
+allowedEnv:
+  - TANGLE_API_KEY
+  - EVAL_TARGET_BASE_URL
+  - EVAL_THRESHOLD
+  - EVAL_INTEGRITY
+  - EVAL_LLM_BASE_URL
+  - EVAL_LLM_API_KEY
+  - EVAL_LLM_PROVIDER
+version: 0.1.0
+---
+
+## Role
+
+You are the eval harness for this project. You read scenarios from
+`scenarios/`, run them against an agent under test (HTTP target or
+in-process function), grade each turn with judges from `judges/`, and
+emit a scorecard at `.evolve/scorecard.json`.
+
+You do **not** modify the agent under test. You do not write to its
+filesystem. You produce one artifact: the scorecard. Operators read it
+locally and CI gates regressions on it.
+
+## How you work
+
+The harness is a thin shell over `@tangle-network/agent-eval@^0.23.0`
+(already in `dependencies`). Reuse its primitives — do not reinvent.
+
+- **Scenarios** are typed objects (`Scenario` from agent-eval). Author
+  one file per scenario in `scenarios/*.ts`. Each scenario declares
+  `id`, `persona`, `turns[]`, `dimensions[]`, and `artifactChecks[]`.
+- **Judges** are `JudgeFn` callables. Author one file per judge in
+  `judges/*.ts`. Use `createCustomJudge`, `createSemanticConceptJudge`,
+  or `createIntentMatchJudge` from agent-eval. Calibrate against a
+  golden set with `calibrateJudge` before relying on the score.
+- **Runner** lives in `src/eval/runner.ts` (smoke-test path) and
+  `src/eval/campaign.ts` (campaign path).
+  - `runHarness` (smoke) loads scenarios, runs each via
+    `runTestGradedScenario`, persists traces to `FileSystemTraceStore`,
+    and aggregates `scorecard.json`. Capture-integrity directives 1–3
+    fire by default; directive 4 (analyst hook) is unavailable on this
+    path because `runTestGradedScenario` constructs its own emitter.
+  - `runCampaign` (sweep) wraps `runEvalCampaign` and wires all four
+    capture-integrity directives by construction. Use it when you have
+    variants × seeds and need a paired-evidence verdict.
+- **Regression gate** lives in `src/eval/regression/` (composed from
+  the `eval:regression` layer). It compares two scorecards via
+  `bootstrapCi` + `welchsTTest` + `compareToBaseline` and exits 0
+  (PROMOTE) / 1 (REVERT) / 2 (HOLD).
+
+## Capture integrity is REQUIRED for launch-grade adoption
+
+Every run wires:
+1. **`RawProviderSink`** — `FileSystemRawProviderSink` per scenario.
+2. **`assertLlmRoute`** at preflight (when `EVAL_LLM_BASE_URL` is set).
+3. **`assertRunCaptured`** after every run — surfaces issues on the
+   outcome row by default (`EVAL_INTEGRITY=log`).
+4. **`onRunComplete` hooks** — campaign path only.
+
+Skipping a directive means the run is descriptive, not anchoring — a
+launch reviewer can't distinguish "we measured a real win" from "we
+measured nothing on the wrong route." Document the reason inline if
+you skip one.
+
+## Operator commands
+
+```bash
+pnpm install
+pnpm eval                 # run all scenarios, write .evolve/scorecard.json
+pnpm eval -- --target http://127.0.0.1:8787  # custom target
+pnpm eval:compare baseline.json head.json    # diff two scorecards
+pnpm eval:gate baseline.json head.json       # CI gate (exit 0/1/2)
+```
+
+CI runs the gate on every PR — see `.github/workflows/eval.yml`. The
+workflow uploads `scorecard.json` as an artifact and fails the build
+on REVERT.
+
+## Adding scenarios
+
+1. Drop a file in `scenarios/your-id.scenario.ts` exporting a default
+   `Scenario`.
+2. Run `pnpm eval` locally — verify the scorecard moves in the right
+   direction.
+3. Open a PR. The CI gate compares head against `origin/main`'s
+   scorecard with statistical rigor (95% bootstrap CI, Benjamini–
+   Hochberg correction across dimensions).
+
+## Adding judges
+
+1. Drop a file in `judges/your-id.judge.ts` exporting a `JudgeFn`.
+2. Calibrate it: produce a small golden set (5–20 hand-graded
+   scenarios), call `calibrateJudge(...)` from agent-eval, and commit
+   the calibration report under `.evolve/agent-eval/judge-calibrations/`.
+3. Wire it into `src/eval/runner.ts`'s judge list.
+
+A judge that has not been calibrated against goldens **must not** gate
+CI. Use it as a signal only.
+
+## What you will NOT do
+
+- Mutate the agent under test (read-only HTTP / in-process call only).
+- Bypass the regression gate (no `--no-verify`, no `continue-on-error`,
+  no silent fallbacks). If the gate is wrong, fix the gate, not the
+  bypass.
+- Score on uncalibrated judges as if they were ground truth.
+- Author scenarios without a thesis. Every scenario answers a specific
+  question about the agent. If you can't articulate the question, the
+  scenario isn't earning its CI time.

package/registry/families/agent-eval-harness-ts/files/README.md

@@ -0,0 +1,138 @@
+# {{projectName}}
+
+Standalone eval harness wrapping `@tangle-network/agent-eval`. Runs
+scenarios + LLM-as-judge rubrics against an agent under test and
+emits a statistical-quality scorecard.
+
+This README is for humans. The agent-readable spec is in
+[`AGENTS.md`](./AGENTS.md) — that's what auto-loaded into in-sandbox
+agents (Claude Code, OpenCode) when they extend this bundle.
+
+## Quick start
+
+```bash
+pnpm install
+pnpm eval                                       # run all scenarios
+pnpm eval -- --target http://127.0.0.1:8787     # custom target URL
+pnpm eval:compare scorecard-a.json scorecard-b.json
+pnpm eval:gate baseline.json head.json          # CI regression gate
+```
+
+Outputs land in `.evolve/scorecard.json` (latest) and
+`.evolve/agent-eval/traces/` (per-run JSONL).
+
+## Layout
+
+- `src/eval/runner.ts` — smoke-test entrypoint (`pnpm eval`). Loads
+  scenarios, runs each via `runTestGradedScenario`, persists traces
+  to `FileSystemTraceStore`, aggregates a scorecard. Capture
+  integrity (agent-eval 0.21+) is wired by default — see below.
+- `src/eval/campaign.ts` — **launch-decision-grade** entrypoint
+  (agent-eval 0.22+). Wraps `runEvalCampaign` for sweeps over
+  (variants × scenarios × seeds) with paired bootstrap CIs +
+  anytime-valid sequential verdicts. The four capture-integrity
+  directives are wired by construction — the caller can't skip one
+  without rewriting the function. Import + call from your own
+  script when you have variants to compare.
+- `src/eval/scorecard.ts` — produces `scorecard.json` matching the
+  project-level dashboard shape (timestamp, aggregate, flows[]).
+- `src/eval/cli.ts` — `eval-harness run | compare`.
+- `scenarios/` — author one `Scenario` per file.
+- `judges/` — author one `JudgeFn` per file. Calibrate before
+  gating.
+
+## Capture integrity (agent-eval 0.21+)
+
+Every `pnpm eval` run wires the four directives from the agent-eval
+skill's "Capture integrity" section by default:
+
+1. **`RawProviderSink`** — a `FileSystemRawProviderSink` per scenario
+   under `.evolve/agent-eval/raw-events/<scenarioId>/`. Headers + body
+   credentials auto-redacted. LLM judges that explicitly call `callLlm`
+   pick the sink up off `globalThis.__agentEvalRawSink`.
+2. **`assertLlmRoute`** at preflight — fires when `EVAL_LLM_BASE_URL`
+   is set. Fails loud if the sweep would silently fall back to the
+   public router or run unauthenticated.
+3. **`assertRunCaptured`** after each scenario — verifies the run wrote
+   spans + (when raw events are present) every `LlmSpan` has a matching
+   raw `request` event. Default `EVAL_INTEGRITY=log` surfaces issues
+   on the outcome row; `EVAL_INTEGRITY=strict` fails the scenario;
+   `EVAL_INTEGRITY=off` disables the wiring.
+4. **`onRunComplete` hooks** — supported on the campaign path; pass
+   `traceAnalystOnRunComplete(...)` via `onRunComplete` in
+   `runCampaign` opts. The smoke-test path uses
+   `runTestGradedScenario` which constructs its own emitter — wire
+   the analyst out-of-band or migrate to `runCampaign` when a hook is
+   load-bearing.
+
+Env knobs:
+
+| Var | Default | Effect |
+| --- | ------- | ------ |
+| `EVAL_INTEGRITY` | `log` | `off` \| `log` \| `strict` capture-integrity policy. |
+| `EVAL_LLM_BASE_URL` | unset | Triggers `assertLlmRoute` preflight. |
+| `EVAL_LLM_API_KEY` | unset | Auth for the LLM judge route (falls back to `TANGLE_API_KEY`). |
+| `EVAL_LLM_PROVIDER` | unset | Pin the expected provider (`openai`, `anthropic`, …). |
+
+## Campaign mode (agent-eval 0.22+)
+
+When the question is "does variant A beat variant B over scenarios ×
+seeds?" — i.e. a launch-decision sweep — use `src/eval/campaign.ts`:
+
+```ts
+import { runCampaign } from './src/eval/campaign.js'
+
+const result = await runCampaign({
+  campaignId: 'prompt-v2-vs-baseline',
+  commitSha: process.env.GIT_SHA!,
+  comparator: 'baseline',
+  variants: [
+    { id: 'baseline', payload: { systemPrompt: '...' } },
+    { id: 'v2',       payload: { systemPrompt: '...' } },
+  ],
+  scenarios: [{ scenarioId: 'math-1' }, { scenarioId: 'tools-a' }],
+  seeds: [0, 1, 2],
+  llmOpts: { baseUrl: process.env.EVAL_LLM_BASE_URL!, apiKey: process.env.TANGLE_API_KEY! },
+  runner: async (ctx) => {
+    await ctx.emitter.startRun({ scenarioId: ctx.scenarioId, variantId: ctx.variantId })
+    // ... call the agent under test, score the response ...
+    await ctx.emitter.endRun({ pass, score })
+    return { pass, score, costUsd, tokenUsage, model, promptHash, configHash }
+  },
+})
+
+// result.runs       → RunRecord[] (canonical, hashed, costed)
+// result.report     → researchReport with paired-bootstrap verdicts
+// result.integrityReports → per-run capture-integrity reports
+```
+
+The campaign emits a `RunRecord[]` that is the exchange currency
+across the rest of agent-eval (research report, replay cache, RL
+bridge, predictive-validity). Skip this entrypoint and you're back to
+re-implementing the same shape ad hoc.
+
+## Composed layers
+
+This bundle is composed with three base layers from
+`registry/layers/agent-eval/`:
+
+- `agent-eval:scenarios` — scenario loader + types
+- `agent-eval:judge-rubric` — multi-dimensional rubric runner
+- `agent-eval:regression` — bootstrap-CI / Welch's t-test gate +
+  Benjamini–Hochberg correction
+
+## CI
+
+`.github/workflows/eval.yml` runs the harness on every PR, uploads
+the scorecard as an artifact, and fails the build on a REVERT
+verdict from `pnpm eval:gate` against `origin/main`'s scorecard.
+
+## Extending
+
+- Need a different judge dimension? Add a file in `judges/`. Use
+  `createCustomJudge` from agent-eval.
+- Need a custom regression policy? Edit
+  `src/eval/regression/gate.ts` — it's composed from the
+  `eval:regression` layer; override the thresholds in your scaffold.
+- Need to gate on a specific dimension? The gate accepts
+  `--dimension <name>` and runs the comparison only on that slice.

package/registry/families/agent-eval-harness-ts/files/agent.json

@@ -0,0 +1,32 @@
+{
+  "name": "eval-harness",
+  "description": "Operator-facing eval harness bundle. Runs scenarios + judges + regression gates against an agent under test (HTTP target or in-process function). Emits scorecard.json into .evolve/ for the project dashboard. Read-only against the router by default — does not need write capabilities. When deployed to a sandbox via the harness-aware deploy script, this agent reads scenarios/, judges/, and writes scorecards back to .evolve/scorecard.json.",
+  "version": "0.1.0",
+  "tags": [
+    "agent-eval",
+    "eval-harness",
+    "node",
+    "typescript"
+  ],
+  "prompt": {
+    "systemPromptFile": "AGENTS.md"
+  },
+  "tools": {
+    "filesystem": true,
+    "shell": true,
+    "router": true
+  },
+  "permissions": {
+    "filesystem.read": "allow",
+    "filesystem.write": "ask",
+    "shell.exec": "ask",
+    "network": "deny"
+  },
+  "resources": {
+    "files": [
+      { "source": "scenarios/" },
+      { "source": "judges/" },
+      { "source": "README.md" }
+    ]
+  }
+}

package/registry/families/agent-eval-harness-ts/files/judges/example.judge.ts

@@ -0,0 +1,23 @@
+// Example LLM-as-judge — copy this to make new ones.
+//
+// `createCustomJudge` from agent-eval wraps a system prompt and returns a
+// JudgeFn. The runner invokes the judge for each ScenarioResult; the judge
+// calls TCloud and returns one JudgeScore per dimension.
+//
+// Calibrate before relying on the score in CI gates — see AGENTS.md.
+
+import { createCustomJudge, type JudgeFn } from '@tangle-network/agent-eval'
+
+const judge: JudgeFn = createCustomJudge(
+  'example-coherence',
+  `You are a strict evaluator. Score the agent's response on a single
+dimension: "coherence" (does the response stay on topic, follow the
+user's question, and form coherent sentences?). Output a number from 0
+to 1. Provide one sentence of reasoning. Be terse.`,
+  {
+    model: 'claude-sonnet-4-5',
+    temperature: 0,
+  },
+)
+
+export default judge

package/registry/families/agent-eval-harness-ts/files/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "{{projectName}}",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "eval": "tsx src/eval/cli.ts run",
+    "eval:compare": "tsx src/eval/cli.ts compare",
+    "eval:gate": "tsx src/eval/regression/cli.ts",
+    "typecheck": "tsc --noEmit",
+    "build": "tsc"
+  },
+  "dependencies": {
+    "@tangle-network/agent-eval": "^0.23.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.7.0",
+    "tsx": "^4.19.0",
+    "typescript": "^5.9.0"
+  },
+  "pnpm": {
+    "minimumReleaseAge": 4320,
+    "minimumReleaseAgeExclude": []
+  }
+}

package/registry/families/agent-eval-harness-ts/files/scenarios/example.scenario.ts

@@ -0,0 +1,34 @@
+// Example scenario — copy this to make new ones.
+//
+// A scenario is a typed assertion against the agent under test. The runner
+// loads every `*.scenario.ts` in this directory and grades each one.
+
+import type { Scenario } from '@tangle-network/agent-eval'
+
+const scenario: Scenario = {
+  id: 'example/health-and-greet',
+  persona: 'first-time-user',
+  label: 'Agent responds to a basic greeting',
+  thesis:
+    'A new user sends a single greeting message; the agent must respond with non-empty content within 30s. This is the smoke test that catches "the agent is wired up" regressions before any quality grading.',
+  dimensions: ['responsiveness', 'basic-coherence'],
+  turns: [
+    {
+      user: 'Hello — can you tell me what you do?',
+      expectedBehaviors: [
+        'responds with non-empty content',
+        'does not error out',
+        'self-describes its role',
+      ],
+    },
+  ],
+  artifactChecks: [
+    {
+      type: 'generation_produced',
+      target: 'response',
+      description: 'Agent emitted a non-empty response.',
+    },
+  ],
+}
+
+export default scenario