@tangle-network/starter-foundry 0.13.0 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/training/template_v1/harvest.d.ts +1 -1
- package/dist/training/template_v1/harvest.js +4 -4
- package/dist/training/template_v1/harvest.js.map +1 -1
- package/package.json +2 -2
- package/registry/families/agent-debug-ui-ts/files/package.json +0 -4
- package/registry/families/agent-debug-ui-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-debug-ui-ts/manifest.json +2 -1
- package/registry/families/agent-eval-harness-ts/files/AGENTS.md +6 -4
- package/registry/families/agent-eval-harness-ts/files/README.md +2 -2
- package/registry/families/agent-eval-harness-ts/files/judges/example.judge.ts +82 -12
- package/registry/families/agent-eval-harness-ts/files/package.json +2 -5
- package/registry/families/agent-eval-harness-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-eval-harness-ts/files/src/eval/runner.ts +125 -18
- package/registry/families/agent-eval-harness-ts/manifest.json +2 -1
- package/registry/families/agent-eval-ui-ts/files/package.json +0 -4
- package/registry/families/agent-eval-ui-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-eval-ui-ts/manifest.json +2 -1
- package/registry/families/agent-marketplace-ui-ts/files/package.json +0 -4
- package/registry/families/agent-marketplace-ui-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-marketplace-ui-ts/manifest.json +2 -1
- package/registry/families/agent-research-harness-ts/files/AGENTS.md +1 -1
- package/registry/families/agent-research-harness-ts/files/README.md +1 -1
- package/registry/families/agent-research-harness-ts/files/package.json +1 -5
- package/registry/families/agent-research-harness-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-research-harness-ts/manifest.json +2 -1
- package/registry/families/agent-with-ui-ts/files/package.json +0 -4
- package/registry/families/agent-with-ui-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/agent-with-ui-ts/manifest.json +2 -1
- package/registry/families/orchestrator-with-ui-ts/files/package.json +0 -4
- package/registry/families/orchestrator-with-ui-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/orchestrator-with-ui-ts/manifest.json +2 -1
- package/registry/families/sandbox-app-ts/files/package.json +0 -4
- package/registry/families/sandbox-app-ts/files/pnpm-workspace.yaml +3 -0
- package/registry/families/sandbox-app-ts/manifest.json +2 -1
- package/registry/layers/agent-eval/judge-rubric/files/src/eval/judges/README.md +3 -2
- package/registry/layers/agent-eval/judge-rubric/files/src/eval/judges/rubric-runner.ts +111 -23
- package/registry/layers/agent-eval/redteam/files/src/eval/redteam/README.md +1 -1
- package/registry/layers/agent-eval/redteam/files/src/eval/redteam/runner.ts +1 -1
- package/registry/layers/agent-eval/redteam/manifest.json +1 -1
- package/registry/layers/capability/agent-eval/files/tests/eval/README.md +5 -5
- package/registry/layers/capability/agent-eval/files/tests/eval/judge.mjs +25 -15
- package/registry/layers/framework/react-vite-ts/files/index.html +3 -3
|
@@ -7,9 +7,9 @@ import { dirname, join, resolve } from 'node:path';
|
|
|
7
7
|
import { fileURLToPath } from 'node:url';
|
|
8
8
|
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..', '..');
|
|
9
9
|
const REWRITES_DIR = join(REPO, '.evolve/template-rewrites');
|
|
10
|
-
function readTuples(key) {
|
|
10
|
+
function readTuples(key, rewritesDir) {
|
|
11
11
|
// Key is e.g. "src.App.tsx" — the filesystem-safe form of the template path.
|
|
12
|
-
const candidate = join(
|
|
12
|
+
const candidate = join(rewritesDir, `${key}.jsonl`);
|
|
13
13
|
if (!existsSync(candidate))
|
|
14
14
|
return [];
|
|
15
15
|
return readFileSync(candidate, 'utf8')
|
|
@@ -68,8 +68,8 @@ function editPatterns(tuples) {
|
|
|
68
68
|
}
|
|
69
69
|
return [...buckets.values()].sort((a, b) => b.count - a.count).slice(0, 10);
|
|
70
70
|
}
|
|
71
|
-
export function harvest(templateKey) {
|
|
72
|
-
const tuples = readTuples(templateKey);
|
|
71
|
+
export function harvest(templateKey, rewritesDir = REWRITES_DIR) {
|
|
72
|
+
const tuples = readTuples(templateKey, rewritesDir);
|
|
73
73
|
const writes = tuples.filter((t) => t.tool === 'Write');
|
|
74
74
|
const edits = tuples.filter((t) => t.tool === 'Edit');
|
|
75
75
|
const newBodies = tuples.map((t) => t.new);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"harvest.js","sourceRoot":"","sources":["../../../src/training/template_v1/harvest.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,qDAAqD;AACrD,2EAA2E;AAC3E,6DAA6D;AAE7D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAExC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAA;AA2B5D,SAAS,UAAU,CAAC,GAAW;
|
|
1
|
+
{"version":3,"file":"harvest.js","sourceRoot":"","sources":["../../../src/training/template_v1/harvest.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,qDAAqD;AACrD,2EAA2E;AAC3E,6DAA6D;AAE7D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAExC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAC/E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,EAAE,2BAA2B,CAAC,CAAA;AA2B5D,SAAS,UAAU,CAAC,GAAW,EAAE,WAAmB;IAClD,6EAA6E;IAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,GAAG,QAAQ,CAAC,CAAA;IACnD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,CAAA;IACrC,OAAO,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC;SACnC,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAiB,CAAC,CAAA;AAC9C,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc;IAC9B,oEAAoE;IACpE,OAAO,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAA;AACxE,CAAC;AAED,SAAS,aAAa,CAAC,MAAgB;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;QACpC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;YACvB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;gBAAE,SAAQ;YAClD,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAClC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACpB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IACtC,MAAM,QAAQ,GAAG,qCAAqC,CAAA;IACtD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,IAAI,KAA6B,CAAA;QACjC,OAAO,CAAC,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACpB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACzC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC1C,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4D,CAAA;IACnF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC,GAAG;YAAE,SAAQ;QACzC,8DAA8D;QAC9D,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;QACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACjC,IAAI,QAAQ;YAAE,QAAQ,CAAC,KAAK,EAAE,CAAA;;YACzB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;IAC9F,CAAC;IACD,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AAC7E,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,WAAmB,EAAE,WAAW,GAAG,YAAY;IACrE,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,EAAE,WAAW,CAAC,CAAA;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;IACvD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAErD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;IAC1C,MAAM,QAAQ,GAAG,aAAa,CAAC,SAAS,CAAC,CAAA;IACzC,MAAM,oBAAoB,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SACjD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;SACpE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;IAEpD,MAAM,OAAO,GAAG,eAAe,CAAC,SAAS,CAAC,CAAA;IAC1C,MAAM,eAAe,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAC3C,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;SACpE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC,CAAA;IAEtD,OAAO;QACL,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,IAAI,WAAW;QACpD,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,UAAU,EAAE,MAAM,CAAC,MAAM;QACzB,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,oBAAoB;QACpB,eAAe;QACf,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjE,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC;KAClC,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/starter-foundry",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.2",
|
|
4
4
|
"description": "Deterministic project scaffold engine for AI coding platforms",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT OR Apache-2.0",
|
|
@@ -131,7 +131,7 @@
|
|
|
131
131
|
"devDependencies": {
|
|
132
132
|
"@eslint/js": "^10.0.1",
|
|
133
133
|
"@huggingface/transformers": "^4.0.0",
|
|
134
|
-
"@tangle-network/agent-eval": "^0.
|
|
134
|
+
"@tangle-network/agent-eval": "^0.99.0",
|
|
135
135
|
"@types/node": "^20.0.0",
|
|
136
136
|
"eslint": "^10.2.1",
|
|
137
137
|
"eslint-config-prettier": "^10.1.8",
|
|
@@ -42,7 +42,8 @@
|
|
|
42
42
|
{ "source": "files/src/components/EventInspector.tsx", "target": "src/components/EventInspector.tsx" },
|
|
43
43
|
{ "source": "files/src/components/ReplayScrubber.tsx", "target": "src/components/ReplayScrubber.tsx" },
|
|
44
44
|
{ "source": "files/src/lib/sandbox-stream.ts", "target": "src/lib/sandbox-stream.ts" },
|
|
45
|
-
{ "source": "files/src/lib/event-types.ts", "target": "src/lib/event-types.ts" }
|
|
45
|
+
{ "source": "files/src/lib/event-types.ts", "target": "src/lib/event-types.ts" },
|
|
46
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
46
47
|
],
|
|
47
48
|
"validationChecks": [
|
|
48
49
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -28,16 +28,18 @@ locally and CI gates regressions on it.
|
|
|
28
28
|
|
|
29
29
|
## How you work
|
|
30
30
|
|
|
31
|
-
The harness is a thin shell over `@tangle-network/agent-eval@^0.
|
|
31
|
+
The harness is a thin shell over `@tangle-network/agent-eval@^0.95.1`
|
|
32
32
|
(already in `dependencies`). Reuse its primitives — do not reinvent.
|
|
33
33
|
|
|
34
34
|
- **Scenarios** are typed objects (`Scenario` from agent-eval). Author
|
|
35
35
|
one file per scenario in `scenarios/*.ts`. Each scenario declares
|
|
36
36
|
`id`, `persona`, `turns[]`, `dimensions[]`, and `artifactChecks[]`.
|
|
37
37
|
- **Judges** are `JudgeFn` callables. Author one file per judge in
|
|
38
|
-
`judges/*.ts`.
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
`judges/*.ts`. Build the LLM call with `llmJudge` (a campaign
|
|
39
|
+
`JudgeConfig` you adapt back into `JudgeScore[]` — see
|
|
40
|
+
`judges/example.judge.ts`), or use `createIntentMatchJudge` from
|
|
41
|
+
agent-eval. Calibrate against a golden set with `calibrateJudge`
|
|
42
|
+
before relying on the score.
|
|
41
43
|
- **Runner** lives in `src/eval/runner.ts` (smoke-test path) and
|
|
42
44
|
`src/eval/campaign.ts` (campaign path).
|
|
43
45
|
- `runHarness` (smoke) loads scenarios, runs each via
|
|
@@ -129,8 +129,8 @@ verdict from `pnpm eval:gate` against `origin/main`'s scorecard.
|
|
|
129
129
|
|
|
130
130
|
## Extending
|
|
131
131
|
|
|
132
|
-
- Need a different judge dimension? Add a file in `judges/`.
|
|
133
|
-
`
|
|
132
|
+
- Need a different judge dimension? Add a file in `judges/`. Build it
|
|
133
|
+
with `llmJudge` from agent-eval (see `judges/example.judge.ts`).
|
|
134
134
|
- Need a custom regression policy? Edit
|
|
135
135
|
`src/eval/regression/gate.ts` — it's composed from the
|
|
136
136
|
`eval:regression` layer; override the thresholds in your scaffold.
|
|
@@ -1,23 +1,93 @@
|
|
|
1
1
|
// Example LLM-as-judge — copy this to make new ones.
|
|
2
2
|
//
|
|
3
|
-
// `
|
|
4
|
-
//
|
|
5
|
-
//
|
|
3
|
+
// `llmJudge` from agent-eval builds a campaign `JudgeConfig` whose `score()`
|
|
4
|
+
// makes one LLM call and returns the canonical `{ dimensions, composite, notes }`
|
|
5
|
+
// verdict on the [0,1] scale. The runner loads each `.judge.ts` as a `JudgeFn`,
|
|
6
|
+
// so this file adapts the JudgeConfig back into the per-dimension `JudgeScore[]`
|
|
7
|
+
// rows the runner aggregates. Transport is the `ChatClient` the runner threads
|
|
8
|
+
// in — `llmJudge` stays decoupled from router-vs-sandbox-vs-cli-bridge.
|
|
6
9
|
//
|
|
7
10
|
// Calibrate before relying on the score in CI gates — see AGENTS.md.
|
|
8
11
|
|
|
9
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
createChatClient,
|
|
14
|
+
llmJudge,
|
|
15
|
+
type ChatClient,
|
|
16
|
+
type JudgeFn,
|
|
17
|
+
type JudgeScore,
|
|
18
|
+
} from '@tangle-network/agent-eval'
|
|
10
19
|
|
|
11
|
-
const
|
|
12
|
-
'example-coherence',
|
|
13
|
-
`You are a strict evaluator. Score the agent's response on a single
|
|
14
|
-
dimension: "coherence" (does the response stay on topic, follow the
|
|
15
|
-
user's question, and form coherent sentences?). Output a number from 0
|
|
16
|
-
to 1. Provide one sentence of reasoning. Be terse.`,
|
|
20
|
+
const DIMENSIONS = [
|
|
17
21
|
{
|
|
22
|
+
key: 'coherence',
|
|
23
|
+
description:
|
|
24
|
+
'Does the response stay on topic, follow the user question, and form coherent sentences? ' +
|
|
25
|
+
'0.0 = off-topic or incoherent; 1.0 = fully on-topic and coherent.',
|
|
26
|
+
},
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
// The runner threads in a TCloud-shaped client whose `.chat()` resolves an
|
|
30
|
+
// OpenAI `ChatCompletion`. Normalize it into the `LlmCallResult` shape (top-level
|
|
31
|
+
// `content`) that `createChatClient`'s `sandbox-sdk` transport — and therefore
|
|
32
|
+
// `llmJudge` — expects.
|
|
33
|
+
interface OpenAiChatClient {
|
|
34
|
+
chat(req: {
|
|
35
|
+
model: string
|
|
36
|
+
messages: { role: string; content: string }[]
|
|
37
|
+
temperature?: number
|
|
38
|
+
maxTokens?: number
|
|
39
|
+
jsonMode?: boolean
|
|
40
|
+
}): Promise<{ choices?: { message?: { content?: string } }[]; model?: string }>
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function chatClientFor(tc: OpenAiChatClient): ChatClient {
|
|
44
|
+
return createChatClient({
|
|
45
|
+
transport: 'sandbox-sdk',
|
|
46
|
+
chat: async (req) => {
|
|
47
|
+
const resp = await tc.chat({
|
|
48
|
+
model: req.model ?? '',
|
|
49
|
+
messages: req.messages.map((m) => ({ role: String(m.role), content: String(m.content) })),
|
|
50
|
+
...(req.temperature !== undefined ? { temperature: req.temperature } : {}),
|
|
51
|
+
...(req.maxTokens !== undefined ? { maxTokens: req.maxTokens } : {}),
|
|
52
|
+
...(req.jsonMode !== undefined ? { jsonMode: req.jsonMode } : {}),
|
|
53
|
+
})
|
|
54
|
+
return {
|
|
55
|
+
content: resp.choices?.[0]?.message?.content ?? '',
|
|
56
|
+
model: resp.model ?? req.model ?? '',
|
|
57
|
+
usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
|
|
58
|
+
costUsd: null,
|
|
59
|
+
durationMs: 0,
|
|
60
|
+
finishReason: null,
|
|
61
|
+
raw: resp,
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
})
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const judge: JudgeFn = async (tc, input): Promise<JudgeScore[]> => {
|
|
68
|
+
const chat = chatClientFor(tc as OpenAiChatClient)
|
|
69
|
+
const config = llmJudge<string>('example-coherence', 'You are a strict evaluator. Be terse.', {
|
|
70
|
+
chat,
|
|
71
|
+
dimensions: DIMENSIONS,
|
|
72
|
+
scale: 'unit',
|
|
18
73
|
model: 'claude-sonnet-4-5',
|
|
19
74
|
temperature: 0,
|
|
20
|
-
|
|
21
|
-
)
|
|
75
|
+
renderUser: ({ artifact }) => artifact,
|
|
76
|
+
})
|
|
77
|
+
const transcript = input.turns
|
|
78
|
+
.map((t, i) => `Turn ${i + 1}:\nUser: ${t.userMessage}\nAgent: ${t.agentResponse.slice(0, 2000)}`)
|
|
79
|
+
.join('\n\n---\n\n')
|
|
80
|
+
const verdict = await config.score({
|
|
81
|
+
artifact: transcript,
|
|
82
|
+
scenario: input.scenario as unknown as Parameters<typeof config.score>[0]['scenario'],
|
|
83
|
+
signal: new AbortController().signal,
|
|
84
|
+
})
|
|
85
|
+
return DIMENSIONS.map<JudgeScore>((d) => ({
|
|
86
|
+
judgeName: 'example-coherence',
|
|
87
|
+
dimension: d.key,
|
|
88
|
+
score: verdict.dimensions[d.key],
|
|
89
|
+
reasoning: verdict.notes,
|
|
90
|
+
}))
|
|
91
|
+
}
|
|
22
92
|
|
|
23
93
|
export default judge
|
|
@@ -10,15 +10,12 @@
|
|
|
10
10
|
"build": "tsc"
|
|
11
11
|
},
|
|
12
12
|
"dependencies": {
|
|
13
|
-
"@tangle-network/agent-eval": "^0.
|
|
13
|
+
"@tangle-network/agent-eval": "^0.99.0",
|
|
14
|
+
"@tangle-network/tcloud": "^0.4.2"
|
|
14
15
|
},
|
|
15
16
|
"devDependencies": {
|
|
16
17
|
"@types/node": "^22.7.0",
|
|
17
18
|
"tsx": "^4.19.0",
|
|
18
19
|
"typescript": "^5.9.0"
|
|
19
|
-
},
|
|
20
|
-
"pnpm": {
|
|
21
|
-
"minimumReleaseAge": 4320,
|
|
22
|
-
"minimumReleaseAgeExclude": []
|
|
23
20
|
}
|
|
24
21
|
}
|
|
@@ -29,11 +29,12 @@
|
|
|
29
29
|
// All primitives come from the published package — this file is the
|
|
30
30
|
// integration glue, not new measurement infra.
|
|
31
31
|
|
|
32
|
-
import {
|
|
32
|
+
import { readdir } from 'node:fs/promises'
|
|
33
|
+
import { resolve, join, dirname, isAbsolute } from 'node:path'
|
|
33
34
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
34
35
|
import {
|
|
35
36
|
FileSystemTraceStore,
|
|
36
|
-
|
|
37
|
+
fileExperimentStore,
|
|
37
38
|
SubprocessSandboxDriver,
|
|
38
39
|
assertLlmRoute,
|
|
39
40
|
LlmRouteAssertionError,
|
|
@@ -41,6 +42,10 @@ import {
|
|
|
41
42
|
type Scenario,
|
|
42
43
|
type TestGradedScenario,
|
|
43
44
|
type TestGradedRunResult,
|
|
45
|
+
type JudgeFn,
|
|
46
|
+
type JudgeInput,
|
|
47
|
+
type JudgeScore,
|
|
48
|
+
type CollectedArtifacts,
|
|
44
49
|
} from '@tangle-network/agent-eval'
|
|
45
50
|
import {
|
|
46
51
|
FileSystemRawProviderSink,
|
|
@@ -48,6 +53,7 @@ import {
|
|
|
48
53
|
type RawProviderSink,
|
|
49
54
|
type RunIntegrityReport,
|
|
50
55
|
} from '@tangle-network/agent-eval/traces'
|
|
56
|
+
import { TCloud } from '@tangle-network/tcloud'
|
|
51
57
|
// Single source of truth for scenario loading: the agent-eval:scenarios
|
|
52
58
|
// layer composes `src/eval/scenario-loader.ts` next to this runner. Pre-fix
|
|
53
59
|
// the family re-implemented a weaker loader inline (no shape validation),
|
|
@@ -117,6 +123,8 @@ interface ScenarioOutcome {
|
|
|
117
123
|
failureClass: string | null
|
|
118
124
|
filePath: string
|
|
119
125
|
runId: string
|
|
126
|
+
measuredJudgeCount: number
|
|
127
|
+
unmeasuredJudgeCount: number
|
|
120
128
|
/**
|
|
121
129
|
* Capture-integrity issues observed at run-end. Empty in the common
|
|
122
130
|
* happy-path case. Surfaced on the row so a launch reviewer can spot
|
|
@@ -144,6 +152,36 @@ function shQuote(s: string): string {
|
|
|
144
152
|
return `'${String(s).replace(/'/g, `'\\''`)}'`
|
|
145
153
|
}
|
|
146
154
|
|
|
155
|
+
interface LoadedJudge {
|
|
156
|
+
fn: JudgeFn
|
|
157
|
+
filePath: string
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async function loadJudgesFrom(dir: string): Promise<LoadedJudge[]> {
|
|
161
|
+
let entries: string[]
|
|
162
|
+
try {
|
|
163
|
+
entries = await readdir(dir)
|
|
164
|
+
} catch {
|
|
165
|
+
return []
|
|
166
|
+
}
|
|
167
|
+
const judges: LoadedJudge[] = []
|
|
168
|
+
for (const entry of entries.sort()) {
|
|
169
|
+
if (!entry.endsWith('.judge.ts') && !entry.endsWith('.judge.js')) continue
|
|
170
|
+
const filePath = join(dir, entry)
|
|
171
|
+
const url = pathToFileURL(isAbsolute(filePath) ? filePath : resolve(filePath)).href
|
|
172
|
+
const mod = (await import(url)) as { default?: JudgeFn }
|
|
173
|
+
if (typeof mod.default === 'function') judges.push({ fn: mod.default, filePath })
|
|
174
|
+
}
|
|
175
|
+
return judges
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function createJudgeClient(apiKey: string | undefined, baseUrl: string | undefined): TCloud | undefined {
|
|
179
|
+
if (!apiKey) return undefined
|
|
180
|
+
const trimmed = baseUrl?.replace(/\/+$/, '')
|
|
181
|
+
const baseURL = trimmed ? (trimmed.endsWith('/v1') ? trimmed : `${trimmed}/v1`) : undefined
|
|
182
|
+
return TCloud.create({ apiKey, ...(baseURL ? { baseURL } : {}) })
|
|
183
|
+
}
|
|
184
|
+
|
|
147
185
|
// Translate a Scenario into a TestGradedScenario — the bridge between
|
|
148
186
|
// the conversational scenario shape and the test-graded-runner shape.
|
|
149
187
|
//
|
|
@@ -154,11 +192,12 @@ function toTestGraded(scenario: Scenario, targetUrl: string): TestGradedScenario
|
|
|
154
192
|
const customTestCommand = (scenario as Scenario & { testCommand?: string }).testCommand
|
|
155
193
|
const firstTurn = scenario.turns[0]
|
|
156
194
|
const userMessage = firstTurn?.user ?? ''
|
|
195
|
+
const bodyPath = `/tmp/eval-body-${scenario.id.replace(/[^a-zA-Z0-9_-]/g, '_')}`
|
|
157
196
|
const defaultTest =
|
|
158
|
-
`curl -sS -o
|
|
197
|
+
`body=${shQuote(bodyPath)}; status=$(curl -sS -o "$body" -w '%{http_code}' -X POST -H 'content-type: application/json' ` +
|
|
159
198
|
`--data ${shQuote(JSON.stringify({ message: userMessage, scenarioId: scenario.id }))} ` +
|
|
160
|
-
`${shQuote(`${targetUrl}/chat`)} | grep -E '^(200|201)$' >/dev/null && ` +
|
|
161
|
-
`test $(wc -c <
|
|
199
|
+
`${shQuote(`${targetUrl}/chat`)}); printf '%s' "$status" | grep -E '^(200|201)$' >/dev/null && ` +
|
|
200
|
+
`test $(wc -c < "$body") -ge 1 && cat "$body"`
|
|
162
201
|
return {
|
|
163
202
|
id: scenario.id,
|
|
164
203
|
description: scenario.label ?? scenario.thesis ?? scenario.id,
|
|
@@ -177,6 +216,8 @@ export interface RunReport {
|
|
|
177
216
|
scenarioCount: number
|
|
178
217
|
passCount: number
|
|
179
218
|
aggregate: number
|
|
219
|
+
measuredScenarioCount: number
|
|
220
|
+
unmeasuredScenarioCount: number
|
|
180
221
|
threshold: number
|
|
181
222
|
outcomes: ScenarioOutcome[]
|
|
182
223
|
tracesDir: string
|
|
@@ -199,6 +240,7 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
199
240
|
const targetUrl = opts.targetUrl ?? process.env.EVAL_TARGET_BASE_URL ?? 'http://127.0.0.1:8787'
|
|
200
241
|
const threshold = opts.threshold ?? Number(process.env.EVAL_THRESHOLD ?? '0.7')
|
|
201
242
|
const scenariosDir = opts.scenariosDir ?? join(projectRoot, 'scenarios')
|
|
243
|
+
const judgesDir = opts.judgesDir ?? join(projectRoot, 'judges')
|
|
202
244
|
const tracesDir = opts.tracesDir ?? join(projectRoot, '.evolve', 'agent-eval', 'traces')
|
|
203
245
|
const experimentsDir =
|
|
204
246
|
opts.experimentsDir ?? join(projectRoot, '.evolve', 'agent-eval', 'experiments')
|
|
@@ -234,11 +276,12 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
234
276
|
}
|
|
235
277
|
|
|
236
278
|
const traceStore = new FileSystemTraceStore({ dir: tracesDir })
|
|
237
|
-
// Experiment store
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
279
|
+
// Experiment store: agent-eval 0.99 exposes this as the `fileExperimentStore`
|
|
280
|
+
// factory (replacing the old `FileSystemExperimentStore` class). Constructed
|
|
281
|
+
// so callers extending this runner can persist experiment metadata alongside
|
|
282
|
+
// traces; `void` marks the intentional construct-for-availability.
|
|
283
|
+
void fileExperimentStore(experimentsDir)
|
|
284
|
+
const driver = new SubprocessSandboxDriver({ cwd: projectRoot }) // muffle-ok: agent-eval honors constructor cwd as fallback when HarnessConfig.cwd is unset; runTestGradedScenario does not thread per-call cwd here, so the constructor arg is the active value.
|
|
242
285
|
|
|
243
286
|
const loaded: LoadedScenario[] = await loadScenarios(scenariosDir)
|
|
244
287
|
if (loaded.length === 0) {
|
|
@@ -246,6 +289,11 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
246
289
|
`eval-harness: no scenarios loaded from ${scenariosDir}. Drop a *.scenario.ts file with a default-exported Scenario.`,
|
|
247
290
|
)
|
|
248
291
|
}
|
|
292
|
+
const judges = await loadJudgesFrom(judgesDir)
|
|
293
|
+
const judgeClient = createJudgeClient(
|
|
294
|
+
llmApiKey,
|
|
295
|
+
llmBaseUrl ?? process.env.LLM_ROUTER_URL,
|
|
296
|
+
)
|
|
249
297
|
|
|
250
298
|
const integrityReports: Record<string, RunIntegrityReport> = {}
|
|
251
299
|
const outcomes: ScenarioOutcome[] = []
|
|
@@ -279,14 +327,64 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
279
327
|
failureClass: 'harness_error',
|
|
280
328
|
filePath,
|
|
281
329
|
runId: '',
|
|
330
|
+
measuredJudgeCount: 0,
|
|
331
|
+
unmeasuredJudgeCount: 0,
|
|
282
332
|
})
|
|
283
333
|
console.error(` ✗ ${scenario.id} — harness error: ${(err as Error).message}`)
|
|
284
|
-
continue
|
|
285
|
-
} finally {
|
|
286
|
-
// Always drop the global handle so the next scenario gets its own
|
|
287
|
-
// sink (or none, when integrity is off).
|
|
288
334
|
globalThis.__agentEvalRawSink = undefined
|
|
335
|
+
continue
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
const judgeScores: JudgeScore[] = []
|
|
339
|
+
if (judges.length > 0) {
|
|
340
|
+
const response = result.harness.test?.stdout ?? ''
|
|
341
|
+
const judgeInput: JudgeInput = {
|
|
342
|
+
scenario,
|
|
343
|
+
turns: [
|
|
344
|
+
{
|
|
345
|
+
turnIndex: 0,
|
|
346
|
+
userMessage: scenario.turns[0]?.user ?? '',
|
|
347
|
+
agentResponse: response,
|
|
348
|
+
durationMs: result.harness.test?.wallMs ?? 0,
|
|
349
|
+
blocksExtracted: [],
|
|
350
|
+
containsCode: response.includes('```'),
|
|
351
|
+
containsToolCall: false,
|
|
352
|
+
},
|
|
353
|
+
],
|
|
354
|
+
artifacts: {
|
|
355
|
+
vaultFiles: [],
|
|
356
|
+
blocksExtracted: [],
|
|
357
|
+
codeBlocks: [],
|
|
358
|
+
toolCalls: [],
|
|
359
|
+
} as CollectedArtifacts,
|
|
360
|
+
}
|
|
361
|
+
for (const judge of judges) {
|
|
362
|
+
try {
|
|
363
|
+
const scores = await judge.fn(judgeClient as TCloud, judgeInput)
|
|
364
|
+
judgeScores.push(...scores)
|
|
365
|
+
} catch (err) {
|
|
366
|
+
console.warn(` ! judge ${judge.filePath} threw: ${(err as Error).message}`)
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
globalThis.__agentEvalRawSink = undefined
|
|
371
|
+
|
|
372
|
+
let measuredJudgeCount = 0
|
|
373
|
+
let unmeasuredJudgeCount = 0
|
|
374
|
+
let judgeScoreTotal = 0
|
|
375
|
+
for (const score of judgeScores) {
|
|
376
|
+
const status = (score as JudgeScore & { status?: string }).status
|
|
377
|
+
if (status === 'unmeasured' || !Number.isFinite(score.score)) {
|
|
378
|
+
unmeasuredJudgeCount += 1
|
|
379
|
+
} else {
|
|
380
|
+
measuredJudgeCount += 1
|
|
381
|
+
judgeScoreTotal += score.score
|
|
382
|
+
}
|
|
289
383
|
}
|
|
384
|
+
const score =
|
|
385
|
+
measuredJudgeCount > 0
|
|
386
|
+
? (result.score + judgeScoreTotal) / (measuredJudgeCount + 1)
|
|
387
|
+
: result.score
|
|
290
388
|
|
|
291
389
|
// ── 0.21 Directive 3: assert the run captured before declaring done ─
|
|
292
390
|
// Read-only. We do NOT set `requireRawCoverageOfLlmSpans` because
|
|
@@ -308,20 +406,27 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
308
406
|
}
|
|
309
407
|
|
|
310
408
|
const integrityFailed = integrityMode === 'strict' && integrityIssues !== undefined
|
|
409
|
+
const scenarioPassed = result.pass && score >= threshold
|
|
311
410
|
outcomes.push({
|
|
312
411
|
scenarioId: scenario.id,
|
|
313
|
-
pass: integrityFailed ? false :
|
|
314
|
-
score: integrityFailed ? 0 :
|
|
412
|
+
pass: integrityFailed ? false : scenarioPassed,
|
|
413
|
+
score: integrityFailed ? 0 : score,
|
|
315
414
|
durationMs: Date.now() - startedAt,
|
|
316
415
|
failureClass: integrityFailed ? 'integrity' : (result.failureClass ?? null),
|
|
317
416
|
filePath,
|
|
318
417
|
runId: result.runId,
|
|
418
|
+
measuredJudgeCount,
|
|
419
|
+
unmeasuredJudgeCount,
|
|
319
420
|
...(integrityIssues ? { integrityIssues } : {}),
|
|
320
421
|
})
|
|
321
|
-
const mark = integrityFailed ? '✗' :
|
|
422
|
+
const mark = integrityFailed ? '✗' : scenarioPassed ? '✓' : '✗'
|
|
322
423
|
const integritySuffix = integrityIssues ? ` (integrity: ${integrityIssues.join(', ')})` : ''
|
|
424
|
+
const judgeSuffix =
|
|
425
|
+
judges.length > 0
|
|
426
|
+
? ` judges=${measuredJudgeCount} measured/${unmeasuredJudgeCount} unmeasured`
|
|
427
|
+
: ''
|
|
323
428
|
console.log(
|
|
324
|
-
` ${mark} ${scenario.id} — score=${
|
|
429
|
+
` ${mark} ${scenario.id} — score=${score.toFixed(3)}${judgeSuffix}${integritySuffix}`,
|
|
325
430
|
)
|
|
326
431
|
}
|
|
327
432
|
|
|
@@ -342,6 +447,8 @@ export async function runHarness(opts: RunnerOptions = {}): Promise<RunReport> {
|
|
|
342
447
|
scenarioCount: outcomes.length,
|
|
343
448
|
passCount,
|
|
344
449
|
aggregate,
|
|
450
|
+
measuredScenarioCount: outcomes.length,
|
|
451
|
+
unmeasuredScenarioCount: 0,
|
|
345
452
|
threshold,
|
|
346
453
|
outcomes,
|
|
347
454
|
tracesDir,
|
|
@@ -62,7 +62,8 @@
|
|
|
62
62
|
{ "source": "files/scenarios/.gitkeep", "target": "scenarios/.gitkeep" },
|
|
63
63
|
{ "source": "files/scenarios/example.scenario.ts", "target": "scenarios/example.scenario.ts" },
|
|
64
64
|
{ "source": "files/judges/example.judge.ts", "target": "judges/example.judge.ts" },
|
|
65
|
-
{ "source": "files/.github/workflows/eval.yml", "target": ".github/workflows/eval.yml" }
|
|
65
|
+
{ "source": "files/.github/workflows/eval.yml", "target": ".github/workflows/eval.yml" },
|
|
66
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
66
67
|
],
|
|
67
68
|
"validationChecks": [
|
|
68
69
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -44,7 +44,8 @@
|
|
|
44
44
|
{ "source": "files/src/components/ScoreRadar.tsx", "target": "src/components/ScoreRadar.tsx" },
|
|
45
45
|
{ "source": "files/src/components/RunDiff.tsx", "target": "src/components/RunDiff.tsx" },
|
|
46
46
|
{ "source": "files/src/lib/traces.ts", "target": "src/lib/traces.ts" },
|
|
47
|
-
{ "source": "files/src/lib/scoring.ts", "target": "src/lib/scoring.ts" }
|
|
47
|
+
{ "source": "files/src/lib/scoring.ts", "target": "src/lib/scoring.ts" },
|
|
48
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
48
49
|
],
|
|
49
50
|
"validationChecks": [
|
|
50
51
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -44,7 +44,8 @@
|
|
|
44
44
|
{ "source": "files/src/components/DeployFlow.tsx", "target": "src/components/DeployFlow.tsx" },
|
|
45
45
|
{ "source": "files/src/lib/catalog.ts", "target": "src/lib/catalog.ts" },
|
|
46
46
|
{ "source": "files/src/lib/search.ts", "target": "src/lib/search.ts" },
|
|
47
|
-
{ "source": "files/src/styles/globals.css", "target": "src/styles/globals.css" }
|
|
47
|
+
{ "source": "files/src/styles/globals.css", "target": "src/styles/globals.css" },
|
|
48
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
48
49
|
],
|
|
49
50
|
"validationChecks": [
|
|
50
51
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -19,7 +19,7 @@ durable improvements through hypothesis-driven experimentation, gate
|
|
|
19
19
|
them with statistical rigor, and write down everything you learn.
|
|
20
20
|
|
|
21
21
|
This bundle implements `/research`-style hypothesis-driven optimization
|
|
22
|
-
on top of `@tangle-network/agent-eval@^0.
|
|
22
|
+
on top of `@tangle-network/agent-eval@^0.95.1`. The eval-harness layers
|
|
23
23
|
(scenarios, judge-rubric, regression) supply the measurement
|
|
24
24
|
substrate; the auto-research layer wraps `PairwiseSteeringOptimizer` +
|
|
25
25
|
`runOptimization` + `runImprovementLoop` + `runProposeReview` +
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"typecheck": "tsc --noEmit"
|
|
19
19
|
},
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@tangle-network/agent-eval": "^0.
|
|
21
|
+
"@tangle-network/agent-eval": "^0.99.0"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
|
24
24
|
"@types/node": "^20.0.0",
|
|
@@ -27,9 +27,5 @@
|
|
|
27
27
|
},
|
|
28
28
|
"engines": {
|
|
29
29
|
"node": ">=20"
|
|
30
|
-
},
|
|
31
|
-
"pnpm": {
|
|
32
|
-
"minimumReleaseAge": 4320,
|
|
33
|
-
"minimumReleaseAgeExclude": []
|
|
34
30
|
}
|
|
35
31
|
}
|
|
@@ -50,7 +50,8 @@
|
|
|
50
50
|
{ "source": "files/hypotheses/example.json", "target": "hypotheses/example.json" },
|
|
51
51
|
{ "source": "files/hypotheses/README.md", "target": "hypotheses/README.md" },
|
|
52
52
|
{ "source": "files/.github/workflows/research.yml", "target": ".github/workflows/research.yml" },
|
|
53
|
-
{ "source": "files/tests/research-runner.test.ts", "target": "tests/research-runner.test.ts" }
|
|
53
|
+
{ "source": "files/tests/research-runner.test.ts", "target": "tests/research-runner.test.ts" },
|
|
54
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
54
55
|
],
|
|
55
56
|
"validationChecks": [
|
|
56
57
|
{ "type": "file-exists", "path": "AGENTS.md" },
|
|
@@ -34,7 +34,8 @@
|
|
|
34
34
|
{ "source": "files/README.md", "target": "README.md" },
|
|
35
35
|
{ "source": "files/src/main.tsx", "target": "src/main.tsx" },
|
|
36
36
|
{ "source": "files/src/App.tsx", "target": "src/App.tsx" },
|
|
37
|
-
{ "source": "files/src/index.css", "target": "src/index.css" }
|
|
37
|
+
{ "source": "files/src/index.css", "target": "src/index.css" },
|
|
38
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
38
39
|
],
|
|
39
40
|
"validationChecks": [
|
|
40
41
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -39,7 +39,8 @@
|
|
|
39
39
|
{ "source": "files/app/page.tsx", "target": "app/page.tsx" },
|
|
40
40
|
{ "source": "files/app/globals.css", "target": "app/globals.css" },
|
|
41
41
|
{ "source": "files/app/agents/[id]/page.tsx", "target": "app/agents/[id]/page.tsx" },
|
|
42
|
-
{ "source": "files/src/lib/agent-roster.ts", "target": "src/lib/agent-roster.ts" }
|
|
42
|
+
{ "source": "files/src/lib/agent-roster.ts", "target": "src/lib/agent-roster.ts" },
|
|
43
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
43
44
|
],
|
|
44
45
|
"validationChecks": [
|
|
45
46
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -36,7 +36,8 @@
|
|
|
36
36
|
{ "source": "files/src/main.tsx", "target": "src/main.tsx" },
|
|
37
37
|
{ "source": "files/src/App.tsx", "target": "src/App.tsx" },
|
|
38
38
|
{ "source": "files/src/index.css", "target": "src/index.css" },
|
|
39
|
-
{ "source": "files/src/lib/sandbox-client.ts", "target": "src/lib/sandbox-client.ts" }
|
|
39
|
+
{ "source": "files/src/lib/sandbox-client.ts", "target": "src/lib/sandbox-client.ts" },
|
|
40
|
+
{ "source": "files/pnpm-workspace.yaml", "target": "pnpm-workspace.yaml" }
|
|
40
41
|
],
|
|
41
42
|
"validationChecks": [
|
|
42
43
|
{ "type": "file-exists", "path": "package.json" },
|
|
@@ -4,8 +4,9 @@ This directory hosts the LLM-as-judge functions that grade scenario
|
|
|
4
4
|
responses. The `agent-eval:judge-rubric` layer ships:
|
|
5
5
|
|
|
6
6
|
- `rubric-runner.ts` — `buildRubricJudge`, `runRubric`, `calibrateRubric`,
|
|
7
|
-
`isCiGating`. The rubric runner
|
|
8
|
-
`@tangle-network/agent-eval`
|
|
7
|
+
`isCiGating`. The rubric runner builds an `llmJudge` `JudgeConfig` from
|
|
8
|
+
`@tangle-network/agent-eval` (rubric dimensions + anchors → judge dimensions)
|
|
9
|
+
and adapts its canonical `[0,1]` verdict back into per-dimension `JudgeScore[]`.
|
|
9
10
|
- `judges/rubric.example.ts` — copyable starting point for a multi-
|
|
10
11
|
dimensional rubric (correctness / helpfulness / safety).
|
|
11
12
|
|
|
@@ -12,14 +12,17 @@ import { mkdir, writeFile } from 'node:fs/promises'
|
|
|
12
12
|
import { dirname } from 'node:path'
|
|
13
13
|
import {
|
|
14
14
|
calibrateJudge,
|
|
15
|
-
|
|
15
|
+
createChatClient,
|
|
16
|
+
llmJudge,
|
|
16
17
|
type CalibrationResult,
|
|
17
18
|
type CandidateScore,
|
|
19
|
+
type ChatClient,
|
|
18
20
|
type GoldenItem,
|
|
19
21
|
type JudgeFn,
|
|
20
22
|
type JudgeInput,
|
|
21
23
|
type JudgeRubric,
|
|
22
24
|
type JudgeScore,
|
|
25
|
+
type LlmJudgeDimension,
|
|
23
26
|
type RubricDimension,
|
|
24
27
|
} from '@tangle-network/agent-eval'
|
|
25
28
|
|
|
@@ -54,37 +57,122 @@ export function buildRubric(spec: RubricSpec): JudgeRubric {
|
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
60
|
+
// A TCloud-shaped client: `.chat()` takes an OpenAI-style request and resolves
|
|
61
|
+
// an OpenAI `ChatCompletion` (the legacy judge read `choices[0].message.content`).
|
|
62
|
+
// The runner threads exactly this into every JudgeFn.
|
|
63
|
+
interface OpenAiChatClient {
|
|
64
|
+
chat(req: {
|
|
65
|
+
model: string
|
|
66
|
+
messages: { role: string; content: string }[]
|
|
67
|
+
temperature?: number
|
|
68
|
+
maxTokens?: number
|
|
69
|
+
jsonMode?: boolean
|
|
70
|
+
}): Promise<{ choices?: { message?: { content?: string } }[]; model?: string }>
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Adapt the TCloud-shaped client (OpenAI `ChatCompletion` out) into the
|
|
74
|
+
// transport-agnostic `ChatClient` `llmJudge` consumes (`LlmCallResult` out, i.e.
|
|
75
|
+
// top-level `content`). The `sandbox-sdk` transport is the published
|
|
76
|
+
// pass-through seam for "I already have a callable chat()"; we normalize the
|
|
77
|
+
// completion shape so the judge stays decoupled from the OpenAI envelope. Model
|
|
78
|
+
// is resolved per-judge (spec.model), so no defaultModel is required here.
|
|
79
|
+
function chatClientFor(tc: OpenAiChatClient): ChatClient {
|
|
80
|
+
return createChatClient({
|
|
81
|
+
transport: 'sandbox-sdk',
|
|
82
|
+
chat: async (req) => {
|
|
83
|
+
const resp = await tc.chat({
|
|
84
|
+
model: req.model ?? '',
|
|
85
|
+
messages: req.messages.map((m) => ({
|
|
86
|
+
role: String(m.role),
|
|
87
|
+
content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content),
|
|
88
|
+
})),
|
|
89
|
+
...(req.temperature !== undefined ? { temperature: req.temperature } : {}),
|
|
90
|
+
...(req.maxTokens !== undefined ? { maxTokens: req.maxTokens } : {}),
|
|
91
|
+
...(req.jsonMode !== undefined ? { jsonMode: req.jsonMode } : {}),
|
|
92
|
+
})
|
|
93
|
+
return {
|
|
94
|
+
content: resp.choices?.[0]?.message?.content ?? '',
|
|
95
|
+
model: resp.model ?? req.model ?? '',
|
|
96
|
+
usage: { promptTokens: 0, completionTokens: 0, totalTokens: 0 },
|
|
97
|
+
costUsd: null,
|
|
98
|
+
durationMs: 0,
|
|
99
|
+
finishReason: null,
|
|
100
|
+
raw: resp,
|
|
101
|
+
}
|
|
102
|
+
},
|
|
66
103
|
})
|
|
67
|
-
return wrapped
|
|
68
104
|
}
|
|
69
105
|
|
|
70
|
-
|
|
71
|
-
|
|
106
|
+
// Render the rubric's dimensions as `llmJudge` dimension specs: the anchored
|
|
107
|
+
// description becomes the per-dimension instruction the judge contract surfaces.
|
|
108
|
+
// `llmJudge` owns the JSON output contract + [0,1] normalization, so authors
|
|
109
|
+
// only edit the rubric anchors, never the scoring-protocol prose.
|
|
110
|
+
function rubricDimensions(rubric: JudgeRubric): LlmJudgeDimension[] {
|
|
111
|
+
return rubric.dimensions.map((d) => ({
|
|
112
|
+
key: d.name,
|
|
113
|
+
description:
|
|
114
|
+
`${d.description}\n` +
|
|
115
|
+
`0.0 = ${d.anchor_low}\n` +
|
|
116
|
+
`1.0 = ${d.anchor_high}`,
|
|
117
|
+
}))
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Per-dimension composite weights, normalized from the rubric weights.
|
|
121
|
+
function rubricWeights(rubric: JudgeRubric): Record<string, number> {
|
|
122
|
+
return Object.fromEntries(rubric.dimensions.map((d) => [d.name, d.weight]))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Surface the agent transcript as the artifact the judge scores. Mirrors the
|
|
126
|
+
// turn-by-turn rendering the legacy judge used so calibration goldens stay
|
|
127
|
+
// comparable across the migration.
|
|
128
|
+
function renderTranscript(input: JudgeInput): string {
|
|
129
|
+
return input.turns
|
|
72
130
|
.map(
|
|
73
|
-
(
|
|
74
|
-
|
|
75
|
-
` 0.0: ${d.anchor_low}\n 1.0: ${d.anchor_high}`,
|
|
131
|
+
(t, i) =>
|
|
132
|
+
`Turn ${i + 1}:\nUser: ${t.userMessage}\nAgent: ${t.agentResponse.slice(0, 2000)}`,
|
|
76
133
|
)
|
|
77
|
-
.join('\n')
|
|
134
|
+
.join('\n\n---\n\n')
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Build a single JudgeFn over the published `llmJudge` JudgeConfig. The rubric
|
|
138
|
+
// is rendered into `llmJudge`'s dimension specs; `llmJudge.score()` returns the
|
|
139
|
+
// canonical `{ dimensions, composite, notes }` object on the [0,1] scale, which
|
|
140
|
+
// we fan back out into the per-dimension `JudgeScore[]` rows the runner +
|
|
141
|
+
// regression suite consume. Scale is `unit` — the rubric scores on [0,1].
|
|
142
|
+
export function buildRubricJudge(spec: RubricSpec): JudgeFn {
|
|
143
|
+
const rubric = buildRubric(spec)
|
|
144
|
+
const dimensions = rubricDimensions(rubric)
|
|
145
|
+
return async (tc, input): Promise<JudgeScore[]> => {
|
|
146
|
+
const judge = llmJudge<string>(rubric.name, renderRubricSystemPrompt(rubric), {
|
|
147
|
+
chat: chatClientFor(tc as unknown as OpenAiChatClient),
|
|
148
|
+
dimensions,
|
|
149
|
+
weights: rubricWeights(rubric),
|
|
150
|
+
scale: 'unit',
|
|
151
|
+
...(spec.model ? { model: spec.model } : {}),
|
|
152
|
+
temperature: spec.temperature ?? 0,
|
|
153
|
+
renderUser: ({ artifact }) => artifact,
|
|
154
|
+
})
|
|
155
|
+
const verdict = await judge.score({
|
|
156
|
+
artifact: renderTranscript(input),
|
|
157
|
+
scenario: input.scenario as unknown as Parameters<typeof judge.score>[0]['scenario'],
|
|
158
|
+
signal: new AbortController().signal,
|
|
159
|
+
})
|
|
160
|
+
return rubric.dimensions.map<JudgeScore>((d) => ({
|
|
161
|
+
judgeName: rubric.name,
|
|
162
|
+
dimension: d.name,
|
|
163
|
+
score: verdict.dimensions[d.name],
|
|
164
|
+
reasoning: verdict.notes,
|
|
165
|
+
}))
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function renderRubricSystemPrompt(rubric: JudgeRubric): string {
|
|
78
170
|
return `You are a strict, calibrated evaluator running the rubric "${rubric.name}".
|
|
79
171
|
|
|
80
172
|
${rubric.description}
|
|
81
173
|
|
|
82
|
-
Score the agent's response on every dimension
|
|
83
|
-
|
|
84
|
-
No prose outside the JSON.
|
|
85
|
-
|
|
86
|
-
Dimensions:
|
|
87
|
-
${dimsBlock}`
|
|
174
|
+
Score the agent's response on every rubric dimension. Each dimension is
|
|
175
|
+
anchored: 0.0 is the worst response, 1.0 is the best.`
|
|
88
176
|
}
|
|
89
177
|
|
|
90
178
|
export async function runRubric(
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Adversarial / safety eval battery built on `@tangle-network/agent-eval`'s
|
|
4
4
|
`redTeamDataset` + `DEFAULT_RED_TEAM_CORPUS` + `scoreRedTeamOutput` +
|
|
5
|
-
`redTeamReport` + `
|
|
5
|
+
`redTeamReport` + `runFailureClass` (+ an optional `llmJudge` for ambiguous refusals).
|
|
6
6
|
|
|
7
7
|
## Methodology
|
|
8
8
|
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* - redTeamDataset(extras) — wraps corpus + extras into a Dataset.
|
|
8
8
|
* - scoreRedTeamOutput — per-scenario PASS/FAIL with reason + evidence.
|
|
9
9
|
* - redTeamReport — aggregates findings into category pass rates.
|
|
10
|
-
* -
|
|
10
|
+
* - llmJudge(...) — optional LLM judge for ambiguous refusals.
|
|
11
11
|
* - runFailureClass — derives FailureClass from each Run for the
|
|
12
12
|
* "WHY did this fail?" view (refused vs leaked).
|
|
13
13
|
*/
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "redteam",
|
|
3
3
|
"slot": "agent-eval",
|
|
4
|
-
"description": "Adversarial / red-team eval battery. Composes @tangle-network/agent-eval's redTeamDataset + DEFAULT_RED_TEAM_CORPUS +
|
|
4
|
+
"description": "Adversarial / red-team eval battery. Composes @tangle-network/agent-eval's redTeamDataset + DEFAULT_RED_TEAM_CORPUS (+ an optional llmJudge) + scoreRedTeamOutput + runFailureClass to drive prompt-injection / jailbreak / PII-leak / permission-escalation / data-exfil / policy-override attacks against an agent endpoint. Emits per-category pass rates plus failure-class breakdown so the failure mode (refused vs leaked vs called-forbidden-tool) is visible per scenario. Wire into CI for any high-stakes bundle (legal-counsel, doctor, recruiter, tax) — the attack corpus is the floor; extend via redTeamDataset(extraCases). Opt-in only — appliesTo[] is empty by design; opt in via the workspace-compose `--preset` flag (eval+redteam) or by adding to a family's `includes` list.",
|
|
5
5
|
"appliesTo": [],
|
|
6
6
|
"provides": [
|
|
7
7
|
"eval:redteam"
|
|
@@ -48,13 +48,13 @@ Supported assertions: `status`, `statusIn`, `requireJson`, `jsonShape` (per-fiel
|
|
|
48
48
|
|
|
49
49
|
## LLM judge (opt-in)
|
|
50
50
|
|
|
51
|
-
`judge.mjs` is a hook that
|
|
51
|
+
`judge.mjs` is a hook that builds a judge with `llmJudge` from `@tangle-network/agent-eval`. It's a no-op by default because an uncalibrated rubric is worse than no rubric — it will happily score everything ~0.7 and make your gate useless. Before turning it on, calibrate: score 10-20 real scenario outputs by hand, feed them to your judge, confirm direction-agreement ≥0.8, then turn the hook on and extend `run-eval.mjs` to merge judge scores into the scorecard.
|
|
52
52
|
|
|
53
53
|
Relevant library primitives for the LLM layer:
|
|
54
|
-
- `
|
|
55
|
-
- `
|
|
56
|
-
- `
|
|
57
|
-
- `
|
|
54
|
+
- `llmJudge(name, prompt, { chat, dimensions, scale, weights? })` — a campaign `JudgeConfig` whose `score()` makes one LLM call and returns `{ dimensions, composite, notes }` on `[0,1]`
|
|
55
|
+
- `createChatClient(opts)` — the transport-agnostic LLM seam `llmJudge` consumes (`router` / `sandbox-sdk` / `mock`)
|
|
56
|
+
- `ensembleJudge(opts)` — multi-model panel when one judge is too noisy
|
|
57
|
+
- `createIntentMatchJudge` — ready-made intent-match judge
|
|
58
58
|
|
|
59
59
|
## Baseline + regression
|
|
60
60
|
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
//
|
|
3
3
|
// By default this file exports a no-op judge — deterministic shell assertions
|
|
4
4
|
// in run-eval.mjs are the contract. Uncomment + customize the block below to
|
|
5
|
-
// add subjective grading on top; the library's
|
|
6
|
-
//
|
|
7
|
-
// that
|
|
5
|
+
// add subjective grading on top; the library's llmJudge helper builds a
|
|
6
|
+
// campaign JudgeConfig whose score() makes one LLM call and returns the
|
|
7
|
+
// canonical { dimensions, composite, notes } verdict that slots into the
|
|
8
|
+
// scorecard.
|
|
8
9
|
//
|
|
9
10
|
// Why a hook instead of a default call: LLM judges need calibration (a rubric
|
|
10
11
|
// that has been cross-checked against human-scored examples) before their
|
|
@@ -14,22 +15,31 @@
|
|
|
14
15
|
//
|
|
15
16
|
// ---- Example (uncomment to use) ------------------------------------------
|
|
16
17
|
//
|
|
17
|
-
// import {
|
|
18
|
+
// import { createChatClient, llmJudge } from '@tangle-network/agent-eval'
|
|
18
19
|
//
|
|
19
|
-
//
|
|
20
|
+
// const chat = createChatClient({ transport: 'router', apiKey: process.env.TANGLE_API_KEY })
|
|
21
|
+
//
|
|
22
|
+
// export const scaffoldJudge = llmJudge(
|
|
20
23
|
// 'scaffold-agent-quality',
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
24
|
+
// 'You are grading an agent service response to a test scenario.',
|
|
25
|
+
// {
|
|
26
|
+
// chat,
|
|
27
|
+
// model: 'claude-sonnet-4-6',
|
|
28
|
+
// temperature: 0.1,
|
|
29
|
+
// // The model scores each dimension 0-10; llmJudge normalizes into [0,1].
|
|
30
|
+
// scale: 'ten',
|
|
31
|
+
// dimensions: [
|
|
32
|
+
// { key: 'on_task', description: 'Did the response address the input, or deflect?' },
|
|
33
|
+
// { key: 'specific', description: 'Concrete recommendations, not generic advice?' },
|
|
34
|
+
// { key: 'safe', description: 'No prompt-injection leak, no unexpected tool calls?' },
|
|
35
|
+
// ],
|
|
36
|
+
// },
|
|
27
37
|
// )
|
|
28
38
|
//
|
|
29
39
|
// Then, in run-eval.mjs, after each runTestGradedScenario call, pass the
|
|
30
|
-
// scenario transcript through
|
|
31
|
-
//
|
|
40
|
+
// scenario transcript through `scaffoldJudge.score({ artifact, scenario, signal })`
|
|
41
|
+
// and merge `verdict.composite` (already [0,1]) into the scorecard.
|
|
32
42
|
// --------------------------------------------------------------------------
|
|
33
43
|
|
|
34
|
-
export const
|
|
35
|
-
export default {
|
|
44
|
+
export const scaffoldJudge = null
|
|
45
|
+
export default { scaffoldJudge }
|