@tangle-network/starter-foundry 0.7.0 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/held-out-validation.json +2 -2
- package/dist/cli.js +0 -0
- package/dist/eval/scaffold-bridge.d.ts +99 -2
- package/dist/eval/scaffold-bridge.js +252 -59
- package/dist/eval/scaffold-bridge.js.map +1 -1
- package/dist/lib/compose.js +41 -2
- package/dist/lib/compose.js.map +1 -1
- package/dist/lib/promoter-gates.d.ts +114 -0
- package/dist/lib/promoter-gates.js +426 -0
- package/dist/lib/promoter-gates.js.map +1 -0
- package/dist/lib/prompt-e2e.d.ts +1 -0
- package/dist/lib/prompt-e2e.js +20 -6
- package/dist/lib/prompt-e2e.js.map +1 -1
- package/dist/lib/template-quality.js +20 -4
- package/dist/lib/template-quality.js.map +1 -1
- package/package.json +20 -19
- package/registry/families/fraud-ops-console/files/src/main.tsx +11 -8
- package/registry/families/fraud-ops-console/manifest.json +13 -17
- package/registry/families/kyc-onboarding/files/index.html +1 -1
- package/registry/families/kyc-onboarding/files/vite.config.ts +0 -1
- package/registry/families/kyc-onboarding/manifest.json +12 -12
- package/registry/families/polymarket-portfolio-hedging/files/src/main.tsx +11 -8
- package/registry/families/polymarket-portfolio-hedging/manifest.json +7 -6
- package/registry/layers/capability/agent-eval/files/.github/workflows/eval.yml +60 -0
- package/registry/layers/capability/agent-eval/files/scripts/eval-baseline.mjs +87 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/README.md +76 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/judge.mjs +35 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/run-eval.mjs +204 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/scenarios.json +47 -0
- package/registry/layers/capability/agent-eval/manifest.json +73 -0
- package/registry/layers/framework/forge-foundation/files/foundry.toml +14 -0
- package/registry/layers/framework/tangle-blueprint/files/foundry.toml +8 -0
- package/dist/lib/planner/zk-preferences.d.ts +0 -9
- package/dist/lib/planner/zk-preferences.js +0 -91
- package/dist/lib/planner/zk-preferences.js.map +0 -1
- /package/registry/families/kyc-onboarding/files/src/{main.ts → main.tsx} +0 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Optional LLM-judge layer, wired via @tangle-network/agent-eval primitives.
|
|
2
|
+
//
|
|
3
|
+
// By default this file exports a no-op judge — deterministic shell assertions
|
|
4
|
+
// in run-eval.mjs are the contract. Uncomment + customize the block below to
|
|
5
|
+
// add subjective grading on top; the library's createCustomJudge /
|
|
6
|
+
// createDomainExpertJudge / coherenceJudge helpers give you judge signatures
|
|
7
|
+
// that slot into BenchmarkRunner + JudgeRunner.
|
|
8
|
+
//
|
|
9
|
+
// Why a hook instead of a default call: LLM judges need calibration (a rubric
|
|
10
|
+
// that has been cross-checked against human-scored examples) before their
|
|
11
|
+
// numbers are load-bearing. Shipping an uncalibrated rubric at 0.7 threshold
|
|
12
|
+
// bakes the "judge passes anything" failure mode into every scaffold. You opt
|
|
13
|
+
// in; you calibrate; you turn it on.
|
|
14
|
+
//
|
|
15
|
+
// ---- Example (uncomment to use) ------------------------------------------
|
|
16
|
+
//
|
|
17
|
+
// import { createCustomJudge } from '@tangle-network/agent-eval'
|
|
18
|
+
//
|
|
19
|
+
// export const llmJudge = createCustomJudge(
|
|
20
|
+
// 'scaffold-agent-quality',
|
|
21
|
+
// `You are grading an agent service's response to a test scenario. Score 0-10 on:
|
|
22
|
+
// 1. on_task (0-10): did the response address the input, or deflect?
|
|
23
|
+
// 2. specific (0-10): concrete recommendations, not generic advice?
|
|
24
|
+
// 3. safe (0-10): no prompt-injection leak, no unexpected tool calls surfaced?
|
|
25
|
+
// Return ONLY JSON: [{"dimension":"on_task","score":N,"reasoning":"...","evidence":"..."},...]`,
|
|
26
|
+
// { model: 'claude-sonnet-4-6', temperature: 0.1 },
|
|
27
|
+
// )
|
|
28
|
+
//
|
|
29
|
+
// Then, in run-eval.mjs, after each runTestGradedScenario call, pass the
|
|
30
|
+
// scenario transcript through the judge via JudgeRunner (see the package
|
|
31
|
+
// README) and merge judge scores into the scorecard.
|
|
32
|
+
// --------------------------------------------------------------------------
|
|
33
|
+
|
|
34
|
+
export const llmJudge = null
|
|
35
|
+
export default { llmJudge }
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Eval runner built on @tangle-network/agent-eval primitives.
|
|
3
|
+
//
|
|
4
|
+
// Lifecycle:
|
|
5
|
+
// 1. Spawn the agent via `pnpm start` (or --no-spawn to use one already up).
|
|
6
|
+
// 2. Wait for /health.
|
|
7
|
+
// 3. For each scenarios.json entry, build a TestGradedScenario whose
|
|
8
|
+
// HarnessConfig.testCommand is a curl-based shell assertion that exits 0
|
|
9
|
+
// on a pass and non-zero on a fail.
|
|
10
|
+
// 4. Delegate each scenario to `runTestGradedScenario` from
|
|
11
|
+
// @tangle-network/agent-eval — it spawns the testCommand via
|
|
12
|
+
// SubprocessSandboxDriver, emits a Run through TraceEmitter, and stores
|
|
13
|
+
// it in the InMemoryTraceStore for later aggregation.
|
|
14
|
+
// 5. After all scenarios, read runs out of the store, write a scorecard
|
|
15
|
+
// to .evolve/eval/latest.json + a timestamped copy, and exit with a
|
|
16
|
+
// non-zero code when the weighted aggregate falls below --threshold.
|
|
17
|
+
//
|
|
18
|
+
// Every primitive used here comes from the published package; if you want
|
|
19
|
+
// richer grading (LLM judge, BenchmarkRunner, ConvergenceTracker, etc.)
|
|
20
|
+
// import more of its surface — this file is intentionally a thin shell
|
|
21
|
+
// around the library.
|
|
22
|
+
//
|
|
23
|
+
// Usage:
|
|
24
|
+
// node tests/eval/run-eval.mjs
|
|
25
|
+
// node tests/eval/run-eval.mjs --no-spawn --base http://localhost:3000
|
|
26
|
+
// node tests/eval/run-eval.mjs --threshold 0.8
|
|
27
|
+
|
|
28
|
+
import { spawn } from 'node:child_process'
|
|
29
|
+
import { readFile, writeFile, mkdir } from 'node:fs/promises'
|
|
30
|
+
import { existsSync } from 'node:fs'
|
|
31
|
+
import { resolve, dirname } from 'node:path'
|
|
32
|
+
import { fileURLToPath } from 'node:url'
|
|
33
|
+
import {
|
|
34
|
+
InMemoryTraceStore,
|
|
35
|
+
SubprocessSandboxDriver,
|
|
36
|
+
runTestGradedScenario,
|
|
37
|
+
} from '@tangle-network/agent-eval'
|
|
38
|
+
|
|
39
|
+
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..')
|
|
40
|
+
const argv = process.argv.slice(2)
|
|
41
|
+
const arg = (flag, fallback) => {
|
|
42
|
+
const i = argv.indexOf(flag)
|
|
43
|
+
return i >= 0 && argv[i + 1] ? argv[i + 1] : fallback
|
|
44
|
+
}
|
|
45
|
+
const NO_SPAWN = argv.includes('--no-spawn')
|
|
46
|
+
const PORT = Number(process.env.EVAL_PORT ?? '3100')
|
|
47
|
+
const BASE = arg('--base', `http://127.0.0.1:${PORT}`)
|
|
48
|
+
const THRESHOLD = Number(arg('--threshold', process.env.EVAL_THRESHOLD ?? '0.7'))
|
|
49
|
+
const START_CMD = process.env.EVAL_START_CMD ?? 'pnpm start'
|
|
50
|
+
|
|
51
|
+
async function loadScenarios() {
|
|
52
|
+
const raw = await readFile(resolve(ROOT, 'tests/eval/scenarios.json'), 'utf8')
|
|
53
|
+
return JSON.parse(raw)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function waitForReady(baseUrl, timeoutMs = 30_000) {
|
|
57
|
+
const deadline = Date.now() + timeoutMs
|
|
58
|
+
while (Date.now() < deadline) {
|
|
59
|
+
try {
|
|
60
|
+
const res = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(1500) })
|
|
61
|
+
if (res.ok) return true
|
|
62
|
+
} catch { /* retry */ }
|
|
63
|
+
await new Promise((r) => setTimeout(r, 250))
|
|
64
|
+
}
|
|
65
|
+
return false
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function shQuote(s) {
|
|
69
|
+
return `'${String(s).replace(/'/g, `'\\''`)}'`
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Translate a scenarios.json entry into a shell command that exits 0 on
|
|
73
|
+
// assertion-pass and non-zero on fail. Keeping the assertion shape small and
|
|
74
|
+
// inspectable so users can edit scenarios.json without touching this file.
|
|
75
|
+
function buildTestCommand(scenario, baseUrl) {
|
|
76
|
+
const path = scenario.path ?? '/'
|
|
77
|
+
const url = `${baseUrl}${path}`
|
|
78
|
+
const checks = []
|
|
79
|
+
const expect = scenario.assert ?? {}
|
|
80
|
+
const curlBase = scenario.kind === 'http-post'
|
|
81
|
+
? `curl -sS -o /tmp/eval-body -w '%{http_code}' -X POST -H 'content-type: application/json' --data ${shQuote(JSON.stringify(scenario.body ?? {}))} ${shQuote(url)}`
|
|
82
|
+
: `curl -sS -o /tmp/eval-body -w '%{http_code}' ${shQuote(url)}`
|
|
83
|
+
checks.push(`status=$(${curlBase})`)
|
|
84
|
+
if (expect.status !== undefined) {
|
|
85
|
+
checks.push(`test "$status" = "${expect.status}" || { echo "status $status != ${expect.status}"; exit 1; }`)
|
|
86
|
+
}
|
|
87
|
+
if (Array.isArray(expect.statusIn)) {
|
|
88
|
+
const pattern = expect.statusIn.map((s) => `"$status" = "${s}"`).join(' -o ')
|
|
89
|
+
checks.push(`test ${pattern} || { echo "status $status not in ${expect.statusIn.join(',')}"; exit 1; }`)
|
|
90
|
+
}
|
|
91
|
+
if (expect.requireJson) {
|
|
92
|
+
checks.push(`jq -e . /tmp/eval-body > /dev/null || { echo "response not JSON"; exit 1; }`)
|
|
93
|
+
}
|
|
94
|
+
if (expect.jsonShape) {
|
|
95
|
+
for (const [key, types] of Object.entries(expect.jsonShape)) {
|
|
96
|
+
const typesArr = Array.isArray(types) ? types : [types]
|
|
97
|
+
const typeChecks = typesArr.map((t) => {
|
|
98
|
+
if (t === 'null') return `(. == null)`
|
|
99
|
+
if (t === 'undefined') return `(has(${shQuote(key)}) | not)`
|
|
100
|
+
if (t === 'array') return `(type == "array")`
|
|
101
|
+
return `(type == ${shQuote(t)})`
|
|
102
|
+
}).join(' or ')
|
|
103
|
+
checks.push(
|
|
104
|
+
`jq -e ${shQuote(`.${key} | ${typeChecks}`)} /tmp/eval-body > /dev/null || { echo "field ${key} did not match ${typesArr.join('|')}"; exit 1; }`,
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (expect.minResponseChars !== undefined) {
|
|
109
|
+
checks.push(`test $(wc -c < /tmp/eval-body) -ge ${expect.minResponseChars} || { echo "response too short"; exit 1; }`)
|
|
110
|
+
}
|
|
111
|
+
return checks.join(' && ') + ' && echo "ok"'
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function runAgentSubprocess() {
|
|
115
|
+
const [cmd, ...args] = START_CMD.split(' ')
|
|
116
|
+
const env = { ...process.env, PORT: String(PORT) }
|
|
117
|
+
const child = spawn(cmd, args, { cwd: ROOT, env, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
118
|
+
let stderr = ''
|
|
119
|
+
child.stderr.on('data', (d) => { stderr += String(d) })
|
|
120
|
+
return { child, getStderr: () => stderr }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async function main() {
|
|
124
|
+
const { scenarios } = await loadScenarios()
|
|
125
|
+
let agentProc = null
|
|
126
|
+
try {
|
|
127
|
+
if (!NO_SPAWN) {
|
|
128
|
+
agentProc = await runAgentSubprocess()
|
|
129
|
+
const ready = await waitForReady(BASE)
|
|
130
|
+
if (!ready) {
|
|
131
|
+
console.error(`[run-eval] agent did not respond on ${BASE} within 30s`)
|
|
132
|
+
console.error(agentProc.getStderr().slice(-1200))
|
|
133
|
+
process.exit(2)
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const store = new InMemoryTraceStore()
|
|
138
|
+
const driver = new SubprocessSandboxDriver({ cwd: ROOT })
|
|
139
|
+
const results = []
|
|
140
|
+
for (const s of scenarios) {
|
|
141
|
+
const testCommand = buildTestCommand(s, BASE)
|
|
142
|
+
const scenario = {
|
|
143
|
+
id: s.id,
|
|
144
|
+
description: s.description,
|
|
145
|
+
harness: {
|
|
146
|
+
testCommand,
|
|
147
|
+
cwd: ROOT,
|
|
148
|
+
timeoutMs: s.timeoutMs ?? 15_000,
|
|
149
|
+
},
|
|
150
|
+
tags: { weight: String(s.weight ?? 1) },
|
|
151
|
+
}
|
|
152
|
+
const res = await runTestGradedScenario(scenario, store, { driver })
|
|
153
|
+
results.push({
|
|
154
|
+
id: s.id,
|
|
155
|
+
description: s.description ?? null,
|
|
156
|
+
passed: res.pass,
|
|
157
|
+
score: res.score,
|
|
158
|
+
failureClass: res.failureClass,
|
|
159
|
+
weight: s.weight ?? 1,
|
|
160
|
+
exitCode: res.harness.test?.exitCode ?? null,
|
|
161
|
+
stderr: (res.harness.test?.stderr ?? '').slice(-400),
|
|
162
|
+
runId: res.runId,
|
|
163
|
+
})
|
|
164
|
+
const mark = res.pass ? '✓' : '✗'
|
|
165
|
+
const reason = res.pass
|
|
166
|
+
? 'ok'
|
|
167
|
+
: (res.harness.test?.stdout?.trim() || res.harness.test?.stderr?.trim() || res.failureClass || 'fail')
|
|
168
|
+
console.log(` ${mark} ${s.id} — ${reason}`)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
const totalWeight = results.reduce((a, r) => a + r.weight, 0) || 1
|
|
172
|
+
const weighted = results.reduce((a, r) => a + r.score * r.weight, 0)
|
|
173
|
+
const aggregate = weighted / totalWeight
|
|
174
|
+
const scorecard = {
|
|
175
|
+
timestamp: new Date().toISOString(),
|
|
176
|
+
baseUrl: BASE,
|
|
177
|
+
aggregate,
|
|
178
|
+
passCount: results.filter((r) => r.passed).length,
|
|
179
|
+
totalCount: results.length,
|
|
180
|
+
threshold: THRESHOLD,
|
|
181
|
+
results,
|
|
182
|
+
tracedRuns: (await store.listRuns?.()) ?? results.map((r) => r.runId),
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const outDir = resolve(ROOT, '.evolve/eval')
|
|
186
|
+
if (!existsSync(outDir)) await mkdir(outDir, { recursive: true })
|
|
187
|
+
await writeFile(resolve(outDir, 'latest.json'), JSON.stringify(scorecard, null, 2))
|
|
188
|
+
await writeFile(
|
|
189
|
+
resolve(outDir, `${scorecard.timestamp.replace(/[:.]/g, '-')}.json`),
|
|
190
|
+
JSON.stringify(scorecard, null, 2),
|
|
191
|
+
)
|
|
192
|
+
console.log(`\naggregate=${aggregate.toFixed(3)} pass=${scorecard.passCount}/${scorecard.totalCount} threshold=${THRESHOLD}`)
|
|
193
|
+
process.exitCode = aggregate < THRESHOLD ? 1 : 0
|
|
194
|
+
} finally {
|
|
195
|
+
if (agentProc?.child && !agentProc.child.killed) {
|
|
196
|
+
try { agentProc.child.kill('SIGTERM') } catch {}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
main().catch((err) => {
|
|
202
|
+
console.error('[run-eval] fatal:', err)
|
|
203
|
+
process.exit(2)
|
|
204
|
+
})
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"description": "Starter eval scenarios. Edit freely — the schema is flat and the runner only cares about fields it sees. Each scenario describes one turn: what to send to the agent and what the response should look like.",
|
|
4
|
+
"scenarios": [
|
|
5
|
+
{
|
|
6
|
+
"id": "smoke/health-ok",
|
|
7
|
+
"description": "Service is up and returns JSON on /health.",
|
|
8
|
+
"kind": "http-get",
|
|
9
|
+
"path": "/health",
|
|
10
|
+
"assert": {
|
|
11
|
+
"status": 200,
|
|
12
|
+
"jsonShape": { "status": ["string"] }
|
|
13
|
+
},
|
|
14
|
+
"weight": 1
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"id": "behavior/handles-simple-task",
|
|
18
|
+
"description": "Agent accepts a plain-language task and returns a non-empty draft or reply.",
|
|
19
|
+
"kind": "http-post",
|
|
20
|
+
"path": "/run",
|
|
21
|
+
"body": { "task": "Summarize the benefits of evaluating agents continuously." },
|
|
22
|
+
"assert": {
|
|
23
|
+
"status": 200,
|
|
24
|
+
"jsonShape": { "draft": ["string", "null"], "messages": ["array", "undefined"] },
|
|
25
|
+
"minResponseChars": 40
|
|
26
|
+
},
|
|
27
|
+
"fallbackPath": "/",
|
|
28
|
+
"weight": 2
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "behavior/rejects-missing-task",
|
|
32
|
+
"description": "Agent input validation — missing task field returns 4xx, not 500 or silent success.",
|
|
33
|
+
"kind": "http-post",
|
|
34
|
+
"path": "/run",
|
|
35
|
+
"body": {},
|
|
36
|
+
"assert": {
|
|
37
|
+
"statusIn": [400, 422],
|
|
38
|
+
"requireJson": true
|
|
39
|
+
},
|
|
40
|
+
"fallbackPath": "/",
|
|
41
|
+
"fallbackAssert": {
|
|
42
|
+
"statusIn": [200, 400, 404, 405]
|
|
43
|
+
},
|
|
44
|
+
"weight": 1
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "agent-eval",
|
|
3
|
+
"description": "Ships a reproducible agent-eval harness into the scaffold: starter scenarios, a deterministic + optional LLM-judge runner that talks to the agent over HTTP, a baseline/regression workflow, and a CI gate. Any scaffold that layers this in can answer `is this agent improving or regressing?` from minute zero, no separate eval repo to bootstrap.",
|
|
4
|
+
"appliesTo": [
|
|
5
|
+
"agent-service-ts",
|
|
6
|
+
"agent-swarm-ts",
|
|
7
|
+
"multimodal-agent",
|
|
8
|
+
"voice-first-agent",
|
|
9
|
+
"vision-first-agent"
|
|
10
|
+
],
|
|
11
|
+
"defaults": {
|
|
12
|
+
"evalPort": "3100",
|
|
13
|
+
"evalThreshold": "0.7"
|
|
14
|
+
},
|
|
15
|
+
"packageDeps": {
|
|
16
|
+
"dependencies": {
|
|
17
|
+
"@tangle-network/agent-eval": "^0.7.0"
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
"files": [
|
|
21
|
+
{
|
|
22
|
+
"source": "files/tests/eval/scenarios.json",
|
|
23
|
+
"target": "tests/eval/scenarios.json"
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"source": "files/tests/eval/run-eval.mjs",
|
|
27
|
+
"target": "tests/eval/run-eval.mjs"
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"source": "files/tests/eval/judge.mjs",
|
|
31
|
+
"target": "tests/eval/judge.mjs"
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"source": "files/tests/eval/README.md",
|
|
35
|
+
"target": "tests/eval/README.md"
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"source": "files/.github/workflows/eval.yml",
|
|
39
|
+
"target": ".github/workflows/eval.yml"
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"source": "files/scripts/eval-baseline.mjs",
|
|
43
|
+
"target": "scripts/eval-baseline.mjs"
|
|
44
|
+
}
|
|
45
|
+
],
|
|
46
|
+
"contextHints": {
|
|
47
|
+
"commands": [
|
|
48
|
+
"node tests/eval/run-eval.mjs",
|
|
49
|
+
"node scripts/eval-baseline.mjs --write"
|
|
50
|
+
],
|
|
51
|
+
"extensionPoints": [
|
|
52
|
+
"tests/eval/scenarios.json",
|
|
53
|
+
"tests/eval/judge.mjs"
|
|
54
|
+
]
|
|
55
|
+
},
|
|
56
|
+
"keywords": [
|
|
57
|
+
"agent eval",
|
|
58
|
+
"agent evaluation",
|
|
59
|
+
"eval harness",
|
|
60
|
+
"llm judge",
|
|
61
|
+
"llm as judge",
|
|
62
|
+
"quality gate",
|
|
63
|
+
"agent quality",
|
|
64
|
+
"scoring pipeline",
|
|
65
|
+
"benchmark agent",
|
|
66
|
+
"eval integrated",
|
|
67
|
+
"measurable agent"
|
|
68
|
+
],
|
|
69
|
+
"tieredKeywords": {
|
|
70
|
+
"tier1": ["agent eval", "eval harness", "quality gate"],
|
|
71
|
+
"tier2": ["llm judge", "scoring pipeline", "benchmark agent", "eval integrated"]
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -13,4 +13,18 @@ remappings = [
|
|
|
13
13
|
"forge-std/=lib/forge-std/src/"
|
|
14
14
|
]
|
|
15
15
|
|
|
16
|
+
# forge lint — tuned so agent-written DEX/swap/vault scaffolds survive the
|
|
17
|
+
# VB "lint" layer without silencing real bug classes. `info` severity is
|
|
18
|
+
# naming-convention noise (`mixed-case-variable`,
|
|
19
|
+
# `screaming-snake-case-immutable`) that every real-world swap/vault
|
|
20
|
+
# contract trips on. `test/**` and `script/**` are fixture code where
|
|
21
|
+
# bytes32 casts + raw arithmetic are routine and not real bugs.
|
|
22
|
+
#
|
|
23
|
+
# History: pre-R1 this config was absent → forge lint ran default strict
|
|
24
|
+
# → 14/14 dex-swap/ethereum-l1 buildouts failed on unsafe-typecast in
|
|
25
|
+
# test files. Closing the cluster that dominated buildout_pass_rate.
|
|
26
|
+
[lint]
|
|
27
|
+
severity = ["high", "med", "gas"]
|
|
28
|
+
ignore = ["test/**/*", "script/**/*"]
|
|
29
|
+
|
|
16
30
|
# See more at https://book.getfoundry.sh/reference/config/overview
|
|
@@ -15,3 +15,11 @@ remappings_version = false
|
|
|
15
15
|
|
|
16
16
|
[dependencies]
|
|
17
17
|
tnt-core = "0.10.6"
|
|
18
|
+
|
|
19
|
+
# forge lint — match forge-foundation's ruleset so blueprint scaffolds
|
|
20
|
+
# don't trip the VB "lint" layer on info-severity naming conventions.
|
|
21
|
+
# See registry/layers/framework/forge-foundation/files/foundry.toml for
|
|
22
|
+
# the shipped-incident rationale (R1: 14/14 dex-swap failures).
|
|
23
|
+
[lint]
|
|
24
|
+
severity = ["high", "med", "gas"]
|
|
25
|
+
ignore = ["contracts/test/**/*", "contracts/script/**/*"]
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pick a zk capability for a prompt that mentions a zk primitive generically.
|
|
3
|
-
*
|
|
4
|
-
* @param text - the prompt text; hashed for deterministic selection.
|
|
5
|
-
* @param group - which preference group to consult (today: only "zkvm").
|
|
6
|
-
* @returns the capability ID (e.g. "capability:zkvm-risczero") or null when
|
|
7
|
-
* the group is unknown or has no weights.
|
|
8
|
-
*/
|
|
9
|
-
export declare function pickZkCapability(text: string, group: string): string | null;
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
// Operator-controlled weighted pick for ambiguous zk capability selection.
|
|
2
|
-
// When a prompt mentions a zk primitive generically (e.g. "zkvm", "zk proof")
|
|
3
|
-
// without naming a specific framework, resolve via weights in
|
|
4
|
-
// .evolve/capability-preferences.json.
|
|
5
|
-
//
|
|
6
|
-
// Selection is DETERMINISTIC per prompt — we hash the prompt text into an
|
|
7
|
-
// integer and use weighted-interval selection. Same prompt → same pick across
|
|
8
|
-
// runs (no session-to-session variance), but different prompts cycle through
|
|
9
|
-
// the weighted distribution.
|
|
10
|
-
//
|
|
11
|
-
// Explicit prompts bypass this entirely; see implicit-caps.ts for the
|
|
12
|
-
// explicit-first branches.
|
|
13
|
-
import { readFileSync, existsSync } from 'node:fs';
|
|
14
|
-
import { dirname, join, resolve } from 'node:path';
|
|
15
|
-
import { fileURLToPath } from 'node:url';
|
|
16
|
-
// Repo-rooted path so this resolves correctly from test fixtures + production.
|
|
17
|
-
// We re-resolve per call so test overrides via STARTER_FOUNDRY_REPO_OVERRIDE
|
|
18
|
-
// work.
|
|
19
|
-
function preferencesPath() {
|
|
20
|
-
const override = process.env.STARTER_FOUNDRY_REPO_OVERRIDE;
|
|
21
|
-
if (override)
|
|
22
|
-
return join(override, '.evolve/capability-preferences.json');
|
|
23
|
-
// __filename equivalent for ESM — resolve from dist/lib/planner/zk-preferences.js
|
|
24
|
-
// up to the repo root. src/lib/planner/zk-preferences.ts compiles to the same
|
|
25
|
-
// relative structure.
|
|
26
|
-
return resolve(dirname(fileURLToPath(import.meta.url)), '../../../.evolve/capability-preferences.json');
|
|
27
|
-
}
|
|
28
|
-
function loadPreferences() {
|
|
29
|
-
const path = preferencesPath();
|
|
30
|
-
if (!existsSync(path))
|
|
31
|
-
return null;
|
|
32
|
-
try {
|
|
33
|
-
return JSON.parse(readFileSync(path, 'utf8'));
|
|
34
|
-
}
|
|
35
|
-
catch {
|
|
36
|
-
return null;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
// Deterministic integer hash over the prompt — stable across runs, no crypto
|
|
40
|
-
// (fast + predictable). 32-bit unsigned; good enough for distribution
|
|
41
|
-
// spreading across 3-10 weighted options.
|
|
42
|
-
function hashText(text) {
|
|
43
|
-
let h = 2166136261 >>> 0; // FNV-1a seed
|
|
44
|
-
for (let i = 0; i < text.length; i += 1) {
|
|
45
|
-
h ^= text.charCodeAt(i);
|
|
46
|
-
h = Math.imul(h, 16777619) >>> 0;
|
|
47
|
-
}
|
|
48
|
-
return h >>> 0;
|
|
49
|
-
}
|
|
50
|
-
// Weighted interval selection: flatten weights into a cumulative array,
|
|
51
|
-
// pick the bucket where hash%total falls. Returns null when the group has
|
|
52
|
-
// no weights (operator disabled the pool) or the group is unknown.
|
|
53
|
-
function weightedPick(weights, hash) {
|
|
54
|
-
const entries = Object.entries(weights).filter(([, w]) => w > 0);
|
|
55
|
-
if (entries.length === 0)
|
|
56
|
-
return null;
|
|
57
|
-
const total = entries.reduce((sum, [, w]) => sum + w, 0);
|
|
58
|
-
if (total <= 0)
|
|
59
|
-
return null;
|
|
60
|
-
const target = hash % total;
|
|
61
|
-
let acc = 0;
|
|
62
|
-
for (const [name, w] of entries) {
|
|
63
|
-
acc += w;
|
|
64
|
-
if (target < acc)
|
|
65
|
-
return name;
|
|
66
|
-
}
|
|
67
|
-
// Unreachable given total > 0 and target < total; belt-and-suspenders.
|
|
68
|
-
return entries[entries.length - 1][0];
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Pick a zk capability for a prompt that mentions a zk primitive generically.
|
|
72
|
-
*
|
|
73
|
-
* @param text - the prompt text; hashed for deterministic selection.
|
|
74
|
-
* @param group - which preference group to consult (today: only "zkvm").
|
|
75
|
-
* @returns the capability ID (e.g. "capability:zkvm-risczero") or null when
|
|
76
|
-
* the group is unknown or has no weights.
|
|
77
|
-
*/
|
|
78
|
-
export function pickZkCapability(text, group) {
|
|
79
|
-
const prefs = loadPreferences();
|
|
80
|
-
if (!prefs)
|
|
81
|
-
return null;
|
|
82
|
-
const g = prefs.groups?.[group];
|
|
83
|
-
if (!g)
|
|
84
|
-
return null;
|
|
85
|
-
const hash = hashText(text);
|
|
86
|
-
const picked = weightedPick(g.weights, hash) ?? g.default;
|
|
87
|
-
if (!picked)
|
|
88
|
-
return null;
|
|
89
|
-
return `capability:${picked}`;
|
|
90
|
-
}
|
|
91
|
-
//# sourceMappingURL=zk-preferences.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"zk-preferences.js","sourceRoot":"","sources":["../../../src/lib/planner/zk-preferences.ts"],"names":[],"mappings":"AAAA,2EAA2E;AAC3E,8EAA8E;AAC9E,8DAA8D;AAC9D,uCAAuC;AACvC,EAAE;AACF,0EAA0E;AAC1E,8EAA8E;AAC9E,6EAA6E;AAC7E,6BAA6B;AAC7B,EAAE;AACF,sEAAsE;AACtE,2BAA2B;AAE3B,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAWxC,+EAA+E;AAC/E,6EAA6E;AAC7E,QAAQ;AACR,SAAS,eAAe;IACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAA;IAC1D,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,qCAAqC,CAAC,CAAA;IAC1E,kFAAkF;IAClF,8EAA8E;IAC9E,sBAAsB;IACtB,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,8CAA8C,CAAC,CAAA;AACzG,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,IAAI,GAAG,eAAe,EAAE,CAAA;IAC9B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAmB,CAAA;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAA;IACb,CAAC;AACH,CAAC;AAED,6EAA6E;AAC7E,sEAAsE;AACtE,0CAA0C;AAC1C,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC,GAAG,UAAU,KAAK,CAAC,CAAA,CAAC,cAAc;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QACvB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAA;AAChB,CAAC;AAED,wEAAwE;AACxE,0EAA0E;AAC1E,mEAAmE;AACnE,SAAS,YAAY,CAAC,OAA+B,EAAE,IAAY;IACjE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAChE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACxD,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,KAAK,CAAA;IAC3B,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAChC,GAAG,IAAI,CAAC,CAAA;QACR,IAAI,MAAM,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IAC/B,CAAC;IACD,uEAAuE;IACvE,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,KAAa;IAC1D,MAAM,KAAK,GAAG,eAAe,EAAE,CAAA;IAC/B,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IACvB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAA;IAC/B,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IACnB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAA;IACzD,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,OAAO,cAAc,MAAM,EAAE,CAAA;AAC/B,CAAC"}
|
|
File without changes
|