@tangle-network/starter-foundry 0.7.0 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/held-out-validation.json +2 -2
- package/dist/cli.js +0 -0
- package/dist/eval/scaffold-bridge.d.ts +99 -2
- package/dist/eval/scaffold-bridge.js +252 -59
- package/dist/eval/scaffold-bridge.js.map +1 -1
- package/dist/lib/compose.js +41 -2
- package/dist/lib/compose.js.map +1 -1
- package/dist/lib/promoter-gates.d.ts +114 -0
- package/dist/lib/promoter-gates.js +426 -0
- package/dist/lib/promoter-gates.js.map +1 -0
- package/dist/lib/prompt-e2e.d.ts +1 -0
- package/dist/lib/prompt-e2e.js +20 -6
- package/dist/lib/prompt-e2e.js.map +1 -1
- package/dist/lib/template-quality.js +20 -4
- package/dist/lib/template-quality.js.map +1 -1
- package/package.json +20 -19
- package/registry/families/fraud-ops-console/files/src/main.tsx +11 -8
- package/registry/families/fraud-ops-console/manifest.json +13 -17
- package/registry/families/kyc-onboarding/files/index.html +1 -1
- package/registry/families/kyc-onboarding/files/vite.config.ts +0 -1
- package/registry/families/kyc-onboarding/manifest.json +12 -12
- package/registry/families/polymarket-portfolio-hedging/files/src/main.tsx +11 -8
- package/registry/families/polymarket-portfolio-hedging/manifest.json +7 -6
- package/registry/layers/capability/agent-eval/files/.github/workflows/eval.yml +60 -0
- package/registry/layers/capability/agent-eval/files/scripts/eval-baseline.mjs +87 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/README.md +76 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/judge.mjs +35 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/run-eval.mjs +204 -0
- package/registry/layers/capability/agent-eval/files/tests/eval/scenarios.json +47 -0
- package/registry/layers/capability/agent-eval/manifest.json +73 -0
- package/registry/layers/framework/forge-foundation/files/foundry.toml +14 -0
- package/registry/layers/framework/tangle-blueprint/files/foundry.toml +8 -0
- package/dist/lib/planner/zk-preferences.d.ts +0 -9
- package/dist/lib/planner/zk-preferences.js +0 -91
- package/dist/lib/planner/zk-preferences.js.map +0 -1
- /package/registry/families/kyc-onboarding/files/src/{main.ts → main.tsx} +0 -0
package/dist/lib/prompt-e2e.js
CHANGED
|
@@ -73,21 +73,25 @@ async function loadWorkspaceValidationEvidence(runDir, plan) {
|
|
|
73
73
|
};
|
|
74
74
|
}
|
|
75
75
|
function matchesExpectation(result, expected) {
|
|
76
|
+
// Previously `if (!expected) return true` silently inflated routeAccuracy
|
|
77
|
+
// for corpora without expectations (vibecoder). The three-valued shape
|
|
78
|
+
// forces the summary layer to exclude no-expectation scenarios from the
|
|
79
|
+
// accuracy denominator rather than treating them as auto-pass.
|
|
76
80
|
if (!expected)
|
|
77
|
-
return true;
|
|
81
|
+
return { matched: true, hasExpectation: false };
|
|
78
82
|
if (expected.kind !== result.kind)
|
|
79
|
-
return false;
|
|
83
|
+
return { matched: false, hasExpectation: true };
|
|
80
84
|
if (expected.kind === 'starter') {
|
|
81
85
|
const route = result.route;
|
|
82
86
|
const sameFamily = expected.family === route.family;
|
|
83
87
|
const sameSlots = Object.entries(expected.slots ?? {}).every(([slotName, layerId]) => route.slots[slotName] === layerId);
|
|
84
|
-
return sameFamily && sameSlots;
|
|
88
|
+
return { matched: sameFamily && sameSlots, hasExpectation: true };
|
|
85
89
|
}
|
|
86
90
|
const route = result.route;
|
|
87
91
|
const samePrimary = expected.primaryProjectId === route.primaryProjectId;
|
|
88
92
|
const sameProjects = (expected.projectIds ?? []).every((projectId) => route.projectIds.includes(projectId));
|
|
89
93
|
const sameFamilies = Object.entries(expected.projectFamilies ?? {}).every(([projectId, family]) => route.projectFamilies[projectId] === family);
|
|
90
|
-
return samePrimary && sameProjects && sameFamilies;
|
|
94
|
+
return { matched: samePrimary && sameProjects && sameFamilies, hasExpectation: true };
|
|
91
95
|
}
|
|
92
96
|
export async function runPromptCorpus({ corpusPath, outDir = null, }) {
|
|
93
97
|
const corpus = await readJson(path.resolve(corpusPath));
|
|
@@ -126,7 +130,8 @@ export async function runPromptCorpus({ corpusPath, outDir = null, }) {
|
|
|
126
130
|
const validationEvidence = plan.kind === 'starter'
|
|
127
131
|
? await loadStarterValidationEvidence(benchmarkRunDir)
|
|
128
132
|
: await loadWorkspaceValidationEvidence(benchmarkRunDir, plan);
|
|
129
|
-
const
|
|
133
|
+
const match = matchesExpectation({ kind: plan.kind, route }, scenario.expected);
|
|
134
|
+
const routeOk = match.matched;
|
|
130
135
|
const validationOk = benchmark.summary.passRate === 1;
|
|
131
136
|
const primaryArtifactHit = benchmark.summary.primaryArtifactHitRate === 1;
|
|
132
137
|
results[index] = {
|
|
@@ -138,6 +143,7 @@ export async function runPromptCorpus({ corpusPath, outDir = null, }) {
|
|
|
138
143
|
confidence: plan.confidence,
|
|
139
144
|
route,
|
|
140
145
|
routeOk,
|
|
146
|
+
hasExpectation: match.hasExpectation,
|
|
141
147
|
validationOk,
|
|
142
148
|
primaryArtifactHit,
|
|
143
149
|
validationEvidence,
|
|
@@ -156,7 +162,15 @@ export async function runPromptCorpus({ corpusPath, outDir = null, }) {
|
|
|
156
162
|
results,
|
|
157
163
|
summary: {
|
|
158
164
|
passRate: results.filter((result) => result.ok).length / results.length,
|
|
159
|
-
routeAccuracy
|
|
165
|
+
// routeAccuracy denominator excludes no-expectation scenarios —
|
|
166
|
+
// previously silent-passed via `if (!expected) return true` in the
|
|
167
|
+
// matcher (Gen 9 muffled-gate audit).
|
|
168
|
+
routeAccuracy: (() => {
|
|
169
|
+
const checkable = results.filter((r) => r.hasExpectation);
|
|
170
|
+
if (checkable.length === 0)
|
|
171
|
+
return 1;
|
|
172
|
+
return checkable.filter((r) => r.routeOk).length / checkable.length;
|
|
173
|
+
})(),
|
|
160
174
|
validationPassRate: results.filter((result) => result.validationOk).length / results.length,
|
|
161
175
|
primaryArtifactHitRate: results.filter((result) => result.primaryArtifactHit).length / results.length,
|
|
162
176
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompt-e2e.js","sourceRoot":"","sources":["../../src/lib/prompt-e2e.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;
|
|
1
|
+
{"version":3,"file":"prompt-e2e.js","sourceRoot":"","sources":["../../src/lib/prompt-e2e.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAyDnD,SAAS,yBAAyB,CAAC,gBAAmC;IAKpE,IAAI,cAAc,GAAG,KAAK,CAAA;IAC1B,IAAI,YAAY,GAAG,KAAK,CAAA;IACxB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;IAE/B,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAChC,cAAc,GAAG,IAAI,CAAA;YACrB,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAA;YAC9B,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACvB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACpC,YAAY,GAAG,IAAI,CAAA;YACnB,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YACpB,SAAQ;QACV,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB;YAAE,SAAQ;QAE9C,MAAM,CAAC,GAAG,GAAG,SAAS,EAAE,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAA;QACtD,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC5C,IAAI,GAAG,KAAK,MAAM,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;YAAE,SAAQ;QAEjE,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtG,YAAY,GAAG,IAAI,CAAA;YACnB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACd,SAAQ;QACV,CAAC;QAED,cAAc,GAAG,IAAI,CAAA;QACrB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAChB,CAAC;IAED,OAAO;QACL,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY;QACjF,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE;QACxB,MAAM,EAAE,gBAAgB,CAAC,MAAM;KAChC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,6BAA6B,CAAC,MAAc;IACzD,MAAM,aAAa,GAAG,MAAM,QAAQ,CAClC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,EAAE,qBAAqB,CAAC,CAC7D,CAAA;IACD,MAAM,OAAO,GAAG,yBAAyB,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAA;IACzE,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,QAAQ,EAAE;YACR;gBACE,EAAE,EAAE,MAAM;gBACV,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,MAAM;gBACvC,GAAG,OAAO;aACX;SACF;KACF,CAAA;AACH,CAAC;AAED,KAAK,UAAU,+BAA+B,CAC5C,MAAc,EACd,IAAkE;IAElE,MAAM,QAAQ,GAAG,EAAE,CAAA;IAEnB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACzC,MAAM,aAAa,GAAG,MAAM,QAAQ,CAClC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,kBAAkB,EAAE,qBAAqB,CAAC,CAC3E,CAAA;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,EAAE,EAAE,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,IAAI;YAC9B,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC,MAAM;YACvC,GAAG,yBAAyB,CAAC,aAAa,CAAC,gBAAgB,CAAC;SAC7D,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAA;IAC9D,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,OAAO;QACtE,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QACxE,QAAQ;KACT,CAAA;AACH,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAuD,EACvD,QAAyB;IAEzB,0EAA0E;IAC1E,uEAAuE;IACvE,wEAAwE;IACxE,+DAA+D;IAC/D,IAAI,CAAC,QAAQ;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,CAAA;IAE9D,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI;QAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,CAAA;IAElF,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAqB,CAAA;QAC1C,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAA;QACnD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,KAAK,CAC1D,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,OAAO,CAC3D,CAAA;QACD,OAAO,EAAE,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,cAAc,EAAE,IAAI,EAAE,CAAA;IACnE,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,KAAuB,CAAA;IAC5C,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,KAAK,KAAK,CAAC,gBAAgB,CAAA;IACxE,MAAM,YAAY,GAAG,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAA;IAC3G,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,KAAK,CACvE,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,SAAS,CAAC,KAAK,MAAM,CACrE,CAAA;IACD,OAAO,EAAE,OAAO,EAAE,WAAW,IAAI,YAAY,IAAI,YAAY,EAAE,cAAc,EAAE,IAAI,EAAE,CAAA;AACvF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EACpC,UAAU,EACV,MAAM,GAAG,IAAI,GAId;IAiBC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAS,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAC/D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,aAAa,CAAC,4BAA4B,CAAC,CAAC,CAAA;IAE7E,mFAAmF;IACnF,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAAC,CAAA;IACrB,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAClE,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC,CAAA;IAE9E,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAA;YAC1B,IAAI,CAAC,IAAI;gBAAE,MAAK;YAChB,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAI,CAAA;YAEhC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAA;YAC9C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC;gBAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,IAAI;aAClC,CAAC,CAAA;YAEF,IAAI,SAA2E,CAAA;YAC/E,IAAI,KAAY,CAAA;YAEhB,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,MAAM,gBAAgB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBAChF,KAAK,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,CAAA;YACpE,CAAC;iBAAM,CAAC;gBACN,SAAS,GAAG,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;gBAClF,KAAK,GAAG;oBACN,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,gBAAgB,IAAI,EAAE;oBAC9D,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;oBACjE,eAAe,EAAE,MAAM,CAAC,WAAW,CACjC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAC7E;iBACF,CAAA;YACH,CAAC;YAED,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YACpD,MAAM,kBAAkB,GACtB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACrB,CAAC,CAAC,MAAM,6BAA6B,CAAC,eAAe,CAAC;gBACtD,CAAC,CAAC,MAAM,+BAA+B,CAAC,eAAe,EAAE,IAAI,CAAC,CAAA;YAElE,MAAM,KAAK,GAAG,kBAAkB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAA;YAC7B,MAAM,YAAY,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAA;YACrD,MAAM,kBAAkB,GAAG,SAAS,CAAC,OAAO,CAAC,sBAAsB,KAAK,CAAC,CAAA;YAEzE,OAAO,CAAC,KAAK,CAAC,GAAG;gBACf,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,IAAI;gBACjC,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,KAAK;gBACL,OAAO;gBACP,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,YAAY;gBACZ,kBAAkB;gBAClB,kBAAkB;gBAClB,SAAS,EAAE,SAAkB;gBAC7B,EAAE,EAAE,OAAO,IAAI,YAAY,IAAI,kBAAkB;aAClD,CAAA;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;QAEjG,MAAM,MAAM,GAAG;YACb,aAAa,EAAE,CAAU;YACzB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,OAAO;YACP,OAAO,EAAE;gBACP,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;gBACvE,gEAAgE;gBAChE,mEAAmE;gBACnE,sCAAsC;gBACtC,aAAa,EAAE,CAAC,GAAG,EAAE;oBACnB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAA;oBACzD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;wBAAE,OAAO,CAAC,CAAA;oBACpC,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;gBACrE,CAAC,CAAC,EAAE;gBACJ,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;gBAC3F,sBAAsB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;aACtG;SACF,CAAA;QAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC,EAAE,MAAM,CAAC,CAAA;QAErE,OAAO;YACL,MAAM,EAAE,OAAO;YACf,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,wBAAwB,CAAC;YACxD,MAAM;SACP,CAAA;IACH,CAAC;YAAS,CAAC;QACT,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,SAAS,CAAC,OAAO,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -15,11 +15,16 @@ const WEIGHTS = {
|
|
|
15
15
|
freshness: 0.05,
|
|
16
16
|
};
|
|
17
17
|
function phaseOk(audit, name) {
|
|
18
|
+
// Three-valued return distinguishes "phase ran and passed" from "phase
|
|
19
|
+
// was skipped" (e.g. no-tsconfig) so callers can price skips explicitly
|
|
20
|
+
// rather than getting silent-pass credit. Previous `if (p.skipped) return true`
|
|
21
|
+
// let families game typecheckOk*0.15 of the quality score by omitting
|
|
22
|
+
// tsconfig — the muffled-gate shape Gen 9 closes.
|
|
18
23
|
const p = audit.phases?.find((ph) => ph.phase === name);
|
|
19
24
|
if (!p)
|
|
20
25
|
return null;
|
|
21
26
|
if (p.skipped)
|
|
22
|
-
return
|
|
27
|
+
return 'skipped';
|
|
23
28
|
return p.ok === true;
|
|
24
29
|
}
|
|
25
30
|
function phaseMs(audit, name) {
|
|
@@ -49,9 +54,20 @@ export function scoreVersion(input) {
|
|
|
49
54
|
const installOk = phaseOk(audit, 'install');
|
|
50
55
|
// Build: if phase absent (not run), give full credit so the bar isn't
|
|
51
56
|
// retroactively applied to v1 snapshots that predated --build.
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
57
|
+
// Skipped phase gets partial credit (0.5), matching validationScore's
|
|
58
|
+
// treatment of `skipped: 'none-declared'`. Gen 9 split silent-pass
|
|
59
|
+
// (previously skipped→1.0) from real pass so families can't inflate
|
|
60
|
+
// quality by omitting their tsconfig.
|
|
61
|
+
const phaseCredit = (v, absentCredit) => {
|
|
62
|
+
if (v === null)
|
|
63
|
+
return absentCredit;
|
|
64
|
+
if (v === 'skipped')
|
|
65
|
+
return 0.5;
|
|
66
|
+
return v ? 1 : 0;
|
|
67
|
+
};
|
|
68
|
+
const buildComponent = phaseCredit(buildOk, 1);
|
|
69
|
+
const tscComponent = phaseCredit(tscOk, 0);
|
|
70
|
+
const installComponent = phaseCredit(installOk, 0);
|
|
55
71
|
const richness = Math.min(1, fileCount / 10);
|
|
56
72
|
const shotsUsed = summary?.shotsUsed ?? 0;
|
|
57
73
|
const firstShotBonus = shotsUsed === 1 ? 1 : shotsUsed <= 3 ? 0.5 : shotsUsed <= 10 ? 0.2 : 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template-quality.js","sourceRoot":"","sources":["../../src/lib/template-quality.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,EAAE;AACF,sEAAsE;AACtE,yEAAyE;AACzE,yEAAyE;AA8DzE,MAAM,OAAO,GAAG;IACd,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,IAAI;IACnB,WAAW,EAAE,IAAI;IACjB,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,IAAI;IACd,cAAc,EAAE,IAAI;IACpB,SAAS,EAAE,IAAI;CACP,CAAA;AAEV,SAAS,OAAO,CAAC,KAAkB,EAAE,IAAY;IAC/C,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;IACvD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IACnB,IAAI,CAAC,CAAC,OAAO;QAAE,OAAO,
|
|
1
|
+
{"version":3,"file":"template-quality.js","sourceRoot":"","sources":["../../src/lib/template-quality.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,yEAAyE;AACzE,wEAAwE;AACxE,EAAE;AACF,sEAAsE;AACtE,yEAAyE;AACzE,yEAAyE;AA8DzE,MAAM,OAAO,GAAG;IACd,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,IAAI;IACnB,WAAW,EAAE,IAAI;IACjB,SAAS,EAAE,IAAI;IACf,QAAQ,EAAE,IAAI;IACd,cAAc,EAAE,IAAI;IACpB,SAAS,EAAE,IAAI;CACP,CAAA;AAEV,SAAS,OAAO,CAAC,KAAkB,EAAE,IAAY;IAC/C,uEAAuE;IACvE,wEAAwE;IACxE,gFAAgF;IAChF,sEAAsE;IACtE,kDAAkD;IAClD,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;IACvD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IACnB,IAAI,CAAC,CAAC,OAAO;QAAE,OAAO,SAAS,CAAA;IAC/B,OAAO,CAAC,CAAC,EAAE,KAAK,IAAI,CAAA;AACtB,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,IAAY;IAC/C,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;IACvD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,UAAU,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAA;IACvD,OAAO,CAAC,CAAC,UAAU,CAAA;AACrB,CAAC;AAED,SAAS,eAAe,CAAC,KAAkB;IACzC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,KAAK,kBAAkB,CAAC,CAAA;IACrE,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAA;IAChB,IAAI,CAAC,CAAC,OAAO,KAAK,eAAe;QAAE,OAAO,GAAG,CAAA,CAAE,4CAA4C;IAC3F,IAAI,CAAC,CAAC,OAAO;QAAE,OAAO,CAAC,CAAA;IACvB,IAAI,OAAO,CAAC,CAAC,WAAW,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;QACjG,OAAO,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,WAAW,CAAA;IACvC,CAAC;IACD,OAAO,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB;IAC5C,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,KAAK,CAAA;IAEtD,MAAM,UAAU,GAAG,eAAe,CAAC,KAAK,CAAC,CAAA;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC,CAAA;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;IAE3C,sEAAsE;IACtE,+DAA+D;IAC/D,sEAAsE;IACtE,mEAAmE;IACnE,oEAAoE;IACpE,sCAAsC;IACtC,MAAM,WAAW,GAAG,CAAC,CAAkC,EAAE,YAAoB,EAAU,EAAE;QACvF,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,YAAY,CAAA;QACnC,IAAI,CAAC,KAAK,SAAS;YAAE,OAAO,GAAG,CAAA;QAC/B,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAClB,CAAC,CAAA;IACD,MAAM,cAAc,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC9C,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC1C,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;IAElD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,CAAC,CAAA;IACzC,MAAM,cAAc,GAClB,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;IAExE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;IACjG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,GAAG,CAAC,CAAC,CAAA;IAE7D,MAAM,UAAU,GAAG;QACjB,cAAc,EAAE,OAAO,CAAC,cAAc,GAAG,UAAU;QACnD,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,cAAc;QACrD,WAAW,EAAE,OAAO,CAAC,WAAW,GAAG,YAAY;QAC/C,SAAS,EAAE,OAAO,CAAC,SAAS,GAAG,gBAAgB;QAC/C,QAAQ,EAAE,OAAO,CAAC,QAAQ,GAAG,QAAQ;QACrC,cAAc,EAAE,OAAO,CAAC,cAAc,GAAG,cAAc;QACvD,SAAS,EAAE,OAAO,CAAC,SAAS,GAAG,SAAS;KACzC,CAAA;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAElE,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,UAAU;QACV,UAAU,EAAE;YACV,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC;YACpC,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,WAAW,CAAC;YAClC,YAAY,EAAE,OAAO,CAAC,KAAK,EAAE,kBAAkB,CAAC;YAChD,SAAS;YACT,SAAS;YACT,OAAO;SACR;KACF,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/starter-foundry",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.3",
|
|
4
4
|
"description": "Deterministic project scaffold engine for AI coding platforms",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -35,25 +35,9 @@
|
|
|
35
35
|
"bin": {
|
|
36
36
|
"starter-foundry": "./dist/cli.js"
|
|
37
37
|
},
|
|
38
|
-
"engines": {
|
|
39
|
-
"node": ">=20"
|
|
40
|
-
},
|
|
41
|
-
"devDependencies": {
|
|
42
|
-
"@tangle-network/agent-eval": "^0.7.0",
|
|
43
|
-
"@types/node": "^20.0.0",
|
|
44
|
-
"knip": "^6.6.0",
|
|
45
|
-
"typescript": "^5.7.0"
|
|
46
|
-
},
|
|
47
|
-
"dependencies": {
|
|
48
|
-
"@ax-llm/ax": "^19.0.45",
|
|
49
|
-
"@huggingface/transformers": "^4.0.0",
|
|
50
|
-
"@tangle-network/tcloud": "^0.4.2"
|
|
51
|
-
},
|
|
52
|
-
"optionalDependencies": {
|
|
53
|
-
"@opentelemetry/api": "^1.9.0"
|
|
54
|
-
},
|
|
55
38
|
"scripts": {
|
|
56
39
|
"build": "tsc",
|
|
40
|
+
"prepublishOnly": "tsc",
|
|
57
41
|
"pretest": "node scripts/check-toolchain.mjs && node scripts/ensure-test-fixtures.mjs",
|
|
58
42
|
"test": "tsc && tsc -p tsconfig.test.json && node --test --test-concurrency=1 dist-tests/*.test.js",
|
|
59
43
|
"catalog": "node dist/cli.js catalog",
|
|
@@ -109,5 +93,22 @@
|
|
|
109
93
|
"sla:rollup": "node scripts/sla-rollup.mjs",
|
|
110
94
|
"rollback:version": "node scripts/rollback-version.mjs",
|
|
111
95
|
"gen:types": "node scripts/gen-types-from-schemas.mjs"
|
|
96
|
+
},
|
|
97
|
+
"engines": {
|
|
98
|
+
"node": ">=20"
|
|
99
|
+
},
|
|
100
|
+
"devDependencies": {
|
|
101
|
+
"@tangle-network/agent-eval": "^0.7.2",
|
|
102
|
+
"@types/node": "^20.0.0",
|
|
103
|
+
"knip": "^6.6.0",
|
|
104
|
+
"typescript": "^5.7.0"
|
|
105
|
+
},
|
|
106
|
+
"dependencies": {
|
|
107
|
+
"@ax-llm/ax": "^19.0.45",
|
|
108
|
+
"@huggingface/transformers": "^4.0.0",
|
|
109
|
+
"@tangle-network/tcloud": "^0.4.2"
|
|
110
|
+
},
|
|
111
|
+
"optionalDependencies": {
|
|
112
|
+
"@opentelemetry/api": "^1.9.0"
|
|
112
113
|
}
|
|
113
|
-
}
|
|
114
|
+
}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import App from './App'
|
|
1
|
+
import { StrictMode } from 'react'
|
|
2
|
+
import { createRoot } from 'react-dom/client'
|
|
3
|
+
import App from './App'
|
|
4
4
|
|
|
5
|
-
const rootElement = document.getElementById('root')
|
|
6
|
-
if (rootElement)
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
const rootElement = document.getElementById('root')
|
|
6
|
+
if (!rootElement) throw new Error('missing #root — index.html must contain <div id="root">')
|
|
7
|
+
|
|
8
|
+
createRoot(rootElement).render(
|
|
9
|
+
<StrictMode>
|
|
10
|
+
<App />
|
|
11
|
+
</StrictMode>,
|
|
12
|
+
)
|
|
@@ -81,17 +81,15 @@
|
|
|
81
81
|
},
|
|
82
82
|
"keywords": [
|
|
83
83
|
"fraud detection",
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
93
|
-
"performance",
|
|
94
|
-
"scalability"
|
|
84
|
+
"fraud ops",
|
|
85
|
+
"fraud ops console",
|
|
86
|
+
"risk scoring dashboard",
|
|
87
|
+
"alert triage",
|
|
88
|
+
"kyc alerts",
|
|
89
|
+
"fraud alerts",
|
|
90
|
+
"aml monitoring",
|
|
91
|
+
"transaction risk scoring",
|
|
92
|
+
"chargeback ops"
|
|
95
93
|
],
|
|
96
94
|
"tieredKeywords": {
|
|
97
95
|
"tier1": [
|
|
@@ -101,12 +99,10 @@
|
|
|
101
99
|
"kyc alerts"
|
|
102
100
|
],
|
|
103
101
|
"tier2": [
|
|
104
|
-
"
|
|
105
|
-
"
|
|
106
|
-
"
|
|
107
|
-
"
|
|
108
|
-
"performance",
|
|
109
|
-
"scalability"
|
|
102
|
+
"alert triage",
|
|
103
|
+
"aml monitoring",
|
|
104
|
+
"chargeback ops",
|
|
105
|
+
"transaction risk scoring"
|
|
110
106
|
]
|
|
111
107
|
},
|
|
112
108
|
"buildHints": {
|
|
@@ -33,8 +33,8 @@
|
|
|
33
33
|
"target": "index.html"
|
|
34
34
|
},
|
|
35
35
|
{
|
|
36
|
-
"source": "files/src/main.
|
|
37
|
-
"target": "src/main.
|
|
36
|
+
"source": "files/src/main.tsx",
|
|
37
|
+
"target": "src/main.tsx"
|
|
38
38
|
},
|
|
39
39
|
{
|
|
40
40
|
"source": "files/README.md",
|
|
@@ -80,16 +80,16 @@
|
|
|
80
80
|
"extensionPoints": []
|
|
81
81
|
},
|
|
82
82
|
"keywords": [
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"
|
|
91
|
-
"
|
|
92
|
-
"
|
|
83
|
+
"kyc",
|
|
84
|
+
"kyc onboarding",
|
|
85
|
+
"identity verification",
|
|
86
|
+
"document verification",
|
|
87
|
+
"liveness check",
|
|
88
|
+
"aml compliance",
|
|
89
|
+
"sumsub",
|
|
90
|
+
"persona identity",
|
|
91
|
+
"passkey onboarding",
|
|
92
|
+
"know your customer"
|
|
93
93
|
],
|
|
94
94
|
"tieredKeywords": {
|
|
95
95
|
"tier1": [
|
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
import
|
|
2
|
-
import
|
|
3
|
-
import App from './App'
|
|
1
|
+
import { StrictMode } from 'react'
|
|
2
|
+
import { createRoot } from 'react-dom/client'
|
|
3
|
+
import App from './App'
|
|
4
4
|
|
|
5
|
-
const rootElement = document.getElementById('root')
|
|
6
|
-
if (rootElement)
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
5
|
+
const rootElement = document.getElementById('root')
|
|
6
|
+
if (!rootElement) throw new Error('missing #root — index.html must contain <div id="root">')
|
|
7
|
+
|
|
8
|
+
createRoot(rootElement).render(
|
|
9
|
+
<StrictMode>
|
|
10
|
+
<App />
|
|
11
|
+
</StrictMode>,
|
|
12
|
+
)
|
|
@@ -80,12 +80,13 @@
|
|
|
80
80
|
"extensionPoints": []
|
|
81
81
|
},
|
|
82
82
|
"keywords": [
|
|
83
|
-
"
|
|
84
|
-
"
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
83
|
+
"polymarket",
|
|
84
|
+
"prediction market",
|
|
85
|
+
"prediction market hedging",
|
|
86
|
+
"portfolio hedging",
|
|
87
|
+
"event contract portfolio",
|
|
88
|
+
"event contract hedging",
|
|
89
|
+
"polymarket portfolio"
|
|
89
90
|
],
|
|
90
91
|
"tieredKeywords": {
|
|
91
92
|
"tier1": [
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
name: Agent eval
|
|
2
|
+
|
|
3
|
+
# Runs the scaffold-level agent-eval harness on every PR + push.
|
|
4
|
+
# Deterministic assertions are the contract. The optional LLM judge runs
|
|
5
|
+
# only when TOGETHER_API_KEY is configured as a repo secret — its score is
|
|
6
|
+
# recorded but does NOT gate the build (rubrics need calibration before
|
|
7
|
+
# they can be load-bearing).
|
|
8
|
+
|
|
9
|
+
on:
|
|
10
|
+
push:
|
|
11
|
+
branches: [main]
|
|
12
|
+
pull_request:
|
|
13
|
+
branches: [main]
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: eval-${{ github.ref }}
|
|
17
|
+
cancel-in-progress: true
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
eval:
|
|
21
|
+
name: agent-eval
|
|
22
|
+
runs-on: ubuntu-latest
|
|
23
|
+
timeout-minutes: 10
|
|
24
|
+
steps:
|
|
25
|
+
- uses: actions/checkout@v4
|
|
26
|
+
|
|
27
|
+
- uses: pnpm/action-setup@v4
|
|
28
|
+
with:
|
|
29
|
+
version: 10
|
|
30
|
+
|
|
31
|
+
- uses: actions/setup-node@v4
|
|
32
|
+
with:
|
|
33
|
+
node-version: '22'
|
|
34
|
+
cache: 'pnpm'
|
|
35
|
+
|
|
36
|
+
- name: install
|
|
37
|
+
run: pnpm install --frozen-lockfile
|
|
38
|
+
|
|
39
|
+
- name: run agent-eval
|
|
40
|
+
env:
|
|
41
|
+
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
|
42
|
+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
|
43
|
+
run: node tests/eval/run-eval.mjs --threshold 0.7
|
|
44
|
+
|
|
45
|
+
- name: regression check against baseline
|
|
46
|
+
if: always()
|
|
47
|
+
run: |
|
|
48
|
+
if [ -f .evolve/eval/baseline.json ]; then
|
|
49
|
+
node scripts/eval-baseline.mjs --check --tolerance 0.02
|
|
50
|
+
else
|
|
51
|
+
echo "no baseline pinned — skipping regression check"
|
|
52
|
+
fi
|
|
53
|
+
|
|
54
|
+
- name: upload scorecard
|
|
55
|
+
if: always()
|
|
56
|
+
uses: actions/upload-artifact@v4
|
|
57
|
+
with:
|
|
58
|
+
name: agent-eval-scorecard
|
|
59
|
+
path: .evolve/eval/
|
|
60
|
+
retention-days: 30
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Baseline + regression-diff tool for tests/eval/.
|
|
3
|
+
//
|
|
4
|
+
// node scripts/eval-baseline.mjs # print latest vs baseline diff
|
|
5
|
+
// node scripts/eval-baseline.mjs --write # pin current latest as baseline
|
|
6
|
+
// node scripts/eval-baseline.mjs --check --tolerance 0.02
|
|
7
|
+
// exit 1 if aggregate regressed by more than tolerance
|
|
8
|
+
//
|
|
9
|
+
// Reads: .evolve/eval/latest.json, .evolve/eval/baseline.json
|
|
10
|
+
// Writes: .evolve/eval/baseline.json (with --write)
|
|
11
|
+
|
|
12
|
+
import { readFile, writeFile } from 'node:fs/promises'
|
|
13
|
+
import { existsSync } from 'node:fs'
|
|
14
|
+
import { resolve, dirname } from 'node:path'
|
|
15
|
+
import { fileURLToPath } from 'node:url'
|
|
16
|
+
|
|
17
|
+
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
|
18
|
+
const LATEST = resolve(ROOT, '.evolve/eval/latest.json')
|
|
19
|
+
const BASELINE = resolve(ROOT, '.evolve/eval/baseline.json')
|
|
20
|
+
const argv = process.argv.slice(2)
|
|
21
|
+
const arg = (flag, fallback) => {
|
|
22
|
+
const i = argv.indexOf(flag)
|
|
23
|
+
return i >= 0 && argv[i + 1] ? argv[i + 1] : fallback
|
|
24
|
+
}
|
|
25
|
+
const WRITE = argv.includes('--write')
|
|
26
|
+
const CHECK = argv.includes('--check')
|
|
27
|
+
const TOLERANCE = Number(arg('--tolerance', '0.02'))
|
|
28
|
+
|
|
29
|
+
async function loadOrNull(path) {
|
|
30
|
+
if (!existsSync(path)) return null
|
|
31
|
+
try {
|
|
32
|
+
return JSON.parse(await readFile(path, 'utf8'))
|
|
33
|
+
} catch (err) {
|
|
34
|
+
console.error(`[eval-baseline] could not parse ${path}:`, err.message)
|
|
35
|
+
return null
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function main() {
|
|
40
|
+
const latest = await loadOrNull(LATEST)
|
|
41
|
+
if (!latest) {
|
|
42
|
+
console.error(`no .evolve/eval/latest.json — run: node tests/eval/run-eval.mjs`)
|
|
43
|
+
process.exit(2)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (WRITE) {
|
|
47
|
+
await writeFile(BASELINE, JSON.stringify(latest, null, 2))
|
|
48
|
+
console.log(`baseline pinned at aggregate=${latest.aggregate.toFixed(3)} (${latest.passCount}/${latest.totalCount})`)
|
|
49
|
+
return
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const baseline = await loadOrNull(BASELINE)
|
|
53
|
+
if (!baseline) {
|
|
54
|
+
console.log(`no baseline yet — run: node scripts/eval-baseline.mjs --write`)
|
|
55
|
+
console.log(`latest: aggregate=${latest.aggregate.toFixed(3)} (${latest.passCount}/${latest.totalCount})`)
|
|
56
|
+
return
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const delta = latest.aggregate - baseline.aggregate
|
|
60
|
+
const sign = delta >= 0 ? '+' : ''
|
|
61
|
+
console.log(`baseline: ${baseline.aggregate.toFixed(3)} (${baseline.passCount}/${baseline.totalCount}, ${baseline.timestamp})`)
|
|
62
|
+
console.log(`latest: ${latest.aggregate.toFixed(3)} (${latest.passCount}/${latest.totalCount}, ${latest.timestamp})`)
|
|
63
|
+
console.log(`delta: ${sign}${delta.toFixed(3)} (tolerance ${TOLERANCE})`)
|
|
64
|
+
|
|
65
|
+
const baselineById = Object.fromEntries(baseline.results.map((r) => [r.id, r]))
|
|
66
|
+
const flips = []
|
|
67
|
+
for (const r of latest.results) {
|
|
68
|
+
const prev = baselineById[r.id]
|
|
69
|
+
if (!prev) continue
|
|
70
|
+
if (prev.passed && !r.passed) flips.push({ id: r.id, reason: r.reason, direction: 'regress' })
|
|
71
|
+
if (!prev.passed && r.passed) flips.push({ id: r.id, reason: r.reason, direction: 'improve' })
|
|
72
|
+
}
|
|
73
|
+
if (flips.length) {
|
|
74
|
+
console.log(`scenario flips (${flips.length}):`)
|
|
75
|
+
for (const f of flips) console.log(` ${f.direction === 'regress' ? '✗' : '✓'} ${f.id} — ${f.reason}`)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (CHECK && delta < -TOLERANCE) {
|
|
79
|
+
console.error(`REGRESSION — delta ${delta.toFixed(3)} below tolerance -${TOLERANCE}`)
|
|
80
|
+
process.exit(1)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
main().catch((err) => {
|
|
85
|
+
console.error('[eval-baseline] fatal:', err)
|
|
86
|
+
process.exit(2)
|
|
87
|
+
})
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Agent eval
|
|
2
|
+
|
|
3
|
+
This scaffold ships with a reproducible eval built on [`@tangle-network/agent-eval`](https://www.npmjs.com/package/@tangle-network/agent-eval). Every scenario is a `TestGradedScenario`; the library spawns it via `SubprocessSandboxDriver`, captures it in `InMemoryTraceStore`, and scores it via the sandbox harness. You get structured `Run` records for free.
|
|
4
|
+
|
|
5
|
+
## What's here
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
tests/eval/
|
|
9
|
+
scenarios.json ← edit first. Each entry becomes a TestGradedScenario.
|
|
10
|
+
run-eval.mjs ← thin shell over agent-eval's runTestGradedScenario.
|
|
11
|
+
judge.mjs ← optional LLM-judge hook (no-op by default — calibrate first).
|
|
12
|
+
scripts/
|
|
13
|
+
eval-baseline.mjs ← pin a baseline; re-runs regress against it.
|
|
14
|
+
.github/workflows/
|
|
15
|
+
eval.yml ← PR gate.
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Loop
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pnpm install
|
|
22
|
+
node tests/eval/run-eval.mjs # boots agent via `pnpm start`, runs scenarios
|
|
23
|
+
node scripts/eval-baseline.mjs --write # pin current score as baseline
|
|
24
|
+
# edit agent code
|
|
25
|
+
node tests/eval/run-eval.mjs # scorecard diffs vs baseline when --check
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Scenario shape
|
|
29
|
+
|
|
30
|
+
Each entry in `scenarios.json` is one HTTP call. `run-eval.mjs` translates it into a shell assertion (`curl` + `jq`) and hands it to `runTestGradedScenario`. The library spawns, captures exit code + stderr, emits a `Run`, and returns `{ pass, score, failureClass }`.
|
|
31
|
+
|
|
32
|
+
```json
|
|
33
|
+
{
|
|
34
|
+
"id": "behavior/handles-task",
|
|
35
|
+
"kind": "http-post",
|
|
36
|
+
"path": "/run",
|
|
37
|
+
"body": { "task": "Summarize X." },
|
|
38
|
+
"assert": {
|
|
39
|
+
"status": 200,
|
|
40
|
+
"jsonShape": { "draft": ["string"] },
|
|
41
|
+
"minResponseChars": 40
|
|
42
|
+
},
|
|
43
|
+
"weight": 2
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Supported assertions: `status`, `statusIn`, `requireJson`, `jsonShape` (per-field `string|number|boolean|null|array|object|undefined`), `minResponseChars`.
|
|
48
|
+
|
|
49
|
+
## LLM judge (opt-in)
|
|
50
|
+
|
|
51
|
+
`judge.mjs` is a hook that imports `createCustomJudge` from `@tangle-network/agent-eval`. It's a no-op by default because an uncalibrated rubric is worse than no rubric — it will happily score everything ~0.7 and make your gate useless. Before turning it on, calibrate: score 10-20 real scenario outputs by hand, feed them to your judge, confirm direction-agreement ≥0.8, then turn the hook on and extend `run-eval.mjs` to merge judge scores into the scorecard.
|
|
52
|
+
|
|
53
|
+
Relevant library primitives for the LLM layer:
|
|
54
|
+
- `createCustomJudge(name, systemPrompt, opts?)` — rubric-backed judge
|
|
55
|
+
- `createDomainExpertJudge(domain)` — default persona-based rubric
|
|
56
|
+
- `coherenceJudge` / `codeExecutionJudge` / `defaultJudges(domain)` — ready-made judges
|
|
57
|
+
- `JudgeRunner` — runs a set of judges over a Run and merges scores
|
|
58
|
+
|
|
59
|
+
## Baseline + regression
|
|
60
|
+
|
|
61
|
+
`scripts/eval-baseline.mjs --write` pins `.evolve/eval/latest.json` as `.evolve/eval/baseline.json`. `--check --tolerance 0.02` fails non-zero when the aggregate regresses beyond tolerance. CI uses this.
|
|
62
|
+
|
|
63
|
+
## Going beyond HTTP
|
|
64
|
+
|
|
65
|
+
The current runner assumes an HTTP agent (`/health` + scenario endpoints). If your agent is stdio-based, websocket-only, or queue-backed: replace the `buildTestCommand` helper in `run-eval.mjs` with one that emits the right shell assertion (stdio pipe, wscat, redis-cli). Everything downstream stays: `runTestGradedScenario` doesn't care what transport you use, only that the shell command exits 0/non-zero.
|
|
66
|
+
|
|
67
|
+
## The store
|
|
68
|
+
|
|
69
|
+
`run-eval.mjs` uses `InMemoryTraceStore`. For persistence (benchmarks across sessions, CI artifact inspection), swap to `FileSystemTraceStore`:
|
|
70
|
+
|
|
71
|
+
```js
|
|
72
|
+
import { FileSystemTraceStore } from '@tangle-network/agent-eval'
|
|
73
|
+
const store = new FileSystemTraceStore({ root: '.evolve/eval/traces' })
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Every `Run` object the library emits (scenarioId, pass, score, failureClass, timing, exit codes, stderr) is then on disk, replayable, and consumable by the library's `scoreProject` / `BenchmarkRunner` / `ConvergenceTracker` / `formatBenchmarkReport` for richer reporting.
|