claudeye 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of claudeye might be problematic. Click here for more details.
- package/.next/standalone/.next/BUILD_ID +1 -1
- package/.next/standalone/.next/app-path-routes-manifest.json +1 -0
- package/.next/standalone/.next/build-manifest.json +5 -5
- package/.next/standalone/.next/routes-manifest.json +9 -0
- package/.next/standalone/.next/server/app/_global-error/page/build-manifest.json +3 -3
- package/.next/standalone/.next/server/app/_global-error/page.js +1 -1
- package/.next/standalone/.next/server/app/_global-error/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_global-error.html +2 -2
- package/.next/standalone/.next/server/app/_global-error.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/__PAGE__.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_full.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_head.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_index.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found/page/build-manifest.json +3 -3
- package/.next/standalone/.next/server/app/_not-found/page.js +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_not-found.html +2 -2
- package/.next/standalone/.next/server/app/_not-found.rsc +5 -5
- package/.next/standalone/.next/server/app/_not-found.segments/_full.segment.rsc +5 -5
- package/.next/standalone/.next/server/app/_not-found.segments/_head.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found.segments/_index.segment.rsc +5 -5
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route/app-paths-manifest.json +3 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route/build-manifest.json +11 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route/server-reference-manifest.json +4 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js +6 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js.map +5 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js.nft.json +1 -0
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route_client-reference-manifest.js +2 -0
- package/.next/standalone/.next/server/app/icon.png/route.js +2 -1
- package/.next/standalone/.next/server/app/icon.png/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page/build-manifest.json +3 -3
- package/.next/standalone/.next/server/app/page.js +1 -1
- package/.next/standalone/.next/server/app/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page/build-manifest.json +3 -3
- package/.next/standalone/.next/server/app/project/[name]/page.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/build-manifest.json +3 -3
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/react-loadable-manifest.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/server-reference-manifest.json +35 -5
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js +4 -3
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app-paths-manifest.json +1 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__f408c708._.js +21 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__fde83e67._.js +3 -0
- package/.next/standalone/.next/server/chunks/ce889_server_app_api_download_[project]_[session]_route_actions_bbdd823f.js +3 -0
- package/.next/standalone/.next/server/chunks/node_modules_next_dist_esm_build_templates_app-route_64175717.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__070e2009._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0a745465._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__bc37261c._.js → [root-of-the-server]__14f58da3._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__164d9311._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__2822fd21._.js +6 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__31b4c2fd._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__45656df2._.js → [root-of-the-server]__4e339665._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__55018089._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__6313e929._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__7e21395a._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__ee388ee0._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/_0b4924bd._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/{node_modules_9e089768._.js → _1404b353._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/_3d21dde5._.js +8 -0
- package/.next/standalone/.next/server/chunks/ssr/_fd9b1ff7._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/{node_modules_next_dist_7769b563._.js → node_modules_next_dist_esm_eedfc1fd._.js} +2 -2
- package/.next/standalone/.next/server/middleware-build-manifest.js +3 -3
- package/.next/standalone/.next/server/pages/404.html +2 -2
- package/.next/standalone/.next/server/pages/500.html +2 -2
- package/.next/standalone/.next/server/server-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.json +35 -5
- package/.next/standalone/.next/static/chunks/0e266948a26a3cdf.js +1 -0
- package/.next/standalone/.next/static/chunks/2774382cf796393c.js +4 -0
- package/.next/standalone/.next/static/chunks/6189ca16caad4352.js +3 -0
- package/.next/standalone/.next/static/chunks/8111dbe882e31821.js +1 -0
- package/.next/standalone/.next/static/chunks/{5a424275276f2bb9.js → bdeaeb8c9876394b.js} +1 -1
- package/.next/standalone/.next/static/chunks/cdbb6932218650fd.js +1 -0
- package/.next/standalone/.next/static/chunks/ea03555bb726c073.css +1 -0
- package/.next/standalone/.next/static/chunks/f091501564eb2ea3.js +4 -0
- package/.next/standalone/.next/static/chunks/{turbopack-2315171089e56fad.js → turbopack-fc1f23734a087d36.js} +1 -1
- package/.next/standalone/README.md +528 -41
- package/.next/standalone/app/actions/run-enrichments.ts +26 -5
- package/.next/standalone/app/actions/run-evals.ts +26 -5
- package/.next/standalone/app/actions/run-subagent-enrichments.ts +89 -0
- package/.next/standalone/app/actions/run-subagent-evals.ts +88 -0
- package/.next/standalone/app/api/download/[project]/[session]/route.ts +49 -0
- package/.next/standalone/app/components/copy-button.tsx +37 -0
- package/.next/standalone/app/components/enrichment-results-panel.tsx +33 -13
- package/.next/standalone/app/components/eval-results-panel.tsx +33 -13
- package/.next/standalone/app/components/log-viewer/entry-row.tsx +43 -14
- package/.next/standalone/app/components/log-viewer/queue-divider.tsx +50 -7
- package/.next/standalone/app/components/log-viewer/tool-input-output.tsx +13 -3
- package/.next/standalone/app/components/project-list.tsx +11 -11
- package/.next/standalone/app/components/raw-log-viewer.tsx +80 -11
- package/.next/standalone/app/components/refresh-button.tsx +79 -0
- package/.next/standalone/app/components/sessions-list.tsx +23 -14
- package/.next/standalone/app/project/[name]/session/[sessionId]/page.tsx +23 -12
- package/.next/standalone/bin/claudeye.mjs +112 -25
- package/.next/standalone/components/navbar.tsx +2 -0
- package/.next/standalone/dist/app.js +10 -4
- package/.next/standalone/dist/condition-registry.js +20 -0
- package/.next/standalone/dist/enrich-registry.js +26 -3
- package/.next/standalone/dist/enrich-runner.js +68 -13
- package/.next/standalone/dist/registry.js +26 -3
- package/.next/standalone/dist/runner.js +78 -20
- package/.next/standalone/dist/server-spawn.js +58 -34
- package/.next/standalone/lib/cache/hash.ts +67 -0
- package/.next/standalone/lib/cache/index.ts +9 -0
- package/.next/standalone/lib/cache/local-backend.ts +81 -0
- package/.next/standalone/lib/cache/manager.ts +127 -0
- package/.next/standalone/lib/cache/types.ts +19 -0
- package/.next/standalone/lib/evals/app.ts +30 -7
- package/.next/standalone/lib/evals/condition-registry.ts +26 -0
- package/.next/standalone/lib/evals/enrich-registry.ts +29 -3
- package/.next/standalone/lib/evals/enrich-runner.ts +68 -14
- package/.next/standalone/lib/evals/enrich-types.ts +6 -1
- package/.next/standalone/lib/evals/index.ts +3 -1
- package/.next/standalone/lib/evals/registry.ts +29 -4
- package/.next/standalone/lib/evals/runner.ts +77 -20
- package/.next/standalone/lib/evals/server-spawn.ts +67 -41
- package/.next/standalone/lib/evals/types.ts +16 -0
- package/.next/standalone/lib/log-format.ts +22 -1
- package/.next/standalone/package-lock.json +244 -308
- package/.next/standalone/package.json +1 -1
- package/.next/standalone/scripts/dev.ts +3 -1
- package/.next/standalone/scripts/parse-script-args.ts +30 -2
- package/.next/standalone/scripts/start.ts +3 -1
- package/.next/standalone/tsconfig.tsbuildinfo +1 -1
- package/README.md +528 -41
- package/bin/claudeye.mjs +112 -25
- package/dist/app.d.ts +17 -3
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +10 -4
- package/dist/app.js.map +1 -1
- package/dist/condition-registry.d.ts +9 -0
- package/dist/condition-registry.d.ts.map +1 -0
- package/dist/condition-registry.js +20 -0
- package/dist/condition-registry.js.map +1 -0
- package/dist/enrich-registry.d.ts +5 -1
- package/dist/enrich-registry.d.ts.map +1 -1
- package/dist/enrich-registry.js +26 -3
- package/dist/enrich-registry.js.map +1 -1
- package/dist/enrich-runner.d.ts +3 -3
- package/dist/enrich-runner.d.ts.map +1 -1
- package/dist/enrich-runner.js +68 -13
- package/dist/enrich-runner.js.map +1 -1
- package/dist/enrich-types.d.ts +6 -1
- package/dist/enrich-types.d.ts.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/registry.d.ts +5 -2
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +26 -3
- package/dist/registry.js.map +1 -1
- package/dist/runner.d.ts +2 -2
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +78 -20
- package/dist/runner.js.map +1 -1
- package/dist/server-spawn.d.ts +2 -1
- package/dist/server-spawn.d.ts.map +1 -1
- package/dist/server-spawn.js +58 -34
- package/dist/server-spawn.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__24a1e50a._.js +0 -21
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__32f115c9._.js +0 -6
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__476a1712._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__4ddcabf2._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__ad593585._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__afd8e13b._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__dd7ee810._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__ff3004de._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_53472598._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_863b6ca8._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_f7347c74._.js +0 -5
- package/.next/standalone/.next/static/chunks/2243ff2814e7a781.js +0 -3
- package/.next/standalone/.next/static/chunks/50531467396cea91.css +0 -1
- package/.next/standalone/.next/static/chunks/8f288c01f8d7ef2d.js +0 -1
- package/.next/standalone/.next/static/chunks/abab1b00b2788443.js +0 -4
- package/.next/standalone/.next/static/chunks/d250d7f6f0a8c325.js +0 -1
- package/.next/standalone/.next/static/chunks/d7a572a8b7eb1ec8.js +0 -1
- package/.next/standalone/.next/static/chunks/fb1b0b9da3f03023.js +0 -4
- /package/.next/standalone/.next/static/{LoGIEEP4cORCqcFv-Ywg0 → 5JsV7rfAEOIwNOQPaX3UP}/_buildManifest.js +0 -0
- /package/.next/standalone/.next/static/{LoGIEEP4cORCqcFv-Ywg0 → 5JsV7rfAEOIwNOQPaX3UP}/_clientMiddlewareManifest.json +0 -0
- /package/.next/standalone/.next/static/{LoGIEEP4cORCqcFv-Ywg0 → 5JsV7rfAEOIwNOQPaX3UP}/_ssgManifest.js +0 -0
package/dist/registry.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.registerEval = registerEval;
|
|
4
4
|
exports.getRegisteredEvals = getRegisteredEvals;
|
|
5
|
+
exports.getSessionScopedEvals = getSessionScopedEvals;
|
|
6
|
+
exports.getSubagentScopedEvals = getSubagentScopedEvals;
|
|
7
|
+
exports.hasSubagentEvals = hasSubagentEvals;
|
|
5
8
|
exports.hasEvals = hasEvals;
|
|
6
9
|
exports.clearEvals = clearEvals;
|
|
7
10
|
const REGISTRY_KEY = "__CLAUDEYE_EVAL_REGISTRY__";
|
|
@@ -12,20 +15,40 @@ function getRegistry() {
|
|
|
12
15
|
}
|
|
13
16
|
return g[REGISTRY_KEY];
|
|
14
17
|
}
|
|
15
|
-
function registerEval(name, fn) {
|
|
18
|
+
function registerEval(name, fn, condition, scope = 'session', subagentType) {
|
|
16
19
|
const registry = getRegistry();
|
|
20
|
+
const entry = { name, fn, scope };
|
|
21
|
+
if (condition)
|
|
22
|
+
entry.condition = condition;
|
|
23
|
+
if (subagentType)
|
|
24
|
+
entry.subagentType = subagentType;
|
|
17
25
|
// Replace if an eval with the same name already exists
|
|
18
26
|
const idx = registry.findIndex((e) => e.name === name);
|
|
19
27
|
if (idx >= 0) {
|
|
20
|
-
registry[idx] =
|
|
28
|
+
registry[idx] = entry;
|
|
21
29
|
}
|
|
22
30
|
else {
|
|
23
|
-
registry.push(
|
|
31
|
+
registry.push(entry);
|
|
24
32
|
}
|
|
25
33
|
}
|
|
26
34
|
function getRegisteredEvals() {
|
|
27
35
|
return getRegistry();
|
|
28
36
|
}
|
|
37
|
+
function getSessionScopedEvals() {
|
|
38
|
+
return getRegistry().filter((e) => e.scope === 'session' || e.scope === 'both');
|
|
39
|
+
}
|
|
40
|
+
function getSubagentScopedEvals(subagentType) {
|
|
41
|
+
return getRegistry().filter((e) => {
|
|
42
|
+
if (e.scope !== 'subagent' && e.scope !== 'both')
|
|
43
|
+
return false;
|
|
44
|
+
if (e.subagentType && subagentType && e.subagentType !== subagentType)
|
|
45
|
+
return false;
|
|
46
|
+
return true;
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
function hasSubagentEvals() {
|
|
50
|
+
return getRegistry().some((e) => e.scope === 'subagent' || e.scope === 'both');
|
|
51
|
+
}
|
|
29
52
|
function hasEvals() {
|
|
30
53
|
return getRegistry().length > 0;
|
|
31
54
|
}
|
package/dist/registry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../lib/evals/registry.ts"],"names":[],"mappings":";;AAqBA,
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../lib/evals/registry.ts"],"names":[],"mappings":";;AAqBA,oCAkBC;AAED,gDAEC;AAED,sDAEC;AAED,wDAMC;AAED,4CAEC;AAED,4BAEC;AAED,gCAGC;AA7DD,MAAM,YAAY,GAAG,4BAA4B,CAAC;AAMlD,SAAS,WAAW;IAClB,MAAM,CAAC,GAAG,UAAgC,CAAC;IAC3C,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC;QACrB,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC;IACD,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC;AACzB,CAAC;AAED,SAAgB,YAAY,CAC1B,IAAY,EACZ,EAAgB,EAChB,SAA6B,EAC7B,QAAmB,SAAS,EAC5B,YAAqB;IAErB,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,KAAK,GAAmB,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;IAClD,IAAI,SAAS;QAAE,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;IAC3C,IAAI,YAAY;QAAE,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;IACpD,uDAAuD;IACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACvD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QACb,QAAQ,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB;IAChC,OAAO,WAAW,EAAE,CAAC;AACvB,CAAC;AAED,SAAgB,qBAAqB;IACnC,OAAO,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;AAClF,CAAC;AAED,SAAgB,sBAAsB,CAAC,YAAqB;IAC1D,OAAO,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM;YAAE,OAAO,KAAK,CAAC;QAC/D,IAAI,CAAC,CAAC,YAAY,IAAI,YAAY,IAAI,CAAC,CAAC,YAAY,KAAK,YAAY;YAAE,OAAO,KAAK,CAAC;QACpF,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,gBAAgB;IAC9B,OAAO,WAAW,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;AACjF,CAAC;AAED,SAAgB,QAAQ;IACtB,OAAO,WAAW,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AAClC,CAAC;AAED,SAAgB,UAAU;IACxB,MAAM,CAAC,GAAG,UAAgC,CAAC;IAC3C,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC;AACvB,CAAC"}
|
package/dist/runner.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import type { EvalLogEntry, EvalLogStats, EvalRunSummary } from "./types";
|
|
2
|
-
export declare function runAllEvals(entries: EvalLogEntry[], stats: EvalLogStats, projectName: string, sessionId: string): Promise<EvalRunSummary>;
|
|
1
|
+
import type { EvalContext, EvalLogEntry, EvalLogStats, EvalRunSummary, RegisteredEval } from "./types";
|
|
2
|
+
export declare function runAllEvals(entries: EvalLogEntry[], stats: EvalLogStats, projectName: string, sessionId: string, evalsToRun?: RegisteredEval[], contextOverrides?: Partial<EvalContext>): Promise<EvalRunSummary>;
|
|
3
3
|
//# sourceMappingURL=runner.d.ts.map
|
package/dist/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../lib/evals/runner.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../lib/evals/runner.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAiB,cAAc,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAOtH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,YAAY,EAAE,EACvB,KAAK,EAAE,YAAY,EACnB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,cAAc,EAAE,EAC7B,gBAAgB,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GACtC,OAAO,CAAC,cAAc,CAAC,CA+FzB"}
|
package/dist/runner.js
CHANGED
|
@@ -6,52 +6,110 @@ exports.runAllEvals = runAllEvals;
|
|
|
6
6
|
* Each eval is individually try/caught so one failure doesn't block others.
|
|
7
7
|
*/
|
|
8
8
|
const registry_1 = require("./registry");
|
|
9
|
+
const condition_registry_1 = require("./condition-registry");
|
|
9
10
|
function clampScore(score) {
|
|
10
11
|
if (score === undefined || score === null)
|
|
11
12
|
return 1;
|
|
12
13
|
return Math.max(0, Math.min(1, score));
|
|
13
14
|
}
|
|
14
|
-
async function runAllEvals(entries, stats, projectName, sessionId) {
|
|
15
|
-
const registeredEvals = (0, registry_1.getRegisteredEvals)();
|
|
15
|
+
async function runAllEvals(entries, stats, projectName, sessionId, evalsToRun, contextOverrides) {
|
|
16
|
+
const registeredEvals = evalsToRun !== null && evalsToRun !== void 0 ? evalsToRun : (0, registry_1.getRegisteredEvals)();
|
|
16
17
|
const results = [];
|
|
17
18
|
const overallStart = performance.now();
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
const context = Object.assign({ entries, stats, projectName, sessionId, scope: 'session' }, contextOverrides);
|
|
20
|
+
// Check global condition first
|
|
21
|
+
const globalCondition = (0, condition_registry_1.getGlobalCondition)();
|
|
22
|
+
let globalSkip = false;
|
|
23
|
+
if (globalCondition) {
|
|
20
24
|
try {
|
|
21
|
-
const result = await
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
message: result.message,
|
|
28
|
-
metadata: result.metadata,
|
|
29
|
-
durationMs,
|
|
30
|
-
});
|
|
25
|
+
const result = await globalCondition(context);
|
|
26
|
+
if (!result)
|
|
27
|
+
globalSkip = true;
|
|
28
|
+
}
|
|
29
|
+
catch (_a) {
|
|
30
|
+
globalSkip = true;
|
|
31
31
|
}
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
}
|
|
33
|
+
if (globalSkip) {
|
|
34
|
+
// All evals skipped due to global condition
|
|
35
|
+
for (const { name } of registeredEvals) {
|
|
34
36
|
results.push({
|
|
35
37
|
name,
|
|
36
38
|
pass: false,
|
|
37
39
|
score: 0,
|
|
38
|
-
durationMs,
|
|
39
|
-
|
|
40
|
+
durationMs: 0,
|
|
41
|
+
skipped: true,
|
|
40
42
|
});
|
|
41
43
|
}
|
|
42
44
|
}
|
|
45
|
+
else {
|
|
46
|
+
for (const { name, fn, condition } of registeredEvals) {
|
|
47
|
+
// Check per-eval condition
|
|
48
|
+
if (condition) {
|
|
49
|
+
try {
|
|
50
|
+
const shouldRun = await condition(context);
|
|
51
|
+
if (!shouldRun) {
|
|
52
|
+
results.push({
|
|
53
|
+
name,
|
|
54
|
+
pass: false,
|
|
55
|
+
score: 0,
|
|
56
|
+
durationMs: 0,
|
|
57
|
+
skipped: true,
|
|
58
|
+
});
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
results.push({
|
|
64
|
+
name,
|
|
65
|
+
pass: false,
|
|
66
|
+
score: 0,
|
|
67
|
+
durationMs: 0,
|
|
68
|
+
error: `Condition error: ${err instanceof Error ? err.message : String(err)}`,
|
|
69
|
+
});
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const start = performance.now();
|
|
74
|
+
try {
|
|
75
|
+
const result = await fn(context);
|
|
76
|
+
const durationMs = Math.round(performance.now() - start);
|
|
77
|
+
results.push({
|
|
78
|
+
name,
|
|
79
|
+
pass: result.pass,
|
|
80
|
+
score: clampScore(result.score),
|
|
81
|
+
message: result.message,
|
|
82
|
+
metadata: result.metadata,
|
|
83
|
+
durationMs,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
catch (err) {
|
|
87
|
+
const durationMs = Math.round(performance.now() - start);
|
|
88
|
+
results.push({
|
|
89
|
+
name,
|
|
90
|
+
pass: false,
|
|
91
|
+
score: 0,
|
|
92
|
+
durationMs,
|
|
93
|
+
error: err instanceof Error ? err.message : String(err),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
43
98
|
const totalDurationMs = Math.round(performance.now() - overallStart);
|
|
44
99
|
let passCount = 0;
|
|
45
100
|
let failCount = 0;
|
|
46
101
|
let errorCount = 0;
|
|
102
|
+
let skippedCount = 0;
|
|
47
103
|
for (const r of results) {
|
|
48
|
-
if (r.
|
|
104
|
+
if (r.skipped)
|
|
105
|
+
skippedCount++;
|
|
106
|
+
else if (r.error)
|
|
49
107
|
errorCount++;
|
|
50
108
|
else if (r.pass)
|
|
51
109
|
passCount++;
|
|
52
110
|
else
|
|
53
111
|
failCount++;
|
|
54
112
|
}
|
|
55
|
-
return { results, totalDurationMs, passCount, failCount, errorCount };
|
|
113
|
+
return { results, totalDurationMs, passCount, failCount, errorCount, skippedCount };
|
|
56
114
|
}
|
|
57
115
|
//# sourceMappingURL=runner.js.map
|
package/dist/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../lib/evals/runner.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../lib/evals/runner.ts"],"names":[],"mappings":";;AAaA,kCAsGC;AAnHD;;;GAGG;AACH,yCAAgD;AAChD,6DAA0D;AAG1D,SAAS,UAAU,CAAC,KAAyB;IAC3C,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;AACzC,CAAC;AAEM,KAAK,UAAU,WAAW,CAC/B,OAAuB,EACvB,KAAmB,EACnB,WAAmB,EACnB,SAAiB,EACjB,UAA6B,EAC7B,gBAAuC;IAEvC,MAAM,eAAe,GAAG,UAAU,aAAV,UAAU,cAAV,UAAU,GAAI,IAAA,6BAAkB,GAAE,CAAC;IAC3D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,YAAY,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACvC,MAAM,OAAO,mBAAkB,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,IAAK,gBAAgB,CAAE,CAAC;IAE/G,+BAA+B;IAC/B,MAAM,eAAe,GAAG,IAAA,uCAAkB,GAAE,CAAC;IAC7C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,eAAe,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM;gBAAE,UAAU,GAAG,IAAI,CAAC;QACjC,CAAC;QAAC,WAAM,CAAC;YACP,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,UAAU,EAAE,CAAC;QACf,4CAA4C;QAC5C,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI;gBACJ,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,CAAC;gBACR,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,eAAe,EAAE,CAAC;YACtD,2BAA2B;YAC3B,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,CAAC;oBAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;wBACf,OAAO,CAAC,IAAI,CAAC;4BACX,IAAI;4BACJ,IAAI,EAAE,KAAK;4BACX,KAAK,EAAE,CAAC;4BACR,UAAU,EAAE,CAAC;4BACb,OAAO,EAAE,IAAI;yBACd,CAAC,CAAC;wBACH,SAAS;oBACX,CAAC;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI;wBACJ,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,UAAU,EAAE,CAAC;wBACb,KAAK,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;qBAC9E,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;YACH,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;gBACjC,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI;oBACJ,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,UAAU;iBACX,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI;oBACJ,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,UAAU;oBACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC,CAAC;IACrE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,OAAO;YAAE,YAAY,EAAE,CAAC;aACzB,IAAI,CAAC,CAAC,KAAK;YAAE,UAAU,EAAE,CAAC;aAC1B,IAAI,CAAC,CAAC,IAAI;YAAE,SAAS,EAAE,CAAC;;YACxB,SAAS,EAAE,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;AACtF,CAAC"}
|
package/dist/server-spawn.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
interface SpawnOptions {
|
|
2
2
|
open?: boolean;
|
|
3
|
+
host?: string;
|
|
3
4
|
}
|
|
4
|
-
export declare function findAvailablePort(preferred: number): Promise<number>;
|
|
5
|
+
export declare function findAvailablePort(preferred: number, host?: string): Promise<number>;
|
|
5
6
|
export declare function openBrowser(url: string): void;
|
|
6
7
|
export declare function waitForServer(url: string, timeoutMs?: number, intervalMs?: number): Promise<void>;
|
|
7
8
|
export declare function spawnServer(preferredPort: number, options: SpawnOptions): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-spawn.d.ts","sourceRoot":"","sources":["../lib/evals/server-spawn.ts"],"names":[],"mappings":"AAWA,UAAU,YAAY;IACpB,IAAI,CAAC,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"server-spawn.d.ts","sourceRoot":"","sources":["../lib/evals/server-spawn.ts"],"names":[],"mappings":"AAWA,UAAU,YAAY;IACpB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAiBD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,SAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAiBxF;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAa7C;AAED,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAS,EAAE,UAAU,SAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAWpG;AAqDD,wBAAsB,WAAW,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CA0C7F"}
|
package/dist/server-spawn.js
CHANGED
|
@@ -14,20 +14,34 @@ const node_net_1 = require("node:net");
|
|
|
14
14
|
const node_path_1 = require("node:path");
|
|
15
15
|
const node_fs_1 = require("node:fs");
|
|
16
16
|
const node_os_1 = require("node:os");
|
|
17
|
-
function
|
|
18
|
-
|
|
17
|
+
function getNetworkAddress() {
|
|
18
|
+
for (const addrs of Object.values((0, node_os_1.networkInterfaces)())) {
|
|
19
|
+
for (const iface of addrs) {
|
|
20
|
+
if (iface.family === "IPv4" && !iface.internal) {
|
|
21
|
+
return iface.address;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return "0.0.0.0";
|
|
26
|
+
}
|
|
27
|
+
function resolveBindAddress(host) {
|
|
28
|
+
return host === "localhost" ? "127.0.0.1" : host;
|
|
29
|
+
}
|
|
30
|
+
function findAvailablePort(preferred, host = "localhost") {
|
|
31
|
+
const bindAddr = resolveBindAddress(host);
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
19
33
|
const srv = (0, node_net_1.createServer)();
|
|
20
|
-
srv.listen(preferred,
|
|
21
|
-
srv.close(() =>
|
|
34
|
+
srv.listen(preferred, bindAddr, () => {
|
|
35
|
+
srv.close(() => resolve(preferred));
|
|
22
36
|
});
|
|
23
37
|
srv.on("error", () => {
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
const addr =
|
|
38
|
+
const fallback = (0, node_net_1.createServer)();
|
|
39
|
+
fallback.listen(0, bindAddr, () => {
|
|
40
|
+
const addr = fallback.address();
|
|
27
41
|
const port = typeof addr === "object" && addr ? addr.port : 0;
|
|
28
|
-
|
|
42
|
+
fallback.close(() => resolve(port));
|
|
29
43
|
});
|
|
30
|
-
|
|
44
|
+
fallback.on("error", reject);
|
|
31
45
|
});
|
|
32
46
|
});
|
|
33
47
|
}
|
|
@@ -81,41 +95,52 @@ function resolveServerInfo() {
|
|
|
81
95
|
// Fallback to dev mode
|
|
82
96
|
return { mode: "dev" };
|
|
83
97
|
}
|
|
84
|
-
|
|
85
|
-
const port = await findAvailablePort(preferredPort);
|
|
86
|
-
if (port !== preferredPort) {
|
|
87
|
-
console.log(`Port ${preferredPort} is busy, using port ${port} instead.`);
|
|
88
|
-
}
|
|
89
|
-
const url = `http://localhost:${port}`;
|
|
90
|
-
const evalsModule = process.argv[1] ? (0, node_path_1.resolve)(process.argv[1]) : undefined;
|
|
98
|
+
function logServerInfo(host, port, localUrl, evalsModule) {
|
|
91
99
|
console.log(`Starting Claudeye dashboard...`);
|
|
92
100
|
if (evalsModule) {
|
|
93
101
|
console.log(` Evals: ${evalsModule}`);
|
|
94
102
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
env.CLAUDEYE_EVALS_MODULE = evalsModule;
|
|
103
|
+
if (host === "0.0.0.0") {
|
|
104
|
+
console.log(` Local: ${localUrl}`);
|
|
105
|
+
console.log(` Network: http://${getNetworkAddress()}:${port}`);
|
|
99
106
|
}
|
|
100
|
-
|
|
101
|
-
|
|
107
|
+
else {
|
|
108
|
+
console.log(` URL: http://${host}:${port}`);
|
|
109
|
+
}
|
|
110
|
+
console.log();
|
|
111
|
+
}
|
|
112
|
+
function spawnChildProcess(serverInfo, env, port) {
|
|
102
113
|
if (serverInfo.mode === "standalone") {
|
|
103
|
-
|
|
114
|
+
return (0, node_child_process_1.spawn)(process.execPath, [serverInfo.script], {
|
|
104
115
|
cwd: serverInfo.cwd,
|
|
105
116
|
stdio: "inherit",
|
|
106
117
|
env,
|
|
107
118
|
});
|
|
108
119
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
120
|
+
return (0, node_child_process_1.spawn)("npx", ["next", "dev", "--port", String(port)], {
|
|
121
|
+
stdio: "inherit",
|
|
122
|
+
shell: true,
|
|
123
|
+
env,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
async function spawnServer(preferredPort, options) {
|
|
127
|
+
var _a;
|
|
128
|
+
const host = (_a = options.host) !== null && _a !== void 0 ? _a : "localhost";
|
|
129
|
+
const port = await findAvailablePort(preferredPort, host);
|
|
130
|
+
if (port !== preferredPort) {
|
|
131
|
+
console.log(`Port ${preferredPort} is busy, using port ${port} instead.`);
|
|
132
|
+
}
|
|
133
|
+
const localUrl = `http://localhost:${port}`;
|
|
134
|
+
const evalsModule = process.argv[1] ? (0, node_path_1.resolve)(process.argv[1]) : undefined;
|
|
135
|
+
logServerInfo(host, port, localUrl, evalsModule);
|
|
136
|
+
const env = Object.assign(Object.assign({}, process.env), { PORT: String(port), HOSTNAME: host });
|
|
137
|
+
if (evalsModule) {
|
|
138
|
+
env.CLAUDEYE_EVALS_MODULE = evalsModule;
|
|
115
139
|
}
|
|
140
|
+
const child = spawnChildProcess(resolveServerInfo(), env, port);
|
|
116
141
|
if (options.open) {
|
|
117
|
-
waitForServer(
|
|
118
|
-
console.warn(`\nServer did not respond within 15 s.\nOpen manually: ${
|
|
142
|
+
waitForServer(localUrl).then(() => openBrowser(localUrl)).catch(() => {
|
|
143
|
+
console.warn(`\nServer did not respond within 15 s.\nOpen manually: ${localUrl}\n`);
|
|
119
144
|
});
|
|
120
145
|
}
|
|
121
146
|
// Forward signals for clean shutdown
|
|
@@ -125,12 +150,11 @@ async function spawnServer(preferredPort, options) {
|
|
|
125
150
|
};
|
|
126
151
|
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
127
152
|
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
128
|
-
|
|
153
|
+
// Block until the child exits — server runs until killed
|
|
154
|
+
await new Promise(() => {
|
|
129
155
|
child.on("exit", (code) => {
|
|
130
156
|
process.exit(code !== null && code !== void 0 ? code : 0);
|
|
131
157
|
});
|
|
132
|
-
// Never resolves — server runs until killed
|
|
133
|
-
void res;
|
|
134
158
|
});
|
|
135
159
|
}
|
|
136
160
|
//# sourceMappingURL=server-spawn.js.map
|
package/dist/server-spawn.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server-spawn.js","sourceRoot":"","sources":["../lib/evals/server-spawn.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"server-spawn.js","sourceRoot":"","sources":["../lib/evals/server-spawn.ts"],"names":[],"mappings":";;AA+BA,8CAiBC;AAED,kCAaC;AAED,sCAWC;AAqDD,kCA0CC;AA3KD;;;;GAIG;AACH,2DAAqD;AACrD,uCAAwC;AACxC,yCAA6C;AAC7C,qCAAqC;AACrC,qCAAsD;AAOtD,SAAS,iBAAiB;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAA,2BAAiB,GAAE,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,KAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;gBAC/C,OAAO,KAAK,CAAC,OAAO,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;AACnD,CAAC;AAED,SAAgB,iBAAiB,CAAC,SAAiB,EAAE,IAAI,GAAG,WAAW;IACrE,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,IAAA,uBAAY,GAAE,CAAC;QAC3B,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,EAAE;YACnC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,MAAM,QAAQ,GAAG,IAAA,uBAAY,GAAE,CAAC;YAChC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE;gBAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9D,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACtC,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,WAAW,CAAC,GAAW;IACrC,MAAM,IAAI,GAAG,IAAA,kBAAQ,GAAE,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,IAAA,6BAAQ,EAAC,SAAS,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACjD,CAAC;aAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YAC5B,IAAA,6BAAQ,EAAC,oBAAoB,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,IAAA,6BAAQ,EAAC,aAAa,GAAG,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAAC,WAAM,CAAC;QACP,yCAAyC;IAC3C,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,SAAS,GAAG,KAAM,EAAE,UAAU,GAAG,GAAG;IACnF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACxC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACxD,OAAO;QACT,CAAC;QAAC,WAAM,CAAC;YACP,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;AAC9C,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB;IACxB,2CAA2C;IAC3C,6DAA6D;IAC7D,iEAAiE;IACjE,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,gBAAgB,GAAG,IAAA,mBAAO,EAAC,GAAG,EAAE,OAAO,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QAC1E,IAAI,IAAA,oBAAU,EAAC,gBAAgB,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,GAAG,EAAE,IAAA,mBAAO,EAAC,GAAG,EAAE,OAAO,EAAE,YAAY,CAAC,EAAE,CAAC;QACpG,CAAC;QACD,GAAG,GAAG,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,uBAAuB;IACvB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,IAAY,EAAE,QAAgB,EAAE,WAAoB;IACvF,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;IAC9C,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,eAAe,WAAW,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,eAAe,QAAQ,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,sBAAsB,iBAAiB,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;IACnE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,UAAgD,EAAE,GAAsB,EAAE,IAAY;IAC/G,IAAI,UAAU,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QACrC,OAAO,IAAA,0BAAK,EAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE;YAClD,GAAG,EAAE,UAAU,CAAC,GAAG;YACnB,KAAK,EAAE,SAAS;YAChB,GAAG;SACJ,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAA,0BAAK,EAAC,KAAK,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE;QAC3D,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,IAAI;QACX,GAAG;KACJ,CAAC,CAAC;AACL,CAAC;AAEM,KAAK,UAAU,WAAW,CAAC,aAAqB,EAAE,OAAqB;;IAC5E,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,IAAI,mCAAI,WAAW,CAAC;IACzC,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;IAC1D,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,QAAQ,aAAa,wBAAwB,IAAI,WAAW,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,QAAQ,GAAG,oBAAoB,IAAI,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAA,mBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;IAEjD,MAAM,GAAG,mCACJ,OAAO,CAAC,GAAG,KACd,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAClB,QAAQ,EAAE,IAAI,GACf,CAAC;IACF,IAAI,WAAW,EAAE,CAAC;QAChB,GAAG,CAAC,qBAAqB,GAAG,WAAW,CAAC;IAC1C,CAAC;IAED,MAAM,KAAK,GAAG,iBAAiB,CAAC,iBAAiB,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IAEhE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,aAAa,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;YACnE,OAAO,CAAC,IAAI,CAAC,yDAAyD,QAAQ,IAAI,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC;IAED,qCAAqC;IACrC,MAAM,QAAQ,GAAG,CAAC,MAAsB,EAAQ,EAAE;QAChD,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;IAClD,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IAEjD,yDAAyD;IACzD,MAAM,IAAI,OAAO,CAAQ,GAAG,EAAE;QAC5B,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,OAAO,CAAC,IAAI,CAAC,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -34,12 +34,19 @@ export interface EvalLogStats {
|
|
|
34
34
|
duration: string;
|
|
35
35
|
models: string[];
|
|
36
36
|
}
|
|
37
|
+
/** Scope for eval/enrichment registration. */
|
|
38
|
+
export type EvalScope = 'session' | 'subagent' | 'both';
|
|
37
39
|
/** Context passed to each eval function. */
|
|
38
40
|
export interface EvalContext {
|
|
39
41
|
entries: EvalLogEntry[];
|
|
40
42
|
stats: EvalLogStats;
|
|
41
43
|
projectName: string;
|
|
42
44
|
sessionId: string;
|
|
45
|
+
scope: 'session' | 'subagent';
|
|
46
|
+
subagentId?: string;
|
|
47
|
+
subagentType?: string;
|
|
48
|
+
subagentDescription?: string;
|
|
49
|
+
parentSessionId?: string;
|
|
43
50
|
}
|
|
44
51
|
/** Result returned by an eval function. */
|
|
45
52
|
export interface EvalResult {
|
|
@@ -50,10 +57,15 @@ export interface EvalResult {
|
|
|
50
57
|
}
|
|
51
58
|
/** An eval function signature. */
|
|
52
59
|
export type EvalFunction = (context: EvalContext) => EvalResult | Promise<EvalResult>;
|
|
60
|
+
/** A condition function that gates eval/enrichment execution. */
|
|
61
|
+
export type ConditionFunction = (context: EvalContext) => boolean | Promise<boolean>;
|
|
53
62
|
/** An eval function stored in the registry. */
|
|
54
63
|
export interface RegisteredEval {
|
|
55
64
|
name: string;
|
|
56
65
|
fn: EvalFunction;
|
|
66
|
+
condition?: ConditionFunction;
|
|
67
|
+
scope: EvalScope;
|
|
68
|
+
subagentType?: string;
|
|
57
69
|
}
|
|
58
70
|
/** Result of running a single eval. */
|
|
59
71
|
export interface EvalRunResult {
|
|
@@ -64,6 +76,7 @@ export interface EvalRunResult {
|
|
|
64
76
|
metadata?: Record<string, unknown>;
|
|
65
77
|
durationMs: number;
|
|
66
78
|
error?: string;
|
|
79
|
+
skipped?: boolean;
|
|
67
80
|
}
|
|
68
81
|
/** Summary of running all registered evals. */
|
|
69
82
|
export interface EvalRunSummary {
|
|
@@ -72,5 +85,6 @@ export interface EvalRunSummary {
|
|
|
72
85
|
passCount: number;
|
|
73
86
|
failCount: number;
|
|
74
87
|
errorCount: number;
|
|
88
|
+
skippedCount: number;
|
|
75
89
|
}
|
|
76
90
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/evals/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,0EAA0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wEAAwE;AACxE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,4CAA4C;AAC5C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,KAAK,EAAE,YAAY,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../lib/evals/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,0EAA0E;AAC1E,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAAC;QACrC,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wEAAwE;AACxE,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,8CAA8C;AAC9C,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,MAAM,CAAC;AAExD,4CAA4C;AAC5C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,KAAK,EAAE,YAAY,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,SAAS,GAAG,UAAU,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,2CAA2C;AAC3C,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,kCAAkC;AAClC,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAEtF,iEAAiE;AACjE,MAAM,MAAM,iBAAiB,GAAG,CAAC,OAAO,EAAE,WAAW,KAAK,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAErF,+CAA+C;AAC/C,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,YAAY,CAAC;IACjB,SAAS,CAAC,EAAE,iBAAiB,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,uCAAuC;AACvC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,+CAA+C;AAC/C,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,aAAa,EAAE,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB"}
|