@sentry/warden 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mcp.json +8 -0
- package/README.md +4 -0
- package/agents.toml +7 -0
- package/conductor.json +8 -0
- package/dist/cli/fix.d.ts +3 -1
- package/dist/cli/fix.d.ts.map +1 -1
- package/dist/cli/fix.js +91 -59
- package/dist/cli/fix.js.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +29 -8
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/output/ink-runner.d.ts +6 -17
- package/dist/cli/output/ink-runner.d.ts.map +1 -1
- package/dist/cli/output/ink-runner.js +138 -122
- package/dist/cli/output/ink-runner.js.map +1 -1
- package/dist/cli/output/jsonl.d.ts +211 -58
- package/dist/cli/output/jsonl.d.ts.map +1 -1
- package/dist/cli/output/jsonl.js +81 -4
- package/dist/cli/output/jsonl.js.map +1 -1
- package/dist/cli/output/reporter.d.ts +7 -3
- package/dist/cli/output/reporter.d.ts.map +1 -1
- package/dist/cli/output/reporter.js +15 -1
- package/dist/cli/output/reporter.js.map +1 -1
- package/dist/cli/output/tasks.d.ts +10 -1
- package/dist/cli/output/tasks.d.ts.map +1 -1
- package/dist/cli/output/tasks.js +145 -32
- package/dist/cli/output/tasks.js.map +1 -1
- package/dist/cli/terminal.d.ts +7 -1
- package/dist/cli/terminal.d.ts.map +1 -1
- package/dist/cli/terminal.js +31 -9
- package/dist/cli/terminal.js.map +1 -1
- package/dist/config/schema.d.ts +3 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +3 -2
- package/dist/config/schema.js.map +1 -1
- package/dist/evals/index.d.ts +22 -0
- package/dist/evals/index.d.ts.map +1 -0
- package/dist/evals/index.js +92 -0
- package/dist/evals/index.js.map +1 -0
- package/dist/evals/judge.d.ts +12 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +171 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/runner.d.ts +18 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +133 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/{examples → evals}/setup.d.ts.map +1 -1
- package/dist/evals/setup.js.map +1 -0
- package/dist/evals/types.d.ts +166 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/evals/types.js +134 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/output/dedup.d.ts.map +1 -1
- package/dist/output/dedup.js +29 -57
- package/dist/output/dedup.js.map +1 -1
- package/dist/output/github-checks.d.ts.map +1 -1
- package/dist/output/github-checks.js +37 -8
- package/dist/output/github-checks.js.map +1 -1
- package/dist/output/renderer.d.ts.map +1 -1
- package/dist/output/renderer.js +16 -1
- package/dist/output/renderer.js.map +1 -1
- package/dist/output/stale.d.ts +1 -0
- package/dist/output/stale.d.ts.map +1 -1
- package/dist/output/stale.js +14 -8
- package/dist/output/stale.js.map +1 -1
- package/dist/sdk/analyze.d.ts +11 -0
- package/dist/sdk/analyze.d.ts.map +1 -1
- package/dist/sdk/analyze.js +99 -19
- package/dist/sdk/analyze.js.map +1 -1
- package/dist/sdk/extract.d.ts +54 -0
- package/dist/sdk/extract.d.ts.map +1 -1
- package/dist/sdk/extract.js +175 -2
- package/dist/sdk/extract.js.map +1 -1
- package/dist/sdk/model-pricing.json +6 -12
- package/dist/sdk/prompt.d.ts +2 -0
- package/dist/sdk/prompt.d.ts.map +1 -1
- package/dist/sdk/prompt.js +18 -8
- package/dist/sdk/prompt.js.map +1 -1
- package/dist/sdk/runner.d.ts +2 -2
- package/dist/sdk/runner.d.ts.map +1 -1
- package/dist/sdk/runner.js +1 -1
- package/dist/sdk/runner.js.map +1 -1
- package/dist/sdk/types.d.ts +11 -3
- package/dist/sdk/types.d.ts.map +1 -1
- package/dist/sdk/types.js +1 -1
- package/dist/sdk/types.js.map +1 -1
- package/dist/sentry.d.ts +15 -0
- package/dist/sentry.d.ts.map +1 -1
- package/dist/sentry.js +40 -1
- package/dist/sentry.js.map +1 -1
- package/dist/types/index.d.ts +19 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +25 -0
- package/dist/types/index.js.map +1 -1
- package/dist/utils/async.d.ts +14 -0
- package/dist/utils/async.d.ts.map +1 -1
- package/dist/utils/async.js +33 -0
- package/dist/utils/async.js.map +1 -1
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/index.js.map +1 -1
- package/evals/README.md +154 -0
- package/evals/bug-detection.yaml +56 -0
- package/evals/fixtures/ignores-style-issues/utils.ts +48 -0
- package/evals/fixtures/missing-await/cache.ts +45 -0
- package/evals/fixtures/null-property-access/handler.ts +36 -0
- package/evals/fixtures/off-by-one/paginator.ts +38 -0
- package/evals/fixtures/sql-injection/api.ts +59 -0
- package/evals/fixtures/stale-closure/counter.tsx +33 -0
- package/evals/fixtures/wrong-comparison/validator.ts +52 -0
- package/evals/fixtures/xss-reflected/server.ts +55 -0
- package/evals/precision.yaml +15 -0
- package/evals/security-scanning.yaml +24 -0
- package/evals/skills/bug-detection.md +33 -0
- package/evals/skills/precision.md +18 -0
- package/evals/skills/security-scanning.md +32 -0
- package/package.json +4 -2
- package/dist/examples/index.d.ts +0 -50
- package/dist/examples/index.d.ts.map +0 -1
- package/dist/examples/index.js +0 -104
- package/dist/examples/index.js.map +0 -1
- package/dist/examples/setup.js.map +0 -1
- /package/dist/{examples → evals}/setup.d.ts +0 -0
- /package/dist/{examples → evals}/setup.js +0 -0
package/dist/cli/terminal.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../src/cli/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,cAAc,EACd,aAAa,EACb,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,eAAe,GAAuC;IAC1D,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI;IACxB,IAAI,EAAE,KAAK,CAAC,GAAG;IACf,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,GAAG,EAAE,KAAK,CAAC,KAAK;IAChB,IAAI,EAAE,KAAK,CAAC,IAAI;CACjB,CAAC;AAOF;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACxC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAgB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEhD,+BAA+B;IAC/B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,6BAA6B;IAC7B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,kFAAkF;IACpF,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAElD,kCAAkC;IAClC,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAgB;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEpD,sEAAsE;IACtE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/G,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,aAAa;IACb,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEvC,4CAA4C;IAC5C,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAmB,EAAE,IAAgB;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEpG,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,WAAW;QAClB,IAAI;KACL,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEtB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,EAAE,CAAC;QACZ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC/C,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC1B,mDAAmD;YACnD,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,OAAO,GAAG,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,MAAmB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAEjD,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;IAEzD,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,iBAAiB,YAAY,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACxH,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,KAAK,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsB,EAAE,IAAiB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,qEAAqE;IACrE,MAAM,UAAU,GAAe,IAAI,IAAI;QACrC,KAAK,EAAE,IAAI;QACX,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAsB;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;QAC5C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,oBAAoB,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACrF,wBAAwB,EAAE,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACjG,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAsB,EAAE,QAA4B;IAC1F,IAAI,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9B,GAAG,MAAM;QACT,QAAQ,EAAE,wBAAwB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC9D,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE;YACP,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM,EAC5E,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,MAAM,EAC1E,CAAC,CACF;gBACD,GAAG,EAAE,OAAO,CAAC,MAAM,CACjB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,MAAM,EACvE,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;aACF;YACD,KAAK,EAAE,UAAU;SAClB;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../src/cli/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,cAAc,EACd,aAAa,EACb,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,eAAe,GAAuC;IAC1D,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI;IACxB,IAAI,EAAE,KAAK,CAAC,GAAG;IACf,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,GAAG,EAAE,KAAK,CAAC,KAAK;IAChB,IAAI,EAAE,KAAK,CAAC,IAAI;CACjB,CAAC;AAOF;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACxC,CAAC;AACH,CAAC;AAOD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAgB,EAAE,OAAuB;IACjE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEhD,+BAA+B;IAC/B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,6BAA6B;IAC7B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,kFAAkF;IACpF,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,mBAAmB,CAAC,MAAM,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;YACnF,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAElD,4EAA4E;IAC5E,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAgB;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEpD,sEAAsE;IACtE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/G,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,aAAa;IACb,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACnD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;YACnF,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEvC,4CAA4C;IAC5C,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAmB,EAAE,IAAgB,EAAE,OAAuB;IACvF,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEpG,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,WAAW;QAClB,IAAI;KACL,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEtB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,EAAE,CAAC;QACZ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC1B,mDAAmD;YACnD,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,OAAO,GAAG,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,MAAmB,EAAE,YAAuB,SAAS,CAAC,MAAM;IACjF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAEjD,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;IAEzD,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,iBAAiB,YAAY,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACxH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACtF,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,KAAK,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsB,EAAE,IAAiB,EAAE,OAAuB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,qEAAqE;IACrE,MAAM,UAAU,GAAe,IAAI,IAAI;QACrC,KAAK,EAAE,IAAI;QACX,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAsB;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;QAC5C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,oBAAoB,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACrF,wBAAwB,EAAE,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACjG,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAsB,EAAE,QAA4B;IAC1F,IAAI,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9B,GAAG,MAAM;QACT,QAAQ,EAAE,wBAAwB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC9D,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE;YACP,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM,EAC5E,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,MAAM,EAC1E,CAAC,CACF;gBACD,GAAG,EAAE,OAAO,CAAC,MAAM,CACjB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,MAAM,EACvE,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;aACF;YACD,KAAK,EAAE,UAAU;SAClB;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -59,7 +59,6 @@ export declare const SkillDefinitionSchema: z.ZodObject<{
|
|
|
59
59
|
WebSearch: "WebSearch";
|
|
60
60
|
}>>>;
|
|
61
61
|
}, z.core.$strip>>;
|
|
62
|
-
outputSchema: z.ZodOptional<z.ZodString>;
|
|
63
62
|
rootDir: z.ZodOptional<z.ZodString>;
|
|
64
63
|
}, z.core.$strip>;
|
|
65
64
|
export type SkillDefinition = z.infer<typeof SkillDefinitionSchema>;
|
|
@@ -208,6 +207,7 @@ export declare const ChunkingConfigSchema: z.ZodObject<{
|
|
|
208
207
|
maxGapLines: z.ZodDefault<z.ZodNumber>;
|
|
209
208
|
maxChunkSize: z.ZodDefault<z.ZodNumber>;
|
|
210
209
|
}, z.core.$strip>>;
|
|
210
|
+
maxContextFiles: z.ZodDefault<z.ZodNumber>;
|
|
211
211
|
}, z.core.$strip>;
|
|
212
212
|
export type ChunkingConfig = z.infer<typeof ChunkingConfigSchema>;
|
|
213
213
|
export declare const DefaultsSchema: z.ZodObject<{
|
|
@@ -249,6 +249,7 @@ export declare const DefaultsSchema: z.ZodObject<{
|
|
|
249
249
|
maxGapLines: z.ZodDefault<z.ZodNumber>;
|
|
250
250
|
maxChunkSize: z.ZodDefault<z.ZodNumber>;
|
|
251
251
|
}, z.core.$strip>>;
|
|
252
|
+
maxContextFiles: z.ZodDefault<z.ZodNumber>;
|
|
252
253
|
}, z.core.$strip>>;
|
|
253
254
|
batchDelayMs: z.ZodOptional<z.ZodNumber>;
|
|
254
255
|
}, z.core.$strip>;
|
|
@@ -294,6 +295,7 @@ export declare const WardenConfigSchema: z.ZodObject<{
|
|
|
294
295
|
maxGapLines: z.ZodDefault<z.ZodNumber>;
|
|
295
296
|
maxChunkSize: z.ZodDefault<z.ZodNumber>;
|
|
296
297
|
}, z.core.$strip>>;
|
|
298
|
+
maxContextFiles: z.ZodDefault<z.ZodNumber>;
|
|
297
299
|
}, z.core.$strip>>;
|
|
298
300
|
batchDelayMs: z.ZodOptional<z.ZodNumber>;
|
|
299
301
|
}, z.core.$strip>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,cAAc;;;;;;;;;EASzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;iBAG3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAG1D,eAAO,MAAM,qBAAqB
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,cAAc;;;;;;;;;EASzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;iBAG3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAG1D,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAOhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAGpE,eAAO,MAAM,oBAAoB;;;;iBAO/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,iBAAiB;;;;EAAgD,CAAC;AAC/E,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8B9B,CAAC;AACF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAG9D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAuB5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,kBAAkB;;iBAG7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAG9D,eAAO,MAAM,iBAAiB;;;;;;;iBAK5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,oBAAoB;;;;iBAO/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;iBAO/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAwBzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgC3B,CAAC;AACL,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC"}
|
package/dist/config/schema.js
CHANGED
|
@@ -22,7 +22,6 @@ export const SkillDefinitionSchema = z.object({
|
|
|
22
22
|
description: z.string(),
|
|
23
23
|
prompt: z.string(),
|
|
24
24
|
tools: ToolConfigSchema.optional(),
|
|
25
|
-
outputSchema: z.string().optional(),
|
|
26
25
|
/** Directory where the skill was loaded from, for resolving resources (scripts/, references/, assets/) */
|
|
27
26
|
rootDir: z.string().optional(),
|
|
28
27
|
});
|
|
@@ -93,7 +92,7 @@ export const SkillConfigSchema = z.object({
|
|
|
93
92
|
});
|
|
94
93
|
// Runner configuration
|
|
95
94
|
export const RunnerConfigSchema = z.object({
|
|
96
|
-
/** Max concurrent
|
|
95
|
+
/** Max concurrent file analyses across all skills (default: 4) */
|
|
97
96
|
concurrency: z.number().int().positive().optional(),
|
|
98
97
|
});
|
|
99
98
|
// File pattern for chunking configuration
|
|
@@ -118,6 +117,8 @@ export const ChunkingConfigSchema = z.object({
|
|
|
118
117
|
filePatterns: z.array(FilePatternSchema).optional(),
|
|
119
118
|
/** Coalescing options for merging nearby hunks */
|
|
120
119
|
coalesce: CoalesceConfigSchema.optional(),
|
|
120
|
+
/** Max number of "other files" to list in hunk prompts for PR context. 0 disables the section entirely. Default: 50 */
|
|
121
|
+
maxContextFiles: z.number().int().nonnegative().default(50),
|
|
121
122
|
});
|
|
122
123
|
// Default configuration that skills inherit from
|
|
123
124
|
export const DefaultsSchema = z.object({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC;IACnC,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,WAAW;CACZ,CAAC,CAAC;AAGH,gCAAgC;AAChC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAGH,mBAAmB;AACnB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE;IAClC,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC;IACnC,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,WAAW;CACZ,CAAC,CAAC;AAGH,gCAAgC;AAChC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAGH,mBAAmB;AACnB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE;IAClC,0GAA0G;IAC1G,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAGH,kCAAkC;AAClC,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,oEAAoE;IACpE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,0DAA0D;IAC1D,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACvC,wDAAwD;IACxD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;CAClD,CAAC,CAAC;AAGH,uCAAuC;AACvC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;AAG/E,8DAA8D;AAC9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,2EAA2E;IAC3E,IAAI,EAAE,iBAAiB;IACvB,yDAAyD;IACzD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,kDAAkD;IAClD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mEAAmE;IACnE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qDAAqD;IACrD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,0EAA0E;IAC1E,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC,MAAM,CACP,CAAC,IAAI,EAAE,EAAE;IACP,4CAA4C;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,EACD;IACE,OAAO,EAAE,+CAA+C;IACxD,IAAI,EAAE,CAAC,SAAS,CAAC;CAClB,CACF,CAAC;AAGF,6CAA6C;AAC7C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,+BAA+B;IAC/B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,+BAA+B;IAC/B,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,yFAAyF;IACzF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,iDAAiD;IACjD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mEAAmE;IACnE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qDAAqD;IACrD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,yGAAyG;IACzG,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,8FAA8F;IAC9F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,uFAAuF;IACvF,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAGH,uBAAuB;AACvB,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,kEAAkE;IAClE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAGH,0CAA0C;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,+DAA+D;IAC/D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,kFAAkF;IAClF,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;CACjE,CAAC,CAAC;AAGH,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,6CAA6C;IAC7C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,yDAAyD;IACzD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACvD,8DAA8D;IAC9D,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACxD,CAAC,CAAC;AAGH,iEAAiE;AACjE,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,+CAA+C;IAC/C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE;IACnD,kDAAkD;IAClD,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IACzC,uHAAuH;IACvH,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;CAC5D,CAAC,CAAC;AAGH,iDAAiD;AACjD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,sDAAsD;IACtD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,qDAAqD;IACrD,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,8DAA8D;IAC9D,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mFAAmF;IACnF,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qEAAqE;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,sEAAsE;IACtE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,6EAA6E;IAC7E,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,+CAA+C;IAC/C,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,6GAA6G;IAC7G,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,qEAAqE;IACrE,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IACzC,+FAA+F;IAC/F,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAGH,iCAAiC;AACjC,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE;IACnC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,WAAW,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EAAE,0BAA0B,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACxE,IAAI,EAAE,CAAC,QAAQ,CAAC;SACjB,CAAC,CAAC;IACL,CAAC;IAED,sCAAsC;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC9E,GAAG,CAAC,QAAQ,CAAC;wBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;wBAC3B,OAAO,EAAE,qDAAqD;wBAC9D,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,CAAC;qBAC7B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { EvalFile, EvalMeta } from './types.js';
|
|
2
|
+
export type { EvalMeta };
|
|
3
|
+
/**
|
|
4
|
+
* Discover all YAML eval files in the evals directory.
|
|
5
|
+
* Returns absolute paths to .yaml files, sorted alphabetically.
|
|
6
|
+
*/
|
|
7
|
+
export declare function discoverEvalFiles(baseDir?: string): string[];
|
|
8
|
+
/**
|
|
9
|
+
* Load and validate a YAML eval file.
|
|
10
|
+
*/
|
|
11
|
+
export declare function loadEvalFile(filePath: string): EvalFile;
|
|
12
|
+
/**
|
|
13
|
+
* Resolve all eval scenarios from a YAML file into executable EvalMeta objects.
|
|
14
|
+
* Resolves relative paths for skills and fixtures against the evals directory.
|
|
15
|
+
*/
|
|
16
|
+
export declare function resolveEvalMetas(evalFile: EvalFile, yamlPath: string): EvalMeta[];
|
|
17
|
+
/**
|
|
18
|
+
* Discover and load all evals from YAML files. Returns a flat list of
|
|
19
|
+
* resolved EvalMeta objects ready for execution.
|
|
20
|
+
*/
|
|
21
|
+
export declare function discoverEvals(baseDir?: string): EvalMeta[];
|
|
22
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErD,YAAY,EAAE,QAAQ,EAAE,CAAC;AASzB;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAkB5D;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CA0BvD;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE,CAcjF;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAW1D"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
|
2
|
+
import { join, basename } from 'node:path';
|
|
3
|
+
import { parse as parseYaml } from 'yaml';
|
|
4
|
+
import { EvalFileSchema } from './types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Get the default evals directory path.
|
|
7
|
+
*/
|
|
8
|
+
function getEvalsDir() {
|
|
9
|
+
return join(import.meta.dirname, '..', '..', 'evals');
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Discover all YAML eval files in the evals directory.
|
|
13
|
+
* Returns absolute paths to .yaml files, sorted alphabetically.
|
|
14
|
+
*/
|
|
15
|
+
export function discoverEvalFiles(baseDir) {
|
|
16
|
+
const evalsDir = baseDir ?? getEvalsDir();
|
|
17
|
+
if (!existsSync(evalsDir)) {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
let entries;
|
|
21
|
+
try {
|
|
22
|
+
entries = readdirSync(evalsDir);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
return entries
|
|
28
|
+
.filter((e) => e.endsWith('.yaml') || e.endsWith('.yml'))
|
|
29
|
+
.map((e) => join(evalsDir, e))
|
|
30
|
+
.sort();
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Load and validate a YAML eval file.
|
|
34
|
+
*/
|
|
35
|
+
export function loadEvalFile(filePath) {
|
|
36
|
+
if (!existsSync(filePath)) {
|
|
37
|
+
throw new Error(`Eval file not found: ${filePath}`);
|
|
38
|
+
}
|
|
39
|
+
let content;
|
|
40
|
+
try {
|
|
41
|
+
content = readFileSync(filePath, 'utf-8');
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
throw new Error(`Failed to read ${filePath}: ${error}`);
|
|
45
|
+
}
|
|
46
|
+
let parsed;
|
|
47
|
+
try {
|
|
48
|
+
parsed = parseYaml(content);
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
throw new Error(`Failed to parse YAML in ${filePath}: ${error}`);
|
|
52
|
+
}
|
|
53
|
+
const validated = EvalFileSchema.safeParse(parsed);
|
|
54
|
+
if (!validated.success) {
|
|
55
|
+
const issues = validated.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join(', ');
|
|
56
|
+
throw new Error(`Invalid eval file ${filePath}: ${issues}`);
|
|
57
|
+
}
|
|
58
|
+
return validated.data;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Resolve all eval scenarios from a YAML file into executable EvalMeta objects.
|
|
62
|
+
* Resolves relative paths for skills and fixtures against the evals directory.
|
|
63
|
+
*/
|
|
64
|
+
export function resolveEvalMetas(evalFile, yamlPath) {
|
|
65
|
+
const evalsDir = join(yamlPath, '..');
|
|
66
|
+
const category = basename(yamlPath).replace(/\.ya?ml$/, '');
|
|
67
|
+
return evalFile.evals.map((scenario) => ({
|
|
68
|
+
name: scenario.name,
|
|
69
|
+
category,
|
|
70
|
+
given: scenario.given,
|
|
71
|
+
skillPath: join(evalsDir, evalFile.skill),
|
|
72
|
+
filePaths: scenario.files.map((f) => join(evalsDir, f)),
|
|
73
|
+
model: scenario.model ?? evalFile.model,
|
|
74
|
+
should_find: scenario.should_find,
|
|
75
|
+
should_not_find: scenario.should_not_find,
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Discover and load all evals from YAML files. Returns a flat list of
|
|
80
|
+
* resolved EvalMeta objects ready for execution.
|
|
81
|
+
*/
|
|
82
|
+
export function discoverEvals(baseDir) {
|
|
83
|
+
const yamlFiles = discoverEvalFiles(baseDir);
|
|
84
|
+
const allEvals = [];
|
|
85
|
+
for (const yamlPath of yamlFiles) {
|
|
86
|
+
const evalFile = loadEvalFile(yamlPath);
|
|
87
|
+
const metas = resolveEvalMetas(evalFile, yamlPath);
|
|
88
|
+
allEvals.push(...metas);
|
|
89
|
+
}
|
|
90
|
+
return allEvals;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAK5C;;GAEG;AACH,SAAS,WAAW;IAClB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAgB;IAChD,MAAM,QAAQ,GAAG,OAAO,IAAI,WAAW,EAAE,CAAC;IAE1C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;SAC7B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,SAAS,CAAC,IAAI,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAkB,EAAE,QAAgB;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAE5D,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ;QACR,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC;QACzC,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACvD,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK;QACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAe,EAAE,CAAC;IAEhC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Finding } from '../types/index.js';
|
|
2
|
+
import type { EvalMeta, JudgeResponse } from './types.js';
|
|
3
|
+
import type { UsageStats } from '../types/index.js';
|
|
4
|
+
export interface JudgeResult {
|
|
5
|
+
response: JudgeResponse;
|
|
6
|
+
usage: UsageStats;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Run the LLM judge to evaluate agent findings against eval assertions.
|
|
10
|
+
*/
|
|
11
|
+
export declare function runJudge(meta: EvalMeta, findings: Finding[], apiKey: string): Promise<JudgeResult>;
|
|
12
|
+
//# sourceMappingURL=judge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAIjD,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAMpD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,aAAa,CAAC;IACxB,KAAK,EAAE,UAAU,CAAC;CACnB;AA6ED;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,QAAQ,EACd,QAAQ,EAAE,OAAO,EAAE,EACnB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,CAAC,CAkFtB"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import { apiUsageToStats } from '../sdk/pricing.js';
|
|
3
|
+
import { emptyUsage } from '../sdk/usage.js';
|
|
4
|
+
import { extractJson } from '../sdk/haiku.js';
|
|
5
|
+
import { DEFAULT_EVAL_MODEL, JudgeResponseSchema } from './types.js';
|
|
6
|
+
const JUDGE_MODEL = DEFAULT_EVAL_MODEL;
|
|
7
|
+
const JUDGE_MAX_TOKENS = 4096;
|
|
8
|
+
const JUDGE_TIMEOUT_MS = 60_000;
|
|
9
|
+
/**
|
|
10
|
+
* Build the judge prompt that evaluates agent findings against BDD assertions.
|
|
11
|
+
*/
|
|
12
|
+
function buildJudgePrompt(meta, findings) {
|
|
13
|
+
const findingsBlock = findings.length > 0
|
|
14
|
+
? findings.map((f, i) => {
|
|
15
|
+
const loc = f.location ? ` (${f.location.path}:${f.location.startLine})` : '';
|
|
16
|
+
return ` [${i}] severity=${f.severity} confidence=${f.confidence ?? 'unset'}${loc}\n title: ${f.title}\n description: ${f.description}`;
|
|
17
|
+
}).join('\n\n')
|
|
18
|
+
: ' (no findings reported)';
|
|
19
|
+
const shouldFindBlock = meta.should_find.map((e, i) => {
|
|
20
|
+
const sev = e.severity ? ` [expected severity: ${e.severity}]` : '';
|
|
21
|
+
const req = e.required ? ' (REQUIRED)' : ' (optional)';
|
|
22
|
+
return ` [${i}] ${e.finding}${sev}${req}`;
|
|
23
|
+
}).join('\n');
|
|
24
|
+
const shouldNotFindBlock = meta.should_not_find.length > 0
|
|
25
|
+
? meta.should_not_find.map((a, i) => ` [${i}] ${a}`).join('\n')
|
|
26
|
+
: ' (none)';
|
|
27
|
+
return `You are an eval judge for a code analysis tool called Warden. Your job is to
|
|
28
|
+
determine whether the tool's findings match the expected behavioral outcomes.
|
|
29
|
+
|
|
30
|
+
## Scenario
|
|
31
|
+
Given: ${meta.given}
|
|
32
|
+
|
|
33
|
+
## Agent Findings
|
|
34
|
+
${findingsBlock}
|
|
35
|
+
|
|
36
|
+
## Should Find (what the agent SHOULD have detected)
|
|
37
|
+
${shouldFindBlock}
|
|
38
|
+
|
|
39
|
+
## Should Not Find (what the agent should NOT have reported)
|
|
40
|
+
${shouldNotFindBlock}
|
|
41
|
+
|
|
42
|
+
## Instructions
|
|
43
|
+
|
|
44
|
+
For each "should find" assertion, determine if ANY of the agent's findings satisfy it.
|
|
45
|
+
A finding satisfies an assertion if it describes the same issue, even if worded differently.
|
|
46
|
+
The severity hint is guidance, not a strict requirement: if the finding describes the
|
|
47
|
+
right issue at a close severity level, it still counts as met.
|
|
48
|
+
|
|
49
|
+
For each "should not find" assertion, determine if ANY of the agent's findings violate it.
|
|
50
|
+
A violation means the agent reported something it should not have.
|
|
51
|
+
|
|
52
|
+
## Response Format
|
|
53
|
+
|
|
54
|
+
Respond with ONLY a JSON object. No explanation, no markdown fences.
|
|
55
|
+
|
|
56
|
+
{
|
|
57
|
+
"expectations": [
|
|
58
|
+
{
|
|
59
|
+
"met": true,
|
|
60
|
+
"matchedFindingIndex": 0,
|
|
61
|
+
"reasoning": "Finding [0] identifies the null access bug on user.name"
|
|
62
|
+
}
|
|
63
|
+
],
|
|
64
|
+
"antiExpectations": [
|
|
65
|
+
{
|
|
66
|
+
"violated": false,
|
|
67
|
+
"violatingFindingIndex": null,
|
|
68
|
+
"reasoning": "No findings report style issues"
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
Requirements:
|
|
74
|
+
- "expectations" array must have exactly ${meta.should_find.length} entries (one per should_find, in order)
|
|
75
|
+
- "antiExpectations" array must have exactly ${meta.should_not_find.length} entries (one per should_not_find, in order)
|
|
76
|
+
- "matchedFindingIndex" is the index of the matched finding, or null if no match
|
|
77
|
+
- "violatingFindingIndex" is the index of the violating finding, or null if no violation
|
|
78
|
+
- Keep reasoning to one sentence`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Run the LLM judge to evaluate agent findings against eval assertions.
|
|
82
|
+
*/
|
|
83
|
+
export async function runJudge(meta, findings, apiKey) {
|
|
84
|
+
const client = new Anthropic({ apiKey, timeout: JUDGE_TIMEOUT_MS });
|
|
85
|
+
const prompt = buildJudgePrompt(meta, findings);
|
|
86
|
+
const messages = [
|
|
87
|
+
{ role: 'user', content: prompt },
|
|
88
|
+
];
|
|
89
|
+
let response;
|
|
90
|
+
try {
|
|
91
|
+
response = await client.messages.create({
|
|
92
|
+
model: JUDGE_MODEL,
|
|
93
|
+
max_tokens: JUDGE_MAX_TOKENS,
|
|
94
|
+
messages,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
// On API failure, return a judge response marking everything as failed
|
|
99
|
+
const failedExpectations = meta.should_find.map(() => ({
|
|
100
|
+
met: false,
|
|
101
|
+
matchedFindingIndex: null,
|
|
102
|
+
reasoning: `Judge API call failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
103
|
+
}));
|
|
104
|
+
const failedAntiExpectations = meta.should_not_find.map(() => ({
|
|
105
|
+
violated: false,
|
|
106
|
+
violatingFindingIndex: null,
|
|
107
|
+
reasoning: 'Judge API call failed, assuming no violation',
|
|
108
|
+
}));
|
|
109
|
+
return {
|
|
110
|
+
response: { expectations: failedExpectations, antiExpectations: failedAntiExpectations },
|
|
111
|
+
usage: emptyUsage(),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
const usage = apiUsageToStats(JUDGE_MODEL, response.usage);
|
|
115
|
+
const textBlock = response.content.find((b) => b.type === 'text');
|
|
116
|
+
if (!textBlock) {
|
|
117
|
+
return {
|
|
118
|
+
response: buildFallbackResponse(meta, 'No text in judge response'),
|
|
119
|
+
usage,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const jsonStr = extractJson(textBlock.text);
|
|
123
|
+
if (!jsonStr) {
|
|
124
|
+
return {
|
|
125
|
+
response: buildFallbackResponse(meta, 'No JSON found in judge response'),
|
|
126
|
+
usage,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
const parsed = JSON.parse(jsonStr);
|
|
130
|
+
const validated = JudgeResponseSchema.safeParse(parsed);
|
|
131
|
+
if (!validated.success) {
|
|
132
|
+
return {
|
|
133
|
+
response: buildFallbackResponse(meta, `Judge response validation failed: ${validated.error.message}`),
|
|
134
|
+
usage,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// Validate array lengths match assertions
|
|
138
|
+
const judgeResp = validated.data;
|
|
139
|
+
if (judgeResp.expectations.length !== meta.should_find.length) {
|
|
140
|
+
return {
|
|
141
|
+
response: buildFallbackResponse(meta, `Judge returned ${judgeResp.expectations.length} verdicts, expected ${meta.should_find.length}`),
|
|
142
|
+
usage,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (judgeResp.antiExpectations.length !== meta.should_not_find.length) {
|
|
146
|
+
return {
|
|
147
|
+
response: buildFallbackResponse(meta, `Judge returned ${judgeResp.antiExpectations.length} anti-verdicts, expected ${meta.should_not_find.length}`),
|
|
148
|
+
usage,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
return { response: judgeResp, usage };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Build a fallback judge response when parsing fails.
|
|
155
|
+
* Marks all assertions as not met with the error reason.
|
|
156
|
+
*/
|
|
157
|
+
function buildFallbackResponse(meta, reason) {
|
|
158
|
+
return {
|
|
159
|
+
expectations: meta.should_find.map(() => ({
|
|
160
|
+
met: false,
|
|
161
|
+
matchedFindingIndex: null,
|
|
162
|
+
reasoning: reason,
|
|
163
|
+
})),
|
|
164
|
+
antiExpectations: meta.should_not_find.map(() => ({
|
|
165
|
+
violated: false,
|
|
166
|
+
violatingFindingIndex: null,
|
|
167
|
+
reasoning: 'Judge failed, assuming no violation',
|
|
168
|
+
})),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=judge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGrE,MAAM,WAAW,GAAG,kBAAkB,CAAC;AACvC,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAOhC;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAc,EAAE,QAAmB;IAC3D,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpB,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,OAAO,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,eAAe,CAAC,CAAC,UAAU,IAAI,OAAO,GAAG,GAAG,kBAAkB,CAAC,CAAC,KAAK,wBAAwB,CAAC,CAAC,WAAW,EAAE,CAAC;QACrJ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACjB,CAAC,CAAC,0BAA0B,CAAC;IAE/B,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpD,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC;QACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;IAC7C,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,kBAAkB,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAChE,CAAC,CAAC,UAAU,CAAC;IAEf,OAAO;;;;SAIA,IAAI,CAAC,KAAK;;;EAGjB,aAAa;;;EAGb,eAAe;;;EAGf,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CAkCuB,IAAI,CAAC,WAAW,CAAC,MAAM;+CACnB,IAAI,CAAC,eAAe,CAAC,MAAM;;;iCAGzC,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAc,EACd,QAAmB,EACnB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAEpE,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAA6B;QACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,CAAC;IAEF,IAAI,QAA2B,CAAC;IAChC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,gBAAgB;YAC5B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uEAAuE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACrD,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;SAC9F,CAAC,CAAC,CAAC;QACJ,MAAM,sBAAsB,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7D,QAAQ,EAAE,KAAK;YACf,qBAAqB,EAAE,IAAI;YAC3B,SAAS,EAAE,8CAA8C;SAC1D,CAAC,CAAC,CAAC;QACJ,OAAO;YACL,QAAQ,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE;YACxF,KAAK,EAAE,UAAU,EAAE;SACpB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAE3D,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CACrC,CAAC,CAAC,EAA4B,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CACnD,CAAC;IAEF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,2BAA2B,CAAC;YAClE,KAAK;SACN,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,iCAAiC,CAAC;YACxE,KAAK;SACN,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,qCAAqC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACrG,KAAK;SACN,CAAC;IACJ,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC;IACjC,IAAI,SAAS,CAAC,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC9D,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,SAAS,CAAC,YAAY,CAAC,MAAM,uBAAuB,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACtI,KAAK;SACN,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,gBAAgB,CAAC,MAAM,KAAK,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;QACtE,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,SAAS,CAAC,gBAAgB,CAAC,MAAM,4BAA4B,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;YACnJ,KAAK;SACN,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,IAAc,EAAE,MAAc;IAC3D,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACxC,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAChD,QAAQ,EAAE,KAAK;YACf,qBAAqB,EAAE,IAAI;YAC3B,SAAS,EAAE,qCAAqC;SACjD,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { EvalMeta, EvalResult } from './types.js';
|
|
2
|
+
export interface RunEvalOptions {
|
|
3
|
+
/** Anthropic API key */
|
|
4
|
+
apiKey: string;
|
|
5
|
+
/** Override the model from the YAML spec */
|
|
6
|
+
model?: string;
|
|
7
|
+
/** Enable verbose logging */
|
|
8
|
+
verbose?: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Run a single eval scenario end-to-end.
|
|
12
|
+
*
|
|
13
|
+
* The only thing mocked is the GitHub event payload (no real PR).
|
|
14
|
+
* Everything else runs for real: git repo, diff parsing, SDK invocation,
|
|
15
|
+
* agent with Read/Grep tools, finding extraction, LLM judge.
|
|
16
|
+
*/
|
|
17
|
+
export declare function runEval(meta: EvalMeta, options: RunEvalOptions): Promise<EvalResult>;
|
|
18
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,WAAW,cAAc;IAC7B,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAyDD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,UAAU,CAAC,CA6ErB"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { basename, join, dirname } from 'node:path';
|
|
2
|
+
import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { execGitNonInteractive } from '../utils/exec.js';
|
|
5
|
+
import { buildLocalEventContext } from '../cli/context.js';
|
|
6
|
+
import { resolveSkillAsync } from '../skills/loader.js';
|
|
7
|
+
import { runSkill } from '../sdk/runner.js';
|
|
8
|
+
import { runJudge } from './judge.js';
|
|
9
|
+
import { evalPassed } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Set up a temporary git repository for an eval scenario.
|
|
12
|
+
*
|
|
13
|
+
* Creates a real git repo with an empty `main` commit (the base) and an
|
|
14
|
+
* `eval` branch containing fixture files + the skill definition. This gives
|
|
15
|
+
* the agent a real git environment to explore with Read/Grep and produces
|
|
16
|
+
* real git diffs for the pipeline to parse.
|
|
17
|
+
*/
|
|
18
|
+
function setupEvalRepo(meta, log) {
|
|
19
|
+
const tmpDir = mkdtempSync(join(tmpdir(), `warden-eval-${meta.name}-`));
|
|
20
|
+
try {
|
|
21
|
+
const git = (args) => execGitNonInteractive(args, { cwd: tmpDir });
|
|
22
|
+
git(['init', '--initial-branch=main']);
|
|
23
|
+
git(['config', 'user.email', 'eval@warden.dev']);
|
|
24
|
+
git(['config', 'user.name', 'Warden Eval']);
|
|
25
|
+
git(['commit', '--allow-empty', '-m', 'initial commit']);
|
|
26
|
+
git(['checkout', '-b', 'eval']);
|
|
27
|
+
// Copy fixture files, preserving their parent directory name
|
|
28
|
+
for (const srcPath of meta.filePaths) {
|
|
29
|
+
const destDir = join(tmpDir, basename(dirname(srcPath)));
|
|
30
|
+
mkdirSync(destDir, { recursive: true });
|
|
31
|
+
copyFileSync(srcPath, join(destDir, basename(srcPath)));
|
|
32
|
+
}
|
|
33
|
+
// Copy skill into repo. If it lives in a directory (skill-name/SKILL.md),
|
|
34
|
+
// copy the whole directory to preserve resource subdirs (scripts/, references/).
|
|
35
|
+
// For flat .md files, just copy the single file.
|
|
36
|
+
const skillSrcDir = dirname(meta.skillPath);
|
|
37
|
+
const skillMarker = join(skillSrcDir, 'SKILL.md');
|
|
38
|
+
const skillDestDir = join(tmpDir, '.warden', 'skills');
|
|
39
|
+
mkdirSync(skillDestDir, { recursive: true });
|
|
40
|
+
if (existsSync(skillMarker)) {
|
|
41
|
+
// Directory-format skill: copy entire directory to preserve resources
|
|
42
|
+
const skillDirName = basename(skillSrcDir);
|
|
43
|
+
cpSync(skillSrcDir, join(skillDestDir, skillDirName), { recursive: true });
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
copyFileSync(meta.skillPath, join(skillDestDir, basename(meta.skillPath)));
|
|
47
|
+
}
|
|
48
|
+
git(['add', '.']);
|
|
49
|
+
git(['commit', '-m', `add fixture: ${meta.name}`]);
|
|
50
|
+
log(`Repo ready: ${tmpDir} (${meta.filePaths.length} file(s))`);
|
|
51
|
+
return tmpDir;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
// Clean up on partial failure so we don't leak temp directories
|
|
55
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Run a single eval scenario end-to-end.
|
|
61
|
+
*
|
|
62
|
+
* The only thing mocked is the GitHub event payload (no real PR).
|
|
63
|
+
* Everything else runs for real: git repo, diff parsing, SDK invocation,
|
|
64
|
+
* agent with Read/Grep tools, finding extraction, LLM judge.
|
|
65
|
+
*/
|
|
66
|
+
export async function runEval(meta, options) {
|
|
67
|
+
const startTime = Date.now();
|
|
68
|
+
const name = `${meta.category}/${meta.name}`;
|
|
69
|
+
const logs = [];
|
|
70
|
+
const log = (msg) => {
|
|
71
|
+
logs.push(`[${Date.now() - startTime}ms] ${msg}`);
|
|
72
|
+
if (options.verbose) {
|
|
73
|
+
console.log(` [eval:${name}] ${msg}`);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
if (meta.filePaths.length === 0) {
|
|
77
|
+
throw new Error(`No fixture files specified for eval: ${name}`);
|
|
78
|
+
}
|
|
79
|
+
log(`Fixture file(s): ${meta.filePaths.map((f) => f.split('/').pop()).join(', ')}`);
|
|
80
|
+
let repoDir;
|
|
81
|
+
try {
|
|
82
|
+
repoDir = setupEvalRepo(meta, log);
|
|
83
|
+
const context = buildLocalEventContext({
|
|
84
|
+
base: 'main',
|
|
85
|
+
head: 'eval',
|
|
86
|
+
cwd: repoDir,
|
|
87
|
+
defaultBranch: 'main',
|
|
88
|
+
});
|
|
89
|
+
log(`Context built: ${context.pullRequest?.files.length ?? 0} file(s) from git diff`);
|
|
90
|
+
// Resolve skill from where setupEvalRepo placed it
|
|
91
|
+
const skillSrcDir = dirname(meta.skillPath);
|
|
92
|
+
const isDirectorySkill = existsSync(join(skillSrcDir, 'SKILL.md'));
|
|
93
|
+
const skillPath = isDirectorySkill
|
|
94
|
+
? join(repoDir, '.warden', 'skills', basename(skillSrcDir))
|
|
95
|
+
: join(repoDir, '.warden', 'skills', basename(meta.skillPath));
|
|
96
|
+
const skill = await resolveSkillAsync(skillPath);
|
|
97
|
+
log(`Skill resolved: ${skill.name}`);
|
|
98
|
+
const model = options.model ?? meta.model;
|
|
99
|
+
log(`Running skill with model: ${model}`);
|
|
100
|
+
const report = await runSkill(skill, context, {
|
|
101
|
+
apiKey: options.apiKey,
|
|
102
|
+
model,
|
|
103
|
+
verbose: options.verbose,
|
|
104
|
+
parallel: false,
|
|
105
|
+
});
|
|
106
|
+
log(`Skill complete: ${report.findings.length} finding(s)`);
|
|
107
|
+
for (const finding of report.findings) {
|
|
108
|
+
const loc = finding.location ? ` (${finding.location.path}:${finding.location.startLine})` : '';
|
|
109
|
+
log(` [${finding.severity}] ${finding.title}${loc}`);
|
|
110
|
+
}
|
|
111
|
+
log('Running judge...');
|
|
112
|
+
const judgeResult = await runJudge(meta, report.findings, options.apiKey);
|
|
113
|
+
const passed = evalPassed(meta, judgeResult.response);
|
|
114
|
+
log(`Result: ${passed ? 'PASS' : 'FAIL'}`);
|
|
115
|
+
return {
|
|
116
|
+
name,
|
|
117
|
+
meta,
|
|
118
|
+
passed,
|
|
119
|
+
report,
|
|
120
|
+
judgeResponse: judgeResult.response,
|
|
121
|
+
logs,
|
|
122
|
+
durationMs: Date.now() - startTime,
|
|
123
|
+
skillUsage: report.usage,
|
|
124
|
+
judgeUsage: judgeResult.usage,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
if (repoDir) {
|
|
129
|
+
rmSync(repoDir, { recursive: true, force: true });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=runner.js.map
|