@sentry/warden 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/conductor.json +8 -0
- package/dist/cli/fix.d.ts +3 -1
- package/dist/cli/fix.d.ts.map +1 -1
- package/dist/cli/fix.js +91 -59
- package/dist/cli/fix.js.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +37 -9
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/output/formatters.d.ts.map +1 -1
- package/dist/cli/output/formatters.js +3 -8
- package/dist/cli/output/formatters.js.map +1 -1
- package/dist/cli/output/ink-runner.d.ts +6 -17
- package/dist/cli/output/ink-runner.d.ts.map +1 -1
- package/dist/cli/output/ink-runner.js +124 -115
- package/dist/cli/output/ink-runner.js.map +1 -1
- package/dist/cli/output/jsonl.d.ts +7 -1
- package/dist/cli/output/jsonl.d.ts.map +1 -1
- package/dist/cli/output/jsonl.js +4 -1
- package/dist/cli/output/jsonl.js.map +1 -1
- package/dist/cli/output/reporter.d.ts +7 -3
- package/dist/cli/output/reporter.d.ts.map +1 -1
- package/dist/cli/output/reporter.js +15 -1
- package/dist/cli/output/reporter.js.map +1 -1
- package/dist/cli/output/tasks.d.ts +10 -1
- package/dist/cli/output/tasks.d.ts.map +1 -1
- package/dist/cli/output/tasks.js +127 -28
- package/dist/cli/output/tasks.js.map +1 -1
- package/dist/cli/terminal.d.ts +7 -1
- package/dist/cli/terminal.d.ts.map +1 -1
- package/dist/cli/terminal.js +31 -9
- package/dist/cli/terminal.js.map +1 -1
- package/dist/config/schema.d.ts +0 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +1 -2
- package/dist/config/schema.js.map +1 -1
- package/dist/evals/index.d.ts +22 -0
- package/dist/evals/index.d.ts.map +1 -0
- package/dist/evals/index.js +92 -0
- package/dist/evals/index.js.map +1 -0
- package/dist/evals/judge.d.ts +12 -0
- package/dist/evals/judge.d.ts.map +1 -0
- package/dist/evals/judge.js +171 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/runner.d.ts +18 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +133 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/{examples → evals}/setup.d.ts.map +1 -1
- package/dist/evals/setup.js.map +1 -0
- package/dist/evals/types.d.ts +166 -0
- package/dist/evals/types.d.ts.map +1 -0
- package/dist/evals/types.js +134 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/output/dedup.d.ts.map +1 -1
- package/dist/output/dedup.js +29 -57
- package/dist/output/dedup.js.map +1 -1
- package/dist/output/github-checks.d.ts.map +1 -1
- package/dist/output/github-checks.js +38 -10
- package/dist/output/github-checks.js.map +1 -1
- package/dist/output/renderer.d.ts.map +1 -1
- package/dist/output/renderer.js +16 -1
- package/dist/output/renderer.js.map +1 -1
- package/dist/output/stale.d.ts +1 -0
- package/dist/output/stale.d.ts.map +1 -1
- package/dist/output/stale.js +14 -8
- package/dist/output/stale.js.map +1 -1
- package/dist/sdk/analyze.d.ts +11 -0
- package/dist/sdk/analyze.d.ts.map +1 -1
- package/dist/sdk/analyze.js +185 -27
- package/dist/sdk/analyze.js.map +1 -1
- package/dist/sdk/extract.d.ts +55 -1
- package/dist/sdk/extract.d.ts.map +1 -1
- package/dist/sdk/extract.js +222 -32
- package/dist/sdk/extract.js.map +1 -1
- package/dist/sdk/haiku.d.ts +20 -0
- package/dist/sdk/haiku.d.ts.map +1 -1
- package/dist/sdk/haiku.js +159 -77
- package/dist/sdk/haiku.js.map +1 -1
- package/dist/sdk/model-pricing.json +6 -12
- package/dist/sdk/pricing.d.ts +5 -0
- package/dist/sdk/pricing.d.ts.map +1 -1
- package/dist/sdk/pricing.js +11 -2
- package/dist/sdk/pricing.js.map +1 -1
- package/dist/sdk/prompt.d.ts.map +1 -1
- package/dist/sdk/prompt.js +4 -1
- package/dist/sdk/prompt.js.map +1 -1
- package/dist/sdk/runner.d.ts +2 -2
- package/dist/sdk/runner.d.ts.map +1 -1
- package/dist/sdk/runner.js +1 -1
- package/dist/sdk/runner.js.map +1 -1
- package/dist/sdk/types.d.ts +9 -3
- package/dist/sdk/types.d.ts.map +1 -1
- package/dist/sdk/types.js +1 -1
- package/dist/sdk/types.js.map +1 -1
- package/dist/sdk/usage.d.ts +5 -0
- package/dist/sdk/usage.d.ts.map +1 -1
- package/dist/sdk/usage.js +11 -3
- package/dist/sdk/usage.js.map +1 -1
- package/dist/sentry.d.ts +15 -0
- package/dist/sentry.d.ts.map +1 -1
- package/dist/sentry.js +40 -1
- package/dist/sentry.js.map +1 -1
- package/dist/triggers/matcher.d.ts +2 -2
- package/dist/triggers/matcher.d.ts.map +1 -1
- package/dist/triggers/matcher.js +10 -8
- package/dist/triggers/matcher.js.map +1 -1
- package/dist/types/index.d.ts +19 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +25 -0
- package/dist/types/index.js.map +1 -1
- package/dist/utils/async.d.ts +14 -0
- package/dist/utils/async.d.ts.map +1 -1
- package/dist/utils/async.js +33 -0
- package/dist/utils/async.js.map +1 -1
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js +1 -1
- package/dist/utils/index.js.map +1 -1
- package/evals/README.md +154 -0
- package/evals/bug-detection.yaml +56 -0
- package/evals/fixtures/ignores-style-issues/utils.ts +48 -0
- package/evals/fixtures/missing-await/cache.ts +45 -0
- package/evals/fixtures/null-property-access/handler.ts +36 -0
- package/evals/fixtures/off-by-one/paginator.ts +38 -0
- package/evals/fixtures/sql-injection/api.ts +59 -0
- package/evals/fixtures/stale-closure/counter.tsx +33 -0
- package/evals/fixtures/wrong-comparison/validator.ts +52 -0
- package/evals/fixtures/xss-reflected/server.ts +55 -0
- package/evals/precision.yaml +15 -0
- package/evals/security-scanning.yaml +24 -0
- package/evals/skills/bug-detection.md +33 -0
- package/evals/skills/precision.md +18 -0
- package/evals/skills/security-scanning.md +32 -0
- package/package.json +4 -2
- package/plugins/warden/skills/warden/references/config-schema.md +4 -2
- package/plugins/warden/skills/warden/references/configuration.md +2 -2
- package/dist/examples/index.d.ts +0 -50
- package/dist/examples/index.d.ts.map +0 -1
- package/dist/examples/index.js +0 -104
- package/dist/examples/index.js.map +0 -1
- package/dist/examples/setup.js.map +0 -1
- /package/dist/{examples → evals}/setup.d.ts +0 -0
- /package/dist/{examples → evals}/setup.js +0 -0
package/dist/cli/terminal.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../src/cli/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,cAAc,EACd,aAAa,EACb,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,eAAe,GAAuC;IAC1D,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI;IACxB,IAAI,EAAE,KAAK,CAAC,GAAG;IACf,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,GAAG,EAAE,KAAK,CAAC,KAAK;IAChB,IAAI,EAAE,KAAK,CAAC,IAAI;CACjB,CAAC;AAOF;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACxC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAgB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEhD,+BAA+B;IAC/B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,6BAA6B;IAC7B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,kFAAkF;IACpF,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAElD,kCAAkC;IAClC,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAgB;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEpD,sEAAsE;IACtE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/G,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,aAAa;IACb,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEvC,4CAA4C;IAC5C,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAmB,EAAE,IAAgB;IAC9D,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEpG,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,WAAW;QAClB,IAAI;KACL,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEtB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,EAAE,CAAC;QACZ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;YAC/C,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC1B,mDAAmD;YACnD,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,OAAO,GAAG,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,MAAmB;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAEjD,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;IAEzD,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,iBAAiB,YAAY,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACxH,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,KAAK,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsB,EAAE,IAAiB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,qEAAqE;IACrE,MAAM,UAAU,GAAe,IAAI,IAAI;QACrC,KAAK,EAAE,IAAI;QACX,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAsB;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;QAC5C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,oBAAoB,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACrF,wBAAwB,EAAE,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACjG,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAsB,EAAE,QAA4B;IAC1F,IAAI,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9B,GAAG,MAAM;QACT,QAAQ,EAAE,wBAAwB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC9D,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE;YACP,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM,EAC5E,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,MAAM,EAC1E,CAAC,CACF;gBACD,GAAG,EAAE,OAAO,CAAC,MAAM,CACjB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,MAAM,EACvE,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;aACF;YACD,KAAK,EAAE,UAAU;SAClB;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
|
1
|
+
{"version":3,"file":"terminal.js","sourceRoot":"","sources":["../../src/cli/terminal.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAC7D,OAAO,EACL,mBAAmB,EACnB,mBAAmB,EACnB,mBAAmB,EACnB,wBAAwB,EACxB,cAAc,EACd,aAAa,EACb,cAAc,EACd,eAAe,EACf,SAAS,GACV,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,MAAM,eAAe,GAAuC;IAC1D,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI;IACxB,IAAI,EAAE,KAAK,CAAC,GAAG;IACf,MAAM,EAAE,KAAK,CAAC,MAAM;IACpB,GAAG,EAAE,KAAK,CAAC,KAAK;IAChB,IAAI,EAAE,KAAK,CAAC,IAAI;CACjB,CAAC;AAOF;;;GAGG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,UAAkB;IACxD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChC,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;IACxC,CAAC;AACH,CAAC;AAOD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAgB,EAAE,OAAuB;IACjE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEhD,+BAA+B;IAC/B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,6BAA6B;IAC7B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvF,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACpC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,eAAe;IACf,IAAI,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;QAC7D,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACxD,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAChD,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,kFAAkF;IACpF,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,OAAO,CAAC,mBAAmB,CAAC,MAAM,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,SAAS,CAAC,KAAK,EAAE,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChF,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,mBAAmB,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;YACnF,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IAElD,4EAA4E;IAC5E,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnE,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACpD,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC3D,OAAO,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;YACjC,CAAC;YACD,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,OAAgB;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEpD,sEAAsE;IACtE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,CAAC;IAC3B,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/G,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACpC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjC,aAAa;IACb,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,uBAAuB;IACvB,IAAI,OAAO,CAAC,mBAAmB,EAAE,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACnD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC;YACnF,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,cAAc;IACd,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAEvC,4CAA4C;IAC5C,IAAI,OAAO,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,MAAmB,EAAE,IAAgB,EAAE,OAAuB;IACvF,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAEpG,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC;QAC1B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,WAAW;QAClB,IAAI;KACL,CAAC,CAAC;IAEH,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEtB,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,EAAE,CAAC;QACZ,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC/C,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YACzD,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,GAAG,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,YAAY,GAAG,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAC1B,mDAAmD;YACnD,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,MAAM,EAAE,CAAC;IAEb,OAAO,GAAG,CAAC,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,MAAmB,EAAE,YAAuB,SAAS,CAAC,MAAM;IACjF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACrG,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAEjD,qCAAqC;IACrC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,WAAW,MAAM,OAAO,EAAE,CAAC,CAAC;IAEzD,4CAA4C;IAC5C,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,iBAAiB,YAAY,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IACxH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,iBAAiB,CAAC,IAAI,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACtF,KAAK,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,KAAK,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAsB,EAAE,IAAiB,EAAE,OAAuB;IACrG,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,qEAAqE;IACrE,MAAM,UAAU,GAAe,IAAI,IAAI;QACrC,KAAK,EAAE,IAAI;QACX,aAAa,EAAE,IAAI;QACnB,OAAO,EAAE,EAAE;KACZ,CAAC;IAEF,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,iBAAiB,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,sBAAsB;QACtB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;YACzD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,OAAsB;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAChC,WAAW,EAAE,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW;QAC5C,YAAY,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;QAC/C,oBAAoB,EAAE,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,oBAAoB,IAAI,CAAC,CAAC;QACrF,wBAAwB,EAAE,CAAC,GAAG,CAAC,wBAAwB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC;QACjG,OAAO,EAAE,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO;KACjC,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAsB,EAAE,QAA4B;IAC1F,IAAI,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC;IAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC9B,GAAG,MAAM;QACT,QAAQ,EAAE,wBAAwB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC;KAC9D,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsB;IACrD,MAAM,UAAU,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE;YACP,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACrE,UAAU,EAAE;gBACV,QAAQ,EAAE,OAAO,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM,EAC5E,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,MAAM,EAC1E,CAAC,CACF;gBACD,GAAG,EAAE,OAAO,CAAC,MAAM,CACjB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,CAAC,MAAM,EACvE,CAAC,CACF;gBACD,IAAI,EAAE,OAAO,CAAC,MAAM,CAClB,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,EACxE,CAAC,CACF;aACF;YACD,KAAK,EAAE,UAAU;SAClB;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
package/dist/config/schema.d.ts
CHANGED
|
@@ -59,7 +59,6 @@ export declare const SkillDefinitionSchema: z.ZodObject<{
|
|
|
59
59
|
WebSearch: "WebSearch";
|
|
60
60
|
}>>>;
|
|
61
61
|
}, z.core.$strip>>;
|
|
62
|
-
outputSchema: z.ZodOptional<z.ZodString>;
|
|
63
62
|
rootDir: z.ZodOptional<z.ZodString>;
|
|
64
63
|
}, z.core.$strip>;
|
|
65
64
|
export type SkillDefinition = z.infer<typeof SkillDefinitionSchema>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,cAAc;;;;;;;;;EASzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;iBAG3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAG1D,eAAO,MAAM,qBAAqB
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,cAAc;;;;;;;;;EASzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;iBAG3B,CAAC;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAG1D,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAOhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAGpE,eAAO,MAAM,oBAAoB;;;;iBAO/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,iBAAiB;;;;EAAgD,CAAC;AAC/E,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA8B9B,CAAC;AACF,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAG9D,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAuB5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,kBAAkB;;iBAG7B,CAAC;AACH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAG9D,eAAO,MAAM,iBAAiB;;;;;;;iBAK5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAG5D,eAAO,MAAM,oBAAoB;;;;iBAO/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;iBAK/B,CAAC;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAGlE,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAwBzB,CAAC;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAGtD,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgC3B,CAAC;AACL,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC"}
|
package/dist/config/schema.js
CHANGED
|
@@ -22,7 +22,6 @@ export const SkillDefinitionSchema = z.object({
|
|
|
22
22
|
description: z.string(),
|
|
23
23
|
prompt: z.string(),
|
|
24
24
|
tools: ToolConfigSchema.optional(),
|
|
25
|
-
outputSchema: z.string().optional(),
|
|
26
25
|
/** Directory where the skill was loaded from, for resolving resources (scripts/, references/, assets/) */
|
|
27
26
|
rootDir: z.string().optional(),
|
|
28
27
|
});
|
|
@@ -93,7 +92,7 @@ export const SkillConfigSchema = z.object({
|
|
|
93
92
|
});
|
|
94
93
|
// Runner configuration
|
|
95
94
|
export const RunnerConfigSchema = z.object({
|
|
96
|
-
/** Max concurrent
|
|
95
|
+
/** Max concurrent file analyses across all skills (default: 4) */
|
|
97
96
|
concurrency: z.number().int().positive().optional(),
|
|
98
97
|
});
|
|
99
98
|
// File pattern for chunking configuration
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC;IACnC,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,WAAW;CACZ,CAAC,CAAC;AAGH,gCAAgC;AAChC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAGH,mBAAmB;AACnB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE;IAClC,
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,wCAAwC;AACxC,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,IAAI,CAAC;IACnC,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,UAAU;IACV,WAAW;CACZ,CAAC,CAAC;AAGH,gCAAgC;AAChC,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAGH,mBAAmB;AACnB,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE;IAClC,0GAA0G;IAC1G,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC/B,CAAC,CAAC;AAGH,kCAAkC;AAClC,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,oEAAoE;IACpE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,0DAA0D;IAC1D,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACvC,wDAAwD;IACxD,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC;CAClD,CAAC,CAAC;AAGH,uCAAuC;AACvC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC;AAG/E,8DAA8D;AAC9D,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,2EAA2E;IAC3E,IAAI,EAAE,iBAAiB;IACvB,yDAAyD;IACzD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,kDAAkD;IAClD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mEAAmE;IACnE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qDAAqD;IACrD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,0EAA0E;IAC1E,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC,MAAM,CACP,CAAC,IAAI,EAAE,EAAE;IACP,4CAA4C;IAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,EACD;IACE,OAAO,EAAE,+CAA+C;IACxD,IAAI,EAAE,CAAC,SAAS,CAAC;CAClB,CACF,CAAC;AAGF,6CAA6C;AAC7C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,+BAA+B;IAC/B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,+BAA+B;IAC/B,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,yFAAyF;IACzF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,iDAAiD;IACjD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mEAAmE;IACnE,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qDAAqD;IACrD,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,yGAAyG;IACzG,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,8FAA8F;IAC9F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,uFAAuF;IACvF,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,QAAQ,EAAE;CACjD,CAAC,CAAC;AAGH,uBAAuB;AACvB,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,kEAAkE;IAClE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAGH,0CAA0C;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,+DAA+D;IAC/D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,kFAAkF;IAClF,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;CACjE,CAAC,CAAC;AAGH,oDAAoD;AACpD,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,6CAA6C;IAC7C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,yDAAyD;IACzD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACvD,8DAA8D;IAC9D,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACxD,CAAC,CAAC;AAGH,iEAAiE;AACjE,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,+CAA+C;IAC/C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,QAAQ,EAAE;IACnD,kDAAkD;IAClD,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAGH,iDAAiD;AACjD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,sDAAsD;IACtD,MAAM,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC1C,qDAAqD;IACrD,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD,8DAA8D;IAC9D,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACvC,mFAAmF;IACnF,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACtC,qEAAqE;IACrE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,sEAAsE;IACtE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,6EAA6E;IAC7E,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAChD,+CAA+C;IAC/C,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC3C,6GAA6G;IAC7G,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,qEAAqE;IACrE,QAAQ,EAAE,oBAAoB,CAAC,QAAQ,EAAE;IACzC,+FAA+F;IAC/F,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAGH,iCAAiC;AACjC,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC;KAChC,MAAM,CAAC;IACN,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrB,QAAQ,EAAE,cAAc,CAAC,QAAQ,EAAE;IACnC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9C,MAAM,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,WAAW,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACxE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EAAE,0BAA0B,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACxE,IAAI,EAAE,CAAC,QAAQ,CAAC;SACjB,CAAC,CAAC;IACL,CAAC;IAED,sCAAsC;IACtC,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC9E,GAAG,CAAC,QAAQ,CAAC;wBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;wBAC3B,OAAO,EAAE,qDAAqD;wBAC9D,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,CAAC;qBAC7B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { EvalFile, EvalMeta } from './types.js';
|
|
2
|
+
export type { EvalMeta };
|
|
3
|
+
/**
|
|
4
|
+
* Discover all YAML eval files in the evals directory.
|
|
5
|
+
* Returns absolute paths to .yaml files, sorted alphabetically.
|
|
6
|
+
*/
|
|
7
|
+
export declare function discoverEvalFiles(baseDir?: string): string[];
|
|
8
|
+
/**
|
|
9
|
+
* Load and validate a YAML eval file.
|
|
10
|
+
*/
|
|
11
|
+
export declare function loadEvalFile(filePath: string): EvalFile;
|
|
12
|
+
/**
|
|
13
|
+
* Resolve all eval scenarios from a YAML file into executable EvalMeta objects.
|
|
14
|
+
* Resolves relative paths for skills and fixtures against the evals directory.
|
|
15
|
+
*/
|
|
16
|
+
export declare function resolveEvalMetas(evalFile: EvalFile, yamlPath: string): EvalMeta[];
|
|
17
|
+
/**
|
|
18
|
+
* Discover and load all evals from YAML files. Returns a flat list of
|
|
19
|
+
* resolved EvalMeta objects ready for execution.
|
|
20
|
+
*/
|
|
21
|
+
export declare function discoverEvals(baseDir?: string): EvalMeta[];
|
|
22
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErD,YAAY,EAAE,QAAQ,EAAE,CAAC;AASzB;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAkB5D;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CA0BvD;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE,CAcjF;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,QAAQ,EAAE,CAW1D"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { readFileSync, readdirSync, existsSync } from 'node:fs';
|
|
2
|
+
import { join, basename } from 'node:path';
|
|
3
|
+
import { parse as parseYaml } from 'yaml';
|
|
4
|
+
import { EvalFileSchema } from './types.js';
|
|
5
|
+
/**
|
|
6
|
+
* Get the default evals directory path.
|
|
7
|
+
*/
|
|
8
|
+
function getEvalsDir() {
|
|
9
|
+
return join(import.meta.dirname, '..', '..', 'evals');
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Discover all YAML eval files in the evals directory.
|
|
13
|
+
* Returns absolute paths to .yaml files, sorted alphabetically.
|
|
14
|
+
*/
|
|
15
|
+
export function discoverEvalFiles(baseDir) {
|
|
16
|
+
const evalsDir = baseDir ?? getEvalsDir();
|
|
17
|
+
if (!existsSync(evalsDir)) {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
let entries;
|
|
21
|
+
try {
|
|
22
|
+
entries = readdirSync(evalsDir);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
return entries
|
|
28
|
+
.filter((e) => e.endsWith('.yaml') || e.endsWith('.yml'))
|
|
29
|
+
.map((e) => join(evalsDir, e))
|
|
30
|
+
.sort();
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Load and validate a YAML eval file.
|
|
34
|
+
*/
|
|
35
|
+
export function loadEvalFile(filePath) {
|
|
36
|
+
if (!existsSync(filePath)) {
|
|
37
|
+
throw new Error(`Eval file not found: ${filePath}`);
|
|
38
|
+
}
|
|
39
|
+
let content;
|
|
40
|
+
try {
|
|
41
|
+
content = readFileSync(filePath, 'utf-8');
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
throw new Error(`Failed to read ${filePath}: ${error}`);
|
|
45
|
+
}
|
|
46
|
+
let parsed;
|
|
47
|
+
try {
|
|
48
|
+
parsed = parseYaml(content);
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
throw new Error(`Failed to parse YAML in ${filePath}: ${error}`);
|
|
52
|
+
}
|
|
53
|
+
const validated = EvalFileSchema.safeParse(parsed);
|
|
54
|
+
if (!validated.success) {
|
|
55
|
+
const issues = validated.error.issues.map((i) => `${i.path.join('.')}: ${i.message}`).join(', ');
|
|
56
|
+
throw new Error(`Invalid eval file ${filePath}: ${issues}`);
|
|
57
|
+
}
|
|
58
|
+
return validated.data;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Resolve all eval scenarios from a YAML file into executable EvalMeta objects.
|
|
62
|
+
* Resolves relative paths for skills and fixtures against the evals directory.
|
|
63
|
+
*/
|
|
64
|
+
export function resolveEvalMetas(evalFile, yamlPath) {
|
|
65
|
+
const evalsDir = join(yamlPath, '..');
|
|
66
|
+
const category = basename(yamlPath).replace(/\.ya?ml$/, '');
|
|
67
|
+
return evalFile.evals.map((scenario) => ({
|
|
68
|
+
name: scenario.name,
|
|
69
|
+
category,
|
|
70
|
+
given: scenario.given,
|
|
71
|
+
skillPath: join(evalsDir, evalFile.skill),
|
|
72
|
+
filePaths: scenario.files.map((f) => join(evalsDir, f)),
|
|
73
|
+
model: scenario.model ?? evalFile.model,
|
|
74
|
+
should_find: scenario.should_find,
|
|
75
|
+
should_not_find: scenario.should_not_find,
|
|
76
|
+
}));
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Discover and load all evals from YAML files. Returns a flat list of
|
|
80
|
+
* resolved EvalMeta objects ready for execution.
|
|
81
|
+
*/
|
|
82
|
+
export function discoverEvals(baseDir) {
|
|
83
|
+
const yamlFiles = discoverEvalFiles(baseDir);
|
|
84
|
+
const allEvals = [];
|
|
85
|
+
for (const yamlPath of yamlFiles) {
|
|
86
|
+
const evalFile = loadEvalFile(yamlPath);
|
|
87
|
+
const metas = resolveEvalMetas(evalFile, yamlPath);
|
|
88
|
+
allEvals.push(...metas);
|
|
89
|
+
}
|
|
90
|
+
return allEvals;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/evals/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAK5C;;GAEG;AACH,SAAS,WAAW;IAClB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAgB;IAChD,MAAM,QAAQ,GAAG,OAAO,IAAI,WAAW,EAAE,CAAC;IAE1C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SACxD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;SAC7B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB;IAC3C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACnD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjG,MAAM,IAAI,KAAK,CAAC,qBAAqB,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,SAAS,CAAC,IAAI,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAkB,EAAE,QAAgB;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAE5D,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,QAAQ;QACR,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,SAAS,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC;QACzC,SAAS,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACvD,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK;QACvC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,MAAM,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAe,EAAE,CAAC;IAEhC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACxC,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACnD,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Finding } from '../types/index.js';
|
|
2
|
+
import type { EvalMeta, JudgeResponse } from './types.js';
|
|
3
|
+
import type { UsageStats } from '../types/index.js';
|
|
4
|
+
export interface JudgeResult {
|
|
5
|
+
response: JudgeResponse;
|
|
6
|
+
usage: UsageStats;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Run the LLM judge to evaluate agent findings against eval assertions.
|
|
10
|
+
*/
|
|
11
|
+
export declare function runJudge(meta: EvalMeta, findings: Finding[], apiKey: string): Promise<JudgeResult>;
|
|
12
|
+
//# sourceMappingURL=judge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAIjD,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAMpD,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,aAAa,CAAC;IACxB,KAAK,EAAE,UAAU,CAAC;CACnB;AA6ED;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,IAAI,EAAE,QAAQ,EACd,QAAQ,EAAE,OAAO,EAAE,EACnB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,CAAC,CAkFtB"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import { apiUsageToStats } from '../sdk/pricing.js';
|
|
3
|
+
import { emptyUsage } from '../sdk/usage.js';
|
|
4
|
+
import { extractJson } from '../sdk/haiku.js';
|
|
5
|
+
import { DEFAULT_EVAL_MODEL, JudgeResponseSchema } from './types.js';
|
|
6
|
+
const JUDGE_MODEL = DEFAULT_EVAL_MODEL;
|
|
7
|
+
const JUDGE_MAX_TOKENS = 4096;
|
|
8
|
+
const JUDGE_TIMEOUT_MS = 60_000;
|
|
9
|
+
/**
|
|
10
|
+
* Build the judge prompt that evaluates agent findings against BDD assertions.
|
|
11
|
+
*/
|
|
12
|
+
function buildJudgePrompt(meta, findings) {
|
|
13
|
+
const findingsBlock = findings.length > 0
|
|
14
|
+
? findings.map((f, i) => {
|
|
15
|
+
const loc = f.location ? ` (${f.location.path}:${f.location.startLine})` : '';
|
|
16
|
+
return ` [${i}] severity=${f.severity} confidence=${f.confidence ?? 'unset'}${loc}\n title: ${f.title}\n description: ${f.description}`;
|
|
17
|
+
}).join('\n\n')
|
|
18
|
+
: ' (no findings reported)';
|
|
19
|
+
const shouldFindBlock = meta.should_find.map((e, i) => {
|
|
20
|
+
const sev = e.severity ? ` [expected severity: ${e.severity}]` : '';
|
|
21
|
+
const req = e.required ? ' (REQUIRED)' : ' (optional)';
|
|
22
|
+
return ` [${i}] ${e.finding}${sev}${req}`;
|
|
23
|
+
}).join('\n');
|
|
24
|
+
const shouldNotFindBlock = meta.should_not_find.length > 0
|
|
25
|
+
? meta.should_not_find.map((a, i) => ` [${i}] ${a}`).join('\n')
|
|
26
|
+
: ' (none)';
|
|
27
|
+
return `You are an eval judge for a code analysis tool called Warden. Your job is to
|
|
28
|
+
determine whether the tool's findings match the expected behavioral outcomes.
|
|
29
|
+
|
|
30
|
+
## Scenario
|
|
31
|
+
Given: ${meta.given}
|
|
32
|
+
|
|
33
|
+
## Agent Findings
|
|
34
|
+
${findingsBlock}
|
|
35
|
+
|
|
36
|
+
## Should Find (what the agent SHOULD have detected)
|
|
37
|
+
${shouldFindBlock}
|
|
38
|
+
|
|
39
|
+
## Should Not Find (what the agent should NOT have reported)
|
|
40
|
+
${shouldNotFindBlock}
|
|
41
|
+
|
|
42
|
+
## Instructions
|
|
43
|
+
|
|
44
|
+
For each "should find" assertion, determine if ANY of the agent's findings satisfy it.
|
|
45
|
+
A finding satisfies an assertion if it describes the same issue, even if worded differently.
|
|
46
|
+
The severity hint is guidance, not a strict requirement: if the finding describes the
|
|
47
|
+
right issue at a close severity level, it still counts as met.
|
|
48
|
+
|
|
49
|
+
For each "should not find" assertion, determine if ANY of the agent's findings violate it.
|
|
50
|
+
A violation means the agent reported something it should not have.
|
|
51
|
+
|
|
52
|
+
## Response Format
|
|
53
|
+
|
|
54
|
+
Respond with ONLY a JSON object. No explanation, no markdown fences.
|
|
55
|
+
|
|
56
|
+
{
|
|
57
|
+
"expectations": [
|
|
58
|
+
{
|
|
59
|
+
"met": true,
|
|
60
|
+
"matchedFindingIndex": 0,
|
|
61
|
+
"reasoning": "Finding [0] identifies the null access bug on user.name"
|
|
62
|
+
}
|
|
63
|
+
],
|
|
64
|
+
"antiExpectations": [
|
|
65
|
+
{
|
|
66
|
+
"violated": false,
|
|
67
|
+
"violatingFindingIndex": null,
|
|
68
|
+
"reasoning": "No findings report style issues"
|
|
69
|
+
}
|
|
70
|
+
]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
Requirements:
|
|
74
|
+
- "expectations" array must have exactly ${meta.should_find.length} entries (one per should_find, in order)
|
|
75
|
+
- "antiExpectations" array must have exactly ${meta.should_not_find.length} entries (one per should_not_find, in order)
|
|
76
|
+
- "matchedFindingIndex" is the index of the matched finding, or null if no match
|
|
77
|
+
- "violatingFindingIndex" is the index of the violating finding, or null if no violation
|
|
78
|
+
- Keep reasoning to one sentence`;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Run the LLM judge to evaluate agent findings against eval assertions.
|
|
82
|
+
*/
|
|
83
|
+
export async function runJudge(meta, findings, apiKey) {
|
|
84
|
+
const client = new Anthropic({ apiKey, timeout: JUDGE_TIMEOUT_MS });
|
|
85
|
+
const prompt = buildJudgePrompt(meta, findings);
|
|
86
|
+
const messages = [
|
|
87
|
+
{ role: 'user', content: prompt },
|
|
88
|
+
];
|
|
89
|
+
let response;
|
|
90
|
+
try {
|
|
91
|
+
response = await client.messages.create({
|
|
92
|
+
model: JUDGE_MODEL,
|
|
93
|
+
max_tokens: JUDGE_MAX_TOKENS,
|
|
94
|
+
messages,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
// On API failure, return a judge response marking everything as failed
|
|
99
|
+
const failedExpectations = meta.should_find.map(() => ({
|
|
100
|
+
met: false,
|
|
101
|
+
matchedFindingIndex: null,
|
|
102
|
+
reasoning: `Judge API call failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
103
|
+
}));
|
|
104
|
+
const failedAntiExpectations = meta.should_not_find.map(() => ({
|
|
105
|
+
violated: false,
|
|
106
|
+
violatingFindingIndex: null,
|
|
107
|
+
reasoning: 'Judge API call failed, assuming no violation',
|
|
108
|
+
}));
|
|
109
|
+
return {
|
|
110
|
+
response: { expectations: failedExpectations, antiExpectations: failedAntiExpectations },
|
|
111
|
+
usage: emptyUsage(),
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
const usage = apiUsageToStats(JUDGE_MODEL, response.usage);
|
|
115
|
+
const textBlock = response.content.find((b) => b.type === 'text');
|
|
116
|
+
if (!textBlock) {
|
|
117
|
+
return {
|
|
118
|
+
response: buildFallbackResponse(meta, 'No text in judge response'),
|
|
119
|
+
usage,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
const jsonStr = extractJson(textBlock.text);
|
|
123
|
+
if (!jsonStr) {
|
|
124
|
+
return {
|
|
125
|
+
response: buildFallbackResponse(meta, 'No JSON found in judge response'),
|
|
126
|
+
usage,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
const parsed = JSON.parse(jsonStr);
|
|
130
|
+
const validated = JudgeResponseSchema.safeParse(parsed);
|
|
131
|
+
if (!validated.success) {
|
|
132
|
+
return {
|
|
133
|
+
response: buildFallbackResponse(meta, `Judge response validation failed: ${validated.error.message}`),
|
|
134
|
+
usage,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
// Validate array lengths match assertions
|
|
138
|
+
const judgeResp = validated.data;
|
|
139
|
+
if (judgeResp.expectations.length !== meta.should_find.length) {
|
|
140
|
+
return {
|
|
141
|
+
response: buildFallbackResponse(meta, `Judge returned ${judgeResp.expectations.length} verdicts, expected ${meta.should_find.length}`),
|
|
142
|
+
usage,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (judgeResp.antiExpectations.length !== meta.should_not_find.length) {
|
|
146
|
+
return {
|
|
147
|
+
response: buildFallbackResponse(meta, `Judge returned ${judgeResp.antiExpectations.length} anti-verdicts, expected ${meta.should_not_find.length}`),
|
|
148
|
+
usage,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
return { response: judgeResp, usage };
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Build a fallback judge response when parsing fails.
|
|
155
|
+
* Marks all assertions as not met with the error reason.
|
|
156
|
+
*/
|
|
157
|
+
function buildFallbackResponse(meta, reason) {
|
|
158
|
+
return {
|
|
159
|
+
expectations: meta.should_find.map(() => ({
|
|
160
|
+
met: false,
|
|
161
|
+
matchedFindingIndex: null,
|
|
162
|
+
reasoning: reason,
|
|
163
|
+
})),
|
|
164
|
+
antiExpectations: meta.should_not_find.map(() => ({
|
|
165
|
+
violated: false,
|
|
166
|
+
violatingFindingIndex: null,
|
|
167
|
+
reasoning: 'Judge failed, assuming no violation',
|
|
168
|
+
})),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=judge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/evals/judge.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAGrE,MAAM,WAAW,GAAG,kBAAkB,CAAC;AACvC,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAOhC;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAc,EAAE,QAAmB;IAC3D,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpB,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,OAAO,MAAM,CAAC,cAAc,CAAC,CAAC,QAAQ,eAAe,CAAC,CAAC,UAAU,IAAI,OAAO,GAAG,GAAG,kBAAkB,CAAC,CAAC,KAAK,wBAAwB,CAAC,CAAC,WAAW,EAAE,CAAC;QACrJ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QACjB,CAAC,CAAC,0BAA0B,CAAC;IAE/B,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACpD,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC;QACvD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC;IAC7C,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,MAAM,kBAAkB,GAAG,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAChE,CAAC,CAAC,UAAU,CAAC;IAEf,OAAO;;;;SAIA,IAAI,CAAC,KAAK;;;EAGjB,aAAa;;;EAGb,eAAe;;;EAGf,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2CAkCuB,IAAI,CAAC,WAAW,CAAC,MAAM;+CACnB,IAAI,CAAC,eAAe,CAAC,MAAM;;;iCAGzC,CAAC;AAClC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,IAAc,EACd,QAAmB,EACnB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAEpE,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAEhD,MAAM,QAAQ,GAA6B;QACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,CAAC;IAEF,IAAI,QAA2B,CAAC;IAChC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,gBAAgB;YAC5B,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uEAAuE;QACvE,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACrD,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;SAC9F,CAAC,CAAC,CAAC;QACJ,MAAM,sBAAsB,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7D,QAAQ,EAAE,KAAK;YACf,qBAAqB,EAAE,IAAI;YAC3B,SAAS,EAAE,8CAA8C;SAC1D,CAAC,CAAC,CAAC;QACJ,OAAO;YACL,QAAQ,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,sBAAsB,EAAE;YACxF,KAAK,EAAE,UAAU,EAAE;SACpB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IAE3D,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CACrC,CAAC,CAAC,EAA4B,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CACnD,CAAC;IAEF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,2BAA2B,CAAC;YAClE,KAAK;SACN,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAE5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,iCAAiC,CAAC;YACxE,KAAK;SACN,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAExD,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACvB,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,qCAAqC,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YACrG,KAAK;SACN,CAAC;IACJ,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,IAAI,CAAC;IACjC,IAAI,SAAS,CAAC,YAAY,CAAC,MAAM,KAAK,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC9D,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,SAAS,CAAC,YAAY,CAAC,MAAM,uBAAuB,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACtI,KAAK;SACN,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,CAAC,gBAAgB,CAAC,MAAM,KAAK,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;QACtE,OAAO;YACL,QAAQ,EAAE,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,SAAS,CAAC,gBAAgB,CAAC,MAAM,4BAA4B,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;YACnJ,KAAK;SACN,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,IAAc,EAAE,MAAc;IAC3D,OAAO;QACL,YAAY,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YACxC,GAAG,EAAE,KAAK;YACV,mBAAmB,EAAE,IAAI;YACzB,SAAS,EAAE,MAAM;SAClB,CAAC,CAAC;QACH,gBAAgB,EAAE,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;YAChD,QAAQ,EAAE,KAAK;YACf,qBAAqB,EAAE,IAAI;YAC3B,SAAS,EAAE,qCAAqC;SACjD,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { EvalMeta, EvalResult } from './types.js';
|
|
2
|
+
export interface RunEvalOptions {
|
|
3
|
+
/** Anthropic API key */
|
|
4
|
+
apiKey: string;
|
|
5
|
+
/** Override the model from the YAML spec */
|
|
6
|
+
model?: string;
|
|
7
|
+
/** Enable verbose logging */
|
|
8
|
+
verbose?: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Run a single eval scenario end-to-end.
|
|
12
|
+
*
|
|
13
|
+
* The only thing mocked is the GitHub event payload (no real PR).
|
|
14
|
+
* Everything else runs for real: git repo, diff parsing, SDK invocation,
|
|
15
|
+
* agent with Read/Grep tools, finding extraction, LLM judge.
|
|
16
|
+
*/
|
|
17
|
+
export declare function runEval(meta: EvalMeta, options: RunEvalOptions): Promise<EvalResult>;
|
|
18
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,WAAW,cAAc;IAC7B,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6BAA6B;IAC7B,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAyDD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,UAAU,CAAC,CA6ErB"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { basename, join, dirname } from 'node:path';
|
|
2
|
+
import { copyFileSync, cpSync, mkdirSync, mkdtempSync, rmSync, existsSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { execGitNonInteractive } from '../utils/exec.js';
|
|
5
|
+
import { buildLocalEventContext } from '../cli/context.js';
|
|
6
|
+
import { resolveSkillAsync } from '../skills/loader.js';
|
|
7
|
+
import { runSkill } from '../sdk/runner.js';
|
|
8
|
+
import { runJudge } from './judge.js';
|
|
9
|
+
import { evalPassed } from './types.js';
|
|
10
|
+
/**
|
|
11
|
+
* Set up a temporary git repository for an eval scenario.
|
|
12
|
+
*
|
|
13
|
+
* Creates a real git repo with an empty `main` commit (the base) and an
|
|
14
|
+
* `eval` branch containing fixture files + the skill definition. This gives
|
|
15
|
+
* the agent a real git environment to explore with Read/Grep and produces
|
|
16
|
+
* real git diffs for the pipeline to parse.
|
|
17
|
+
*/
|
|
18
|
+
function setupEvalRepo(meta, log) {
|
|
19
|
+
const tmpDir = mkdtempSync(join(tmpdir(), `warden-eval-${meta.name}-`));
|
|
20
|
+
try {
|
|
21
|
+
const git = (args) => execGitNonInteractive(args, { cwd: tmpDir });
|
|
22
|
+
git(['init', '--initial-branch=main']);
|
|
23
|
+
git(['config', 'user.email', 'eval@warden.dev']);
|
|
24
|
+
git(['config', 'user.name', 'Warden Eval']);
|
|
25
|
+
git(['commit', '--allow-empty', '-m', 'initial commit']);
|
|
26
|
+
git(['checkout', '-b', 'eval']);
|
|
27
|
+
// Copy fixture files, preserving their parent directory name
|
|
28
|
+
for (const srcPath of meta.filePaths) {
|
|
29
|
+
const destDir = join(tmpDir, basename(dirname(srcPath)));
|
|
30
|
+
mkdirSync(destDir, { recursive: true });
|
|
31
|
+
copyFileSync(srcPath, join(destDir, basename(srcPath)));
|
|
32
|
+
}
|
|
33
|
+
// Copy skill into repo. If it lives in a directory (skill-name/SKILL.md),
|
|
34
|
+
// copy the whole directory to preserve resource subdirs (scripts/, references/).
|
|
35
|
+
// For flat .md files, just copy the single file.
|
|
36
|
+
const skillSrcDir = dirname(meta.skillPath);
|
|
37
|
+
const skillMarker = join(skillSrcDir, 'SKILL.md');
|
|
38
|
+
const skillDestDir = join(tmpDir, '.warden', 'skills');
|
|
39
|
+
mkdirSync(skillDestDir, { recursive: true });
|
|
40
|
+
if (existsSync(skillMarker)) {
|
|
41
|
+
// Directory-format skill: copy entire directory to preserve resources
|
|
42
|
+
const skillDirName = basename(skillSrcDir);
|
|
43
|
+
cpSync(skillSrcDir, join(skillDestDir, skillDirName), { recursive: true });
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
copyFileSync(meta.skillPath, join(skillDestDir, basename(meta.skillPath)));
|
|
47
|
+
}
|
|
48
|
+
git(['add', '.']);
|
|
49
|
+
git(['commit', '-m', `add fixture: ${meta.name}`]);
|
|
50
|
+
log(`Repo ready: ${tmpDir} (${meta.filePaths.length} file(s))`);
|
|
51
|
+
return tmpDir;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
// Clean up on partial failure so we don't leak temp directories
|
|
55
|
+
rmSync(tmpDir, { recursive: true, force: true });
|
|
56
|
+
throw error;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Run a single eval scenario end-to-end.
|
|
61
|
+
*
|
|
62
|
+
* The only thing mocked is the GitHub event payload (no real PR).
|
|
63
|
+
* Everything else runs for real: git repo, diff parsing, SDK invocation,
|
|
64
|
+
* agent with Read/Grep tools, finding extraction, LLM judge.
|
|
65
|
+
*/
|
|
66
|
+
export async function runEval(meta, options) {
|
|
67
|
+
const startTime = Date.now();
|
|
68
|
+
const name = `${meta.category}/${meta.name}`;
|
|
69
|
+
const logs = [];
|
|
70
|
+
const log = (msg) => {
|
|
71
|
+
logs.push(`[${Date.now() - startTime}ms] ${msg}`);
|
|
72
|
+
if (options.verbose) {
|
|
73
|
+
console.log(` [eval:${name}] ${msg}`);
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
if (meta.filePaths.length === 0) {
|
|
77
|
+
throw new Error(`No fixture files specified for eval: ${name}`);
|
|
78
|
+
}
|
|
79
|
+
log(`Fixture file(s): ${meta.filePaths.map((f) => f.split('/').pop()).join(', ')}`);
|
|
80
|
+
let repoDir;
|
|
81
|
+
try {
|
|
82
|
+
repoDir = setupEvalRepo(meta, log);
|
|
83
|
+
const context = buildLocalEventContext({
|
|
84
|
+
base: 'main',
|
|
85
|
+
head: 'eval',
|
|
86
|
+
cwd: repoDir,
|
|
87
|
+
defaultBranch: 'main',
|
|
88
|
+
});
|
|
89
|
+
log(`Context built: ${context.pullRequest?.files.length ?? 0} file(s) from git diff`);
|
|
90
|
+
// Resolve skill from where setupEvalRepo placed it
|
|
91
|
+
const skillSrcDir = dirname(meta.skillPath);
|
|
92
|
+
const isDirectorySkill = existsSync(join(skillSrcDir, 'SKILL.md'));
|
|
93
|
+
const skillPath = isDirectorySkill
|
|
94
|
+
? join(repoDir, '.warden', 'skills', basename(skillSrcDir))
|
|
95
|
+
: join(repoDir, '.warden', 'skills', basename(meta.skillPath));
|
|
96
|
+
const skill = await resolveSkillAsync(skillPath);
|
|
97
|
+
log(`Skill resolved: ${skill.name}`);
|
|
98
|
+
const model = options.model ?? meta.model;
|
|
99
|
+
log(`Running skill with model: ${model}`);
|
|
100
|
+
const report = await runSkill(skill, context, {
|
|
101
|
+
apiKey: options.apiKey,
|
|
102
|
+
model,
|
|
103
|
+
verbose: options.verbose,
|
|
104
|
+
parallel: false,
|
|
105
|
+
});
|
|
106
|
+
log(`Skill complete: ${report.findings.length} finding(s)`);
|
|
107
|
+
for (const finding of report.findings) {
|
|
108
|
+
const loc = finding.location ? ` (${finding.location.path}:${finding.location.startLine})` : '';
|
|
109
|
+
log(` [${finding.severity}] ${finding.title}${loc}`);
|
|
110
|
+
}
|
|
111
|
+
log('Running judge...');
|
|
112
|
+
const judgeResult = await runJudge(meta, report.findings, options.apiKey);
|
|
113
|
+
const passed = evalPassed(meta, judgeResult.response);
|
|
114
|
+
log(`Result: ${passed ? 'PASS' : 'FAIL'}`);
|
|
115
|
+
return {
|
|
116
|
+
name,
|
|
117
|
+
meta,
|
|
118
|
+
passed,
|
|
119
|
+
report,
|
|
120
|
+
judgeResponse: judgeResult.response,
|
|
121
|
+
logs,
|
|
122
|
+
durationMs: Date.now() - startTime,
|
|
123
|
+
skillUsage: report.usage,
|
|
124
|
+
judgeUsage: judgeResult.usage,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
if (repoDir) {
|
|
129
|
+
rmSync(repoDir, { recursive: true, force: true });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC3F,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAYxC;;;;;;;GAOG;AACH,SAAS,aAAa,CAAC,IAAc,EAAE,GAA0B;IAC/D,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,eAAe,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAExE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,IAAc,EAAE,EAAE,CAAC,qBAAqB,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAE7E,GAAG,CAAC,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC,CAAC;QACvC,GAAG,CAAC,CAAC,QAAQ,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC,CAAC;QACjD,GAAG,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,CAAC,QAAQ,EAAE,eAAe,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACzD,GAAG,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAEhC,6DAA6D;QAC7D,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC;QAED,0EAA0E;QAC1E,iFAAiF;QACjF,iDAAiD;QACjD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;QAClD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;QACvD,SAAS,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE7C,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,sEAAsE;YACtE,MAAM,YAAY,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3C,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7E,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QAClB,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,gBAAgB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEnD,GAAG,CAAC,eAAe,MAAM,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,WAAW,CAAC,CAAC;QAChE,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gEAAgE;QAChE,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAc,EACd,OAAuB;IAEvB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,MAAM,GAAG,GAAG,CAAC,GAAW,EAAQ,EAAE;QAChC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,OAAO,GAAG,EAAE,CAAC,CAAC;QAClD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,GAAG,CAAC,oBAAoB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEpF,IAAI,OAA2B,CAAC;IAEhC,IAAI,CAAC;QACH,OAAO,GAAG,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAEnC,MAAM,OAAO,GAAG,sBAAsB,CAAC;YACrC,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,OAAO;YACZ,aAAa,EAAE,MAAM;SACtB,CAAC,CAAC;QACH,GAAG,CAAC,kBAAkB,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAEtF,mDAAmD;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5C,MAAM,gBAAgB,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;QACnE,MAAM,SAAS,GAAG,gBAAgB;YAChC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC3D,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACjD,GAAG,CAAC,mBAAmB,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAErC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC;QAC1C,GAAG,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE;YAC5C,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;QAEH,GAAG,CAAC,mBAAmB,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QAC5D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,GAAG,CAAC,MAAM,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,GAAG,CAAC,kBAAkB,CAAC,CAAC;QACxB,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAE1E,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;QACtD,GAAG,CAAC,WAAW,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAE3C,OAAO;YACL,IAAI;YACJ,IAAI;YACJ,MAAM;YACN,MAAM;YACN,aAAa,EAAE,WAAW,CAAC,QAAQ;YACnC,IAAI;YACJ,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,UAAU,EAAE,MAAM,CAAC,KAAK;YACxB,UAAU,EAAE,WAAW,CAAC,KAAK;SAC9B,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../src/
|
|
1
|
+
{"version":3,"file":"setup.d.ts","sourceRoot":"","sources":["../../src/evals/setup.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"setup.js","sourceRoot":"","sources":["../../src/evals/setup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEhD;;;;;;GAMG;AACH,SAAS,WAAW;IAClB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAEnD,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IAErD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9B,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,YAAY,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC;AAED,WAAW,EAAE,CAAC"}
|