chivgent 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +108 -3
- package/README.zh-CN.md +100 -3
- package/dist/auth/credentials.d.ts.map +1 -1
- package/dist/auth/credentials.js +4 -1
- package/dist/auth/credentials.js.map +1 -1
- package/dist/auth/file-credentials.d.ts +8 -0
- package/dist/auth/file-credentials.d.ts.map +1 -1
- package/dist/auth/file-credentials.js +40 -1
- package/dist/auth/file-credentials.js.map +1 -1
- package/dist/cli.js +48 -31
- package/dist/cli.js.map +1 -1
- package/dist/evals/cli.d.ts +3 -0
- package/dist/evals/cli.d.ts.map +1 -0
- package/dist/evals/cli.js +137 -0
- package/dist/evals/cli.js.map +1 -0
- package/dist/evals/fixture.d.ts +13 -0
- package/dist/evals/fixture.d.ts.map +1 -0
- package/dist/evals/fixture.js +18 -0
- package/dist/evals/fixture.js.map +1 -0
- package/dist/evals/graders.d.ts +21 -0
- package/dist/evals/graders.d.ts.map +1 -0
- package/dist/evals/graders.js +175 -0
- package/dist/evals/graders.js.map +1 -0
- package/dist/evals/parse-args.d.ts +18 -0
- package/dist/evals/parse-args.d.ts.map +1 -0
- package/dist/evals/parse-args.js +67 -0
- package/dist/evals/parse-args.js.map +1 -0
- package/dist/evals/report.d.ts +18 -0
- package/dist/evals/report.d.ts.map +1 -0
- package/dist/evals/report.js +103 -0
- package/dist/evals/report.js.map +1 -0
- package/dist/evals/runner.d.ts +43 -0
- package/dist/evals/runner.d.ts.map +1 -0
- package/dist/evals/runner.js +144 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/evals/task.d.ts +27 -0
- package/dist/evals/task.d.ts.map +1 -0
- package/dist/evals/task.js +116 -0
- package/dist/evals/task.js.map +1 -0
- package/dist/prompts.d.ts +12 -0
- package/dist/prompts.d.ts.map +1 -0
- package/dist/prompts.js +31 -0
- package/dist/prompts.js.map +1 -0
- package/dist/providers/deferred-client.d.ts +26 -0
- package/dist/providers/deferred-client.d.ts.map +1 -0
- package/dist/providers/deferred-client.js +50 -0
- package/dist/providers/deferred-client.js.map +1 -0
- package/dist/repl.d.ts +16 -1
- package/dist/repl.d.ts.map +1 -1
- package/dist/repl.js +80 -1
- package/dist/repl.js.map +1 -1
- package/package.json +3 -2
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,OAAO,MAAM,cAAc,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,OAAO,GAGR,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GAEjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,sBAAsB,GACvB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,MAAM,aAAa,GAAG;;;;;;;;;6HASuG,CAAC;AAE9H,MAAM,mBAAmB,GAAG;;;;;uCAKW,CAAC;AAExC,MAAM,mBAAmB,GAAG;;;;;gEAKoC,CAAC;AAEjE,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;;GAOG;AACH,SAAS,mBAAmB;IAC1B,MAAM,OAAO,GAAG,GAAS,EAAE;QACzB,mBAAmB,EAAE,CAAC;IACxB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,OAAO,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAmB,EACnB,aAAa,GAAsB,EAAE;IAErC,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,mBAAmB,GAAG;IAC1B,YAAY;IACZ,aAAa;IACb,WAAW;IACX,YAAY;IACZ,WAAW;IACX,MAAM;CACE,CAAC;AAEX;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAC5B,OAAmB,EACnB,GAAW;IAEX,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG;YACH,KAAK;YACL,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,cAAc,EAAE,KAAK,IAAI,EAAE,CACzB,CAAC,MAAM,kBAAkB,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;gBACrD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC;SACvC,CAAC,CAAC;QACH,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,sEAAsE;QACtE,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,uCAAuC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE;QACpD,iBAAiB,EAAE,CAAC,GAAG,mBAAmB,CAAC;QAC3C,oBAAoB,EAAE,CAAC,GAAG,iBAAiB,CAAC;QAC5C,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,IAAI,CAAC;KACxE,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAA0B;IAClD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAA4B,EAAE,EAAE;QAClD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAQD,KAAK,UAAU,IAAI,CAAC,IAAuB;IACzC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO;QAC3B,CAAC,CAAC,IAAI,gBAAgB,CAAC,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACvD,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,YAAY,CAAC,KAAK,IAAI,IAAI,gBAAgB,EAAE,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAErD,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,2EAA2E;IAC3E,6EAA6E;IAC7E,OAAO;IACP,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK;QAC1B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,MAAM,KAAK,SAAS,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACzE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC;QACtC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,kBAAkB,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,qBAAqB,EAAE,CAAC;gBAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBAC3C,OAAO,CAAC,CAAC;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,mBAAmB,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,IAAI,aAAa,EAAE;QACnB,IAAI,cAAc,EAAE;QACpB,IAAI,YAAY,EAAE;KACnB,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC;QACxC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,YAAY,EAAE,WAAW,EAAE,EAAE,EAAE;YAC/D,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,aAAa,eAAe,uBAAuB,YAAY,QAAQ,WAAW,aAAa,CAChG,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IACH,MAAM,YAAY,GAAkC;QAClD,YAAY,EAAE,iBAAiB,CAC7B,OAAO,EACP,QAAQ,EAAE,yBAAyB,IAAI,EAAE,CAC1C;QACD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG;QACH,KAAK,EAAE;YACL,GAAG,aAAa;YAChB,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,aAAa,EAAE,EAAE,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,QAAQ,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAChE;QACD,SAAS,EAAE,IAAI,cAAc,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QACxE,SAAS,EAAE,OAAO,CAAC,MAAM;QACzB,cAAc;KACf,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;QAC/B,KAAK,EAAE,YAAY;QACnB,GAAG;QACH,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;QACzD,GAAG,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3E,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;KAC1C,CAAC,CAAC;IAEH,OAAO,CAAC,SAAS,CACf,OAAO,CAAC,IAAI;QACV,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACzD,CAAC,CAAC,mBAAmB,CACjB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAClD;YACE,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,gBAAgB,EACd,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;YACjD,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;SACrC,CACF,CACN,CAAC;IAEF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,OAAO,CAAC;YACb,OAAO;YACP,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,0EAA0E;YAC1E,yEAAyE;YACzE,+DAA+D;YAC/D,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YACtD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,OAAO,CAAC;YACrD,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS;gBACxB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,QAAQ,CAAC,kBAAkB,EAAE,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,OAAqB,EACrB,MAAc;IAEd,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX,KAAK,SAAS;gBACZ,OAAO,gBAAgB,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,OAAO,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,CAAC;IACX,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,UAAU,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,SAAS,KAAK,SAAS;YACrB,CAAC,CAAC,4BAA4B,GAAG,KAAK;YACtC,CAAC,CAAC,iCAAiC,SAAS,KAAK,CACpD,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAuC;IACjE,MAAM,KAAK,GAAG,QAAQ,EAAE,eAAe,IAAI,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAiC,QAAQ,EAAE,kBAAkB,IAAI,EAAE,CAAC;IAClF,MAAM,OAAO,GAAG,QAAQ,EAAE,yBAAyB,IAAI,EAAE,CAAC;IAE1D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACpD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,cAAc,CAAC,OAAmB;IACzC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CACzB,OAAmB,EACnB,OAAqB;IAErB,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;QAC/B,OAAO;QACP,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,YAAY,EAAE,cAAc,CAAC,OAAO,CAAC;QACrC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC;KAC7D,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IACE,KAAK,YAAY,gBAAgB;YACjC,KAAK,YAAY,sBAAsB,EACvC,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;QACE,YAAY,OAAO,oBAAoB,OAAO,CAAC,EAAE,EAAE;QACnD,iBAAiB,MAAM,CAAC,UAAU,EAAE;QACpC,iBAAiB,OAAO,CAAC,GAAG,EAAE;QAC9B,iBAAiB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACzE,EAAE;QACF,mCAAmC,OAAO,CAAC,EAAE,EAAE;QAC/C,8DAA8D;QAC9D,yBAAyB;QACzB,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAmB,EACnB,MAAc;IAEd,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,mBAAmB,CAClC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAClD;QACE,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK;QAChC,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;QACjE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;KACrC,CACF,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;QAC/B,UAAU;QACV,OAAO,EAAE,OAAO,CAAC,IAAI;YACnB,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,CAAC,CAAC,QAAQ;KACb,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,MAAM,aAAa,CAAC;gBACzB,MAAM;gBACN,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBACtD,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,UAAU;aACX,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAqB,EACrB,MAAc;IAEd,MAAM,SAAS,GAAG,GAAS,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX,KAAK,SAAS;gBACZ,OAAO,gBAAgB,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX;gBACE,OAAO,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,cAAc,CAC3B,OAAmB,EACnB,KAA+B,EAC/B,GAAW;IAEX,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACtF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IACxB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,2BAA2B,GAAG,GAAG,CAAC;QAC3C,CAAC;QACD,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,oBAAoB,EAAE,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAmB,EACnB,GAAW;IAEX,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,GAAG,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,WAAW,eAAe,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAC7G,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CACb,OAAmB,EACnB,SAAiB,EACjB,OAAgB;IAEhB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;IAC/C,OAAO;QACL,YAAY,OAAO,MAAM,OAAO,CAAC,QAAQ,MAAM,KAAK,EAAE;QACtD,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,EAAE;QACjD,2CAA2C;QAC3C,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,KAAK,UAAU,eAAe;IAC5B,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;AACpE,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;IACjB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC9B,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,OAAO,MAAM,cAAc,CAAC;AAEnC,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,OAAO,GAGR,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AACzE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,kBAAkB,EAClB,gBAAgB,GAEjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAGnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACrE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,aAAa,EACb,sBAAsB,GACvB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAE1E,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B;;;;;;;GAOG;AACH,SAAS,mBAAmB;IAC1B,MAAM,OAAO,GAAG,GAAS,EAAE;QACzB,mBAAmB,EAAE,CAAC;IACxB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;QACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,OAAO,EAAE,CAAC;YACV,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,OAAmB,EACnB,aAAa,GAAsB,EAAE;IAErC,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,CAAC;IAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,mBAAmB,GAAG;IAC1B,YAAY;IACZ,aAAa;IACb,WAAW;IACX,YAAY;IACZ,WAAW;IACX,MAAM;CACE,CAAC;AAEX;;;;;GAKG;AACH,KAAK,UAAU,eAAe,CAC5B,OAAmB,EACnB,GAAW;IAEX,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,UAAU,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC;YAChC,GAAG;YACH,KAAK;YACL,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,cAAc,EAAE,KAAK,IAAI,EAAE,CACzB,CAAC,MAAM,kBAAkB,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,SAAS,CAAC;gBACrD,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC;SACvC,CAAC,CAAC;QACH,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAC5B,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,sEAAsE;QACtE,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,uCAAuC,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE;QACpD,iBAAiB,EAAE,CAAC,GAAG,mBAAmB,CAAC;QAC3C,oBAAoB,EAAE,CAAC,GAAG,iBAAiB,CAAC;QAC5C,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,IAAI,CAAC;KACxE,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,MAA0B;IAClD,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAA4B,EAAE,EAAE;QAClD,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC;AAQD,KAAK,UAAU,IAAI,CAAC,IAAuB;IACzC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAChD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO;QAC3B,CAAC,CAAC,IAAI,gBAAgB,CAAC,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACvD,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,YAAY,CAAC,KAAK,IAAI,IAAI,gBAAgB,EAAE,EAAE,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,OAAO,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAErD,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,2EAA2E;IAC3E,6EAA6E;IAC7E,OAAO;IACP,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK;QAC1B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,MAAM,KAAK,SAAS,CAAC;IACzC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACzE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;IACzD,2EAA2E;IAC3E,4EAA4E;IAC5E,2EAA2E;IAC3E,MAAM,gBAAgB,GACpB,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,IAAI,gBAAgB,KAAK,SAAS,IAAI,CAAC,CAAC,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,gBAAgB,IAAI,CAAC,CAAC;QAC9C,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,iBAAiB,CAC/B,gBAAgB,IAAI,+CAA+C,CACpE,CAAC;IACF,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;QACnC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC;QACtC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,kBAAkB,EAAE,CAAC;QACvB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,qBAAqB,EAAE,CAAC;gBAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBAC3C,OAAO,CAAC,CAAC;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,mBAAmB,EAAE,CAAC;IACxB,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,IAAI,aAAa,EAAE;QACnB,IAAI,cAAc,EAAE;QACpB,IAAI,YAAY,EAAE;KACnB,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,cAAc,CAAC;QACxC,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,YAAY,EAAE,WAAW,EAAE,EAAE,EAAE;YAC/D,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,aAAa,eAAe,uBAAuB,YAAY,QAAQ,WAAW,aAAa,CAChG,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;IACH,MAAM,YAAY,GAAkC;QAClD,YAAY,EAAE,iBAAiB,CAC7B,OAAO,EACP,QAAQ,EAAE,yBAAyB,IAAI,EAAE,CAC1C;QACD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG;QACH,KAAK,EAAE;YACL,GAAG,aAAa;YAChB,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,aAAa,EAAE,EAAE,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,GAAG,CAAC,QAAQ,EAAE,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAChE;QACD,SAAS,EAAE,IAAI,cAAc,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QACxE,SAAS,EAAE,OAAO,CAAC,MAAM;QACzB,cAAc;KACf,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;QAC/B,KAAK,EAAE,YAAY;QACnB,GAAG;QACH,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,GAAG,CAAC,QAAQ,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC;QACzD,GAAG,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3E,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;KAC1C,CAAC,CAAC;IAEH,OAAO,CAAC,SAAS,CACf,OAAO,CAAC,IAAI;QACV,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACzD,CAAC,CAAC,mBAAmB,CACjB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAClD;YACE,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,gBAAgB,EACd,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;YACjD,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;SACrC,CACF,CACN,CAAC;IAEF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,MAAM,GAAW;QACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,GAAG,CAAC;QACtC,KAAK,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK;QACtB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACvB,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YACpE,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC9C,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAChE,CAAC;YACD,kEAAkE;YAClE,yCAAyC;YACzC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChB,OAAO,SAAS,CAAC;QACnB,CAAC;KACF,CAAC;IAEF,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,OAAO,CAAC;YACb,OAAO;YACP,MAAM;YACN,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,0EAA0E;YAC1E,yEAAyE;YACzE,+DAA+D;YAC/D,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YACtD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;YACvE,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3E,GAAG,CAAC,QAAQ,KAAK,SAAS;gBACxB,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,iBAAiB,EAAE,QAAQ,CAAC,kBAAkB,EAAE,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAClC,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,OAAqB,EACrB,MAAc;IAEd,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,GAAS,EAAE;QAC3B,UAAU,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX,KAAK,SAAS;gBACZ,OAAO,gBAAgB,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,OAAO,IAAI,CAAC,CAAC;QACnD,OAAO,CAAC,CAAC;IACX,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,UAAU,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,SAAS,KAAK,SAAS;YACrB,CAAC,CAAC,4BAA4B,GAAG,KAAK;YACtC,CAAC,CAAC,iCAAiC,SAAS,KAAK,CACpD,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAuC;IACjE,MAAM,KAAK,GAAG,QAAQ,EAAE,eAAe,IAAI,EAAE,CAAC;IAC9C,MAAM,QAAQ,GAAiC,QAAQ,EAAE,kBAAkB,IAAI,EAAE,CAAC;IAClF,MAAM,OAAO,GAAG,QAAQ,EAAE,yBAAyB,IAAI,EAAE,CAAC;IAE1D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACpD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,CAAC,IAAI,CAAC,YAAY,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,IAAI,MAAM,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,CAAC;IACD,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,KAAK,CAAC,IAAI,CAAC,YAAY,SAAS,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,cAAc,CAAC,OAAmB;IACzC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CACzB,OAAmB,EACnB,OAAqB;IAErB,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;QAC/B,OAAO;QACP,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,YAAY,EAAE,cAAc,CAAC,OAAO,CAAC;QACrC,SAAS,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC;KAC7D,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,IACE,KAAK,YAAY,gBAAgB;YACjC,KAAK,YAAY,sBAAsB,EACvC,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB;QACE,YAAY,OAAO,oBAAoB,OAAO,CAAC,EAAE,EAAE;QACnD,iBAAiB,MAAM,CAAC,UAAU,EAAE;QACpC,iBAAiB,OAAO,CAAC,GAAG,EAAE;QAC9B,iBAAiB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACzE,EAAE;QACF,mCAAmC,OAAO,CAAC,EAAE,EAAE;QAC/C,8DAA8D;QAC9D,yBAAyB;QACzB,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC7B,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,WAAW;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;IACpC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAmB,EACnB,MAAc;IAEd,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,mBAAmB,CAClC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,EAClD;QACE,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK;QAChC,gBAAgB,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;QACjE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI;KACrC,CACF,CAAC;IACF,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC;QAC/B,UAAU;QACV,OAAO,EAAE,OAAO,CAAC,IAAI;YACnB,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,CAAC,CAAC,QAAQ;KACb,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,MAAM,aAAa,CAAC;gBACzB,MAAM;gBACN,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;gBACtD,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,UAAU;aACX,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAqB,EACrB,MAAc;IAEd,MAAM,SAAS,GAAG,GAAS,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACjD,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX,KAAK,SAAS;gBACZ,OAAO,gBAAgB,CAAC;YAC1B,KAAK,WAAW;gBACd,OAAO,CAAC,CAAC;YACX;gBACE,OAAO,CAAC,CAAC;QACb,CAAC;IACH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,cAAc,CAC3B,OAAmB,EACnB,KAA+B,EAC/B,GAAW;IAEX,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QACtF,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IACxB,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACrB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,2BAA2B,GAAG,GAAG,CAAC;QAC3C,CAAC;QACD,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACxC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,oBAAoB,EAAE,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAmB,EACnB,GAAW;IAEX,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,GAAG,KAAK,CAAC,CAAC;QAC3D,OAAO,CAAC,CAAC;IACX,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,SAAS,KAAK,OAAO,CAAC,WAAW,eAAe,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAC7G,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CACb,OAAmB,EACnB,SAAiB,EACjB,OAAgB,EAChB,gBAAyB;IAEzB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,eAAe,CAAC;IAC/C,MAAM,KAAK,GAAG;QACZ,YAAY,OAAO,MAAM,OAAO,CAAC,QAAQ,MAAM,KAAK,EAAE;QACtD,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,EAAE;KAClD,CAAC;IACF,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,sEAAsE;QACtE,2EAA2E;QAC3E,wEAAwE;QACxE,KAAK,CAAC,IAAI,CACR,EAAE,EACF,kBAAkB,OAAO,CAAC,QAAQ,OAAO,EACzC,oEAAoE,CACrE,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,2CAA2C,EAAE,EAAE,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAC/E,KAAK,UAAU,eAAe;IAC5B,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;QACjC,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,QAAQ,IAAI,KAAK,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;AACpE,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;IACjB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC9B,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,OAAO,IAAI,CAAC,CAAC;IAChD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import process from "node:process";
|
|
5
|
+
import { parseCliArgs, VERSION } from "../cli-options.js";
|
|
6
|
+
import { createConfiguredClient } from "../providers/client.js";
|
|
7
|
+
import { SHELL_SYSTEM_PROMPT, SYSTEM_PROMPT, WRITE_SYSTEM_PROMPT, } from "../prompts.js";
|
|
8
|
+
import { formatFailures, formatTable, toJsonReport } from "./report.js";
|
|
9
|
+
import { MissingCapabilityError, runTask } from "./runner.js";
|
|
10
|
+
import { loadTasks, TaskError } from "./task.js";
|
|
11
|
+
import { parseEvalArgs } from "./parse-args.js";
|
|
12
|
+
const HELP = `chivgent evals ${VERSION}
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
npm run eval -- [options]
|
|
16
|
+
|
|
17
|
+
Options:
|
|
18
|
+
--task NAME Run only this task (repeatable)
|
|
19
|
+
--attempts N Override the attempts each task declares
|
|
20
|
+
--dir PATH Task directory (default: evals)
|
|
21
|
+
--json PATH Also write a JSON report
|
|
22
|
+
--provider NAME Provider to evaluate
|
|
23
|
+
--model MODEL Model to evaluate
|
|
24
|
+
--allow-writes Grant tasks that need to change files
|
|
25
|
+
--allow-shell Grant tasks that need a shell
|
|
26
|
+
-h, --help Show this help
|
|
27
|
+
|
|
28
|
+
Evals need real credentials and cost money. They are not part of npm test:
|
|
29
|
+
a model is nondeterministic, so a task's result is a pass rate over several
|
|
30
|
+
attempts rather than a verdict, and that is not something to gate a merge on.
|
|
31
|
+
`;
|
|
32
|
+
function buildSystemPrompt(capabilities) {
|
|
33
|
+
const sections = [SYSTEM_PROMPT];
|
|
34
|
+
if (capabilities.includes("writes")) {
|
|
35
|
+
sections.push(WRITE_SYSTEM_PROMPT);
|
|
36
|
+
}
|
|
37
|
+
if (capabilities.includes("shell")) {
|
|
38
|
+
sections.push(SHELL_SYSTEM_PROMPT);
|
|
39
|
+
}
|
|
40
|
+
return sections.join("\n");
|
|
41
|
+
}
|
|
42
|
+
function selectTasks(all, wanted) {
|
|
43
|
+
if (wanted.length === 0) {
|
|
44
|
+
return all;
|
|
45
|
+
}
|
|
46
|
+
const known = new Set(all.map((task) => task.name));
|
|
47
|
+
for (const name of wanted) {
|
|
48
|
+
if (!known.has(name)) {
|
|
49
|
+
throw new TaskError(`Unknown task "${name}". Available: ${[...known].join(", ")}.`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return all.filter((task) => wanted.includes(task.name));
|
|
53
|
+
}
|
|
54
|
+
async function main(argv) {
|
|
55
|
+
let options;
|
|
56
|
+
try {
|
|
57
|
+
options = parseEvalArgs(argv);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
61
|
+
return 1;
|
|
62
|
+
}
|
|
63
|
+
if (options.help) {
|
|
64
|
+
process.stdout.write(HELP);
|
|
65
|
+
return 0;
|
|
66
|
+
}
|
|
67
|
+
let tasks;
|
|
68
|
+
try {
|
|
69
|
+
tasks = selectTasks(await loadTasks(path.resolve(options.directory)), options.tasks);
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
73
|
+
return 1;
|
|
74
|
+
}
|
|
75
|
+
if (tasks.length === 0) {
|
|
76
|
+
process.stderr.write(`No tasks found in ${options.directory}.\n`);
|
|
77
|
+
return 1;
|
|
78
|
+
}
|
|
79
|
+
const cliOptions = parseCliArgs([...options.providerArgs], process.env);
|
|
80
|
+
const llm = await createConfiguredClient(cliOptions);
|
|
81
|
+
if (typeof llm === "string") {
|
|
82
|
+
// The same configuration message the main CLI gives, rather than a run of
|
|
83
|
+
// zero scores that looks like the agent failing.
|
|
84
|
+
process.stderr.write(`${llm}\n`);
|
|
85
|
+
return 1;
|
|
86
|
+
}
|
|
87
|
+
const startedAt = new Date().toISOString();
|
|
88
|
+
process.stderr.write(`Evaluating ${cliOptions.provider}/${cliOptions.model ?? "?"} on ${tasks.length} task(s).\n\n`);
|
|
89
|
+
const results = [];
|
|
90
|
+
for (const task of tasks) {
|
|
91
|
+
try {
|
|
92
|
+
results.push(await runTask(task, {
|
|
93
|
+
createClient: () => llm,
|
|
94
|
+
systemPrompt: buildSystemPrompt(task.capabilities),
|
|
95
|
+
capabilities: options.capabilities,
|
|
96
|
+
...(options.attempts === undefined ? {} : { attempts: options.attempts }),
|
|
97
|
+
onAttempt: (result) => {
|
|
98
|
+
process.stderr.write(` ${task.name} ${result.attempt}: ${result.passed ? "pass" : "fail"}\n`);
|
|
99
|
+
},
|
|
100
|
+
}));
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
if (error instanceof MissingCapabilityError) {
|
|
104
|
+
process.stderr.write(`skipped: ${error.message}\n`);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
throw error;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
process.stdout.write(`\n${formatTable(results)}`);
|
|
111
|
+
const failures = formatFailures(results);
|
|
112
|
+
if (failures.length > 0) {
|
|
113
|
+
process.stdout.write(`\n${failures}`);
|
|
114
|
+
}
|
|
115
|
+
if (options.jsonPath !== undefined) {
|
|
116
|
+
const report = toJsonReport(results, {
|
|
117
|
+
provider: cliOptions.provider,
|
|
118
|
+
model: cliOptions.model ?? "unknown",
|
|
119
|
+
version: VERSION,
|
|
120
|
+
startedAt,
|
|
121
|
+
});
|
|
122
|
+
await writeFile(options.jsonPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
123
|
+
process.stderr.write(`\nWrote ${options.jsonPath}\n`);
|
|
124
|
+
}
|
|
125
|
+
// Exit 0 even when tasks fail: a pass rate is a measurement, not a gate.
|
|
126
|
+
return 0;
|
|
127
|
+
}
|
|
128
|
+
main(process.argv.slice(2))
|
|
129
|
+
.then((exitCode) => {
|
|
130
|
+
process.exitCode = exitCode;
|
|
131
|
+
})
|
|
132
|
+
.catch((error) => {
|
|
133
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
134
|
+
process.stderr.write(`Eval runner failed: ${message}\n`);
|
|
135
|
+
process.exitCode = 1;
|
|
136
|
+
});
|
|
137
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,mBAAmB,GACpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,OAAO,EAAmB,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,SAAS,EAA8B,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAoB,MAAM,iBAAiB,CAAC;AAElE,MAAM,IAAI,GAAG,kBAAkB,OAAO;;;;;;;;;;;;;;;;;;;CAmBrC,CAAC;AAGF,SAAS,iBAAiB,CAAC,YAAmC;IAC5D,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAClB,GAAoB,EACpB,MAAyB;IAEzB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,SAAS,CACjB,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAuB;IACzC,IAAI,OAAoB,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CACjB,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAChD,OAAO,CAAC,KAAK,CACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,0EAA0E;QAC1E,iDAAiD;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,GAAG,OAAO,KAAK,CAAC,MAAM,eAAe,CAC/F,CAAC;IAEF,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CACV,MAAM,OAAO,CAAC,IAAI,EAAE;gBAClB,YAAY,EAAE,GAAG,EAAE,CAAC,GAAG;gBACvB,YAAY,EAAE,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;gBAClD,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzE,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;oBACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CACzE,CAAC;gBACJ,CAAC;aACF,CAAC,CACH,CAAC;QACJ,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE;YACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,SAAS;YACpC,OAAO,EAAE,OAAO;YAChB,SAAS;SACV,CAAC,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;IACjB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC9B,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,OAAO,IAAI,CAAC,CAAC;IACzD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A throwaway copy of a task's fixture.
|
|
3
|
+
*
|
|
4
|
+
* Every attempt gets its own: attempts write to the workspace, and reusing one
|
|
5
|
+
* would let attempt 2 start from whatever attempt 1 left behind, which turns a
|
|
6
|
+
* pass rate into a sequence of increasingly different tasks.
|
|
7
|
+
*/
|
|
8
|
+
export interface AttemptWorkspace {
|
|
9
|
+
readonly path: string;
|
|
10
|
+
dispose(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
export declare function createAttemptWorkspace(fixtureDirectory: string, prefix?: string): Promise<AttemptWorkspace>;
|
|
13
|
+
//# sourceMappingURL=fixture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/evals/fixture.ts"],"names":[],"mappings":"AAKA;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,wBAAsB,sBAAsB,CAC1C,gBAAgB,EAAE,MAAM,EACxB,MAAM,SAAmB,GACxB,OAAO,CAAC,gBAAgB,CAAC,CAY3B"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { cp, mkdtemp, rm } from "node:fs/promises";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
export async function createAttemptWorkspace(fixtureDirectory, prefix = "chivgent-eval-") {
|
|
6
|
+
const directory = await mkdtemp(path.join(tmpdir(), prefix));
|
|
7
|
+
if (existsSync(fixtureDirectory)) {
|
|
8
|
+
// A task may legitimately have no fixture: some tasks only ask a question.
|
|
9
|
+
await cp(fixtureDirectory, directory, { recursive: true });
|
|
10
|
+
}
|
|
11
|
+
return {
|
|
12
|
+
path: directory,
|
|
13
|
+
async dispose() {
|
|
14
|
+
await rm(directory, { recursive: true, force: true });
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=fixture.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/evals/fixture.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAc7B,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,gBAAwB,EACxB,MAAM,GAAG,gBAAgB;IAEzB,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,IAAI,UAAU,CAAC,gBAAgB,CAAC,EAAE,CAAC;QACjC,2EAA2E;QAC3E,MAAM,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,CAAC,OAAO;YACX,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { GraderSpec, TaskError } from "./task.js";
|
|
2
|
+
import type { AgentEvent } from "../events.js";
|
|
3
|
+
export interface AttemptFacts {
|
|
4
|
+
/** The workspace as the attempt left it. */
|
|
5
|
+
readonly workspace: string;
|
|
6
|
+
readonly finalAnswer: string;
|
|
7
|
+
readonly turnCount: number;
|
|
8
|
+
readonly status: "completed" | "max_turns" | "aborted" | "error";
|
|
9
|
+
readonly events: readonly AgentEvent[];
|
|
10
|
+
}
|
|
11
|
+
export interface GraderResult {
|
|
12
|
+
readonly passed: boolean;
|
|
13
|
+
/** Why it failed, specific enough to act on. Empty when it passed. */
|
|
14
|
+
readonly reason: string;
|
|
15
|
+
}
|
|
16
|
+
export type Grader = (facts: AttemptFacts) => Promise<GraderResult>;
|
|
17
|
+
export declare const GRADER_TYPES: readonly string[];
|
|
18
|
+
export declare function createGrader(spec: GraderSpec): Grader;
|
|
19
|
+
export declare function describeGrader(spec: GraderSpec): string;
|
|
20
|
+
export type { TaskError };
|
|
21
|
+
//# sourceMappingURL=graders.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../src/evals/graders.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/C,MAAM,WAAW,YAAY;IAC3B,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,OAAO,CAAC;IACjE,QAAQ,CAAC,MAAM,EAAE,SAAS,UAAU,EAAE,CAAC;CACxC;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,sEAAsE;IACtE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;AA4LpE,eAAO,MAAM,YAAY,EAAE,SAAS,MAAM,EAA2B,CAAC;AAEtE,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAQrD;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAMvD;AAED,YAAY,EAAE,SAAS,EAAE,CAAC"}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
const pass = { passed: true, reason: "" };
|
|
4
|
+
function fail(reason) {
|
|
5
|
+
return { passed: false, reason };
|
|
6
|
+
}
|
|
7
|
+
function requireString(spec, field) {
|
|
8
|
+
const value = spec[field];
|
|
9
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
10
|
+
throw new Error(`${spec.type}: "${field}" must be a non-empty string.`);
|
|
11
|
+
}
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
async function readWorkspaceFile(workspace, relativePath) {
|
|
15
|
+
try {
|
|
16
|
+
return await readFile(path.join(workspace, relativePath), "utf8");
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
/** Tool names that produced a result, in call order, errors included. */
|
|
23
|
+
function toolCalls(events) {
|
|
24
|
+
return events
|
|
25
|
+
.filter((event) => event.type === "tool_execution_end")
|
|
26
|
+
.map((event) => event.toolName);
|
|
27
|
+
}
|
|
28
|
+
function successfulToolCalls(events) {
|
|
29
|
+
return events
|
|
30
|
+
.filter((event) => event.type === "tool_execution_end" &&
|
|
31
|
+
event.isError === false)
|
|
32
|
+
.map((event) => event.toolName);
|
|
33
|
+
}
|
|
34
|
+
const factories = {
|
|
35
|
+
"file-contains": (spec) => {
|
|
36
|
+
const file = requireString(spec, "path");
|
|
37
|
+
const text = requireString(spec, "text");
|
|
38
|
+
return async ({ workspace }) => {
|
|
39
|
+
const contents = await readWorkspaceFile(workspace, file);
|
|
40
|
+
if (contents === undefined) {
|
|
41
|
+
return fail(`${file} does not exist`);
|
|
42
|
+
}
|
|
43
|
+
return contents.includes(text)
|
|
44
|
+
? pass
|
|
45
|
+
: fail(`${file} does not contain ${JSON.stringify(text)}`);
|
|
46
|
+
};
|
|
47
|
+
},
|
|
48
|
+
"file-excludes": (spec) => {
|
|
49
|
+
const file = requireString(spec, "path");
|
|
50
|
+
const text = requireString(spec, "text");
|
|
51
|
+
return async ({ workspace }) => {
|
|
52
|
+
const contents = await readWorkspaceFile(workspace, file);
|
|
53
|
+
if (contents === undefined) {
|
|
54
|
+
return fail(`${file} does not exist`);
|
|
55
|
+
}
|
|
56
|
+
if (!contents.includes(text)) {
|
|
57
|
+
return pass;
|
|
58
|
+
}
|
|
59
|
+
// Naming the line makes the failure actionable without opening the file.
|
|
60
|
+
const line = contents.slice(0, contents.indexOf(text)).split("\n").length;
|
|
61
|
+
return fail(`${file} still contains ${JSON.stringify(text)} at line ${line}`);
|
|
62
|
+
};
|
|
63
|
+
},
|
|
64
|
+
"file-exists": (spec) => {
|
|
65
|
+
const file = requireString(spec, "path");
|
|
66
|
+
return async ({ workspace }) => {
|
|
67
|
+
try {
|
|
68
|
+
await stat(path.join(workspace, file));
|
|
69
|
+
return pass;
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return fail(`${file} was not created`);
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
},
|
|
76
|
+
"file-absent": (spec) => {
|
|
77
|
+
const file = requireString(spec, "path");
|
|
78
|
+
return async ({ workspace }) => {
|
|
79
|
+
try {
|
|
80
|
+
await stat(path.join(workspace, file));
|
|
81
|
+
return fail(`${file} still exists`);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return pass;
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
},
|
|
88
|
+
"file-equals": (spec) => {
|
|
89
|
+
const file = requireString(spec, "path");
|
|
90
|
+
const text = requireString(spec, "text");
|
|
91
|
+
return async ({ workspace }) => {
|
|
92
|
+
const contents = await readWorkspaceFile(workspace, file);
|
|
93
|
+
if (contents === undefined) {
|
|
94
|
+
return fail(`${file} does not exist`);
|
|
95
|
+
}
|
|
96
|
+
return contents === text
|
|
97
|
+
? pass
|
|
98
|
+
: fail(`${file} does not match the expected contents byte for byte`);
|
|
99
|
+
};
|
|
100
|
+
},
|
|
101
|
+
"used-tool": (spec) => {
|
|
102
|
+
const name = requireString(spec, "name");
|
|
103
|
+
return async ({ events }) => {
|
|
104
|
+
const used = toolCalls(events);
|
|
105
|
+
return used.includes(name)
|
|
106
|
+
? pass
|
|
107
|
+
: fail(`never called ${name}; called ${used.length === 0 ? "nothing" : [...new Set(used)].join(", ")}`);
|
|
108
|
+
};
|
|
109
|
+
},
|
|
110
|
+
"not-used-tool": (spec) => {
|
|
111
|
+
const name = requireString(spec, "name");
|
|
112
|
+
return async ({ events }) => {
|
|
113
|
+
const used = toolCalls(events);
|
|
114
|
+
const count = used.filter((tool) => tool === name).length;
|
|
115
|
+
return count === 0
|
|
116
|
+
? pass
|
|
117
|
+
: fail(`called ${name} ${count} time(s), which this task forbids`);
|
|
118
|
+
};
|
|
119
|
+
},
|
|
120
|
+
"tool-succeeded": (spec) => {
|
|
121
|
+
const name = requireString(spec, "name");
|
|
122
|
+
return async ({ events }) => {
|
|
123
|
+
return successfulToolCalls(events).includes(name)
|
|
124
|
+
? pass
|
|
125
|
+
: fail(`${name} never returned a successful result`);
|
|
126
|
+
};
|
|
127
|
+
},
|
|
128
|
+
"answer-matches": (spec) => {
|
|
129
|
+
const pattern = requireString(spec, "pattern");
|
|
130
|
+
const flags = typeof spec.flags === "string" ? spec.flags : "i";
|
|
131
|
+
const expression = new RegExp(pattern, flags);
|
|
132
|
+
return async ({ finalAnswer }) => {
|
|
133
|
+
return expression.test(finalAnswer)
|
|
134
|
+
? pass
|
|
135
|
+
: fail(`the answer did not match /${pattern}/${flags}: ${JSON.stringify(finalAnswer.slice(0, 120))}`);
|
|
136
|
+
};
|
|
137
|
+
},
|
|
138
|
+
"answer-excludes": (spec) => {
|
|
139
|
+
const pattern = requireString(spec, "pattern");
|
|
140
|
+
const flags = typeof spec.flags === "string" ? spec.flags : "i";
|
|
141
|
+
const expression = new RegExp(pattern, flags);
|
|
142
|
+
return async ({ finalAnswer }) => {
|
|
143
|
+
return expression.test(finalAnswer)
|
|
144
|
+
? fail(`the answer matched /${pattern}/${flags}, which this task forbids`)
|
|
145
|
+
: pass;
|
|
146
|
+
};
|
|
147
|
+
},
|
|
148
|
+
"max-turns-under": (spec) => {
|
|
149
|
+
const limit = spec.turns;
|
|
150
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
151
|
+
throw new Error("max-turns-under: \"turns\" must be a positive integer.");
|
|
152
|
+
}
|
|
153
|
+
return async ({ turnCount }) => {
|
|
154
|
+
return turnCount <= limit
|
|
155
|
+
? pass
|
|
156
|
+
: fail(`took ${turnCount} turns, over the limit of ${limit}`);
|
|
157
|
+
};
|
|
158
|
+
},
|
|
159
|
+
};
|
|
160
|
+
export const GRADER_TYPES = Object.keys(factories);
|
|
161
|
+
export function createGrader(spec) {
|
|
162
|
+
const factory = factories[spec.type];
|
|
163
|
+
if (factory === undefined) {
|
|
164
|
+
throw new Error(`Unknown grader type "${spec.type}". Known types: ${GRADER_TYPES.join(", ")}.`);
|
|
165
|
+
}
|
|
166
|
+
return factory(spec);
|
|
167
|
+
}
|
|
168
|
+
export function describeGrader(spec) {
|
|
169
|
+
const detail = Object.entries(spec)
|
|
170
|
+
.filter(([key]) => key !== "type")
|
|
171
|
+
.map(([key, value]) => `${key}=${JSON.stringify(value)}`)
|
|
172
|
+
.join(" ");
|
|
173
|
+
return detail.length === 0 ? spec.type : `${spec.type} ${detail}`;
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=graders.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"graders.js","sourceRoot":"","sources":["../../src/evals/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAqB7B,MAAM,IAAI,GAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;AAExD,SAAS,IAAI,CAAC,MAAc;IAC1B,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,aAAa,CAAC,IAAgB,EAAE,KAAa;IACpD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,MAAM,KAAK,+BAA+B,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAiB,EACjB,YAAoB;IAEpB,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,yEAAyE;AACzE,SAAS,SAAS,CAAC,MAA6B;IAC9C,OAAO,MAAM;SACV,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC;SACtD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAE,KAA8B,CAAC,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,mBAAmB,CAAC,MAA6B;IACxD,OAAO,MAAM;SACV,MAAM,CACL,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,IAAI,KAAK,oBAAoB;QAClC,KAA8B,CAAC,OAAO,KAAK,KAAK,CACpD;SACA,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAE,KAA8B,CAAC,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAID,MAAM,SAAS,GAAkC;IAC/C,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC5B,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,qBAAqB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/D,CAAC,CAAC;IACJ,CAAC;IAED,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;YACxC,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,OAAO,IAAI,CAAC;YACd,CAAC;YACD,yEAAyE;YACzE,MAAM,IAAI,GACR,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC/D,OAAO,IAAI,CACT,GAAG,IAAI,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CACjE,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;IAED,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;gBACvC,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC,GAAG,IAAI,kBAAkB,CAAC,CAAC;YACzC,CAAC;QACH,CAAC,CAAC;IACJ,CAAC;IAED,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;gBACvC,OAAO,IAAI,CAAC,GAAG,IAAI,eAAe,CAAC,CAAC;YACtC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC;IACJ,CAAC;IAED,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAC1D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,OAAO,IAAI,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,QAAQ,KAAK,IAAI;gBACtB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,qDAAqD,CAAC,CAAC;QACzE,CAAC,CAAC;IACJ,CAAC;IAED,WAAW,EAAE,CAAC,IAAI,EAAE,EAAE;QACpB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAC/B,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CACF,gBAAgB,IAAI,YAAY,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChG,CAAC;QACR,CAAC,CAAC;IACJ,CAAC;IAED,eAAe,EAAE,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;YAC1D,OAAO,KAAK,KAAK,CAAC;gBAChB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,KAAK,mCAAmC,CAAC,CAAC;QACvE,CAAC,CAAC;IACJ,CAAC;IAED,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACzC,OAAO,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC1B,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,qCAAqC,CAAC,CAAC;QACzD,CAAC,CAAC;IACJ,CAAC;IAED,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC9C,OAAO,KAAK,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;YAC/B,OAAO,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC;gBACjC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CACF,6BAA6B,OAAO,IAAI,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAC9F,CAAC;QACR,CAAC,CAAC;IACJ,CAAC;IAED,iBAAiB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC1B,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC9C,OAAO,KAAK,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE;YAC/B,OAAO,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC;gBACjC,CAAC,CAAC,IAAI,CAAC,uBAAuB,OAAO,IAAI,KAAK,2BAA2B,CAAC;gBAC1E,CAAC,CAAC,IAAI,CAAC;QACX,CAAC,CAAC;IACJ,CAAC;IAED,iBAAiB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;YAC7B,OAAO,SAAS,IAAK,KAAgB;gBACnC,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,IAAI,CAAC,QAAQ,SAAS,6BAA6B,KAAe,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC;IACJ,CAAC;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAsB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AAEtE,MAAM,UAAU,YAAY,CAAC,IAAgB;IAC3C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,wBAAwB,IAAI,CAAC,IAAI,mBAAmB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/E,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAgB;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC;SAChC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,MAAM,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;SACxD,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Capability } from "./task.js";
|
|
2
|
+
/**
|
|
3
|
+
* Argument parsing lives apart from the entry point.
|
|
4
|
+
*
|
|
5
|
+
* cli.ts runs main() when imported, so anything that wants to test the parsing
|
|
6
|
+
* would start an eval run just by importing it.
|
|
7
|
+
*/
|
|
8
|
+
export interface EvalOptions {
|
|
9
|
+
readonly tasks: readonly string[];
|
|
10
|
+
readonly attempts?: number;
|
|
11
|
+
readonly directory: string;
|
|
12
|
+
readonly jsonPath?: string;
|
|
13
|
+
readonly capabilities: readonly Capability[];
|
|
14
|
+
readonly providerArgs: readonly string[];
|
|
15
|
+
readonly help: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare function parseEvalArgs(argv: readonly string[]): EvalOptions;
|
|
18
|
+
//# sourceMappingURL=parse-args.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-args.d.ts","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CA4DlE"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
export function parseEvalArgs(argv) {
|
|
2
|
+
const tasks = [];
|
|
3
|
+
const capabilities = [];
|
|
4
|
+
const providerArgs = [];
|
|
5
|
+
let attempts;
|
|
6
|
+
let directory = "evals";
|
|
7
|
+
let jsonPath;
|
|
8
|
+
let help = false;
|
|
9
|
+
const value = (index, option) => {
|
|
10
|
+
const next = argv[index + 1];
|
|
11
|
+
if (next === undefined || next.startsWith("-")) {
|
|
12
|
+
throw new TypeError(`${option} requires a value.`);
|
|
13
|
+
}
|
|
14
|
+
return next;
|
|
15
|
+
};
|
|
16
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
17
|
+
const argument = argv[index];
|
|
18
|
+
if (argument === "--help" || argument === "-h") {
|
|
19
|
+
help = true;
|
|
20
|
+
}
|
|
21
|
+
else if (argument === "--task") {
|
|
22
|
+
tasks.push(value(index, "--task"));
|
|
23
|
+
index += 1;
|
|
24
|
+
}
|
|
25
|
+
else if (argument === "--attempts") {
|
|
26
|
+
const parsed = Number(value(index, "--attempts"));
|
|
27
|
+
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 50) {
|
|
28
|
+
throw new TypeError("--attempts must be an integer from 1 to 50.");
|
|
29
|
+
}
|
|
30
|
+
attempts = parsed;
|
|
31
|
+
index += 1;
|
|
32
|
+
}
|
|
33
|
+
else if (argument === "--dir") {
|
|
34
|
+
directory = value(index, "--dir");
|
|
35
|
+
index += 1;
|
|
36
|
+
}
|
|
37
|
+
else if (argument === "--json") {
|
|
38
|
+
jsonPath = value(index, "--json");
|
|
39
|
+
index += 1;
|
|
40
|
+
}
|
|
41
|
+
else if (argument === "--allow-writes") {
|
|
42
|
+
capabilities.push("writes");
|
|
43
|
+
}
|
|
44
|
+
else if (argument === "--allow-shell") {
|
|
45
|
+
capabilities.push("shell");
|
|
46
|
+
}
|
|
47
|
+
else if (argument === "--provider" || argument === "--model") {
|
|
48
|
+
// Passed through so the Provider registry resolves them exactly as the
|
|
49
|
+
// real CLI does, including environment fallbacks.
|
|
50
|
+
providerArgs.push(argument, value(index, argument));
|
|
51
|
+
index += 1;
|
|
52
|
+
}
|
|
53
|
+
else if (argument !== undefined) {
|
|
54
|
+
throw new TypeError(`Unknown option: ${argument}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
tasks,
|
|
59
|
+
...(attempts === undefined ? {} : { attempts }),
|
|
60
|
+
directory,
|
|
61
|
+
...(jsonPath === undefined ? {} : { jsonPath }),
|
|
62
|
+
capabilities,
|
|
63
|
+
providerArgs,
|
|
64
|
+
help,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=parse-args.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-args.js","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAkBA,MAAM,UAAU,aAAa,CAAC,IAAuB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAiB,EAAE,CAAC;IACtC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,QAA4B,CAAC;IACjC,IAAI,SAAS,GAAG,OAAO,CAAC;IACxB,IAAI,QAA4B,CAAC;IACjC,IAAI,IAAI,GAAG,KAAK,CAAC;IAEjB,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,MAAc,EAAU,EAAE;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,MAAM,oBAAoB,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC/C,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;gBAC/D,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;YACrE,CAAC;YACD,QAAQ,GAAG,MAAM,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,SAAS,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;YACzC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,QAAQ,KAAK,YAAY,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC/D,uEAAuE;YACvE,kDAAkD;YAClD,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpD,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,IAAI,SAAS,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,SAAS;QACT,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,YAAY;QACZ,YAAY;QACZ,IAAI;KACL,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { TaskResult } from "./runner.js";
|
|
2
|
+
export interface ReportMeta {
|
|
3
|
+
readonly provider: string;
|
|
4
|
+
readonly model: string;
|
|
5
|
+
readonly version: string;
|
|
6
|
+
readonly startedAt: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* The table always shows the sample size next to the score.
|
|
10
|
+
*
|
|
11
|
+
* "4/5" and "80%" carry different information from "passed": a rate without its
|
|
12
|
+
* denominator invites reading one lucky attempt as a result.
|
|
13
|
+
*/
|
|
14
|
+
export declare function formatTable(results: readonly TaskResult[]): string;
|
|
15
|
+
/** Failures, grouped so the same reason across attempts is one line. */
|
|
16
|
+
export declare function formatFailures(results: readonly TaskResult[]): string;
|
|
17
|
+
export declare function toJsonReport(results: readonly TaskResult[], meta: ReportMeta): object;
|
|
18
|
+
//# sourceMappingURL=report.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAwBD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAqClE;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAgBrE;AAED,wBAAgB,YAAY,CAC1B,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,IAAI,EAAE,UAAU,GACf,MAAM,CA4BR"}
|