eqho-eval 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +552 -0
- package/dist/cli/auth-store.d.ts +5 -0
- package/dist/cli/auth-store.d.ts.map +1 -0
- package/dist/cli/auth-store.js +39 -0
- package/dist/cli/auth-store.js.map +1 -0
- package/dist/cli/banner.d.ts +3 -0
- package/dist/cli/banner.d.ts.map +1 -0
- package/dist/cli/banner.js +38 -0
- package/dist/cli/banner.js.map +1 -0
- package/dist/cli/commands/action-eval.d.ts +3 -0
- package/dist/cli/commands/action-eval.d.ts.map +1 -0
- package/dist/cli/commands/action-eval.js +133 -0
- package/dist/cli/commands/action-eval.js.map +1 -0
- package/dist/cli/commands/auth.d.ts +3 -0
- package/dist/cli/commands/auth.d.ts.map +1 -0
- package/dist/cli/commands/auth.js +156 -0
- package/dist/cli/commands/auth.js.map +1 -0
- package/dist/cli/commands/cache.d.ts +3 -0
- package/dist/cli/commands/cache.d.ts.map +1 -0
- package/dist/cli/commands/cache.js +43 -0
- package/dist/cli/commands/cache.js.map +1 -0
- package/dist/cli/commands/ci.d.ts +3 -0
- package/dist/cli/commands/ci.d.ts.map +1 -0
- package/dist/cli/commands/ci.js +124 -0
- package/dist/cli/commands/ci.js.map +1 -0
- package/dist/cli/commands/conversations.d.ts +3 -0
- package/dist/cli/commands/conversations.d.ts.map +1 -0
- package/dist/cli/commands/conversations.js +89 -0
- package/dist/cli/commands/conversations.js.map +1 -0
- package/dist/cli/commands/diff.d.ts +3 -0
- package/dist/cli/commands/diff.d.ts.map +1 -0
- package/dist/cli/commands/diff.js +122 -0
- package/dist/cli/commands/diff.js.map +1 -0
- package/dist/cli/commands/doctor.d.ts +11 -0
- package/dist/cli/commands/doctor.d.ts.map +1 -0
- package/dist/cli/commands/doctor.js +308 -0
- package/dist/cli/commands/doctor.js.map +1 -0
- package/dist/cli/commands/eval.d.ts +3 -0
- package/dist/cli/commands/eval.d.ts.map +1 -0
- package/dist/cli/commands/eval.js +101 -0
- package/dist/cli/commands/eval.js.map +1 -0
- package/dist/cli/commands/init.d.ts +3 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +182 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/list.d.ts +3 -0
- package/dist/cli/commands/list.d.ts.map +1 -0
- package/dist/cli/commands/list.js +80 -0
- package/dist/cli/commands/list.js.map +1 -0
- package/dist/cli/commands/mentions.d.ts +3 -0
- package/dist/cli/commands/mentions.d.ts.map +1 -0
- package/dist/cli/commands/mentions.js +125 -0
- package/dist/cli/commands/mentions.js.map +1 -0
- package/dist/cli/commands/org.d.ts +3 -0
- package/dist/cli/commands/org.d.ts.map +1 -0
- package/dist/cli/commands/org.js +196 -0
- package/dist/cli/commands/org.js.map +1 -0
- package/dist/cli/commands/postcall-eval.d.ts +3 -0
- package/dist/cli/commands/postcall-eval.d.ts.map +1 -0
- package/dist/cli/commands/postcall-eval.js +188 -0
- package/dist/cli/commands/postcall-eval.js.map +1 -0
- package/dist/cli/commands/render.d.ts +3 -0
- package/dist/cli/commands/render.d.ts.map +1 -0
- package/dist/cli/commands/render.js +223 -0
- package/dist/cli/commands/render.js.map +1 -0
- package/dist/cli/commands/results.d.ts +3 -0
- package/dist/cli/commands/results.d.ts.map +1 -0
- package/dist/cli/commands/results.js +128 -0
- package/dist/cli/commands/results.js.map +1 -0
- package/dist/cli/commands/scenarios.d.ts +3 -0
- package/dist/cli/commands/scenarios.d.ts.map +1 -0
- package/dist/cli/commands/scenarios.js +57 -0
- package/dist/cli/commands/scenarios.js.map +1 -0
- package/dist/cli/commands/start.d.ts +3 -0
- package/dist/cli/commands/start.d.ts.map +1 -0
- package/dist/cli/commands/start.js +260 -0
- package/dist/cli/commands/start.js.map +1 -0
- package/dist/cli/commands/status.d.ts +3 -0
- package/dist/cli/commands/status.d.ts.map +1 -0
- package/dist/cli/commands/status.js +133 -0
- package/dist/cli/commands/status.js.map +1 -0
- package/dist/cli/commands/sync.d.ts +3 -0
- package/dist/cli/commands/sync.d.ts.map +1 -0
- package/dist/cli/commands/sync.js +80 -0
- package/dist/cli/commands/sync.js.map +1 -0
- package/dist/cli/commands/view.d.ts +3 -0
- package/dist/cli/commands/view.d.ts.map +1 -0
- package/dist/cli/commands/view.js +29 -0
- package/dist/cli/commands/view.js.map +1 -0
- package/dist/cli/error-handler.d.ts +8 -0
- package/dist/cli/error-handler.d.ts.map +1 -0
- package/dist/cli/error-handler.js +133 -0
- package/dist/cli/error-handler.js.map +1 -0
- package/dist/cli/gateway.d.ts +14 -0
- package/dist/cli/gateway.d.ts.map +1 -0
- package/dist/cli/gateway.js +222 -0
- package/dist/cli/gateway.js.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +194 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/core/action-eval-builder.d.ts +20 -0
- package/dist/core/action-eval-builder.d.ts.map +1 -0
- package/dist/core/action-eval-builder.js +276 -0
- package/dist/core/action-eval-builder.js.map +1 -0
- package/dist/core/agent-fetcher.d.ts +35 -0
- package/dist/core/agent-fetcher.d.ts.map +1 -0
- package/dist/core/agent-fetcher.js +81 -0
- package/dist/core/agent-fetcher.js.map +1 -0
- package/dist/core/api-cache.d.ts +11 -0
- package/dist/core/api-cache.d.ts.map +1 -0
- package/dist/core/api-cache.js +89 -0
- package/dist/core/api-cache.js.map +1 -0
- package/dist/core/config-generator.d.ts +26 -0
- package/dist/core/config-generator.d.ts.map +1 -0
- package/dist/core/config-generator.js +457 -0
- package/dist/core/config-generator.js.map +1 -0
- package/dist/core/conversation-loader.d.ts +21 -0
- package/dist/core/conversation-loader.d.ts.map +1 -0
- package/dist/core/conversation-loader.js +74 -0
- package/dist/core/conversation-loader.js.map +1 -0
- package/dist/core/dataset-loader.d.ts +26 -0
- package/dist/core/dataset-loader.d.ts.map +1 -0
- package/dist/core/dataset-loader.js +121 -0
- package/dist/core/dataset-loader.js.map +1 -0
- package/dist/core/disposition-builder.d.ts +38 -0
- package/dist/core/disposition-builder.d.ts.map +1 -0
- package/dist/core/disposition-builder.js +270 -0
- package/dist/core/disposition-builder.js.map +1 -0
- package/dist/core/eqho-client.d.ts +45 -0
- package/dist/core/eqho-client.d.ts.map +1 -0
- package/dist/core/eqho-client.js +154 -0
- package/dist/core/eqho-client.js.map +1 -0
- package/dist/core/greeting-builder.d.ts +18 -0
- package/dist/core/greeting-builder.d.ts.map +1 -0
- package/dist/core/greeting-builder.js +83 -0
- package/dist/core/greeting-builder.js.map +1 -0
- package/dist/core/postcall-simulator.d.ts +20 -0
- package/dist/core/postcall-simulator.d.ts.map +1 -0
- package/dist/core/postcall-simulator.js +212 -0
- package/dist/core/postcall-simulator.js.map +1 -0
- package/dist/core/prompt-assembler.d.ts +25 -0
- package/dist/core/prompt-assembler.d.ts.map +1 -0
- package/dist/core/prompt-assembler.js +185 -0
- package/dist/core/prompt-assembler.js.map +1 -0
- package/dist/core/promptfoo-runner.d.ts +13 -0
- package/dist/core/promptfoo-runner.d.ts.map +1 -0
- package/dist/core/promptfoo-runner.js +49 -0
- package/dist/core/promptfoo-runner.js.map +1 -0
- package/dist/core/provider-mapper.d.ts +39 -0
- package/dist/core/provider-mapper.d.ts.map +1 -0
- package/dist/core/provider-mapper.js +120 -0
- package/dist/core/provider-mapper.js.map +1 -0
- package/dist/core/template-engine.d.ts +10 -0
- package/dist/core/template-engine.d.ts.map +1 -0
- package/dist/core/template-engine.js +78 -0
- package/dist/core/template-engine.js.map +1 -0
- package/dist/core/tools-builder.d.ts +14 -0
- package/dist/core/tools-builder.d.ts.map +1 -0
- package/dist/core/tools-builder.js +208 -0
- package/dist/core/tools-builder.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/index.js.map +1 -0
- package/dist/types/config.d.ts +100 -0
- package/dist/types/config.d.ts.map +1 -0
- package/dist/types/config.js +2 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/eqho.d.ts +221 -0
- package/dist/types/eqho.d.ts.map +1 -0
- package/dist/types/eqho.js +2 -0
- package/dist/types/eqho.js.map +1 -0
- package/dist/types/helpers.d.ts +9 -0
- package/dist/types/helpers.d.ts.map +1 -0
- package/dist/types/helpers.js +8 -0
- package/dist/types/helpers.js.map +1 -0
- package/package.json +77 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"org.js","sourceRoot":"","sources":["../../../src/cli/commands/org.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEtD,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC;KACzC,WAAW,CAAC,0CAA0C,CAAC;KACvD,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACzC,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IAExB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;YACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;SAC9B,CAAC,CAAC,CAAC;QACJ,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;IAClD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,IAAI,QAAQ,GAAG,CAAC,CAAC;IAC/E,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAC9F,CAAC;IAED,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;QAC5E,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,yEAAyE;AACzE,UAAU;KACP,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,8BAA8B,CAAC;KAC3C,MAAM,CAAC,iBAAiB,EAAE,8BAA8B,CAAC;KACzD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACjD,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IAEjC,MAAM,OAAO,GAAG,QAAQ;QACtB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAE/E,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpF,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CACzB,GAAG,IAAI,CAAC,UAAU,iBAAiB,WAAW,EAAE,EAChD;YACE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,YAAY,EAAE,EAAE;YACzD,OAAO,EAAE,MAAM;SAChB,CACF,CAAC;QAEF,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,aAAoD,CAAC;QAC3E,OAAO,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,kBAAkB,CAAC,CAAC;QAEzD,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3C,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,CAAC;YACpD,OAAO;QACT,CAAC;QAED,OAAO,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CACT,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAC/D,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,GAAmE,CAAC;QAClF,OAAO,EAAE,IAAI,CAAC,8BAA8B,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrF,IAAI,GAAG;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,yEAAyE;AACzE,UAAU;KACP,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,oCAAoC,CAAC;KACjD,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACjD,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IAEjC,MAAM,OAAO,GAAG,QAAQ;QACtB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,oBAAoB,KAAK,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAElF,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAC1B,GAAG,IAAI,CAAC,UAAU,sBAAsB,EACxC,EAAE,MAAM,EAAE,KAAK,EAAE,EACjB;YACE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,YAAY,EAAE,EAAE;YACzD,OAAO,EAAE,MAAM;SAChB,CACF,CAAC;QAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,KAAe,CAAC;QAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,OAA6B,CAAC;QAEvD,QAAQ,CAAC;YACP,GAAG,IAAI;YACP,YAAY,EAAE,QAAQ;YACtB,KAAK;YACL,OAAO,EAAE,OAAO,IAAI,SAAS;SAC9B,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QACxD,OAAO,EAAE,OAAO,CAAC,eAAe,KAAK,EAAE,CAAC,CAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,KAAK,GAAG,GAAmE,CAAC;QAClF,OAAO,EAAE,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC,CAAC;QACjF,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrF,IAAI,GAAG;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,yEAAyE;AACzE,UAAU;KACP,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,sCAAsC,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;IAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IACjD,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IAEjC,MAAM,OAAO,GAAG,QAAQ;QACtB,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAE9E,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAC1B,GAAG,IAAI,CAAC,UAAU,sBAAsB,EACxC,EAAE,EACF;YACE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,YAAY,EAAE,EAAE;YACzD,OAAO,EAAE,MAAM;SAChB,CACF,CAAC;QAEF,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,KAAe,CAAC;QAE1C,QAAQ,CAAC;YACP,GAAG,IAAI;YACP,YAAY,EAAE,QAAQ;YACtB,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,SAAS;SACnB,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,CAAC,wCAAwC,CAAC,CAAC;QAE3D,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,EAAE,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,IAAI,GAAG;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS,iBAAiB;IACxB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IAExB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC,CAAC;QAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,IAAkE,CAAC;AAC5E,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postcall-eval.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/postcall-eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAcpC,eAAO,MAAM,mBAAmB,SAuP5B,CAAC"}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import ora from "ora";
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { EqhoClient, EqhoApiError } from "../../core/eqho-client.js";
|
|
7
|
+
import { fetchCampaignAgents } from "../../core/agent-fetcher.js";
|
|
8
|
+
import { generatePostcallEvalConfig } from "../../core/postcall-simulator.js";
|
|
9
|
+
import { generateDispositionEvalConfig } from "../../core/disposition-builder.js";
|
|
10
|
+
import { loadAuth } from "../auth-store.js";
|
|
11
|
+
import { getProxyConfig } from "../../core/provider-mapper.js";
|
|
12
|
+
import { getId } from "../../types/helpers.js";
|
|
13
|
+
export const postcallEvalCommand = new Command("postcall-eval")
|
|
14
|
+
.description("Generate eval configs for postcall actions and/or dispositions")
|
|
15
|
+
.option("-c, --campaign <id>", "Campaign ID (reads from eqho.config.json if omitted)")
|
|
16
|
+
.option("-a, --agent <id>", "Specific agent ID (default: first agent in campaign)")
|
|
17
|
+
.option("-n, --calls <count>", "Number of real calls to use as test data", "0")
|
|
18
|
+
.option("--disposition", "Generate disposition accuracy eval config", false)
|
|
19
|
+
.option("--postcall", "Generate postcall action eval config (default if no flags)", false)
|
|
20
|
+
.option("--on-call-start", "Generate on-call-start action eval config", false)
|
|
21
|
+
.option("-o, --output <dir>", "Output directory", ".")
|
|
22
|
+
.option("-d, --dir <dir>", "Project directory (for reading eqho.config.json)", ".")
|
|
23
|
+
.action(async (opts) => {
|
|
24
|
+
const auth = loadAuth();
|
|
25
|
+
if (!auth) {
|
|
26
|
+
console.log(chalk.red("Not authenticated. Run: eqho-eval auth"));
|
|
27
|
+
process.exit(1);
|
|
28
|
+
}
|
|
29
|
+
const client = new EqhoClient(auth);
|
|
30
|
+
let campaignId = opts.campaign;
|
|
31
|
+
if (!campaignId) {
|
|
32
|
+
const configPath = path.resolve(opts.dir, "eqho.config.json");
|
|
33
|
+
if (fs.existsSync(configPath)) {
|
|
34
|
+
try {
|
|
35
|
+
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
36
|
+
campaignId = config.campaignId;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// ignore parse errors
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
if (!campaignId) {
|
|
44
|
+
console.log(chalk.red("No campaign ID. Use --campaign <id> or run from an init'd project."));
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
const doDisposition = opts.disposition;
|
|
48
|
+
const doOnCallStart = opts.onCallStart;
|
|
49
|
+
const doPostcall = opts.postcall || (!doDisposition && !doOnCallStart);
|
|
50
|
+
const spinner = ora({ text: "Fetching campaign...", stream: process.stderr }).start();
|
|
51
|
+
let campaign;
|
|
52
|
+
try {
|
|
53
|
+
campaign = await client.getCampaign(campaignId);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
spinner.fail(`Failed to fetch campaign: ${err instanceof EqhoApiError ? err.message : String(err)}`);
|
|
57
|
+
process.exit(1);
|
|
58
|
+
}
|
|
59
|
+
const { agents, errors } = await fetchCampaignAgents(client, campaign, {
|
|
60
|
+
agentId: opts.agent,
|
|
61
|
+
onProgress: (msg) => {
|
|
62
|
+
spinner.text = msg;
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
if (errors.length) {
|
|
66
|
+
for (const e of errors)
|
|
67
|
+
console.log(chalk.yellow(` Warning: ${e}`));
|
|
68
|
+
}
|
|
69
|
+
if (!agents.length) {
|
|
70
|
+
spinner.fail("No agents loaded from campaign.");
|
|
71
|
+
process.exit(1);
|
|
72
|
+
}
|
|
73
|
+
const agent = agents[0].agent;
|
|
74
|
+
const agentSlug = agent.name
|
|
75
|
+
.toLowerCase()
|
|
76
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
77
|
+
.replace(/(^-|-$)/g, "");
|
|
78
|
+
let calls;
|
|
79
|
+
const callCount = parseInt(opts.calls, 10);
|
|
80
|
+
if (callCount > 0) {
|
|
81
|
+
spinner.text = `Fetching ${callCount} real calls...`;
|
|
82
|
+
try {
|
|
83
|
+
const { calls: rawCalls } = await client.listCalls(campaignId, {
|
|
84
|
+
limit: callCount,
|
|
85
|
+
});
|
|
86
|
+
const fullCalls = [];
|
|
87
|
+
for (const call of rawCalls) {
|
|
88
|
+
spinner.text = `Fetching transcript ${fullCalls.length + 1}/${rawCalls.length}...`;
|
|
89
|
+
try {
|
|
90
|
+
const full = await client.getCall(getId(call));
|
|
91
|
+
if (full.transcript?.length)
|
|
92
|
+
fullCalls.push(full);
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
// skip failed fetches
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
calls = fullCalls;
|
|
99
|
+
spinner.text = `Loaded ${fullCalls.length} calls with transcripts`;
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
console.log(chalk.yellow(` Warning: Failed to fetch calls: ${err instanceof EqhoApiError ? err.message : String(err)}`));
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const outDir = path.resolve(opts.output);
|
|
106
|
+
const proxy = getProxyConfig(loadAuth());
|
|
107
|
+
let filesWritten = 0;
|
|
108
|
+
if (doPostcall) {
|
|
109
|
+
spinner.text = "Generating postcall eval config...";
|
|
110
|
+
const postcallDir = path.join(outDir, "postcall-eval");
|
|
111
|
+
const postcallFiles = generatePostcallEvalConfig({
|
|
112
|
+
actions: agent.actions,
|
|
113
|
+
agentName: agent.name,
|
|
114
|
+
agentSlug,
|
|
115
|
+
campaignName: campaign.name,
|
|
116
|
+
calls,
|
|
117
|
+
proxy,
|
|
118
|
+
});
|
|
119
|
+
for (const [filePath, content] of Object.entries(postcallFiles)) {
|
|
120
|
+
const fullPath = path.join(postcallDir, filePath);
|
|
121
|
+
const dir = path.dirname(fullPath);
|
|
122
|
+
if (!fs.existsSync(dir))
|
|
123
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
124
|
+
fs.writeFileSync(fullPath, content, "utf-8");
|
|
125
|
+
}
|
|
126
|
+
filesWritten += Object.keys(postcallFiles).length;
|
|
127
|
+
console.log(chalk.dim(` Postcall eval → ${postcallDir}/ (${Object.keys(postcallFiles).length} files)`));
|
|
128
|
+
}
|
|
129
|
+
if (doDisposition) {
|
|
130
|
+
spinner.text = "Generating disposition eval config...";
|
|
131
|
+
const dispoDir = path.join(outDir, "disposition-eval");
|
|
132
|
+
const dispositions = campaign.dispositions ?? [];
|
|
133
|
+
if (!dispositions.length) {
|
|
134
|
+
console.log(chalk.yellow(" No dispositions configured on this campaign. Generating defaults."));
|
|
135
|
+
}
|
|
136
|
+
const dispoFiles = generateDispositionEvalConfig({
|
|
137
|
+
dispositions,
|
|
138
|
+
campaignName: campaign.name,
|
|
139
|
+
calls,
|
|
140
|
+
proxy,
|
|
141
|
+
});
|
|
142
|
+
for (const [filePath, content] of Object.entries(dispoFiles)) {
|
|
143
|
+
const fullPath = path.join(dispoDir, filePath);
|
|
144
|
+
const dir = path.dirname(fullPath);
|
|
145
|
+
if (!fs.existsSync(dir))
|
|
146
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
147
|
+
fs.writeFileSync(fullPath, content, "utf-8");
|
|
148
|
+
}
|
|
149
|
+
filesWritten += Object.keys(dispoFiles).length;
|
|
150
|
+
console.log(chalk.dim(` Disposition eval → ${dispoDir}/ (${Object.keys(dispoFiles).length} files)`));
|
|
151
|
+
}
|
|
152
|
+
if (doOnCallStart) {
|
|
153
|
+
spinner.text = "Generating on-call-start eval config...";
|
|
154
|
+
const ocsDir = path.join(outDir, "oncallstart-eval");
|
|
155
|
+
const ocsFiles = generatePostcallEvalConfig({
|
|
156
|
+
actions: agent.actions,
|
|
157
|
+
agentName: agent.name,
|
|
158
|
+
agentSlug,
|
|
159
|
+
campaignName: campaign.name,
|
|
160
|
+
calls,
|
|
161
|
+
executionType: "on_call_start",
|
|
162
|
+
proxy,
|
|
163
|
+
});
|
|
164
|
+
for (const [filePath, content] of Object.entries(ocsFiles)) {
|
|
165
|
+
const fullPath = path.join(ocsDir, filePath);
|
|
166
|
+
const dir = path.dirname(fullPath);
|
|
167
|
+
if (!fs.existsSync(dir))
|
|
168
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
169
|
+
fs.writeFileSync(fullPath, content, "utf-8");
|
|
170
|
+
}
|
|
171
|
+
filesWritten += Object.keys(ocsFiles).length;
|
|
172
|
+
console.log(chalk.dim(` On-call-start eval → ${ocsDir}/ (${Object.keys(ocsFiles).length} files)`));
|
|
173
|
+
}
|
|
174
|
+
spinner.succeed(`Generated ${filesWritten} files for ${campaign.name}`);
|
|
175
|
+
console.log();
|
|
176
|
+
console.log(chalk.bold("Next steps:"));
|
|
177
|
+
if (doPostcall) {
|
|
178
|
+
console.log(chalk.dim(" cd postcall-eval && npx promptfoo eval"));
|
|
179
|
+
}
|
|
180
|
+
if (doDisposition) {
|
|
181
|
+
console.log(chalk.dim(" cd disposition-eval && npx promptfoo eval"));
|
|
182
|
+
}
|
|
183
|
+
if (doOnCallStart) {
|
|
184
|
+
console.log(chalk.dim(" cd oncallstart-eval && npx promptfoo eval"));
|
|
185
|
+
}
|
|
186
|
+
console.log(chalk.dim(" npx promptfoo view"));
|
|
187
|
+
});
|
|
188
|
+
//# sourceMappingURL=postcall-eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postcall-eval.js","sourceRoot":"","sources":["../../../src/cli/commands/postcall-eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,mBAAmB,EAAE,MAAM,6BAA6B,CAAC;AAClE,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,mCAAmC,CAAC;AAClF,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAE/D,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAE/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;KAC5D,WAAW,CACV,gEAAgE,CACjE;KACA,MAAM,CACL,qBAAqB,EACrB,sDAAsD,CACvD;KACA,MAAM,CACL,kBAAkB,EAClB,sDAAsD,CACvD;KACA,MAAM,CACL,qBAAqB,EACrB,0CAA0C,EAC1C,GAAG,CACJ;KACA,MAAM,CACL,eAAe,EACf,2CAA2C,EAC3C,KAAK,CACN;KACA,MAAM,CACL,YAAY,EACZ,4DAA4D,EAC5D,KAAK,CACN;KACA,MAAM,CACL,iBAAiB,EACjB,2CAA2C,EAC3C,KAAK,CACN;KACA,MAAM,CAAC,oBAAoB,EAAE,kBAAkB,EAAE,GAAG,CAAC;KACrD,MAAM,CAAC,iBAAiB,EAAE,kDAAkD,EAAE,GAAG,CAAC;KAClF,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACrB,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC;IAE/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAmB,IAAI,CAAC,KAAK,CACvC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CACrC,CAAC;gBACF,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,oEAAoE,CACrE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC;IACvC,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,CAAC;IAEvE,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAEtF,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CACV,6BAA6B,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE;QACrE,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC;QACrB,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,MAAM;YACpB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;IAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI;SACzB,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAE3B,IAAI,KAAK,CAAC;IACV,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,GAAG,YAAY,SAAS,gBAAgB,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,UAAU,EAAE;gBAC7D,KAAK,EAAE,SAAS;aACjB,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,EAAE,CAAC;YACrB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,GAAG,uBAAuB,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;gBACnF,IAAI,CAAC;oBACH,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC/C,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM;wBAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;gBAAC,MAAM,CAAC;oBACP,sBAAsB;gBACxB,CAAC;YACH,CAAC;YACD,KAAK,GAAG,SAAS,CAAC;YAClB,OAAO,CAAC,IAAI,GAAG,UAAU,SAAS,CAAC,MAAM,yBAAyB,CAAC;QACrE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CACV,qCAAqC,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC/F,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC;IACzC,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,GAAG,oCAAoC,CAAC;QACpD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QAEvD,MAAM,aAAa,GAAG,0BAA0B,CAAC;YAC/C,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,SAAS;YACT,YAAY,EAAE,QAAQ,CAAC,IAAI;YAC3B,KAAK;YACL,KAAK;SACN,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAClD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAED,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;QAClD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,qBAAqB,WAAW,MAAM,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,SAAS,CACjF,CACF,CAAC;IACJ,CAAC;IAED,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,GAAG,uCAAuC,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC;QAEjD,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,MAAM,CACV,qEAAqE,CACtE,CACF,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,6BAA6B,CAAC;YAC/C,YAAY;YACZ,YAAY,EAAE,QAAQ,CAAC,IAAI;YAC3B,KAAK;YACL,KAAK;SACN,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAED,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;QAC/C,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,wBAAwB,QAAQ,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,SAAS,CAC9E,CACF,CAAC;IACJ,CAAC;IAED,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,GAAG,yCAAyC,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;QAErD,MAAM,QAAQ,GAAG,0BAA0B,CAAC;YAC1C,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,KAAK,CAAC,IAAI;YACrB,SAAS;YACT,YAAY,EAAE,QAAQ,CAAC,IAAI;YAC3B,KAAK;YACL,aAAa,EAAE,eAAe;YAC9B,KAAK;SACN,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAChE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QAED,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QAC7C,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,0BAA0B,MAAM,MAAM,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,SAAS,CAC5E,CACF,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,OAAO,CACb,aAAa,YAAY,cAAc,QAAQ,CAAC,IAAI,EAAE,CACvD,CAAC;IAEF,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;IACvC,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/render.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqBpC,eAAO,MAAM,aAAa,SAkLtB,CAAC"}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import ora from "ora";
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import yaml from "js-yaml";
|
|
7
|
+
import { EqhoClient, EqhoApiError } from "../../core/eqho-client.js";
|
|
8
|
+
import { fetchCampaignAgents, resolveSystemPromptSections, } from "../../core/agent-fetcher.js";
|
|
9
|
+
import { assemblePrompt } from "../../core/prompt-assembler.js";
|
|
10
|
+
import { loadAuth } from "../auth-store.js";
|
|
11
|
+
function extractUnresolvedMentions(text) {
|
|
12
|
+
const matches = text.match(/\{\{[^}]+\}\}/g) ?? [];
|
|
13
|
+
return [...new Set(matches)].sort();
|
|
14
|
+
}
|
|
15
|
+
export const renderCommand = new Command("render")
|
|
16
|
+
.description("Preview the assembled system prompt, tools, and mentions for a campaign/agent")
|
|
17
|
+
.option("-c, --campaign <id>", "Campaign ID (reads from eqho.config.json if omitted)")
|
|
18
|
+
.option("-a, --agent <id>", "Specific agent ID (default: first agent)")
|
|
19
|
+
.option("--vars <json>", "JSON string or path to JSON file of mention values to hydrate")
|
|
20
|
+
.option("--call-type <type>", "Call type: inbound or outbound", "inbound")
|
|
21
|
+
.option("-f, --format <format>", "Output format: text (default), json, yaml", "text")
|
|
22
|
+
.option("-d, --dir <dir>", "Project directory", ".")
|
|
23
|
+
.action(async (opts, cmd) => {
|
|
24
|
+
const globalJson = cmd.parent?.opts().json;
|
|
25
|
+
if (globalJson)
|
|
26
|
+
opts.format = "json";
|
|
27
|
+
const auth = loadAuth();
|
|
28
|
+
if (!auth) {
|
|
29
|
+
if (globalJson) {
|
|
30
|
+
console.log(JSON.stringify({ error: "Not authenticated", fix: "eqho-eval auth --key <key>" }));
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
console.log(chalk.red("Not authenticated. Run: eqho-eval auth"));
|
|
34
|
+
}
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
37
|
+
const client = new EqhoClient(auth);
|
|
38
|
+
let campaignId = opts.campaign;
|
|
39
|
+
if (!campaignId) {
|
|
40
|
+
const configPath = path.resolve(opts.dir, "eqho.config.json");
|
|
41
|
+
if (fs.existsSync(configPath)) {
|
|
42
|
+
try {
|
|
43
|
+
const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
|
|
44
|
+
campaignId = config.campaignId;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// ignore
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (!campaignId) {
|
|
52
|
+
console.log(chalk.red("No campaign ID. Use --campaign <id> or run from an init'd project."));
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
let leadOverrides;
|
|
56
|
+
if (opts.vars) {
|
|
57
|
+
try {
|
|
58
|
+
if (fs.existsSync(opts.vars)) {
|
|
59
|
+
leadOverrides = JSON.parse(fs.readFileSync(opts.vars, "utf-8"));
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
leadOverrides = JSON.parse(opts.vars);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
console.log(chalk.red(`Failed to parse --vars: ${String(err)}`));
|
|
67
|
+
process.exit(1);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const callType = opts.callType;
|
|
71
|
+
const isStructured = opts.format === "json" || opts.format === "yaml";
|
|
72
|
+
const spinner = ora({
|
|
73
|
+
text: "Fetching campaign and agent data...",
|
|
74
|
+
stream: isStructured ? process.stderr : process.stdout,
|
|
75
|
+
}).start();
|
|
76
|
+
let campaign;
|
|
77
|
+
try {
|
|
78
|
+
campaign = await client.getCampaign(campaignId);
|
|
79
|
+
}
|
|
80
|
+
catch (err) {
|
|
81
|
+
spinner.fail(`Failed to fetch campaign: ${err instanceof EqhoApiError ? err.message : String(err)}`);
|
|
82
|
+
process.exit(1);
|
|
83
|
+
}
|
|
84
|
+
const { agents, errors } = await fetchCampaignAgents(client, campaign, {
|
|
85
|
+
agentId: opts.agent,
|
|
86
|
+
onProgress: (msg) => {
|
|
87
|
+
spinner.text = msg;
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
spinner.stop();
|
|
91
|
+
if (errors.length) {
|
|
92
|
+
for (const e of errors)
|
|
93
|
+
console.log(chalk.yellow(` Warning: ${e}`));
|
|
94
|
+
}
|
|
95
|
+
if (!agents.length) {
|
|
96
|
+
console.log(chalk.red("No agents loaded from campaign."));
|
|
97
|
+
process.exit(1);
|
|
98
|
+
}
|
|
99
|
+
for (const agentData of agents) {
|
|
100
|
+
const { agent } = agentData;
|
|
101
|
+
const { sections, seeder } = resolveSystemPromptSections(agent, campaign, callType);
|
|
102
|
+
const assembled = assemblePrompt({
|
|
103
|
+
agent,
|
|
104
|
+
campaign,
|
|
105
|
+
roles: agent.roles,
|
|
106
|
+
actions: agent.actions,
|
|
107
|
+
scripts: agent.scripts,
|
|
108
|
+
systemPromptSections: sections,
|
|
109
|
+
conversationSeeder: seeder,
|
|
110
|
+
leadOverrides,
|
|
111
|
+
});
|
|
112
|
+
const unresolvedMentions = extractUnresolvedMentions(assembled.systemPrompt);
|
|
113
|
+
if (opts.format === "json") {
|
|
114
|
+
const output = {
|
|
115
|
+
agentName: assembled.agentName,
|
|
116
|
+
agentSlug: assembled.agentSlug,
|
|
117
|
+
callType,
|
|
118
|
+
campaignName: campaign.name,
|
|
119
|
+
systemPrompt: assembled.systemPrompt,
|
|
120
|
+
systemPromptLength: assembled.systemPrompt.length,
|
|
121
|
+
tools: assembled.tools,
|
|
122
|
+
conversationSeeder: assembled.conversationSeeder,
|
|
123
|
+
unresolvedMentions,
|
|
124
|
+
actionMeta: assembled.actionMeta,
|
|
125
|
+
aiSettings: agent.ai_settings ?? null,
|
|
126
|
+
};
|
|
127
|
+
console.log(JSON.stringify(output, null, 2));
|
|
128
|
+
}
|
|
129
|
+
else if (opts.format === "yaml") {
|
|
130
|
+
const output = {
|
|
131
|
+
agentName: assembled.agentName,
|
|
132
|
+
agentSlug: assembled.agentSlug,
|
|
133
|
+
callType,
|
|
134
|
+
campaignName: campaign.name,
|
|
135
|
+
systemPromptLength: assembled.systemPrompt.length,
|
|
136
|
+
systemPrompt: assembled.systemPrompt,
|
|
137
|
+
tools: assembled.tools,
|
|
138
|
+
conversationSeeder: assembled.conversationSeeder,
|
|
139
|
+
unresolvedMentions,
|
|
140
|
+
actionMeta: assembled.actionMeta,
|
|
141
|
+
aiSettings: agent.ai_settings ?? null,
|
|
142
|
+
};
|
|
143
|
+
console.log(yaml.dump(output, { lineWidth: 120, noRefs: true }));
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
printTextFormat(assembled, campaign.name, callType, unresolvedMentions, agent);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
function printTextFormat(assembled, campaignName, callType, unresolvedMentions, agent) {
|
|
151
|
+
const divider = chalk.dim("─".repeat(72));
|
|
152
|
+
console.log();
|
|
153
|
+
console.log(chalk.bold(`Agent: ${assembled.agentName}`) +
|
|
154
|
+
chalk.dim(` (${assembled.agentSlug})`));
|
|
155
|
+
console.log(chalk.dim(`Campaign: ${campaignName}`));
|
|
156
|
+
console.log(chalk.dim(`Call type: ${callType}`));
|
|
157
|
+
console.log();
|
|
158
|
+
// AI Settings
|
|
159
|
+
if (agent.ai_settings && Object.keys(agent.ai_settings).length > 0) {
|
|
160
|
+
console.log(chalk.bold.cyan("AI SETTINGS"));
|
|
161
|
+
console.log(divider);
|
|
162
|
+
for (const [key, val] of Object.entries(agent.ai_settings)) {
|
|
163
|
+
if (val != null) {
|
|
164
|
+
console.log(` ${chalk.dim(key)}: ${String(val)}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
console.log();
|
|
168
|
+
}
|
|
169
|
+
// System Prompt
|
|
170
|
+
console.log(chalk.bold.cyan("SYSTEM PROMPT"));
|
|
171
|
+
console.log(divider +
|
|
172
|
+
chalk.dim(` (${assembled.systemPrompt.length.toLocaleString()} chars)`));
|
|
173
|
+
console.log(assembled.systemPrompt);
|
|
174
|
+
console.log();
|
|
175
|
+
// Unresolved Mentions
|
|
176
|
+
if (unresolvedMentions.length > 0) {
|
|
177
|
+
console.log(chalk.bold.yellow("UNRESOLVED MENTIONS"));
|
|
178
|
+
console.log(divider);
|
|
179
|
+
for (const m of unresolvedMentions) {
|
|
180
|
+
console.log(` ${chalk.yellow(m)}`);
|
|
181
|
+
}
|
|
182
|
+
console.log(chalk.dim("\n Hydrate with: --vars '{\"first_name\":\"John\", ...}'"));
|
|
183
|
+
console.log();
|
|
184
|
+
}
|
|
185
|
+
// Tools
|
|
186
|
+
if (assembled.tools.length > 0) {
|
|
187
|
+
console.log(chalk.bold.cyan("TOOLS"));
|
|
188
|
+
console.log(divider + chalk.dim(` (${assembled.tools.length} tools)`));
|
|
189
|
+
for (const tool of assembled.tools) {
|
|
190
|
+
const meta = assembled.actionMeta?.find((m) => m.slug === tool.function.name);
|
|
191
|
+
const execLabel = meta
|
|
192
|
+
? chalk.dim(` [${meta.executionType}]`)
|
|
193
|
+
: "";
|
|
194
|
+
const params = Object.keys(tool.function.parameters.properties);
|
|
195
|
+
const paramStr = params.length > 0
|
|
196
|
+
? chalk.dim(` (${params.join(", ")})`)
|
|
197
|
+
: chalk.dim(" (no params)");
|
|
198
|
+
console.log(` ${chalk.green(tool.function.name)}${execLabel}${paramStr}`);
|
|
199
|
+
const desc = tool.function.description.slice(0, 100);
|
|
200
|
+
console.log(chalk.dim(` ${desc}${tool.function.description.length > 100 ? "..." : ""}`));
|
|
201
|
+
}
|
|
202
|
+
console.log();
|
|
203
|
+
}
|
|
204
|
+
// Conversation Seeder
|
|
205
|
+
if (assembled.conversationSeeder.length > 0) {
|
|
206
|
+
console.log(chalk.bold.cyan("CONVERSATION SEEDER"));
|
|
207
|
+
console.log(divider);
|
|
208
|
+
for (const msg of assembled.conversationSeeder) {
|
|
209
|
+
const label = msg.role === "user" ? chalk.blue("User") : chalk.green("Agent");
|
|
210
|
+
console.log(` ${label}: ${msg.content.slice(0, 200)}${msg.content.length > 200 ? "..." : ""}`);
|
|
211
|
+
}
|
|
212
|
+
console.log();
|
|
213
|
+
}
|
|
214
|
+
// Summary
|
|
215
|
+
console.log(chalk.bold.cyan("SUMMARY"));
|
|
216
|
+
console.log(divider);
|
|
217
|
+
console.log(` Prompt: ${chalk.white(assembled.systemPrompt.length.toLocaleString())} chars`);
|
|
218
|
+
console.log(` Tools: ${chalk.white(String(assembled.tools.length))}`);
|
|
219
|
+
console.log(` Seeder messages: ${chalk.white(String(assembled.conversationSeeder.length))}`);
|
|
220
|
+
console.log(` Unresolved mentions: ${unresolvedMentions.length > 0 ? chalk.yellow(String(unresolvedMentions.length)) : chalk.green("0")}`);
|
|
221
|
+
console.log();
|
|
222
|
+
}
|
|
223
|
+
//# sourceMappingURL=render.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"render.js","sourceRoot":"","sources":["../../../src/cli/commands/render.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,GAAG,MAAM,KAAK,CAAC;AACtB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,IAAI,MAAM,SAAS,CAAC;AAC3B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACrE,OAAO,EACL,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAI5C,SAAS,yBAAyB,CAAC,IAAY;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IACnD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;KAC/C,WAAW,CACV,+EAA+E,CAChF;KACA,MAAM,CACL,qBAAqB,EACrB,sDAAsD,CACvD;KACA,MAAM,CAAC,kBAAkB,EAAE,0CAA0C,CAAC;KACtE,MAAM,CACL,eAAe,EACf,+DAA+D,CAChE;KACA,MAAM,CACL,oBAAoB,EACpB,gCAAgC,EAChC,SAAS,CACV;KACA,MAAM,CACL,uBAAuB,EACvB,2CAA2C,EAC3C,MAAM,CACP;KACA,MAAM,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,GAAG,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,UAAU,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC;IAC3C,IAAI,UAAU;QAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IAErC,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,UAAU,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,mBAAmB,EAAE,GAAG,EAAE,4BAA4B,EAAE,CAAC,CAAC,CAAC;QACjG,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC;IAE/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC;QAC9D,IAAI,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAmB,IAAI,CAAC,KAAK,CACvC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CACrC,CAAC;gBACF,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACjC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,oEAAoE,CACrE,CACF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,aAAwD,CAAC;IAC7D,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC;YACH,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,aAAa,GAAG,IAAI,CAAC,KAAK,CACxB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CACH,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAEnC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CACpD,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAkC,CAAC;IAEzD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC;IACtE,MAAM,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,EAAE,qCAAqC;QAC3C,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;KACvD,CAAC,CAAC,KAAK,EAAE,CAAC;IAEX,IAAI,QAAQ,CAAC;IACb,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAClD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CACV,6BAA6B,GAAG,YAAY,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACvF,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE;QACrE,OAAO,EAAE,IAAI,CAAC,KAAK;QACnB,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE;YAClB,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC;QACrB,CAAC;KACF,CAAC,CAAC;IAEH,OAAO,CAAC,IAAI,EAAE,CAAC;IAEf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;QAC/B,MAAM,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC;QAE5B,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,2BAA2B,CACtD,KAAK,EACL,QAAQ,EACR,QAAQ,CACT,CAAC;QAEF,MAAM,SAAS,GAAG,cAAc,CAAC;YAC/B,KAAK;YACL,QAAQ;YACR,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,oBAAoB,EAAE,QAAQ;YAC9B,kBAAkB,EAAE,MAAM;YAC1B,aAAa;SACd,CAAC,CAAC;QAEH,MAAM,kBAAkB,GAAG,yBAAyB,CAClD,SAAS,CAAC,YAAY,CACvB,CAAC;QAEF,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG;gBACb,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,QAAQ;gBACR,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,YAAY,EAAE,SAAS,CAAC,YAAY;gBACpC,kBAAkB,EAAE,SAAS,CAAC,YAAY,CAAC,MAAM;gBACjD,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;gBAChD,kBAAkB;gBAClB,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,UAAU,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;aACtC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;aAAM,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG;gBACb,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,SAAS,EAAE,SAAS,CAAC,SAAS;gBAC9B,QAAQ;gBACR,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,kBAAkB,EAAE,SAAS,CAAC,YAAY,CAAC,MAAM;gBACjD,YAAY,EAAE,SAAS,CAAC,YAAY;gBACpC,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;gBAChD,kBAAkB;gBAClB,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,UAAU,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;aACtC,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS,eAAe,CACtB,SAA4C,EAC5C,YAAoB,EACpB,QAAgB,EAChB,kBAA4B,EAC5B,KAAgD;IAEhD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,CAAC,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,SAAS,GAAG,CAAC,CACzC,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,YAAY,EAAE,CAAC,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,cAAc;IACd,IAAI,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3D,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;IAC9C,OAAO,CAAC,GAAG,CACT,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,CAC1E,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,EAAE,CAAC;IAEd,sBAAsB;IACtB,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,kBAAkB,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,CAAC,GAAG,CACT,KAAK,CAAC,GAAG,CACP,2DAA2D,CAC5D,CACF,CAAC;QACF,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,QAAQ;IACR,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC;QACvE,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,SAAS,CAAC,UAAU,EAAE,IAAI,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,IAAI,CACrC,CAAC;YACF,MAAM,SAAS,GAAG,IAAI;gBACpB,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,aAAa,GAAG,CAAC;gBACvC,CAAC,CAAC,EAAE,CAAC;YACP,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CACpC,CAAC;YACF,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;gBACf,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;gBACtC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,SAAS,GAAG,QAAQ,EAAE,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,sBAAsB;IACtB,IAAI,SAAS,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,MAAM,GAAG,IAAI,SAAS,CAAC,kBAAkB,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC9E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC;IAED,UAAU;IACV,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,CAAC,GAAG,CACT,aAAa,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,QAAQ,CACjF,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CACT,sBAAsB,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACjF,CAAC;IACF,OAAO,CAAC,GAAG,CACT,0BAA0B,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAC/H,CAAC;IACF,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"results.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/results.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAsJpC,eAAO,MAAM,cAAc,SAgCvB,CAAC"}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import chalk from "chalk";
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
function loadResults(filePath) {
|
|
6
|
+
const resolved = path.resolve(filePath);
|
|
7
|
+
if (!fs.existsSync(resolved))
|
|
8
|
+
return null;
|
|
9
|
+
try {
|
|
10
|
+
const raw = JSON.parse(fs.readFileSync(resolved, "utf-8"));
|
|
11
|
+
const rawResults = raw.results?.results ?? [];
|
|
12
|
+
const stats = raw.results?.stats ?? {};
|
|
13
|
+
const simplified = rawResults.map((r) => {
|
|
14
|
+
const failedComponent = r.gradingResult?.componentResults?.find((c) => !c.pass);
|
|
15
|
+
return {
|
|
16
|
+
test: r.testCase.description ?? "Unnamed test",
|
|
17
|
+
provider: r.provider.label ?? r.provider.id ?? "unknown",
|
|
18
|
+
pass: r.success,
|
|
19
|
+
score: r.score,
|
|
20
|
+
reason: failedComponent?.reason ?? (r.success ? undefined : r.gradingResult?.reason),
|
|
21
|
+
};
|
|
22
|
+
});
|
|
23
|
+
const pass = stats.successes ?? simplified.filter((r) => r.pass).length;
|
|
24
|
+
const fail = stats.failures ?? simplified.filter((r) => !r.pass).length;
|
|
25
|
+
const errors = stats.errors ?? 0;
|
|
26
|
+
const total = pass + fail + errors;
|
|
27
|
+
const passRate = total > 0 ? `${((pass / total) * 100).toFixed(0)}%` : "N/A";
|
|
28
|
+
return { results: simplified, stats: { pass, fail, errors, total, passRate } };
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function formatMarkdown(data) {
|
|
35
|
+
if (!data)
|
|
36
|
+
return "No eval results found.";
|
|
37
|
+
const { results, stats } = data;
|
|
38
|
+
const providers = [...new Set(results.map((r) => r.provider))];
|
|
39
|
+
const tests = [...new Set(results.map((r) => r.test))];
|
|
40
|
+
const lines = [];
|
|
41
|
+
lines.push(`## Eval Results — ${stats.pass}/${stats.total} passed (${stats.passRate})`);
|
|
42
|
+
lines.push("");
|
|
43
|
+
// Build pivot table
|
|
44
|
+
const header = `| Test | ${providers.join(" | ")} |`;
|
|
45
|
+
const sep = `|------|${providers.map(() => ":---:").join("|")}|`;
|
|
46
|
+
lines.push(header);
|
|
47
|
+
lines.push(sep);
|
|
48
|
+
for (const test of tests) {
|
|
49
|
+
const cells = providers.map((prov) => {
|
|
50
|
+
const r = results.find((x) => x.test === test && x.provider === prov);
|
|
51
|
+
if (!r)
|
|
52
|
+
return "-";
|
|
53
|
+
return r.pass ? "PASS" : "FAIL";
|
|
54
|
+
});
|
|
55
|
+
lines.push(`| ${test} | ${cells.join(" | ")} |`);
|
|
56
|
+
}
|
|
57
|
+
lines.push("");
|
|
58
|
+
lines.push(`**Overall: ${stats.pass}/${stats.total} passed (${stats.passRate})**`);
|
|
59
|
+
// Show failures with reasons
|
|
60
|
+
const failures = results.filter((r) => !r.pass && r.reason);
|
|
61
|
+
if (failures.length > 0) {
|
|
62
|
+
lines.push("");
|
|
63
|
+
lines.push("### Failures");
|
|
64
|
+
lines.push("");
|
|
65
|
+
for (const f of failures) {
|
|
66
|
+
lines.push(`- **${f.test}** (${f.provider}): ${f.reason}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return lines.join("\n");
|
|
70
|
+
}
|
|
71
|
+
function formatTable(data) {
|
|
72
|
+
if (!data)
|
|
73
|
+
return "No eval results found.";
|
|
74
|
+
const { results, stats } = data;
|
|
75
|
+
const providers = [...new Set(results.map((r) => r.provider))];
|
|
76
|
+
const tests = [...new Set(results.map((r) => r.test))];
|
|
77
|
+
const lines = [];
|
|
78
|
+
const maxTestLen = Math.max(6, ...tests.map((t) => t.length));
|
|
79
|
+
const colWidth = Math.max(8, ...providers.map((p) => p.length));
|
|
80
|
+
lines.push(` ${"Test".padEnd(maxTestLen)} ${providers.map((p) => p.padStart(colWidth)).join(" ")}`);
|
|
81
|
+
lines.push(` ${"─".repeat(maxTestLen)} ${providers.map(() => "─".repeat(colWidth)).join(" ")}`);
|
|
82
|
+
for (const test of tests) {
|
|
83
|
+
const cells = providers.map((prov) => {
|
|
84
|
+
const r = results.find((x) => x.test === test && x.provider === prov);
|
|
85
|
+
if (!r)
|
|
86
|
+
return "-".padStart(colWidth);
|
|
87
|
+
const label = r.pass ? chalk.green("PASS") : chalk.red("FAIL");
|
|
88
|
+
const raw = r.pass ? "PASS" : "FAIL";
|
|
89
|
+
return raw.padStart(colWidth - 4) + label.slice(label.indexOf(raw[0]));
|
|
90
|
+
});
|
|
91
|
+
const truncTest = test.length > maxTestLen ? test.slice(0, maxTestLen - 1) + "…" : test.padEnd(maxTestLen);
|
|
92
|
+
lines.push(` ${truncTest} ${cells.join(" ")}`);
|
|
93
|
+
}
|
|
94
|
+
lines.push("");
|
|
95
|
+
lines.push(` ${stats.pass}/${stats.total} passed (${stats.passRate})`);
|
|
96
|
+
return lines.join("\n");
|
|
97
|
+
}
|
|
98
|
+
export const resultsCommand = new Command("results")
|
|
99
|
+
.description("Show eval results in a machine-readable format")
|
|
100
|
+
.option("-f, --file <path>", "Path to eval results JSON", "output/eval-results.json")
|
|
101
|
+
.option("--format <fmt>", "Output format: json, markdown, table", "markdown")
|
|
102
|
+
.action(async (opts, cmd) => {
|
|
103
|
+
const jsonMode = cmd.parent?.opts().json || opts.format === "json";
|
|
104
|
+
const data = loadResults(opts.file);
|
|
105
|
+
if (!data) {
|
|
106
|
+
if (jsonMode) {
|
|
107
|
+
console.log(JSON.stringify({ error: "No eval results found", file: opts.file }));
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
console.log(chalk.red(`No eval results found at: ${opts.file}`));
|
|
111
|
+
console.log(chalk.dim(" Run an eval first: eqho-eval eval"));
|
|
112
|
+
}
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
if (jsonMode) {
|
|
116
|
+
console.log(JSON.stringify(data, null, 2));
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
if (opts.format === "markdown") {
|
|
120
|
+
console.log(formatMarkdown(data));
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
// Default: table
|
|
124
|
+
console.log();
|
|
125
|
+
console.log(formatTable(data));
|
|
126
|
+
console.log();
|
|
127
|
+
});
|
|
128
|
+
//# sourceMappingURL=results.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"results.js","sourceRoot":"","sources":["../../../src/cli/commands/results.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAiC7B,SAAS,WAAW,CAAC,QAAgB;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;QACvE,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;QAEvC,MAAM,UAAU,GAAuB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC1D,MAAM,eAAe,GAAG,CAAC,CAAC,aAAa,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAChF,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW,IAAI,cAAc;gBAC9C,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,IAAI,SAAS;gBACxD,IAAI,EAAE,CAAC,CAAC,OAAO;gBACf,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,MAAM,EAAE,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,EAAE,MAAM,CAAC;aACrF,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACxE,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACxE,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;QAE7E,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,CAAC;IACjF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,IAAoC;IAC1D,IAAI,CAAC,IAAI;QAAE,OAAO,wBAAwB,CAAC;IAE3C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAEhC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;IACxF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,oBAAoB;IACpB,MAAM,MAAM,GAAG,YAAY,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACrD,MAAM,GAAG,GAAG,WAAW,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IACjE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;YACtE,IAAI,CAAC,CAAC;gBAAE,OAAO,GAAG,CAAC;YACnB,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,cAAc,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC;IAEnF,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,QAAQ,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,IAAoC;IACvD,IAAI,CAAC,IAAI;QAAE,OAAO,wBAAwB,CAAC;IAE3C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEhE,KAAK,CAAC,IAAI,CACR,KAAK,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC3F,CAAC;IACF,KAAK,CAAC,IAAI,CACR,KAAK,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACvF,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACnC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;YACtE,IAAI,CAAC,CAAC;gBAAE,OAAO,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/D,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,OAAO,GAAG,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3G,KAAK,CAAC,IAAI,CAAC,KAAK,SAAS,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC;IAExE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,OAAO,CAAC,SAAS,CAAC;KACjD,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,mBAAmB,EAAE,2BAA2B,EAAE,0BAA0B,CAAC;KACpF,MAAM,CAAC,gBAAgB,EAAE,sCAAsC,EAAE,UAAU,CAAC;KAC5E,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IAC1B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC;IACnE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEpC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACnF,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO;IACT,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,iBAAiB;IACjB,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/B,OAAO,CAAC,GAAG,EAAE,CAAC;AAChB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenarios.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/scenarios.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,eAAO,MAAM,gBAAgB,SAwFzB,CAAC"}
|