judgeval 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -68
- package/dist/cjs/common/tracer.js +235 -143
- package/dist/cjs/common/tracer.js.map +1 -1
- package/dist/cjs/constants.js +8 -5
- package/dist/cjs/constants.js.map +1 -1
- package/dist/cjs/data/datasets/eval-dataset-client.js +349 -0
- package/dist/cjs/data/datasets/eval-dataset-client.js.map +1 -0
- package/dist/cjs/data/datasets/eval-dataset.js +405 -0
- package/dist/cjs/data/datasets/eval-dataset.js.map +1 -0
- package/dist/cjs/data/example.js +22 -1
- package/dist/cjs/data/example.js.map +1 -1
- package/dist/cjs/e2etests/eval-operations.test.js +282 -0
- package/dist/cjs/e2etests/eval-operations.test.js.map +1 -0
- package/dist/cjs/e2etests/judgee-traces.test.js +278 -0
- package/dist/cjs/e2etests/judgee-traces.test.js.map +1 -0
- package/dist/cjs/index.js +1 -3
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/judgment-client.js +326 -645
- package/dist/cjs/judgment-client.js.map +1 -1
- package/dist/cjs/scorers/api-scorer.js +56 -48
- package/dist/cjs/scorers/api-scorer.js.map +1 -1
- package/dist/cjs/scorers/base-scorer.js +66 -11
- package/dist/cjs/scorers/base-scorer.js.map +1 -1
- package/dist/esm/common/tracer.js +236 -144
- package/dist/esm/common/tracer.js.map +1 -1
- package/dist/esm/constants.js +7 -4
- package/dist/esm/constants.js.map +1 -1
- package/dist/esm/data/datasets/eval-dataset-client.js +342 -0
- package/dist/esm/data/datasets/eval-dataset-client.js.map +1 -0
- package/dist/esm/data/datasets/eval-dataset.js +375 -0
- package/dist/esm/data/datasets/eval-dataset.js.map +1 -0
- package/dist/esm/data/example.js +22 -1
- package/dist/esm/data/example.js.map +1 -1
- package/dist/esm/e2etests/eval-operations.test.js +254 -0
- package/dist/esm/e2etests/eval-operations.test.js.map +1 -0
- package/dist/esm/e2etests/judgee-traces.test.js +253 -0
- package/dist/esm/e2etests/judgee-traces.test.js.map +1 -0
- package/dist/esm/index.js +0 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/judgment-client.js +328 -647
- package/dist/esm/judgment-client.js.map +1 -1
- package/dist/esm/scorers/api-scorer.js +56 -48
- package/dist/esm/scorers/api-scorer.js.map +1 -1
- package/dist/esm/scorers/base-scorer.js +66 -11
- package/dist/esm/scorers/base-scorer.js.map +1 -1
- package/dist/types/common/tracer.d.ts +27 -14
- package/dist/types/constants.d.ts +4 -4
- package/dist/types/data/datasets/eval-dataset-client.d.ts +39 -0
- package/dist/types/data/datasets/eval-dataset.d.ts +45 -0
- package/dist/types/data/example.d.ts +24 -12
- package/dist/types/e2etests/eval-operations.test.d.ts +5 -0
- package/dist/types/e2etests/judgee-traces.test.d.ts +5 -0
- package/dist/types/index.d.ts +0 -1
- package/dist/types/judgment-client.d.ts +3 -47
- package/dist/types/scorers/api-scorer.d.ts +15 -15
- package/dist/types/scorers/base-scorer.d.ts +53 -10
- package/package.json +2 -1
- package/dist/cjs/scorers/exact-match-scorer.js +0 -84
- package/dist/cjs/scorers/exact-match-scorer.js.map +0 -1
- package/dist/esm/scorers/exact-match-scorer.js +0 -80
- package/dist/esm/scorers/exact-match-scorer.js.map +0 -1
- package/dist/types/scorers/exact-match-scorer.d.ts +0 -10
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-dataset-client.js","sourceRoot":"","sources":["../../../../src/data/datasets/eval-dataset-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,8CAAwD,CAAC,oBAAoB;AAC7E,uDAAgD;AAChD,kDAAyD,CAAC,YAAY;AACtE,gEAAuC,CAAC,mBAAmB;AAC3D,8DAAiC,CAAC,gCAAgC;AAClE,yFAAqD,CAAC,wBAAwB;AAE9E,yEAAyE;AACzE,qDAO4B,CAAC,wBAAwB;AAErD,MAAa,iBAAiB;IAI5B,YAAY,cAAsB,EAAE,cAAsB;QACxD,IAAI,CAAC,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAC5F,IAAI,CAAC,cAAc;YAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QAC3F,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACpC,0CAA0C;QAC1C,4BAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;IACjD,CAAC;IAEM,aAAa,CAAC,WAAsB,EAAE;QAC3C,OAAO,IAAI,6BAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED;;;OAGG;IACU,WAAW;6DACtB,OAAoB,EACpB,KAAa,EACb,WAAmB,EACnB,YAAqB,KAAK;;YAE1B,4BAAM,CAAC,KAAK,CACV,+BAA+B,KAAK,kBAAkB,WAAW,gBAAgB,SAAS,GAAG,CAC9F,CAAC;YACF,IAAI,SAAS,EAAE,CAAC;gBACd,4BAAM,CAAC,IAAI,CAAC,wCAAwC,KAAK,iBAAiB,WAAW,GAAG,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC5C,MAAM,EAAE,WAAW,qBAAM,CAAC,OAAO,CAAC,KAAK,CAAC,qBAAqB,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBAC/G,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,OAAO,GAAG;gBACd,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,WAAW;gBACzB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC1D,SAAS,EAAE,SAAS;aACrB,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,6CAA8B,EAC9B,OAAO,EACP;oBACE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;iBAC/B,CACF,CAAC;gBACF,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;gBAE7D,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC;oBAC9B,OAAO,CAAC,KAAK,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,mCAAI,KAAK,CAAC;oBACzC,OAAO,CAAC,EAAE,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC;oBAC1B,4BAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,iBAAiB,WAAW,GAAG,CAAC,CAAC;oBAC7F,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBAC3D,OAAO,IAAI,CAAC;gBACd,CAAC;qBAAM,CAAC;oBACN,4BAAM,CAAC,KAAK,CACV,qCAAqC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC9E,CAAC;oBACF,WAAW,CAAC,IAAI,EAAE,CAAC;oBACnB,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;gBAC1C,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACU,WAAW,CACtB,KAAa,EACb,WAAmB;;;YAEnB,4BAAM,CAAC,KAAK,CAAC,+BAA+B,KAAK,mBAAmB,WAAW,GAAG,CAAC,CAAC;YACpF,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,wBAAwB;YAE9D,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC5C,MAAM,EAAE,WAAW,qBAAM,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBACjH,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,WAAW,GAAG;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,WAAW;aAC1B,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,6CAA8B,EAC9B,WAAW,EACX;oBACE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;iBAC/B,CACF,CAAC;gBACF,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;gBAE7D,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC;oBAC9B,MAAM,eAAe,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,mCAAI,EAAE,CAAC;oBAEhD,2EAA2E;oBAC3E,OAAO,CAAC,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;wBACjD,MAAM,OAAO,GAAmB;4BAC5B,KAAK,EAAE,KAAK,CAAC,KAAK;4BAClB,YAAY,EAAE,KAAK,CAAC,aAAa;4BACjC,cAAc,EAAE,KAAK,CAAC,eAAe;4BACrC,OAAO,EAAE,KAAK,CAAC,OAAO;4BACtB,gBAAgB,EAAE,KAAK,CAAC,iBAAiB;4BACzC,kBAAkB,EAAE,KAAK,CAAC,mBAAmB;4BAC7C,WAAW,EAAE,KAAK,CAAC,YAAY;4BAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;4BACnC,SAAS,EAAE,KAAK,CAAC,UAAU;4BAC3B,YAAY,EAAE,KAAK,CAAC,aAAa;4BACjC,SAAS,EAAE,KAAK,CAAC,SAAS;4BAC1B,OAAO,EAAE,KAAK,CAAC,OAAO;yBACzB,CAAC;wBACF,OAAO,IAAI,oBAAO,CAAC,OAAO,CAAC,CAAC;oBAChC,CAAC,CAAC,CAAC;oBAEJ,OAAO,CAAC,KAAK,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,mCAAI,KAAK,CAAC;oBACxC,OAAO,CAAC,EAAE,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,EAAE,CAAC;oBAEzB,4BAAM,CAAC,IAAI,CAAC,2CAA2C,KAAK,mBAAmB,WAAW,GAAG,CAAC,CAAC;oBAC/F,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBAC3D,OAAO,OAAO,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACN,4BAAM,CAAC,KAAK,CACV,qCAAqC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAC9E,CAAC;oBACF,WAAW,CAAC,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,2BAA2B,KAAK,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC/E,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;gBAC1C,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KAAA;IAED;;;OAGG;IACU,aAAa,CACxB,KAAa,EACb,WAAmB;;YAEnB,4BAAM,CAAC,KAAK,CAAC,gCAAgC,KAAK,mBAAmB,WAAW,GAAG,CAAC,CAAC;YAErF,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC5C,MAAM,EAAE,YAAY,qBAAM,CAAC,OAAO,CAAC,KAAK,CAAC,uBAAuB,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBAClH,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,WAAW,GAAG;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,WAAW;aAC1B,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,eAAK,CAAC,IAAI,CAAC,+CAAgC,EAAE,WAAW,EAAE;oBAC9D,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;iBAC/B,CAAC,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC3D,4BAAM,CAAC,IAAI,CAAC,4CAA4C,KAAK,mBAAmB,WAAW,GAAG,CAAC,CAAC;gBAChG,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;gBAC5C,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACU,uBAAuB,CAClC,WAAmB;;YAElB,4BAAM,CAAC,KAAK,CAAC,8CAA8C,WAAW,GAAG,CAAC,CAAC;YAE5E,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC5C,MAAM,EAAE,6BAA6B,qBAAM,CAAC,OAAO,CAAC,WAAW,CAAC,SAAS,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBAC3H,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,WAAW,GAAG;gBAClB,YAAY,EAAE,WAAW;aAC1B,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,sDAAuC,EACvC,WAAW,EACX;oBACE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;iBAC/B,CACF,CAAC;gBACF,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,wBAAwB,EAAE,CAAC,CAAC;gBAE7D,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACpD,4BAAM,CAAC,IAAI,CACT,0CAA0C,WAAW,GAAG,CACzD,CAAC;oBACF,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;oBAC3D,OAAO,QAAQ,CAAC,IAAI,CAAC;gBACvB,CAAC;qBAAM,CAAC;oBACN,4BAAM,CAAC,KAAK,CACV,uCAAuC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAChF,CAAC;oBACF,WAAW,CAAC,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,WAAW,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC/F,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC;gBACtD,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KAAA;IAED;;;OAGG;IACU,aAAa,CACxB,KAAa,EACb,QAAmB,EACnB,WAAmB;;YAElB,4BAAM,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,2BAA2B,KAAK,kBAAkB,WAAW,GAAG,CAAC,CAAC;YAE5G,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC5C,MAAM,EAAE,2BAA2B,qBAAM,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBACnH,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEvC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,OAAO,GAAG;gBACd,aAAa,EAAE,KAAK;gBACpB,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzC,YAAY,EAAE,WAAW;aAC1B,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,eAAK,CAAC,IAAI,CAAC,+CAAgC,EAAE,OAAO,EAAE;oBAC1D,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;iBAC/B,CAAC,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC3D,4BAAM,CAAC,IAAI,CAAC,gDAAgD,KAAK,iBAAiB,WAAW,GAAG,CAAC,CAAC;gBAClG,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,eAAe,CAAC,CAAC;gBAC5C,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;KAAA;IAED;;;OAGG;IACU,WAAW,CACtB,KAAa,EACb,WAAmB;;YAElB,4BAAM,CAAC,KAAK,CAAC,sBAAsB,KAAK,mBAAmB,WAAW,YAAY,CAAC,CAAC;YAErF,MAAM,WAAW,GAAG,IAAI,sBAAW,CAAC,SAAS,CAAC;gBAC1C,MAAM,EAAE,aAAa,qBAAM,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,8BAA8B;gBAC9G,eAAe,EAAE,QAAQ;gBACzB,iBAAiB,EAAE,QAAQ;gBAC3B,UAAU,EAAE,IAAI;gBAChB,eAAe,EAAE,KAAK;gBACtB,cAAc,EAAE,IAAI;aACrB,EAAE,sBAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YAEzC,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;YAErD,MAAM,WAAW,GAAG;gBAClB,aAAa,EAAE,KAAK;gBACpB,YAAY,EAAE,WAAW;aAC1B,CAAC;YAEF,IAAI,CAAC;gBACH,WAAW,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAC;gBACzD,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,IAAI,CAC/B,qDAAsC,EACtC,WAAW,EACX;oBACE,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE;oBAC9B,YAAY,EAAE,QAAQ;iBACvB,CACF,CAAC;gBACF,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,qBAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAC3D,4BAAM,CAAC,IAAI,CAAC,8CAA8C,KAAK,mBAAmB,WAAW,GAAG,CAAC,CAAC;gBAClG,OAAO,QAAQ,CAAC;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;gBAC1C,WAAW,CAAC,IAAI,EAAE,CAAC;gBACnB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;KAAA;IAED,yBAAyB;IAEjB,cAAc;QACpB,OAAO;YACL,cAAc,EAAE,kBAAkB;YAClC,eAAe,EAAE,UAAU,IAAI,CAAC,cAAc,EAAE;YAChD,mBAAmB,EAAE,IAAI,CAAC,cAAc;SACzC,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,KAAc,EAAE,OAAe;QACpD,4BAAM,CAAC,KAAK,CAAC,oBAAoB,OAAO,GAAG,CAAC,CAAC;QAC7C,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,KAAmB,CAAC;YACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;YAErC,IAAI,QAAQ,EAAE,CAAC;gBACb,4BAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAClE,4BAAM,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,4BAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC1D,CAAC;qBAAM,IAAI,OAAO,KAAK,aAAa,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC/D,4BAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;gBACtC,CAAC;qBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAClE,4BAAM,CAAC,IAAI,CAAC,GAAG,OAAO,sDAAsD,CAAC,CAAC;gBACjF,CAAC;YACH,CAAC;iBAAM,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBAC9B,4BAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC,sBAAsB;YAC3E,CAAC;iBAAM,CAAC;gBACN,4BAAM,CAAC,KAAK,CAAC,oCAAoC,OAAO,EAAE,CAAC,CAAC,CAAC,sBAAsB;YACrF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,4BAAM,CAAC,KAAK,CAAC,2BAA2B,OAAO,EAAE,CAAC,CAAC,CAAC,sBAAsB;QAC5E,CAAC;IACH,CAAC;CACF;AA/XD,8CA+XC"}
|
|
@@ -0,0 +1,405 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.EvalDataset = void 0;
|
|
30
|
+
const example_js_1 = require("../example.js"); // Import class and options
|
|
31
|
+
const fs = __importStar(require("fs"));
|
|
32
|
+
const path = __importStar(require("path"));
|
|
33
|
+
const papaparse_1 = __importDefault(require("papaparse")); // Added PapaParse
|
|
34
|
+
const js_yaml_1 = __importDefault(require("js-yaml")); // Added js-yaml
|
|
35
|
+
const date_fns_1 = require("date-fns"); // For default filename timestamp
|
|
36
|
+
const ACCEPTABLE_FILE_TYPES = ['json', 'csv', 'yaml'];
|
|
37
|
+
// Define keys that represent lists potentially split by a delimiter in CSV
|
|
38
|
+
// Use camelCase names from Example class/ExampleOptions
|
|
39
|
+
const LIST_LIKE_KEYS = [
|
|
40
|
+
'context',
|
|
41
|
+
'retrievalContext',
|
|
42
|
+
'toolsCalled',
|
|
43
|
+
'expectedTools',
|
|
44
|
+
];
|
|
45
|
+
class EvalDataset {
|
|
46
|
+
// These might not be needed directly in the TS class if client handles auth
|
|
47
|
+
// public judgment_api_key: string;
|
|
48
|
+
// public organization_id: string;
|
|
49
|
+
constructor(examples = []) {
|
|
50
|
+
console.debug(`Initializing EvalDataset with ${examples.length} examples`);
|
|
51
|
+
this.examples = examples;
|
|
52
|
+
this._alias = null;
|
|
53
|
+
this._id = null;
|
|
54
|
+
// Assuming API keys are handled by the client, not stored in the dataset object itself
|
|
55
|
+
// this.judgment_api_key = judgment_api_key || process.env.JUDGMENT_API_KEY || '';
|
|
56
|
+
// this.organization_id = organization_id || process.env.JUDGMENT_ORG_ID || '';
|
|
57
|
+
// if (!this.judgment_api_key) {
|
|
58
|
+
// console.warn("No judgment_api_key provided");
|
|
59
|
+
// }
|
|
60
|
+
}
|
|
61
|
+
addExample(e) {
|
|
62
|
+
this.examples.push(e);
|
|
63
|
+
}
|
|
64
|
+
get length() {
|
|
65
|
+
return this.examples.length;
|
|
66
|
+
}
|
|
67
|
+
get alias() {
|
|
68
|
+
return this._alias;
|
|
69
|
+
}
|
|
70
|
+
set alias(value) {
|
|
71
|
+
this._alias = value;
|
|
72
|
+
}
|
|
73
|
+
get id() {
|
|
74
|
+
return this._id;
|
|
75
|
+
}
|
|
76
|
+
set id(value) {
|
|
77
|
+
this._id = value;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Adds examples from a JSON file.
|
|
81
|
+
* Assumes the JSON file has a top-level key "examples" containing an array of example objects.
|
|
82
|
+
* @param filePath Path to the JSON file.
|
|
83
|
+
*/
|
|
84
|
+
addFromJson(filePath) {
|
|
85
|
+
console.debug(`Loading dataset from JSON file: ${filePath}`);
|
|
86
|
+
try {
|
|
87
|
+
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
|
88
|
+
const payload = JSON.parse(fileContent);
|
|
89
|
+
const examplesFromJson = payload === null || payload === void 0 ? void 0 : payload.examples;
|
|
90
|
+
if (!Array.isArray(examplesFromJson)) {
|
|
91
|
+
throw new Error('Invalid JSON format: "examples" key not found or not an array.');
|
|
92
|
+
}
|
|
93
|
+
console.info(`Adding ${examplesFromJson.length} examples from JSON`);
|
|
94
|
+
examplesFromJson.forEach((e) => {
|
|
95
|
+
// Map snake_case from file to camelCase for ExampleOptions
|
|
96
|
+
const options = {
|
|
97
|
+
input: e.input,
|
|
98
|
+
actualOutput: e.actual_output,
|
|
99
|
+
expectedOutput: e.expected_output,
|
|
100
|
+
context: e.context,
|
|
101
|
+
retrievalContext: e.retrieval_context,
|
|
102
|
+
additionalMetadata: e.additional_metadata,
|
|
103
|
+
toolsCalled: e.tools_called,
|
|
104
|
+
expectedTools: e.expected_tools,
|
|
105
|
+
name: e.name,
|
|
106
|
+
exampleId: e.example_id,
|
|
107
|
+
exampleIndex: e.example_index,
|
|
108
|
+
timestamp: e.timestamp,
|
|
109
|
+
traceId: e.trace_id,
|
|
110
|
+
example: e.example,
|
|
111
|
+
};
|
|
112
|
+
// TODO: Add handling for fields not in ExampleOptions (name, comments, etc.) if needed
|
|
113
|
+
this.addExample(new example_js_1.Example(options));
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
catch (error) {
|
|
117
|
+
if (error.code === 'ENOENT') {
|
|
118
|
+
console.error(`JSON file not found: ${filePath}`);
|
|
119
|
+
throw new Error(`The file ${filePath} was not found.`);
|
|
120
|
+
}
|
|
121
|
+
else if (error instanceof SyntaxError) {
|
|
122
|
+
console.error(`Invalid JSON file: ${filePath}`, error);
|
|
123
|
+
throw new Error(`The file ${filePath} is not a valid JSON file.`);
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
console.error(`Error loading dataset from JSON: ${filePath}`, error);
|
|
127
|
+
throw error; // Re-throw other errors
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Adds examples from a YAML file.
|
|
133
|
+
* Assumes the YAML file has a top-level key "examples" containing an array of example objects.
|
|
134
|
+
* @param filePath Path to the YAML file.
|
|
135
|
+
*/
|
|
136
|
+
addFromYaml(filePath) {
|
|
137
|
+
console.debug(`Loading dataset from YAML file: ${filePath}`);
|
|
138
|
+
try {
|
|
139
|
+
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
|
140
|
+
const payload = js_yaml_1.default.load(fileContent);
|
|
141
|
+
if (!payload) {
|
|
142
|
+
throw new Error('The YAML file is empty.');
|
|
143
|
+
}
|
|
144
|
+
const examplesFromYaml = payload === null || payload === void 0 ? void 0 : payload.examples;
|
|
145
|
+
if (!Array.isArray(examplesFromYaml)) {
|
|
146
|
+
throw new Error('Invalid YAML format: "examples" key not found or not an array.');
|
|
147
|
+
}
|
|
148
|
+
console.info(`Adding ${examplesFromYaml.length} examples from YAML`);
|
|
149
|
+
examplesFromYaml.forEach((e) => {
|
|
150
|
+
// Map snake_case from file to camelCase for ExampleOptions
|
|
151
|
+
const options = {
|
|
152
|
+
input: e.input,
|
|
153
|
+
actualOutput: e.actual_output,
|
|
154
|
+
expectedOutput: e.expected_output,
|
|
155
|
+
context: e.context,
|
|
156
|
+
retrievalContext: e.retrieval_context,
|
|
157
|
+
additionalMetadata: e.additional_metadata,
|
|
158
|
+
toolsCalled: e.tools_called,
|
|
159
|
+
expectedTools: e.expected_tools,
|
|
160
|
+
name: e.name,
|
|
161
|
+
exampleId: e.example_id,
|
|
162
|
+
exampleIndex: e.example_index,
|
|
163
|
+
timestamp: e.timestamp,
|
|
164
|
+
traceId: e.trace_id,
|
|
165
|
+
example: e.example,
|
|
166
|
+
};
|
|
167
|
+
this.addExample(new example_js_1.Example(options));
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
if (error.code === 'ENOENT') {
|
|
172
|
+
console.error(`YAML file not found: ${filePath}`);
|
|
173
|
+
throw new Error(`The file ${filePath} was not found.`);
|
|
174
|
+
}
|
|
175
|
+
else if (error instanceof js_yaml_1.default.YAMLException) {
|
|
176
|
+
console.error(`Invalid YAML file: ${filePath}`, error);
|
|
177
|
+
throw new Error(`The file ${filePath} is not a valid YAML file.`);
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
console.error(`Error loading dataset from YAML: ${filePath}`, error);
|
|
181
|
+
throw error; // Re-throw other errors
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Adds examples from a CSV file.
|
|
187
|
+
* @param filePath Path to the CSV file.
|
|
188
|
+
* @param headerMapping Dictionary mapping Example headers (keys) to custom headers in the CSV (values).
|
|
189
|
+
* @param primaryDelimiter Main delimiter used in CSV file. Defaults to ",".
|
|
190
|
+
* @param secondaryDelimiter Secondary delimiter for list fields (context, retrieval_context, etc.). Defaults to ";".
|
|
191
|
+
*/
|
|
192
|
+
addFromCsv(filePath,
|
|
193
|
+
// headerMapping values are CSV headers, keys need to be ExampleOptions properties (camelCase)
|
|
194
|
+
headerMapping, primaryDelimiter = ',', secondaryDelimiter = ';') {
|
|
195
|
+
console.debug(`Loading dataset from CSV file: ${filePath}`);
|
|
196
|
+
try {
|
|
197
|
+
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
|
198
|
+
// Invert headerMapping for easier lookup: CSV Header -> ExampleOption Key
|
|
199
|
+
const csvHeaderToExampleKeyMap = Object.entries(headerMapping).reduce((acc, [exampleKey, csvHeader]) => {
|
|
200
|
+
if (csvHeader)
|
|
201
|
+
acc[csvHeader] = exampleKey;
|
|
202
|
+
return acc;
|
|
203
|
+
}, {});
|
|
204
|
+
const parseResult = papaparse_1.default.parse(fileContent, {
|
|
205
|
+
header: true,
|
|
206
|
+
delimiter: primaryDelimiter,
|
|
207
|
+
skipEmptyLines: true,
|
|
208
|
+
dynamicTyping: (field) => {
|
|
209
|
+
const exampleKey = csvHeaderToExampleKeyMap[field];
|
|
210
|
+
// Prevent auto-typing traceId
|
|
211
|
+
if (exampleKey === 'traceId')
|
|
212
|
+
return false;
|
|
213
|
+
return true;
|
|
214
|
+
},
|
|
215
|
+
});
|
|
216
|
+
if (parseResult.errors.length > 0) {
|
|
217
|
+
console.error('CSV parsing errors:', parseResult.errors);
|
|
218
|
+
// throw new Error(`Failed to parse CSV file: ${filePath}`);
|
|
219
|
+
}
|
|
220
|
+
const csvData = parseResult.data;
|
|
221
|
+
const newExamples = [];
|
|
222
|
+
csvData.forEach((row, index) => {
|
|
223
|
+
var _a;
|
|
224
|
+
const exampleOptions = {};
|
|
225
|
+
// Need a way to handle the 'example' boolean column if it's not in ExampleOptions
|
|
226
|
+
let isExampleRow = true; // Assume true unless 'example' column says otherwise
|
|
227
|
+
const exampleColumnCsvHeader = headerMapping['example']; // Assuming 'example' key exists in mapping for the boolean
|
|
228
|
+
for (const csvHeader in row) {
|
|
229
|
+
const exampleKey = csvHeaderToExampleKeyMap[csvHeader];
|
|
230
|
+
if (!exampleKey)
|
|
231
|
+
continue;
|
|
232
|
+
let value = row[csvHeader];
|
|
233
|
+
if (value === null || value === undefined || value === '') {
|
|
234
|
+
value = exampleKey === 'additionalMetadata' ? {} : null;
|
|
235
|
+
}
|
|
236
|
+
else {
|
|
237
|
+
value = String(value);
|
|
238
|
+
if (LIST_LIKE_KEYS.includes(exampleKey)) {
|
|
239
|
+
value = value
|
|
240
|
+
.split(secondaryDelimiter)
|
|
241
|
+
.map((item) => item.trim())
|
|
242
|
+
.filter((item) => item !== '');
|
|
243
|
+
}
|
|
244
|
+
else if (exampleKey === 'additionalMetadata') {
|
|
245
|
+
try {
|
|
246
|
+
value = JSON.parse(value);
|
|
247
|
+
}
|
|
248
|
+
catch (e) {
|
|
249
|
+
console.warn(`Row ${index + 2}: Could not parse additionalMetadata "${value}" as JSON. Storing as string.`);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
else if (exampleColumnCsvHeader && csvHeader === exampleColumnCsvHeader) {
|
|
253
|
+
// Check if current CSV header matches the one mapped to the 'example' concept
|
|
254
|
+
const lowerVal = value.toLowerCase();
|
|
255
|
+
isExampleRow = !['false', '0', 'no', ''].includes(lowerVal);
|
|
256
|
+
// We don't store this boolean directly on ExampleOptions unless it has an 'example' field
|
|
257
|
+
continue; // Don't assign this value to exampleOptions map
|
|
258
|
+
}
|
|
259
|
+
else if (exampleKey === 'example') {
|
|
260
|
+
// Check key directly
|
|
261
|
+
const lowerVal = value.toLowerCase();
|
|
262
|
+
isExampleRow = !['false', '0', 'no', ''].includes(lowerVal);
|
|
263
|
+
// Assign to options IF the key exists in headerMapping
|
|
264
|
+
if (headerMapping.example) {
|
|
265
|
+
exampleOptions[exampleKey] = isExampleRow;
|
|
266
|
+
}
|
|
267
|
+
// Don't continue here, let assignment happen below
|
|
268
|
+
}
|
|
269
|
+
else if (exampleKey === 'actualOutput' || exampleKey === 'expectedOutput') {
|
|
270
|
+
// If it contains the delimiter, treat as array, otherwise string
|
|
271
|
+
if (value.includes(secondaryDelimiter)) {
|
|
272
|
+
value = value
|
|
273
|
+
.split(secondaryDelimiter)
|
|
274
|
+
.map((item) => item.trim())
|
|
275
|
+
.filter((item) => item !== '');
|
|
276
|
+
} // else leave as string
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
exampleOptions[exampleKey] = value; // Assign using the camelCase key
|
|
280
|
+
}
|
|
281
|
+
// Only add if it's marked as an example and has required fields
|
|
282
|
+
if (isExampleRow) {
|
|
283
|
+
// Check required fields using camelCase
|
|
284
|
+
if (exampleOptions.input === undefined ||
|
|
285
|
+
exampleOptions.input === null
|
|
286
|
+
// actualOutput is optional in ExampleOptions, so don't check here
|
|
287
|
+
) {
|
|
288
|
+
console.warn(`Row ${index + 2}: Skipping example due to missing 'input'.`);
|
|
289
|
+
}
|
|
290
|
+
else {
|
|
291
|
+
// Add trace_id if missing and mapped
|
|
292
|
+
if (headerMapping.traceId && !exampleOptions.traceId) {
|
|
293
|
+
exampleOptions.traceId = String((_a = row[headerMapping.traceId]) !== null && _a !== void 0 ? _a : null); // Ensure it's a string
|
|
294
|
+
}
|
|
295
|
+
newExamples.push(new example_js_1.Example(exampleOptions));
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
console.info(`Adding ${newExamples.length} examples from CSV`);
|
|
300
|
+
newExamples.forEach((e) => this.addExample(e));
|
|
301
|
+
}
|
|
302
|
+
catch (error) {
|
|
303
|
+
if (error.code === 'ENOENT') {
|
|
304
|
+
console.error(`CSV file not found: ${filePath}`);
|
|
305
|
+
throw new Error(`The file ${filePath} was not found.`);
|
|
306
|
+
}
|
|
307
|
+
else {
|
|
308
|
+
console.error(`Error loading dataset from CSV: ${filePath}`, error);
|
|
309
|
+
throw error; // Re-throw other errors
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* Saves the dataset as a file.
|
|
315
|
+
* @param fileType The file type to save as ('json', 'csv', 'yaml').
|
|
316
|
+
* @param dirPath The directory path to save the file to.
|
|
317
|
+
* @param saveName Optional: The name of the file (without extension). Defaults to a timestamp.
|
|
318
|
+
* @param secondaryDelimiter Optional: The delimiter used for joining list fields in CSV output. Defaults to ";".
|
|
319
|
+
*/
|
|
320
|
+
saveAs(fileType, dirPath, saveName, secondaryDelimiter = ';') {
|
|
321
|
+
if (!ACCEPTABLE_FILE_TYPES.includes(fileType)) {
|
|
322
|
+
throw new TypeError(`Invalid file type: ${fileType}. Please choose from ${ACCEPTABLE_FILE_TYPES.join(', ')}`);
|
|
323
|
+
}
|
|
324
|
+
try {
|
|
325
|
+
// Ensure directory exists
|
|
326
|
+
if (!fs.existsSync(dirPath)) {
|
|
327
|
+
fs.mkdirSync(dirPath, { recursive: true });
|
|
328
|
+
}
|
|
329
|
+
const fileName = saveName !== null && saveName !== void 0 ? saveName : (0, date_fns_1.format)(new Date(), 'yyyyMMdd_HHmmss');
|
|
330
|
+
const completePath = path.join(dirPath, `${fileName}.${fileType}`);
|
|
331
|
+
// Use the Example class's toJSON method which outputs snake_case keys
|
|
332
|
+
const examplesForOutput = this.examples.map(e => e.toJSON());
|
|
333
|
+
console.info(`Saving dataset to ${completePath}`);
|
|
334
|
+
if (fileType === 'json') {
|
|
335
|
+
const jsonContent = JSON.stringify({ examples: examplesForOutput }, null, 4);
|
|
336
|
+
fs.writeFileSync(completePath, jsonContent, 'utf-8');
|
|
337
|
+
}
|
|
338
|
+
else if (fileType === 'yaml') {
|
|
339
|
+
// The toJSON output is already structured correctly for YAML
|
|
340
|
+
const yamlContent = js_yaml_1.default.dump({ examples: examplesForOutput }, { noRefs: true, lineWidth: -1 });
|
|
341
|
+
fs.writeFileSync(completePath, yamlContent, 'utf-8');
|
|
342
|
+
}
|
|
343
|
+
else if (fileType === 'csv') {
|
|
344
|
+
// Define CSV headers using the snake_case keys expected in the output
|
|
345
|
+
const csvHeaders = [
|
|
346
|
+
'input',
|
|
347
|
+
'actual_output',
|
|
348
|
+
'expected_output',
|
|
349
|
+
'context',
|
|
350
|
+
'retrieval_context',
|
|
351
|
+
'additional_metadata',
|
|
352
|
+
'tools_called',
|
|
353
|
+
'expected_tools',
|
|
354
|
+
'name',
|
|
355
|
+
'example_id',
|
|
356
|
+
'example_index',
|
|
357
|
+
'timestamp',
|
|
358
|
+
'trace_id',
|
|
359
|
+
'example',
|
|
360
|
+
];
|
|
361
|
+
// Prepare data for PapaParse using the snake_case keys from toJSON()
|
|
362
|
+
const csvData = examplesForOutput.map((e_json) => {
|
|
363
|
+
const row = {};
|
|
364
|
+
csvHeaders.forEach((header) => {
|
|
365
|
+
const value = e_json[header];
|
|
366
|
+
// Check original camelCase list keys for joining arrays
|
|
367
|
+
if ((header === 'context' ||
|
|
368
|
+
header === 'retrieval_context' ||
|
|
369
|
+
header === 'tools_called' ||
|
|
370
|
+
header === 'expected_tools') && Array.isArray(value)) {
|
|
371
|
+
row[header] = value.join(secondaryDelimiter);
|
|
372
|
+
}
|
|
373
|
+
else if (header === 'additional_metadata' && typeof value === 'object' && value !== null) {
|
|
374
|
+
try {
|
|
375
|
+
row[header] = JSON.stringify(value);
|
|
376
|
+
}
|
|
377
|
+
catch (_a) {
|
|
378
|
+
row[header] = String(value);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
else if (header === 'example') {
|
|
382
|
+
row[header] = value !== null && value !== void 0 ? value : true; // Default to true if null/undefined
|
|
383
|
+
}
|
|
384
|
+
else {
|
|
385
|
+
row[header] = value === undefined ? null : value;
|
|
386
|
+
}
|
|
387
|
+
});
|
|
388
|
+
return row;
|
|
389
|
+
});
|
|
390
|
+
const csvContent = papaparse_1.default.unparse(csvData, {
|
|
391
|
+
columns: csvHeaders,
|
|
392
|
+
header: true,
|
|
393
|
+
delimiter: ',',
|
|
394
|
+
});
|
|
395
|
+
fs.writeFileSync(completePath, csvContent, 'utf-8');
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
catch (error) {
|
|
399
|
+
console.error(`Error saving dataset to ${fileType}:`, error);
|
|
400
|
+
throw error; // Re-throw error
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
exports.EvalDataset = EvalDataset;
|
|
405
|
+
//# sourceMappingURL=eval-dataset.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-dataset.js","sourceRoot":"","sources":["../../../../src/data/datasets/eval-dataset.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAwD,CAAC,2BAA2B;AACpF,uCAAyB;AACzB,2CAA6B;AAC7B,0DAA6B,CAAC,kBAAkB;AAChD,sDAA2B,CAAC,gBAAgB;AAC5C,uCAAkC,CAAC,iCAAiC;AAOpE,MAAM,qBAAqB,GAAmB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AAEtE,2EAA2E;AAC3E,wDAAwD;AACxD,MAAM,cAAc,GAA6B;IAC/C,SAAS;IACT,kBAAkB;IAClB,aAAa;IACb,eAAe;CAChB,CAAC;AAEF,MAAa,WAAW;IAItB,4EAA4E;IAC5E,mCAAmC;IACnC,kCAAkC;IAElC,YAAY,WAAsB,EAAE;QAClC,OAAO,CAAC,KAAK,CAAC,iCAAiC,QAAQ,CAAC,MAAM,WAAW,CAAC,CAAC;QAC3E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QAChB,uFAAuF;QACvF,kFAAkF;QAClF,+EAA+E;QAC/E,gCAAgC;QAChC,kDAAkD;QAClD,IAAI;IACN,CAAC;IAEM,UAAU,CAAC,CAAU;QAC1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,IAAW,MAAM;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED,IAAW,KAAK;QACd,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAW,KAAK,CAAC,KAAoB;QACnC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,IAAW,EAAE;QACX,OAAO,IAAI,CAAC,GAAG,CAAC;IAClB,CAAC;IAED,IAAW,EAAE,CAAC,KAAoB;QAChC,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC;IACnB,CAAC;IAED;;;;OAIG;IACI,WAAW,CAAC,QAAgB;QACjC,OAAO,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxC,MAAM,gBAAgB,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,CAAC;YAE3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,UAAU,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,CAAC;YACrE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAM,EAAE,EAAE;gBAClC,2DAA2D;gBAC3D,MAAM,OAAO,GAAmB;oBAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,YAAY,EAAE,CAAC,CAAC,aAAa;oBAC7B,cAAc,EAAE,CAAC,CAAC,eAAe;oBACjC,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,gBAAgB,EAAE,CAAC,CAAC,iBAAiB;oBACrC,kBAAkB,EAAE,CAAC,CAAC,mBAAmB;oBACzC,WAAW,EAAE,CAAC,CAAC,YAAY;oBAC3B,aAAa,EAAE,CAAC,CAAC,cAAc;oBAC/B,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,UAAU;oBACvB,YAAY,EAAE,CAAC,CAAC,aAAa;oBAC7B,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,OAAO,EAAE,CAAC,CAAC,QAAQ;oBACnB,OAAO,EAAE,CAAC,CAAC,OAAO;iBACnB,CAAC;gBACF,uFAAuF;gBACvF,IAAI,CAAC,UAAU,CAAC,IAAI,oBAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,iBAAiB,CAAC,CAAC;YACzD,CAAC;iBAAM,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,sBAAsB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;gBACvD,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,4BAA4B,CAAC,CAAC;YACpE,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,oCAAoC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;gBACrE,MAAM,KAAK,CAAC,CAAC,wBAAwB;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACI,WAAW,CAAC,QAAgB;QACjC,OAAO,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,OAAO,GAAG,iBAAI,CAAC,IAAI,CAAC,WAAW,CAAyB,CAAC;YAE/D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,gBAAgB,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,QAAQ,CAAC;YAE3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CACb,gEAAgE,CACjE,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,UAAU,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,CAAC;YACrE,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAM,EAAE,EAAE;gBAClC,2DAA2D;gBAC3D,MAAM,OAAO,GAAmB;oBAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,YAAY,EAAE,CAAC,CAAC,aAAa;oBAC7B,cAAc,EAAE,CAAC,CAAC,eAAe;oBACjC,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,gBAAgB,EAAE,CAAC,CAAC,iBAAiB;oBACrC,kBAAkB,EAAE,CAAC,CAAC,mBAAmB;oBACzC,WAAW,EAAE,CAAC,CAAC,YAAY;oBAC3B,aAAa,EAAE,CAAC,CAAC,cAAc;oBAC/B,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,SAAS,EAAE,CAAC,CAAC,UAAU;oBACvB,YAAY,EAAE,CAAC,CAAC,aAAa;oBAC7B,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,OAAO,EAAE,CAAC,CAAC,QAAQ;oBACnB,OAAO,EAAE,CAAC,CAAC,OAAO;iBACnB,CAAC;gBACF,IAAI,CAAC,UAAU,CAAC,IAAI,oBAAO,CAAC,OAAO,CAAC,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;gBAClD,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,iBAAiB,CAAC,CAAC;YACzD,CAAC;iBAAM,IAAI,KAAK,YAAY,iBAAI,CAAC,aAAa,EAAE,CAAC;gBAC/C,OAAO,CAAC,KAAK,CAAC,sBAAsB,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;gBACvD,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,4BAA4B,CAAC,CAAC;YACpE,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,oCAAoC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;gBACrE,MAAM,KAAK,CAAC,CAAC,wBAAwB;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,UAAU,CACf,QAAgB;IAChB,8FAA8F;IAC9F,aAAyD,EACzD,mBAA2B,GAAG,EAC9B,qBAA6B,GAAG;QAEhC,OAAO,CAAC,KAAK,CAAC,kCAAkC,QAAQ,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAEvD,0EAA0E;YAC1E,MAAM,wBAAwB,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,CACnE,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE;gBAC/B,IAAI,SAAS;oBAAE,GAAG,CAAC,SAAS,CAAC,GAAG,UAAkC,CAAC;gBACnE,OAAO,GAAG,CAAC;YACb,CAAC,EACD,EAA0C,CAC3C,CAAC;YAEF,MAAM,WAAW,GAAG,mBAAI,CAAC,KAAK,CAAC,WAAW,EAAE;gBAC1C,MAAM,EAAE,IAAI;gBACZ,SAAS,EAAE,gBAAgB;gBAC3B,cAAc,EAAE,IAAI;gBACpB,aAAa,EAAE,CAAC,KAAsB,EAAE,EAAE;oBACxC,MAAM,UAAU,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;oBACnD,8BAA8B;oBAC9B,IAAI,UAAU,KAAK,SAAS;wBAAE,OAAO,KAAK,CAAC;oBAC3C,OAAO,IAAI,CAAC;gBACd,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,WAAW,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;gBACzD,4DAA4D;YAC9D,CAAC;YAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAA6B,CAAC;YAC1D,MAAM,WAAW,GAAc,EAAE,CAAC;YAElC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;;gBAC7B,MAAM,cAAc,GAA4B,EAAE,CAAC;gBACnD,kFAAkF;gBAClF,IAAI,YAAY,GAAG,IAAI,CAAC,CAAC,qDAAqD;gBAC9E,MAAM,sBAAsB,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,2DAA2D;gBAEpH,KAAK,MAAM,SAAS,IAAI,GAAG,EAAE,CAAC;oBAC5B,MAAM,UAAU,GAAG,wBAAwB,CAAC,SAAS,CAAC,CAAC;oBACvD,IAAI,CAAC,UAAU;wBAAE,SAAS;oBAE1B,IAAI,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC;oBAE3B,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;wBAC1D,KAAK,GAAG,UAAU,KAAK,oBAAoB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;oBAC1D,CAAC;yBAAM,CAAC;wBACN,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;wBAEtB,IAAI,cAAc,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;4BACxC,KAAK,GAAI,KAAgB;iCACtB,KAAK,CAAC,kBAAkB,CAAC;iCACzB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;iCAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;wBACnC,CAAC;6BAAM,IAAI,UAAU,KAAK,oBAAoB,EAAE,CAAC;4BAC/C,IAAI,CAAC;gCACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAe,CAAC,CAAC;4BACtC,CAAC;4BAAC,OAAO,CAAC,EAAE,CAAC;gCACX,OAAO,CAAC,IAAI,CACV,OAAO,KAAK,GAAG,CAAC,yCAAyC,KAAK,+BAA+B,CAC9F,CAAC;4BACJ,CAAC;wBACH,CAAC;6BAAM,IAAI,sBAAsB,IAAI,SAAS,KAAK,sBAAsB,EAAE,CAAC;4BACzE,8EAA8E;4BAC9E,MAAM,QAAQ,GAAI,KAAgB,CAAC,WAAW,EAAE,CAAC;4BACjD,YAAY,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;4BAC5D,0FAA0F;4BAC1F,SAAS,CAAC,gDAAgD;wBAC5D,CAAC;6BAAM,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;4BACpC,qBAAqB;4BACrB,MAAM,QAAQ,GAAI,KAAgB,CAAC,WAAW,EAAE,CAAC;4BACjD,YAAY,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;4BAC5D,uDAAuD;4BACvD,IAAI,aAAa,CAAC,OAAO,EAAE,CAAC;gCACvB,cAAsB,CAAC,UAAU,CAAC,GAAG,YAAY,CAAC;4BACvD,CAAC;4BACD,mDAAmD;wBACrD,CAAC;6BAAM,IAAI,UAAU,KAAK,cAAc,IAAI,UAAU,KAAK,gBAAgB,EAAE,CAAC;4BAC3E,iEAAiE;4BACjE,IAAK,KAAgB,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gCACjD,KAAK,GAAI,KAAgB;qCACtB,KAAK,CAAC,kBAAkB,CAAC;qCACzB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;qCAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;4BACrC,CAAC,CAAC,uBAAuB;wBAC5B,CAAC;oBACJ,CAAC;oBACA,cAAsB,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC,iCAAiC;gBAChF,CAAC;gBAED,gEAAgE;gBAChE,IAAI,YAAY,EAAE,CAAC;oBAChB,wCAAwC;oBACzC,IACE,cAAc,CAAC,KAAK,KAAK,SAAS;wBAClC,cAAc,CAAC,KAAK,KAAK,IAAI;oBAC7B,kEAAkE;sBAClE,CAAC;wBACD,OAAO,CAAC,IAAI,CACV,OAAO,KAAK,GAAG,CAAC,4CAA4C,CAC7D,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,qCAAqC;wBACrC,IAAI,aAAa,CAAC,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,CAAC;4BACnD,cAAc,CAAC,OAAO,GAAG,MAAM,CAAC,MAAA,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,mCAAI,IAAI,CAAC,CAAC,CAAC,uBAAuB;wBAChG,CAAC;wBACD,WAAW,CAAC,IAAI,CAAC,IAAI,oBAAO,CAAC,cAAgC,CAAC,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC,UAAU,WAAW,CAAC,MAAM,oBAAoB,CAAC,CAAC;YAC/D,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,KAAK,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;gBACjD,MAAM,IAAI,KAAK,CAAC,YAAY,QAAQ,iBAAiB,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,mCAAmC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAC;gBACpE,MAAM,KAAK,CAAC,CAAC,wBAAwB;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CACX,QAAsB,EACtB,OAAe,EACf,QAAiB,EACjB,qBAA6B,GAAG;QAEhC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,SAAS,CACjB,sBAAsB,QAAQ,wBAAwB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACzF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,0BAA0B;YAC1B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,QAAQ,GAAG,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,IAAA,iBAAM,EAAC,IAAI,IAAI,EAAE,EAAE,iBAAiB,CAAC,CAAC;YACnE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC,CAAC;YACnE,sEAAsE;YACtE,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YAE7D,OAAO,CAAC,IAAI,CAAC,qBAAqB,YAAY,EAAE,CAAC,CAAC;YAElD,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gBACxB,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC7E,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;gBAC9B,6DAA6D;gBAC9D,MAAM,WAAW,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,iBAAiB,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAChG,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC9B,sEAAsE;gBACtE,MAAM,UAAU,GAAa;oBAC3B,OAAO;oBACP,eAAe;oBACf,iBAAiB;oBACjB,SAAS;oBACT,mBAAmB;oBACnB,qBAAqB;oBACrB,cAAc;oBACd,gBAAgB;oBAChB,MAAM;oBACN,YAAY;oBACZ,eAAe;oBACf,WAAW;oBACX,UAAU;oBACV,SAAS;iBACV,CAAC;gBAEF,qEAAqE;gBACpE,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;oBAC9C,MAAM,GAAG,GAAqD,EAAE,CAAC;oBACjE,UAAU,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;wBAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;wBAC5B,wDAAwD;wBACxD,IAAK,CACE,MAAM,KAAK,SAAS;4BACpB,MAAM,KAAK,mBAAmB;4BAC9B,MAAM,KAAK,cAAc;4BACzB,MAAM,KAAK,gBAAgB,CAC5B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EACzB,CAAC;4BACJ,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;wBAC/C,CAAC;6BAAM,IAAI,MAAM,KAAK,qBAAqB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BAC3F,IAAI,CAAC;gCACF,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;4BACvC,CAAC;4BAAC,WAAM,CAAC;gCACN,GAAG,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;4BAC/B,CAAC;wBACH,CAAC;6BAAM,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;4BAC9B,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,aAAL,KAAK,cAAL,KAAK,GAAI,IAAI,CAAC,CAAC,oCAAoC;wBACrE,CAAC;6BAAM,CAAC;4BACN,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;wBACnD,CAAC;oBACN,CAAC,CAAC,CAAC;oBACJ,OAAO,GAAG,CAAC;gBACZ,CAAC,CAAC,CAAC;gBAEL,MAAM,UAAU,GAAG,mBAAI,CAAC,OAAO,CAAC,OAAO,EAAE;oBACvC,OAAO,EAAE,UAAU;oBACnB,MAAM,EAAE,IAAI;oBACZ,SAAS,EAAE,GAAG;iBACf,CAAC,CAAC;gBACH,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,2BAA2B,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;YAC7D,MAAM,KAAK,CAAC,CAAC,iBAAiB;QAChC,CAAC;IACH,CAAC;CACF;AA3YD,kCA2YC"}
|
package/dist/cjs/data/example.js
CHANGED
|
@@ -11,9 +11,12 @@ class Example {
|
|
|
11
11
|
this.additionalMetadata = options.additionalMetadata;
|
|
12
12
|
this.toolsCalled = options.toolsCalled;
|
|
13
13
|
this.expectedTools = options.expectedTools;
|
|
14
|
+
this.name = options.name;
|
|
14
15
|
this.exampleId = options.exampleId || this.generateUUID();
|
|
15
16
|
this.exampleIndex = options.exampleIndex;
|
|
16
|
-
this.timestamp = options.timestamp;
|
|
17
|
+
this.timestamp = options.timestamp || new Date().toISOString();
|
|
18
|
+
this.traceId = options.traceId;
|
|
19
|
+
this.example = options.example;
|
|
17
20
|
}
|
|
18
21
|
/**
|
|
19
22
|
* Generate a UUID for the example ID
|
|
@@ -44,9 +47,12 @@ class Example {
|
|
|
44
47
|
additional_metadata: this.additionalMetadata,
|
|
45
48
|
tools_called: this.toolsCalled,
|
|
46
49
|
expected_tools: this.expectedTools,
|
|
50
|
+
name: this.name,
|
|
47
51
|
example_id: this.exampleId,
|
|
48
52
|
example_index: this.exampleIndex,
|
|
49
53
|
timestamp: this.timestamp,
|
|
54
|
+
trace_id: this.traceId,
|
|
55
|
+
example: this.example,
|
|
50
56
|
};
|
|
51
57
|
}
|
|
52
58
|
}
|
|
@@ -90,6 +96,10 @@ class ExampleBuilder {
|
|
|
90
96
|
this._expectedTools = expectedTools;
|
|
91
97
|
return this;
|
|
92
98
|
}
|
|
99
|
+
name(name) {
|
|
100
|
+
this._name = name;
|
|
101
|
+
return this;
|
|
102
|
+
}
|
|
93
103
|
exampleId(exampleId) {
|
|
94
104
|
this._exampleId = exampleId;
|
|
95
105
|
return this;
|
|
@@ -102,6 +112,14 @@ class ExampleBuilder {
|
|
|
102
112
|
this._timestamp = timestamp;
|
|
103
113
|
return this;
|
|
104
114
|
}
|
|
115
|
+
traceId(traceId) {
|
|
116
|
+
this._traceId = traceId;
|
|
117
|
+
return this;
|
|
118
|
+
}
|
|
119
|
+
example(example) {
|
|
120
|
+
this._example = example;
|
|
121
|
+
return this;
|
|
122
|
+
}
|
|
105
123
|
build() {
|
|
106
124
|
if (!this._input) {
|
|
107
125
|
throw new Error('Input is required for an Example');
|
|
@@ -115,9 +133,12 @@ class ExampleBuilder {
|
|
|
115
133
|
additionalMetadata: this._additionalMetadata,
|
|
116
134
|
toolsCalled: this._toolsCalled,
|
|
117
135
|
expectedTools: this._expectedTools,
|
|
136
|
+
name: this._name,
|
|
118
137
|
exampleId: this._exampleId,
|
|
119
138
|
exampleIndex: this._exampleIndex,
|
|
120
139
|
timestamp: this._timestamp,
|
|
140
|
+
traceId: this._traceId,
|
|
141
|
+
example: this._example,
|
|
121
142
|
});
|
|
122
143
|
}
|
|
123
144
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../../src/data/example.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../../src/data/example.ts"],"names":[],"mappings":";;;AAoBA,MAAa,OAAO;IAgBlB,YAAY,OAAuB;QACjC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC7C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC,kBAAkB,CAAC;QACrD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACjC,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,OAAO,sCAAsC,CAAC,OAAO,CAAC,OAAO,EAAE,UAAS,CAAC;YACvE,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACjC,MAAM,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC;YAC1C,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,OAAO;QACZ,OAAO,IAAI,cAAc,EAAE,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,eAAe,EAAE,IAAI,CAAC,cAAc;YACpC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,iBAAiB,EAAE,IAAI,CAAC,gBAAgB;YACxC,mBAAmB,EAAE,IAAI,CAAC,kBAAkB;YAC5C,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,IAAI,CAAC,SAAS;YAC1B,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,IAAI,CAAC,OAAO;YACtB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;CACF;AAxED,0BAwEC;AAED;;GAEG;AACH,MAAa,cAAc;IAA3B;QACU,WAAM,GAAW,EAAE,CAAC;IA2G9B,CAAC;IA5FC,KAAK,CAAC,KAAa;QACjB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,YAA+B;QAC1C,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,cAAiC;QAC9C,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,OAAiB;QACvB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gBAAgB,CAAC,gBAA0B;QACzC,IAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,kBAAkB,CAAC,kBAAuC;QACxD,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,CAAC;QAC9C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,WAAqB;QAC/B,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,aAAuB;QACnC,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,IAAY;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,SAAiB;QACzB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,YAAoB;QAC/B,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,SAAiB;QACzB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,OAAe;QACrB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,OAAgB;QACtB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,OAAO,IAAI,OAAO,CAAC;YACjB,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,cAAc,EAAE,IAAI,CAAC,eAAe;YACpC,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,gBAAgB,EAAE,IAAI,CAAC,iBAAiB;YACxC,kBAAkB,EAAE,IAAI,CAAC,mBAAmB;YAC5C,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,aAAa,EAAE,IAAI,CAAC,cAAc;YAClC,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,YAAY,EAAE,IAAI,CAAC,aAAa;YAChC,SAAS,EAAE,IAAI,CAAC,UAAU;YAC1B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC,CAAC;IACL,CAAC;CACF;AA5GD,wCA4GC"}
|