@runtypelabs/sdk 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +398 -0
- package/dist/batch-builder.d.ts +106 -0
- package/dist/batch-builder.d.ts.map +1 -0
- package/dist/batch-builder.js +124 -0
- package/dist/batch-builder.js.map +1 -0
- package/dist/batches-namespace.d.ts +132 -0
- package/dist/batches-namespace.d.ts.map +1 -0
- package/dist/batches-namespace.js +128 -0
- package/dist/batches-namespace.js.map +1 -0
- package/dist/client.d.ts +121 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +485 -0
- package/dist/client.js.map +1 -0
- package/dist/endpoints.d.ts +560 -0
- package/dist/endpoints.d.ts.map +1 -0
- package/dist/endpoints.js +725 -0
- package/dist/endpoints.js.map +1 -0
- package/dist/eval-builder.d.ts +216 -0
- package/dist/eval-builder.d.ts.map +1 -0
- package/dist/eval-builder.js +225 -0
- package/dist/eval-builder.js.map +1 -0
- package/dist/evals-namespace.d.ts +205 -0
- package/dist/evals-namespace.d.ts.map +1 -0
- package/dist/evals-namespace.js +208 -0
- package/dist/evals-namespace.js.map +1 -0
- package/dist/flow-builder.d.ts +620 -0
- package/dist/flow-builder.d.ts.map +1 -0
- package/dist/flow-builder.js +565 -0
- package/dist/flow-builder.js.map +1 -0
- package/dist/flow-result.d.ts +117 -0
- package/dist/flow-result.d.ts.map +1 -0
- package/dist/flow-result.js +175 -0
- package/dist/flow-result.js.map +1 -0
- package/dist/flows-namespace.d.ts +430 -0
- package/dist/flows-namespace.d.ts.map +1 -0
- package/dist/flows-namespace.js +679 -0
- package/dist/flows-namespace.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +76 -0
- package/dist/index.js.map +1 -0
- package/dist/prompts-namespace.d.ts +236 -0
- package/dist/prompts-namespace.d.ts.map +1 -0
- package/dist/prompts-namespace.js +222 -0
- package/dist/prompts-namespace.js.map +1 -0
- package/dist/runtype.d.ts +232 -0
- package/dist/runtype.d.ts.map +1 -0
- package/dist/runtype.js +367 -0
- package/dist/runtype.js.map +1 -0
- package/dist/stream-utils.d.ts +58 -0
- package/dist/stream-utils.d.ts.map +1 -0
- package/dist/stream-utils.js +348 -0
- package/dist/stream-utils.js.map +1 -0
- package/dist/transform.d.ts +21 -0
- package/dist/transform.d.ts.map +1 -0
- package/dist/transform.js +170 -0
- package/dist/transform.js.map +1 -0
- package/dist/types.d.ts +626 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/package.json +61 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"endpoints.js","sourceRoot":"","sources":["../src/endpoints.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AA0CH;;GAEG;AACH,MAAa,aAAa;IACxB,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAAmB;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAA2B,QAAQ,EAAE,MAAM,CAAC,CAAA;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAO,UAAU,EAAE,EAAE,CAAC,CAAA;QAC5D,OAAO,QAAQ,CAAA;IACjB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAuB;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAO,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAgC;QACvD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAO,UAAU,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,UAAU,EAAE,EAAE,CAAC,CAAA;IACjD,CAAC;IAGD;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,EAAU,EAAE,UAAkB;QAClD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,qBAAqB,EAAE;YACzD,UAAU;SACX,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,EAAU;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,UAAU,CAAC,CAAA;IACjD,CAAC;CACF;AAvDD,sCAuDC;AAED;;GAEG;AACH,MAAa,eAAe;IAC1B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAAyC;QAClD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAA6B,UAAU,EAAE,MAAM,CAAC,CAAA;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,YAAY,EAAE,EAAE,CAAC,CAAA;IAClD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAyB;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAS,UAAU,EAAE,IAAI,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAkC;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,YAAY,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACxD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,YAAY,EAAE,EAAE,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,EAAU,EAAE,QAAgB;QAC5C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,gBAAgB,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAA;IACvE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,EAAU;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,EAAU,EAAE,OAAiB;QAC7C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAC7D,CAAC;CACF;AA1DD,0CA0DC;AAED;;GAEG;AACH,MAAa,eAAe;IAC1B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAAyB;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAoC,UAAU,EAAE,MAAM,CAAC,CAAA;IAC/E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAgB,YAAY,EAAE,EAAE,CAAC,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAyB;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAgB,UAAU,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAkC;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAgB,YAAY,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,YAAY,EAAE,EAAE,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,GAAa;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAyF,sBAAsB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;IAClJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAqB;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAmB,oBAAoB,EAAE,IAAI,CAAC,CAAA;IACvE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU,EAAE,MAAgE;QAC3F,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,QAAgB;QAC7C,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,YAAY,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAA;IACzE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,IAAU,EAAE,MAAqD;QAC/E,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAA;QAC/B,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC7B,IAAI,MAAM,EAAE,UAAU,EAAE,CAAC;YACvB,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;QAClD,CAAC;QACD,IAAI,MAAM,EAAE,UAAU,EAAE,CAAC;YACvB,QAAQ,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;QAClD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,MAAwC;QACvD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE;YACjC,GAAG,MAAM;YACT,aAAa,EAAE,IAAI;YACnB,KAAK,EAAE,CAAC;SACT,CAAC,CAAA;IACJ,CAAC;CACF;AAlGD,0CAkGC;AAED;;GAEG;AACH,MAAa,eAAe;IAC1B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAA0D,WAAW,CAAC,CAAA;QAC5G,OAAO,QAAQ,CAAC,OAAO,CAAA;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,aAAa,EAAE,EAAE,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAyB;QACpC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAkC;QACzD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,aAAa,EAAE,EAAE,CAAC,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,EAAU;QAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,YAAY,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,EAAU,EAAE,MAAiE;QAC1F,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,QAAwB,EAAE,MAA+B;QAClF,MAAM,QAAQ,GAAG,QAAQ,KAAK,KAAK;YACjC,CAAC,CAAC,qBAAqB;YACvB,CAAC,CAAC,aAAa,QAAQ,YAAY,CAAA;QACrC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,QAAwB,EAAE,MAA+B;QACrE,MAAM,QAAQ,GAAG,QAAQ,KAAK,KAAK;YACjC,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,aAAa,QAAQ,OAAO,CAAA;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,EAAU;QAC9B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,EAAU,EAAE,IAAS;QACxC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,uBAAuB,CAAC,EAAU;QACtC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAA;IAC7D,CAAC;CACF;AA3GD,0CA2GC;AAED;;GAEG;AACH,MAAa,oBAAoB;IAC/B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAyD,0BAA0B,CAAC,CAAA;QAC1H,iDAAiD;QACjD,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAyC,+BAA+B,CAAC,CAAA;QAC/G,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB;QACvB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAyC,kCAAkC,CAAC,CAAA;QAClH,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAA0C,gBAAgB,CAAC,CAAA;QACjG,iDAAiD;QACjD,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAA;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAA8B;QACzC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAc,gBAAgB,EAAE,IAAI,CAAC,CAAA;IAC9D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAuC;QAC9D,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAc,kBAAkB,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAO,kBAAkB,EAAE,EAAE,CAAC,CAAA;IACzD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,EAAU,EAAE,SAAkB;QAC/C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;IACxE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,EAAU;QACzB,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE,UAAU,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAA;IAChD,CAAC;CACF;AA9ED,oDA8EC;AAED;;GAEG;AACH,MAAa,gBAAgB;IAC3B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,IAAqB;QACjC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAA;IAC5C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,IAAqB;QACvC,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,EAAE;YAC5C,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAA;IACJ,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CAAC,IAKZ;QACC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,kBAAkB,EAAE;gBACnD,MAAM,EAAE,MAAM;gBACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,CAAA;IACnD,CAAC;CACF;AA3CD,4CA2CC;AAED;;GAEG;AACH,MAAa,YAAY;IACvB,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,OAAe;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,OAAa;QAC7C,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE;YACxC,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,EAAE,CAAC;SAC9C,CAAC,CAAA;IACJ,CAAC;CACF;AAnBD,oCAmBC;AAED;;GAEG;AACH,MAAa,aAAa;IACxB,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAc,gBAAgB,CAAC,CAAA;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,IAA0B;QAC5C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAc,gBAAgB,EAAE,IAAI,CAAC,CAAA;IAC7D,CAAC;CACF;AAhBD,sCAgBC;AAED;;GAEG;AACH,MAAa,iBAAiB;IAC5B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,MAA+B;QAC5C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,MAA+B;QACjD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAA;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,MAA+B;QACvD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,2BAA2B,EAAE,MAAM,CAAC,CAAA;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,MAA8B;QAChD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAqB,cAAc,EAAE,MAAM,CAAC,CAAA;IACpE,CAAC;CACF;AA9BD,8CA8BC;AAED;;GAEG;AACH,MAAa,iBAAiB;IAC5B,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAA;IACtD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,EAAE,CAAC,CAAA;IAC7C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAS;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,CAAA;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAS;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,EAAE,CAAC,CAAA;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,UAAgD;QAC5E,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE;YAC7C,MAAM;YACN,UAAU;SACX,CAAC,CAAA;IACJ,CAAC;CACF;AA/CD,8CA+CC;AAED;;GAEG;AACH,MAAa,wBAAwB;IACnC,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAA+B;QACxC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAA;IACtD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAS;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,CAAA;IACrD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAS;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAA;IACvD,CAAC;CACF;AArCD,4DAqCC;AAED;;GAEG;AACH,MAAa,aAAa;IACxB,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,MAAmB;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAA2B,QAAQ,EAAE,MAAM,CAAC,CAAA;IACpE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,EAAU;QAClB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAO,UAAU,EAAE,EAAE,CAAC,CAAA;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,IAAuB;QAClC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAO,QAAQ,EAAE,IAAI,CAAC,CAAA;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,IAAuB;QAC9C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAO,UAAU,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO,CAAC,EAAU,EAAE,IAAwB;QAChD,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAsB,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,CAAA;IAC5E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,EAAU,EAAE,IAAwB;QACtD,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,EAAE;YACvD,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAA;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI,CAAC,EAAU,EAAE,IAAwB;QAC7C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAsB,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;IACzE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,MAAc,EAAE,MAAmB;QACrD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAoC,UAAU,MAAM,aAAa,EAAE,MAAM,CAAC,CAAA;IAClG,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,MAA+B;QAC9C,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,CAAA;IACjD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY;QAChB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAA2B,QAAQ,EAAE;YACzE,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,GAAG;SACX,CAAC,CAAA;QAEF,MAAM,OAAO,GAAQ,QAAQ,CAAA;QAE7B,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;YACjC,OAAO,OAAO,CAAC,IAAI,CAAA;QACrB,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAA;QAChB,CAAC;QAED,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,MAAc,EAAE,IAKrC;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAO,UAAU,MAAM,kBAAkB,EAAE,IAAI,CAAC,CAAA;IACzE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAkC,gBAAgB,CAAC,CAAA;IAC3E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,yBAAyB,CAAC,KAAa,EAAE,QAAgB;QAC7D,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAkC,2BAA2B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;IAC3G,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB,CAAC,MAAc;QACvC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAc,kBAAkB,MAAM,SAAS,CAAC,CAAA;IACxE,CAAC;CACF;AAlID,sCAkIC;AAED;;GAEG;AACH,MAAa,YAAY;IACvB,YAAoB,MAAiB;QAAjB,WAAM,GAAN,MAAM,CAAW;IAAG,CAAC;IAEzC;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAkBpB;QACC,OAAO,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,EAAE;YAC/C,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAA;IACJ,CAAC;CACF;AA/BD,oCA+BC"}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EvalBuilder - Fluent builder for evaluation runs
|
|
3
|
+
*
|
|
4
|
+
* Provides a chainable API for building evaluation configurations
|
|
5
|
+
* that test flows against records with model overrides and comparisons.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { EvalBuilder } from '@runtypelabs/sdk'
|
|
10
|
+
*
|
|
11
|
+
* // Single model eval with overrides
|
|
12
|
+
* const eval1 = await new EvalBuilder()
|
|
13
|
+
* .useFlow('flow_abc123')
|
|
14
|
+
* .forRecordType('test_data')
|
|
15
|
+
* .withModelOverrides([{ stepName: 'Analyze', model: 'gpt-4o' }])
|
|
16
|
+
* .run(apiClient)
|
|
17
|
+
*
|
|
18
|
+
* // Multi-model comparison
|
|
19
|
+
* const eval2 = await new EvalBuilder()
|
|
20
|
+
* .useFlow('flow_abc123')
|
|
21
|
+
* .forRecordType('test_data')
|
|
22
|
+
* .compareModels([
|
|
23
|
+
* { stepName: 'Analyze', model: 'gpt-4o' },
|
|
24
|
+
* { stepName: 'Analyze', model: 'claude-3-opus' },
|
|
25
|
+
* ])
|
|
26
|
+
* .run(apiClient)
|
|
27
|
+
*
|
|
28
|
+
* // Virtual flow eval
|
|
29
|
+
* const eval3 = await new EvalBuilder()
|
|
30
|
+
* .useVirtualFlow(flowBuilder)
|
|
31
|
+
* .forRecordType('test_data')
|
|
32
|
+
* .run(apiClient)
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
import type { FlowBuilder } from './flow-builder';
|
|
36
|
+
export interface ModelOverride {
|
|
37
|
+
/** Name of the step to override */
|
|
38
|
+
stepName: string;
|
|
39
|
+
/** Model to use for this step */
|
|
40
|
+
model: string;
|
|
41
|
+
/** Optional temperature override */
|
|
42
|
+
temperature?: number;
|
|
43
|
+
/** Optional max tokens override */
|
|
44
|
+
maxTokens?: number;
|
|
45
|
+
}
|
|
46
|
+
export interface EvalOptions {
|
|
47
|
+
/** Store evaluation results */
|
|
48
|
+
storeResults?: boolean;
|
|
49
|
+
/** Run evaluations in parallel */
|
|
50
|
+
parallel?: boolean;
|
|
51
|
+
/** Maximum concurrent evaluations */
|
|
52
|
+
concurrency?: number;
|
|
53
|
+
/** Continue on individual record failures */
|
|
54
|
+
continueOnError?: boolean;
|
|
55
|
+
}
|
|
56
|
+
export interface EvalRecord {
|
|
57
|
+
/** Record name */
|
|
58
|
+
name: string;
|
|
59
|
+
/** Record type */
|
|
60
|
+
type: string;
|
|
61
|
+
/** Record metadata */
|
|
62
|
+
metadata: Record<string, any>;
|
|
63
|
+
}
|
|
64
|
+
export interface EvalRequest {
|
|
65
|
+
/** Flow ID (for existing flow) */
|
|
66
|
+
flowId?: string;
|
|
67
|
+
/** Virtual flow definition */
|
|
68
|
+
flow?: {
|
|
69
|
+
name: string;
|
|
70
|
+
steps: any[];
|
|
71
|
+
};
|
|
72
|
+
/** Record type to evaluate against */
|
|
73
|
+
recordType?: string;
|
|
74
|
+
/** Inline records for ad-hoc evaluation */
|
|
75
|
+
records?: EvalRecord[];
|
|
76
|
+
/** Model overrides for evaluation */
|
|
77
|
+
modelOverrides?: ModelOverride[];
|
|
78
|
+
/** Multiple model configurations for comparison */
|
|
79
|
+
modelConfigs?: ModelOverride[][];
|
|
80
|
+
/** Eval options */
|
|
81
|
+
options?: EvalOptions;
|
|
82
|
+
/** Optional filter for records */
|
|
83
|
+
filter?: Record<string, any>;
|
|
84
|
+
/** Optional limit on number of records */
|
|
85
|
+
limit?: number;
|
|
86
|
+
}
|
|
87
|
+
export interface EvalResult {
|
|
88
|
+
evalId: string;
|
|
89
|
+
status: 'running' | 'completed' | 'failed';
|
|
90
|
+
totalRecords: number;
|
|
91
|
+
completedRecords: number;
|
|
92
|
+
failedRecords: number;
|
|
93
|
+
/** Results grouped by model config (for multi-model eval) */
|
|
94
|
+
results?: Array<{
|
|
95
|
+
modelConfig: ModelOverride[];
|
|
96
|
+
records: Array<{
|
|
97
|
+
recordId: string;
|
|
98
|
+
status: 'success' | 'error';
|
|
99
|
+
result?: any;
|
|
100
|
+
error?: string;
|
|
101
|
+
executionTime: number;
|
|
102
|
+
}>;
|
|
103
|
+
}>;
|
|
104
|
+
}
|
|
105
|
+
export declare class EvalBuilder {
|
|
106
|
+
private flowId;
|
|
107
|
+
private virtualFlow;
|
|
108
|
+
private recordType;
|
|
109
|
+
private inlineRecords;
|
|
110
|
+
private modelOverrides;
|
|
111
|
+
private modelConfigs;
|
|
112
|
+
private evalOptions;
|
|
113
|
+
private filterConfig;
|
|
114
|
+
private limitConfig;
|
|
115
|
+
/**
|
|
116
|
+
* Use an existing flow by ID for evaluation
|
|
117
|
+
*/
|
|
118
|
+
useFlow(flowId: string): this;
|
|
119
|
+
/**
|
|
120
|
+
* Use a virtual flow (from FlowBuilder) for evaluation
|
|
121
|
+
*
|
|
122
|
+
* @example
|
|
123
|
+
* ```typescript
|
|
124
|
+
* const flowBuilder = new FlowBuilder()
|
|
125
|
+
* .createFlow({ name: 'Test Flow' })
|
|
126
|
+
* .prompt({ name: 'Analyze', model: 'gpt-4', userPrompt: '...' })
|
|
127
|
+
*
|
|
128
|
+
* const eval = await new EvalBuilder()
|
|
129
|
+
* .useVirtualFlow(flowBuilder)
|
|
130
|
+
* .forRecordType('test_data')
|
|
131
|
+
* .run(apiClient)
|
|
132
|
+
* ```
|
|
133
|
+
*/
|
|
134
|
+
useVirtualFlow(builder: FlowBuilder): this;
|
|
135
|
+
/**
|
|
136
|
+
* Specify the record type to evaluate against
|
|
137
|
+
*/
|
|
138
|
+
forRecordType(recordType: string): this;
|
|
139
|
+
/**
|
|
140
|
+
* Use inline records for ad-hoc evaluation (no record type lookup)
|
|
141
|
+
*
|
|
142
|
+
* @example
|
|
143
|
+
* ```typescript
|
|
144
|
+
* .withRecords([
|
|
145
|
+
* { name: 'Test 1', type: 'test', metadata: { input: 'hello' } },
|
|
146
|
+
* { name: 'Test 2', type: 'test', metadata: { input: 'world' } },
|
|
147
|
+
* ])
|
|
148
|
+
* ```
|
|
149
|
+
*/
|
|
150
|
+
withRecords(records: EvalRecord[]): this;
|
|
151
|
+
/**
|
|
152
|
+
* Set model overrides for single-model evaluation
|
|
153
|
+
*
|
|
154
|
+
* @example
|
|
155
|
+
* ```typescript
|
|
156
|
+
* .withModelOverrides([
|
|
157
|
+
* { stepName: 'Analyze', model: 'gpt-4o', temperature: 0.2 }
|
|
158
|
+
* ])
|
|
159
|
+
* ```
|
|
160
|
+
*/
|
|
161
|
+
withModelOverrides(overrides: ModelOverride[]): this;
|
|
162
|
+
/**
|
|
163
|
+
* Compare multiple model configurations
|
|
164
|
+
*
|
|
165
|
+
* @example
|
|
166
|
+
* ```typescript
|
|
167
|
+
* .compareModels([
|
|
168
|
+
* [{ stepName: 'Analyze', model: 'gpt-4o' }],
|
|
169
|
+
* [{ stepName: 'Analyze', model: 'claude-3-opus' }],
|
|
170
|
+
* [{ stepName: 'Analyze', model: 'gemini-pro' }],
|
|
171
|
+
* ])
|
|
172
|
+
* ```
|
|
173
|
+
*/
|
|
174
|
+
compareModels(configs: ModelOverride[][]): this;
|
|
175
|
+
/**
|
|
176
|
+
* Set evaluation options
|
|
177
|
+
*/
|
|
178
|
+
withOptions(options: EvalOptions): this;
|
|
179
|
+
/**
|
|
180
|
+
* Filter records to evaluate
|
|
181
|
+
*/
|
|
182
|
+
withFilter(filter: Record<string, any>): this;
|
|
183
|
+
/**
|
|
184
|
+
* Limit the number of records to evaluate
|
|
185
|
+
*/
|
|
186
|
+
withLimit(limit: number): this;
|
|
187
|
+
/**
|
|
188
|
+
* Build the eval request configuration
|
|
189
|
+
*/
|
|
190
|
+
build(): EvalRequest;
|
|
191
|
+
/**
|
|
192
|
+
* Execute the evaluation
|
|
193
|
+
*
|
|
194
|
+
* @param client - Client with eval capability
|
|
195
|
+
* @returns EvalResult with evaluation status and results
|
|
196
|
+
*/
|
|
197
|
+
run(client: EvalClient): Promise<EvalResult>;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Interface for clients that can execute evaluations
|
|
201
|
+
*/
|
|
202
|
+
export interface EvalClient {
|
|
203
|
+
eval(config: EvalRequest): Promise<EvalResult>;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* EvalBuilder that is bound to a client for direct execution
|
|
207
|
+
*/
|
|
208
|
+
export declare class ClientEvalBuilder extends EvalBuilder {
|
|
209
|
+
private boundClient;
|
|
210
|
+
constructor(client: EvalClient);
|
|
211
|
+
/**
|
|
212
|
+
* Execute the evaluation using the bound client
|
|
213
|
+
*/
|
|
214
|
+
run(): Promise<EvalResult>;
|
|
215
|
+
}
|
|
216
|
+
//# sourceMappingURL=eval-builder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAkB,MAAM,gBAAgB,CAAA;AAMjE,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,EAAE,MAAM,CAAA;IAChB,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,mCAAmC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,+BAA+B;IAC/B,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,sBAAsB;IACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,8BAA8B;IAC9B,IAAI,CAAC,EAAE;QACL,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,GAAG,EAAE,CAAA;KACb,CAAA;IACD,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,UAAU,EAAE,CAAA;IACtB,qCAAqC;IACrC,cAAc,CAAC,EAAE,aAAa,EAAE,CAAA;IAChC,mDAAmD;IACnD,YAAY,CAAC,EAAE,aAAa,EAAE,EAAE,CAAA;IAChC,mBAAmB;IACnB,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAC5B,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,CAAA;IAC1C,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,KAAK,CAAC;QACd,WAAW,EAAE,aAAa,EAAE,CAAA;QAC5B,OAAO,EAAE,KAAK,CAAC;YACb,QAAQ,EAAE,MAAM,CAAA;YAChB,MAAM,EAAE,SAAS,GAAG,OAAO,CAAA;YAC3B,MAAM,CAAC,EAAE,GAAG,CAAA;YACZ,KAAK,CAAC,EAAE,MAAM,CAAA;YACd,aAAa,EAAE,MAAM,CAAA;SACtB,CAAC,CAAA;KACH,CAAC,CAAA;CACH;AAMD,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,WAAW,CAA4C;IAC/D,OAAO,CAAC,UAAU,CAAoB;IACtC,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,YAAY,CAA+B;IACnD,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,WAAW,CAAoB;IAEvC;;OAEG;IACH,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAM7B;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI;IAY1C;;OAEG;IACH,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAMvC;;;;;;;;;;OAUG;IACH,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI;IAMxC;;;;;;;;;OASG;IACH,kBAAkB,CAAC,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI;IAMpD;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,GAAG,IAAI;IAM/C;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI;IAKvC;;OAEG;IACH,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI;IAK7C;;OAEG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAK9B;;OAEG;IACH,KAAK,IAAI,WAAW;IA2CpB;;;;;OAKG;IACG,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;CAInD;AAMD;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAC/C;AAED;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,OAAO,CAAC,WAAW,CAAY;gBAEnB,MAAM,EAAE,UAAU;IAK9B;;OAEG;IACG,GAAG,IAAI,OAAO,CAAC,UAAU,CAAC;CAIjC"}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* EvalBuilder - Fluent builder for evaluation runs
|
|
4
|
+
*
|
|
5
|
+
* Provides a chainable API for building evaluation configurations
|
|
6
|
+
* that test flows against records with model overrides and comparisons.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { EvalBuilder } from '@runtypelabs/sdk'
|
|
11
|
+
*
|
|
12
|
+
* // Single model eval with overrides
|
|
13
|
+
* const eval1 = await new EvalBuilder()
|
|
14
|
+
* .useFlow('flow_abc123')
|
|
15
|
+
* .forRecordType('test_data')
|
|
16
|
+
* .withModelOverrides([{ stepName: 'Analyze', model: 'gpt-4o' }])
|
|
17
|
+
* .run(apiClient)
|
|
18
|
+
*
|
|
19
|
+
* // Multi-model comparison
|
|
20
|
+
* const eval2 = await new EvalBuilder()
|
|
21
|
+
* .useFlow('flow_abc123')
|
|
22
|
+
* .forRecordType('test_data')
|
|
23
|
+
* .compareModels([
|
|
24
|
+
* { stepName: 'Analyze', model: 'gpt-4o' },
|
|
25
|
+
* { stepName: 'Analyze', model: 'claude-3-opus' },
|
|
26
|
+
* ])
|
|
27
|
+
* .run(apiClient)
|
|
28
|
+
*
|
|
29
|
+
* // Virtual flow eval
|
|
30
|
+
* const eval3 = await new EvalBuilder()
|
|
31
|
+
* .useVirtualFlow(flowBuilder)
|
|
32
|
+
* .forRecordType('test_data')
|
|
33
|
+
* .run(apiClient)
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
37
|
+
exports.ClientEvalBuilder = exports.EvalBuilder = void 0;
|
|
38
|
+
// ============================================================================
|
|
39
|
+
// EvalBuilder Class
|
|
40
|
+
// ============================================================================
|
|
41
|
+
class EvalBuilder {
|
|
42
|
+
constructor() {
|
|
43
|
+
this.evalOptions = {};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Use an existing flow by ID for evaluation
|
|
47
|
+
*/
|
|
48
|
+
useFlow(flowId) {
|
|
49
|
+
this.flowId = flowId;
|
|
50
|
+
this.virtualFlow = undefined;
|
|
51
|
+
return this;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Use a virtual flow (from FlowBuilder) for evaluation
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```typescript
|
|
58
|
+
* const flowBuilder = new FlowBuilder()
|
|
59
|
+
* .createFlow({ name: 'Test Flow' })
|
|
60
|
+
* .prompt({ name: 'Analyze', model: 'gpt-4', userPrompt: '...' })
|
|
61
|
+
*
|
|
62
|
+
* const eval = await new EvalBuilder()
|
|
63
|
+
* .useVirtualFlow(flowBuilder)
|
|
64
|
+
* .forRecordType('test_data')
|
|
65
|
+
* .run(apiClient)
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
useVirtualFlow(builder) {
|
|
69
|
+
const config = builder.build();
|
|
70
|
+
if (config.flow && 'name' in config.flow && 'steps' in config.flow) {
|
|
71
|
+
this.virtualFlow = {
|
|
72
|
+
name: config.flow.name,
|
|
73
|
+
steps: config.flow.steps,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
this.flowId = undefined;
|
|
77
|
+
return this;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Specify the record type to evaluate against
|
|
81
|
+
*/
|
|
82
|
+
forRecordType(recordType) {
|
|
83
|
+
this.recordType = recordType;
|
|
84
|
+
this.inlineRecords = undefined;
|
|
85
|
+
return this;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Use inline records for ad-hoc evaluation (no record type lookup)
|
|
89
|
+
*
|
|
90
|
+
* @example
|
|
91
|
+
* ```typescript
|
|
92
|
+
* .withRecords([
|
|
93
|
+
* { name: 'Test 1', type: 'test', metadata: { input: 'hello' } },
|
|
94
|
+
* { name: 'Test 2', type: 'test', metadata: { input: 'world' } },
|
|
95
|
+
* ])
|
|
96
|
+
* ```
|
|
97
|
+
*/
|
|
98
|
+
withRecords(records) {
|
|
99
|
+
this.inlineRecords = records;
|
|
100
|
+
this.recordType = undefined;
|
|
101
|
+
return this;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Set model overrides for single-model evaluation
|
|
105
|
+
*
|
|
106
|
+
* @example
|
|
107
|
+
* ```typescript
|
|
108
|
+
* .withModelOverrides([
|
|
109
|
+
* { stepName: 'Analyze', model: 'gpt-4o', temperature: 0.2 }
|
|
110
|
+
* ])
|
|
111
|
+
* ```
|
|
112
|
+
*/
|
|
113
|
+
withModelOverrides(overrides) {
|
|
114
|
+
this.modelOverrides = overrides;
|
|
115
|
+
this.modelConfigs = undefined;
|
|
116
|
+
return this;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Compare multiple model configurations
|
|
120
|
+
*
|
|
121
|
+
* @example
|
|
122
|
+
* ```typescript
|
|
123
|
+
* .compareModels([
|
|
124
|
+
* [{ stepName: 'Analyze', model: 'gpt-4o' }],
|
|
125
|
+
* [{ stepName: 'Analyze', model: 'claude-3-opus' }],
|
|
126
|
+
* [{ stepName: 'Analyze', model: 'gemini-pro' }],
|
|
127
|
+
* ])
|
|
128
|
+
* ```
|
|
129
|
+
*/
|
|
130
|
+
compareModels(configs) {
|
|
131
|
+
this.modelConfigs = configs;
|
|
132
|
+
this.modelOverrides = undefined;
|
|
133
|
+
return this;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Set evaluation options
|
|
137
|
+
*/
|
|
138
|
+
withOptions(options) {
|
|
139
|
+
this.evalOptions = { ...this.evalOptions, ...options };
|
|
140
|
+
return this;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Filter records to evaluate
|
|
144
|
+
*/
|
|
145
|
+
withFilter(filter) {
|
|
146
|
+
this.filterConfig = filter;
|
|
147
|
+
return this;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Limit the number of records to evaluate
|
|
151
|
+
*/
|
|
152
|
+
withLimit(limit) {
|
|
153
|
+
this.limitConfig = limit;
|
|
154
|
+
return this;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Build the eval request configuration
|
|
158
|
+
*/
|
|
159
|
+
build() {
|
|
160
|
+
if (!this.flowId && !this.virtualFlow) {
|
|
161
|
+
throw new Error('EvalBuilder: flow is required. Call .useFlow(id) or .useVirtualFlow(builder) first.');
|
|
162
|
+
}
|
|
163
|
+
if (!this.recordType && !this.inlineRecords) {
|
|
164
|
+
throw new Error('EvalBuilder: records are required. Call .forRecordType(type) or .withRecords([...]) first.');
|
|
165
|
+
}
|
|
166
|
+
const request = {};
|
|
167
|
+
if (this.flowId) {
|
|
168
|
+
request.flowId = this.flowId;
|
|
169
|
+
}
|
|
170
|
+
else if (this.virtualFlow) {
|
|
171
|
+
request.flow = this.virtualFlow;
|
|
172
|
+
}
|
|
173
|
+
if (this.recordType) {
|
|
174
|
+
request.recordType = this.recordType;
|
|
175
|
+
}
|
|
176
|
+
else if (this.inlineRecords) {
|
|
177
|
+
request.records = this.inlineRecords;
|
|
178
|
+
}
|
|
179
|
+
if (this.modelOverrides) {
|
|
180
|
+
request.modelOverrides = this.modelOverrides;
|
|
181
|
+
}
|
|
182
|
+
else if (this.modelConfigs) {
|
|
183
|
+
request.modelConfigs = this.modelConfigs;
|
|
184
|
+
}
|
|
185
|
+
if (Object.keys(this.evalOptions).length > 0) {
|
|
186
|
+
request.options = this.evalOptions;
|
|
187
|
+
}
|
|
188
|
+
if (this.filterConfig) {
|
|
189
|
+
request.filter = this.filterConfig;
|
|
190
|
+
}
|
|
191
|
+
if (this.limitConfig !== undefined) {
|
|
192
|
+
request.limit = this.limitConfig;
|
|
193
|
+
}
|
|
194
|
+
return request;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Execute the evaluation
|
|
198
|
+
*
|
|
199
|
+
* @param client - Client with eval capability
|
|
200
|
+
* @returns EvalResult with evaluation status and results
|
|
201
|
+
*/
|
|
202
|
+
async run(client) {
|
|
203
|
+
const config = this.build();
|
|
204
|
+
return client.eval(config);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
exports.EvalBuilder = EvalBuilder;
|
|
208
|
+
/**
|
|
209
|
+
* EvalBuilder that is bound to a client for direct execution
|
|
210
|
+
*/
|
|
211
|
+
class ClientEvalBuilder extends EvalBuilder {
|
|
212
|
+
constructor(client) {
|
|
213
|
+
super();
|
|
214
|
+
this.boundClient = client;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Execute the evaluation using the bound client
|
|
218
|
+
*/
|
|
219
|
+
async run() {
|
|
220
|
+
const config = this.build();
|
|
221
|
+
return this.boundClient.eval(config);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
exports.ClientEvalBuilder = ClientEvalBuilder;
|
|
225
|
+
//# sourceMappingURL=eval-builder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;;;AAkFH,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E,MAAa,WAAW;IAAxB;QAOU,gBAAW,GAAgB,EAAE,CAAA;IAoLvC,CAAC;IAhLC;;OAEG;IACH,OAAO,CAAC,MAAc;QACpB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;QAC5B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAoB;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAA;QAC9B,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YACnE,IAAI,CAAC,WAAW,GAAG;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAc;gBAChC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAc;aAClC,CAAA;QACH,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;QACvB,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,UAAkB;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;QAC5B,IAAI,CAAC,aAAa,GAAG,SAAS,CAAA;QAC9B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;OAUG;IACH,WAAW,CAAC,OAAqB;QAC/B,IAAI,CAAC,aAAa,GAAG,OAAO,CAAA;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;QAC3B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;OASG;IACH,kBAAkB,CAAC,SAA0B;QAC3C,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAA;QAC7B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;;OAWG;IACH,aAAa,CAAC,OAA0B;QACtC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAA;QAC3B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,OAAoB;QAC9B,IAAI,CAAC,WAAW,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,GAAG,OAAO,EAAE,CAAA;QACtD,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,MAA2B;QACpC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAA;QAC1B,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAa;QACrB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;QACxB,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,qFAAqF,CAAC,CAAA;QACxG,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAA;QAC/G,CAAC;QAED,MAAM,OAAO,GAAgB,EAAE,CAAA;QAE/B,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC9B,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC5B,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,CAAA;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAA;QACtC,CAAC;aAAM,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9B,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,CAAA;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAA;QAC9C,CAAC;aAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YAC7B,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAA;QAC1C,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,CAAA;QACpC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,WAAW,CAAA;QAClC,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,GAAG,CAAC,MAAkB;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QAC3B,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC5B,CAAC;CACF;AA3LD,kCA2LC;AAaD;;GAEG;AACH,MAAa,iBAAkB,SAAQ,WAAW;IAGhD,YAAY,MAAkB;QAC5B,KAAK,EAAE,CAAA;QACP,IAAI,CAAC,WAAW,GAAG,MAAM,CAAA;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG;QACP,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAA;QAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACtC,CAAC;CACF;AAfD,8CAeC"}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EvalsNamespace - Static namespace for evaluation operations
|
|
3
|
+
*
|
|
4
|
+
* Provides methods for running evaluations and comparing model performance.
|
|
5
|
+
* Evals can be streamed for real-time results or submitted as batch jobs.
|
|
6
|
+
*/
|
|
7
|
+
import type { RuntypeClient } from './runtype';
|
|
8
|
+
import { FlowResult } from './flow-result';
|
|
9
|
+
import type { StreamCallbacks } from './flow-builder';
|
|
10
|
+
export interface ModelOverride {
|
|
11
|
+
/** Name of the step to override */
|
|
12
|
+
stepName: string;
|
|
13
|
+
/** Model to use for this step */
|
|
14
|
+
model: string;
|
|
15
|
+
/** Optional temperature override */
|
|
16
|
+
temperature?: number;
|
|
17
|
+
/** Optional max tokens override */
|
|
18
|
+
maxTokens?: number;
|
|
19
|
+
}
|
|
20
|
+
export interface EvalRunConfig {
|
|
21
|
+
/** Flow ID to evaluate */
|
|
22
|
+
flowId?: string;
|
|
23
|
+
/** Virtual flow definition (alternative to flowId) */
|
|
24
|
+
flow?: {
|
|
25
|
+
name: string;
|
|
26
|
+
steps: any[];
|
|
27
|
+
};
|
|
28
|
+
/** Record type to evaluate against */
|
|
29
|
+
recordType?: string;
|
|
30
|
+
/** Inline records for ad-hoc evaluation */
|
|
31
|
+
records?: Array<{
|
|
32
|
+
name: string;
|
|
33
|
+
type: string;
|
|
34
|
+
metadata: Record<string, any>;
|
|
35
|
+
}>;
|
|
36
|
+
/** Model overrides for single-model evaluation */
|
|
37
|
+
models?: ModelOverride[];
|
|
38
|
+
/** Multiple model configurations for comparison */
|
|
39
|
+
compareModels?: ModelOverride[][];
|
|
40
|
+
/** Store evaluation results */
|
|
41
|
+
storeResults?: boolean;
|
|
42
|
+
/** Run evaluations in parallel */
|
|
43
|
+
parallel?: boolean;
|
|
44
|
+
/** Maximum concurrent evaluations */
|
|
45
|
+
concurrency?: number;
|
|
46
|
+
/** Continue on individual record failures */
|
|
47
|
+
continueOnError?: boolean;
|
|
48
|
+
/** Optional filter for records */
|
|
49
|
+
filter?: Record<string, any>;
|
|
50
|
+
/** Optional limit on number of records */
|
|
51
|
+
limit?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface EvalStatus {
|
|
54
|
+
evalId: string;
|
|
55
|
+
status: 'queued' | 'running' | 'completed' | 'failed';
|
|
56
|
+
totalRecords: number;
|
|
57
|
+
completedRecords: number;
|
|
58
|
+
failedRecords: number;
|
|
59
|
+
results?: Array<{
|
|
60
|
+
modelConfig: ModelOverride[];
|
|
61
|
+
records: Array<{
|
|
62
|
+
recordId: string;
|
|
63
|
+
status: 'success' | 'error';
|
|
64
|
+
result?: any;
|
|
65
|
+
error?: string;
|
|
66
|
+
executionTime: number;
|
|
67
|
+
}>;
|
|
68
|
+
}>;
|
|
69
|
+
}
|
|
70
|
+
export interface EvalListParams {
|
|
71
|
+
/** Filter by status */
|
|
72
|
+
status?: 'queued' | 'running' | 'completed' | 'failed';
|
|
73
|
+
/** Filter by flow ID */
|
|
74
|
+
flowId?: string;
|
|
75
|
+
/** Pagination limit */
|
|
76
|
+
limit?: number;
|
|
77
|
+
/** Pagination offset */
|
|
78
|
+
offset?: number;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* EvalRunner - Builder returned by Runtype.evals.run()
|
|
82
|
+
*
|
|
83
|
+
* Provides terminal methods for executing evaluations:
|
|
84
|
+
* - .stream() - Execute and stream results in real-time
|
|
85
|
+
* - .submit() - Submit as a batch job for async processing
|
|
86
|
+
*/
|
|
87
|
+
export declare class EvalRunner {
|
|
88
|
+
private getClient;
|
|
89
|
+
private config;
|
|
90
|
+
constructor(getClient: () => RuntypeClient, config: EvalRunConfig);
|
|
91
|
+
/**
|
|
92
|
+
* Execute the evaluation with streaming results
|
|
93
|
+
*
|
|
94
|
+
* Streams evaluation results as they complete.
|
|
95
|
+
* Good for real-time feedback during development/testing.
|
|
96
|
+
*
|
|
97
|
+
* @example
|
|
98
|
+
* ```typescript
|
|
99
|
+
* const result = await Runtype.evals.run({
|
|
100
|
+
* flowId: 'flow_123',
|
|
101
|
+
* recordType: 'test_data',
|
|
102
|
+
* models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
|
|
103
|
+
* }).stream()
|
|
104
|
+
*
|
|
105
|
+
* // Process with callbacks
|
|
106
|
+
* await result.stream({
|
|
107
|
+
* onStepComplete: (result, event) => {
|
|
108
|
+
* console.log('Step completed:', event.name, result)
|
|
109
|
+
* },
|
|
110
|
+
* })
|
|
111
|
+
* ```
|
|
112
|
+
*/
|
|
113
|
+
stream(callbacks?: StreamCallbacks): Promise<FlowResult>;
|
|
114
|
+
/**
|
|
115
|
+
* Submit the evaluation as a batch job
|
|
116
|
+
*
|
|
117
|
+
* Queues the evaluation to run asynchronously.
|
|
118
|
+
* Good for large evaluations or scheduled runs.
|
|
119
|
+
*
|
|
120
|
+
* @example
|
|
121
|
+
* ```typescript
|
|
122
|
+
* const eval = await Runtype.evals.run({
|
|
123
|
+
* flowId: 'flow_123',
|
|
124
|
+
* recordType: 'test_data',
|
|
125
|
+
* compareModels: [
|
|
126
|
+
* [{ stepName: 'Analyze', model: 'gpt-4o' }],
|
|
127
|
+
* [{ stepName: 'Analyze', model: 'claude-3-opus' }],
|
|
128
|
+
* ]
|
|
129
|
+
* }).submit()
|
|
130
|
+
*
|
|
131
|
+
* console.log('Eval queued:', eval.evalId)
|
|
132
|
+
*
|
|
133
|
+
* // Check status later
|
|
134
|
+
* const status = await Runtype.evals.get(eval.evalId)
|
|
135
|
+
* ```
|
|
136
|
+
*/
|
|
137
|
+
submit(): Promise<EvalStatus>;
|
|
138
|
+
/**
|
|
139
|
+
* Build the evaluation payload
|
|
140
|
+
*/
|
|
141
|
+
private buildPayload;
|
|
142
|
+
}
|
|
143
|
+
export declare class EvalsNamespace {
|
|
144
|
+
private getClient;
|
|
145
|
+
constructor(getClient: () => RuntypeClient);
|
|
146
|
+
/**
|
|
147
|
+
* Run an evaluation
|
|
148
|
+
*
|
|
149
|
+
* Returns an EvalRunner with terminal methods:
|
|
150
|
+
* - .stream() - Execute and stream results
|
|
151
|
+
* - .submit() - Submit as batch job
|
|
152
|
+
*
|
|
153
|
+
* @example
|
|
154
|
+
* ```typescript
|
|
155
|
+
* // Single model evaluation with streaming
|
|
156
|
+
* const result = await Runtype.evals.run({
|
|
157
|
+
* flowId: 'flow_123',
|
|
158
|
+
* recordType: 'test_data',
|
|
159
|
+
* models: [{ stepName: 'Analyze', model: 'gpt-4o' }]
|
|
160
|
+
* }).stream()
|
|
161
|
+
*
|
|
162
|
+
* // Multi-model comparison as batch
|
|
163
|
+
* const eval = await Runtype.evals.run({
|
|
164
|
+
* flowId: 'flow_123',
|
|
165
|
+
* recordType: 'test_data',
|
|
166
|
+
* compareModels: [
|
|
167
|
+
* [{ stepName: 'Analyze', model: 'gpt-4o' }],
|
|
168
|
+
* [{ stepName: 'Analyze', model: 'claude-3-opus' }],
|
|
169
|
+
* [{ stepName: 'Analyze', model: 'gemini-pro' }],
|
|
170
|
+
* ]
|
|
171
|
+
* }).submit()
|
|
172
|
+
* ```
|
|
173
|
+
*/
|
|
174
|
+
run(config: EvalRunConfig): EvalRunner;
|
|
175
|
+
/**
|
|
176
|
+
* Get evaluation status by ID
|
|
177
|
+
*
|
|
178
|
+
* @example
|
|
179
|
+
* ```typescript
|
|
180
|
+
* const status = await Runtype.evals.get('eval_123')
|
|
181
|
+
* console.log(status.status, status.completedRecords, '/', status.totalRecords)
|
|
182
|
+
* ```
|
|
183
|
+
*/
|
|
184
|
+
get(evalId: string): Promise<EvalStatus>;
|
|
185
|
+
/**
|
|
186
|
+
* List evaluations
|
|
187
|
+
*
|
|
188
|
+
* @example
|
|
189
|
+
* ```typescript
|
|
190
|
+
* // List all evals
|
|
191
|
+
* const evals = await Runtype.evals.list()
|
|
192
|
+
*
|
|
193
|
+
* // Filter by status
|
|
194
|
+
* const running = await Runtype.evals.list({ status: 'running' })
|
|
195
|
+
*
|
|
196
|
+
* // Filter by flow
|
|
197
|
+
* const flowEvals = await Runtype.evals.list({ flowId: 'flow_123' })
|
|
198
|
+
* ```
|
|
199
|
+
*/
|
|
200
|
+
list(params?: EvalListParams): Promise<{
|
|
201
|
+
data: EvalStatus[];
|
|
202
|
+
total: number;
|
|
203
|
+
}>;
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=evals-namespace.d.ts.map
|