@uipath/flow-tool 1.206.0-preview.303 → 1.207.0-preview.305
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/THIRD-PARTY-NOTICES.md +13 -13
- package/dist/{applicationinsights-web-8m26566f.js → applicationinsights-web-c6epsatg.js} +1 -1
- package/dist/{authoring-az69rn0j.js → authoring-f5ttjy3d.js} +5 -5
- package/dist/{binding-service-zg19e6vy.js → binding-service-qfbtg6qy.js} +37 -34
- package/dist/{browser-strategy-gnftxwsf.js → browser-strategy-1nk8hgcf.js} +1 -1
- package/dist/case-catalog-h4fjn6q3.js +304 -0
- package/dist/{connector-library-1yw5r47w.js → connector-library-844myxz2.js} +20 -6
- package/dist/{connector-service-e99tdk52.js → connector-service-8t5swsf9.js} +156 -76
- package/dist/{conversion-7O2PDOQB-2677ata3.js → conversion-A2KZVSOA-s70m4p50.js} +256 -90
- package/dist/{de-AI7ML6JM-km47f8k4.js → de-B6IFKV43-9sk50xx0.js} +164 -2
- package/dist/{de-2BEKRUMY-x4dx0nn9.js → de-HF4GJ4EK-NX7YTOSY-4f0r23sx.js} +1282 -10
- package/dist/{de-2BEKRUMY-MD7L477H-nan48spf.js → de-HF4GJ4EK-xrrajtht.js} +1277 -15
- package/dist/{de-VQWT6L5Y-mch4qwyv.js → de-IQ44WEAK-ayb953tp.js} +19 -4
- package/dist/{de-3CU65EDB-r7r10am9.js → de-T4OCQPNU-hrqdjart.js} +6 -6
- package/dist/{debug-conversation-trigger-62ebex43.js → debug-conversation-trigger-j20ms2xc.js} +4 -9
- package/dist/{debug-http-client-ykzhg1zx.js → debug-http-client-fypqjmfs.js} +6 -5
- package/dist/{debug-service-xq8vhrkq.js → debug-service-zv5hj0cx.js} +11 -6
- package/dist/{edge-service-ytvdz98e.js → edge-service-706zfmyb.js} +41 -31
- package/dist/{elk.bundled-phy0zc81.js → elk.bundled-pfmz610v.js} +1 -1
- package/dist/{en-LAVDF7IQ-ba7wzsxp.js → en-5JIPZTS5-v0f9wmd2.js} +4 -2
- package/dist/{en-X6T4SP2Y-FPGCJIFV-jgjgvnws.js → en-ETY46SDS-RI6EZCN6-g3n0v4ff.js} +67 -3
- package/dist/{en-X6T4SP2Y-xe57av1w.js → en-ETY46SDS-yfqhqvbx.js} +66 -2
- package/dist/{en-WPCQ4JKF-nr4tr5vp.js → en-JZXY3PYE-9q1q9w7h.js} +4 -4
- package/dist/{en-LQHKFSEY-vv8bbfdf.js → en-LQHKFSEY-j7gk3aaw.js} +1 -1
- package/dist/{en-U2NSIIZN-whenw6wh.js → en-T6VSEF3E-1hq4zfrt.js} +5 -3
- package/dist/{es-SPNKTD5Y-vw5cbgv5.js → es-AQW6PFIU-q2mz247f.js} +164 -2
- package/dist/{es-7ZTQTGKN-TXQBFYRA-n0bw44wg.js → es-IYP2OLUY-7SKYGF6A-jp5kcwrd.js} +1272 -11
- package/dist/{es-7ZTQTGKN-11sj0yjx.js → es-IYP2OLUY-98xsbyd5.js} +1271 -10
- package/dist/{es-MX-S2RFDGDE-x9r0tv30.js → es-MX-A2DFN7CA-q5m62f60.js} +19 -4
- package/dist/{es-MX-IHGRIEWH-23X37OPS-zmw94959.js → es-MX-D2MN3U37-0y5q0nbe.js} +1276 -15
- package/dist/{es-MX-IHGRIEWH-g63p0gb4.js → es-MX-D2MN3U37-D74A6FRL-1r3g7760.js} +1281 -10
- package/dist/{es-MX-7KJZPEQN-1fq0a9jr.js → es-MX-KWITVBWE-w49gknha.js} +164 -2
- package/dist/{es-MX-F7ZEHSNE-gxpm47vv.js → es-MX-YQNHSXDT-fd5ybreb.js} +6 -6
- package/dist/{es-EP563JWQ-1agj5n4e.js → es-TNUPGHGG-886me0sq.js} +19 -4
- package/dist/{es-KHVY67FO-natcvmmm.js → es-YHECPVFB-7jsrxa01.js} +6 -6
- package/dist/extraction-availability-service-h08mngty.js +343 -0
- package/dist/{flow-api-1hr7z6qc.js → flow-api-bmg7beey.js} +15 -12
- package/dist/{flow-debug-service-p5w8gj48.js → flow-debug-service-bppp59pe.js} +232 -156
- package/dist/flow-eval-node-target-hc9n1913.js +31 -0
- package/dist/flow-eval-run-api-g6sphtf3.js +11 -0
- package/dist/{flow-eval-run-service-ej6xg2jd.js → flow-eval-run-service-qdtz7ec7.js} +116 -119
- package/dist/{flow-eval-service-vf73ejad.js → flow-eval-service-mg7gveh3.js} +141 -337
- package/dist/{flow-init-sdk-source-vg1ckdrf.js → flow-init-sdk-source-t2q2w6v7.js} +5 -5
- package/dist/flow-init-service-0m5yp9fe.js +27 -0
- package/dist/flow-io-c8b17vbx.js +29 -0
- package/dist/{flow-migrate-service-fsxzyj9c.js → flow-migrate-service-a8s4rmq6.js} +20 -20
- package/dist/{flow-pack-service-09b385hw.js → flow-pack-service-jhr4rmsy.js} +89 -90
- package/dist/flow-validate-service-hpt8dqyr.js +49 -0
- package/dist/{fr-7LK3QUWT-a94yejcs.js → fr-EPBGOBOP-rs7qy97k.js} +19 -4
- package/dist/{fr-7ZDL4MVO-pd27vwh6.js → fr-NISALSPL-3ANDQPI3-993fw1pz.js} +1282 -10
- package/dist/{fr-7ZDL4MVO-CXMRZDOF-y6pa4ntx.js → fr-NISALSPL-vzfmygd7.js} +1277 -15
- package/dist/{fr-AXJGUT53-hdg3pmd4.js → fr-RNWK2TMC-dpyzj366.js} +164 -2
- package/dist/{fr-I42FODC5-p80x5v4b.js → fr-W3AFCEDL-cjfr0xqp.js} +6 -6
- package/dist/{governance-fetcher-v88cz998.js → governance-fetcher-hkm1hvn0.js} +7 -7
- package/dist/{headless-41t9wwb7.js → headless-23ywyx9h.js} +390 -195
- package/dist/{hitl-service-5symf9as.js → hitl-service-hs54d6tt.js} +11 -11
- package/dist/index-7enyyjxr.js +161 -0
- package/dist/{index-wss1mf5j.js → index-b02kt4sx.js} +114 -68
- package/dist/{index-qjarp6ky.js → index-cg965dhq.js} +34 -27
- package/dist/{index-t2f7scza.js → index-db1djbzm.js} +12 -8
- package/dist/{index-r44xc976.js → index-eyjrdzgx.js} +5 -5
- package/dist/{index-7fet3f0s.js → index-h67w2p9p.js} +3 -3
- package/dist/{index-f5kq7265.js → index-r6crd7tx.js} +6 -4
- package/dist/{index-ng011fzs.js → index-r6y07e14.js} +94 -74
- package/dist/{index-5pzp2qeb.js → index-v3w1930f.js} +18 -18
- package/dist/{index-d4bdedx3.js → index-ynw2efd8.js} +6 -4
- package/dist/{index-2methrrt.js → index-zemcat7k.js} +13 -11
- package/dist/index.js +16 -14
- package/dist/{index.umd.min-ttg1qc0x.js → index.umd.min-5yxdz5j3.js} +1 -1
- package/dist/init.js +18 -17
- package/dist/{inline-entity-resolver-pya7frj9.js → inline-entity-resolver-9esjm2e7.js} +11 -11
- package/dist/input-reader-3zvbsn6a.js +12 -0
- package/dist/integration-service-fetcher-h6er48b7.js +65 -0
- package/dist/{ja-C7GZV6YP-hvtbngr5.js → ja-DH4I3GEF-ywyhhppa.js} +164 -2
- package/dist/{ja-LGAM5HKR-sgrtjqjh.js → ja-HTS2Q4YG-4ahnq47y.js} +1277 -10
- package/dist/{ja-LGAM5HKR-NTPKIZEM-mrdn9rvy.js → ja-HTS2Q4YG-7Q6WUI6H-6e5nzy0t.js} +1278 -11
- package/dist/{ja-WY2W6733-ayr36h1f.js → ja-HWFIXC2V-gg9xvhsv.js} +19 -4
- package/dist/{ja-GI3OUHPZ-28fy60r0.js → ja-WOANIGX4-qnyzk2t8.js} +6 -6
- package/dist/{jmespath-wwym30dp.js → jmespath-dbsejrwn.js} +1 -1
- package/dist/json-input-pj18qjt0.js +17 -0
- package/dist/{known-types-v9sxcq1c.js → known-types-pen5h5kz.js} +4 -4
- package/dist/{ko-7B4SUZB7-2SLI636D-479c9akr.js → ko-3JFGTMNY-VT6UQ3IS-5a2hnf97.js} +1272 -11
- package/dist/{ko-7B4SUZB7-c8wr20sc.js → ko-3JFGTMNY-qnf1bnef.js} +1271 -10
- package/dist/{ko-H2RGQWKJ-9696w762.js → ko-L3BMYD3Q-exfhqfbh.js} +19 -4
- package/dist/{ko-Q24HSSL4-5vn55pte.js → ko-R5BCWVO7-whzgmh6q.js} +164 -2
- package/dist/{ko-RYKAL74T-10m4z8vw.js → ko-RMOVRC7M-whyynn7n.js} +6 -6
- package/dist/library-option-9fqvdh8v.js +14 -0
- package/dist/localization-axzmx1mb.js +45 -0
- package/dist/{maestro-automate-entry-d1v5q5mv.js → maestro-automate-entry-mrap5xmt.js} +2 -2
- package/dist/manifest-hmeqvk74.js +60 -0
- package/dist/node-service-bq7m24js.js +65 -0
- package/dist/{node-strategy-wrjxmbzt.js → node-strategy-2r78gtca.js} +2 -2
- package/dist/{node-sync-service-z3gjn1a6.js → node-sync-service-6v7j8m4p.js} +22 -15
- package/dist/node-xzwyr0mf.js +172 -0
- package/dist/{packager-tool-h44jmvjd.js → packager-tool-0g2grsn8.js} +2 -2
- package/dist/packager-tool-0x8g8b5r.js +25 -0
- package/dist/packager-tool-13mmsykc.js +125 -0
- package/dist/{packager-tool-vgrswa8p.js → packager-tool-1db1wqdy.js} +30 -8
- package/dist/packager-tool-1yew22sn.js +9 -0
- package/dist/packager-tool-2devymwf.js +423 -0
- package/dist/{packager-tool-14rdgz3c.js → packager-tool-3sxg16ft.js} +7 -7
- package/dist/packager-tool-40bxn7j2.js +63 -0
- package/dist/{packager-tool-6an893wm.js → packager-tool-4f4jp7ez.js} +3 -3
- package/dist/{packager-tool-pxq0bwp0.js → packager-tool-4hn7v05x.js} +16 -9
- package/dist/packager-tool-4skf6ak1.js +31 -0
- package/dist/{packager-tool-951nhh3e.js → packager-tool-5n1v6y3q.js} +1 -1
- package/dist/{packager-tool-2b0tn4p4.js → packager-tool-6azsz9h8.js} +7 -7
- package/dist/{packager-tool-54jdtyk4.js → packager-tool-7d6y5paj.js} +11 -11
- package/dist/{packager-tool-ac6rvny7.js → packager-tool-7re5d8aq.js} +1139 -466
- package/dist/{packager-tool-6an563h8.js → packager-tool-7wpb19zf.js} +10 -3
- package/dist/packager-tool-9tv5cyhr.js +16 -0
- package/dist/{packager-tool-7atc775f.js → packager-tool-ajx79m52.js} +1 -1
- package/dist/{packager-tool-96es3aqy.js → packager-tool-asfftv7z.js} +330 -902
- package/dist/{packager-tool-zx162jbb.js → packager-tool-b0prefcm.js} +489 -118
- package/dist/packager-tool-bbq5kbb9.js +318 -0
- package/dist/{packager-tool-xhrm7cgz.js → packager-tool-bcsjm5hv.js} +470 -315
- package/dist/{packager-tool-xdaxdc00.js → packager-tool-bn09xgy8.js} +14 -14
- package/dist/{packager-tool-1qg04b7q.js → packager-tool-bzcphfa8.js} +4 -2
- package/dist/{node-vz9f9wqw.js → packager-tool-cpbsp04w.js} +189 -285
- package/dist/{packager-tool-gdpw0daf.js → packager-tool-dmhmcav3.js} +3773 -2625
- package/dist/{packager-tool-m1y5ah6s.js → packager-tool-f3htkspn.js} +215 -233
- package/dist/{packager-tool-710j2ga4.js → packager-tool-f4pvpaw2.js} +1123 -295
- package/dist/packager-tool-geh2qybm.js +179 -0
- package/dist/{packager-tool-1rhch6b5.js → packager-tool-gjjr2a2w.js} +66 -19
- package/dist/packager-tool-h0gzqmya.js +224 -0
- package/dist/packager-tool-h9f2vjwg.js +40 -0
- package/dist/{packager-tool-ndenq8jp.js → packager-tool-jf4m7sqc.js} +23 -9
- package/dist/{packager-tool-swn9mp69.js → packager-tool-jghqgdmm.js} +228 -487
- package/dist/{packager-tool-64r7e4ek.js → packager-tool-jjnfwfd4.js} +2 -2
- package/dist/packager-tool-kh1taypp.js +41 -0
- package/dist/{packager-tool-pdt4jmnw.js → packager-tool-m2h98d2z.js} +246 -61
- package/dist/packager-tool-m8avdf10.js +1060 -0
- package/dist/{packager-tool-g18z2qds.js → packager-tool-ma1z6nqe.js} +2 -2
- package/dist/{packager-tool-9m5vba43.js → packager-tool-md1vcqh4.js} +233 -14
- package/dist/{packager-tool-4g87fbz6.js → packager-tool-mm6a12y4.js} +8 -8
- package/dist/{packager-tool-2n6eg5eq.js → packager-tool-mvzswn9n.js} +134 -68
- package/dist/packager-tool-p8ncdnsj.js +30 -0
- package/dist/{packager-tool-vdp3s2fb.js → packager-tool-px52tjy2.js} +9 -7
- package/dist/{packager-tool-rfs464c3.js → packager-tool-qf08dexq.js} +584 -651
- package/dist/{packager-tool-m126jzdd.js → packager-tool-qkdbfkev.js} +14 -11
- package/dist/{packager-tool-7mxn3e7j.js → packager-tool-qtz83h01.js} +4 -2
- package/dist/{packager-tool-hk0fnvz5.js → packager-tool-qx8wvedy.js} +1 -1
- package/dist/{packager-tool-q2kwk5bf.js → packager-tool-qyhvgenw.js} +6 -4
- package/dist/{packager-tool-6pyaaxzk.js → packager-tool-rcjtmknf.js} +351 -42
- package/dist/packager-tool-swvgkcq7.js +5973 -0
- package/dist/{packager-tool-fzgfb0jt.js → packager-tool-t0e7fvqz.js} +1228 -759
- package/dist/{packager-tool-p123mtfx.js → packager-tool-tgawhrva.js} +58 -16
- package/dist/{packager-tool-s4zvbgvd.js → packager-tool-tm4d12jb.js} +15328 -7171
- package/dist/{packager-tool-8ra2qxqe.js → packager-tool-v7gxvkpx.js} +13 -13
- package/dist/{packager-tool-yewnsyag.js → packager-tool-vg16h3pk.js} +80 -6
- package/dist/packager-tool-w2eqf95v.js +16 -0
- package/dist/packager-tool-wvf6b31h.js +50 -0
- package/dist/{packager-tool-ejvmkxgg.js → packager-tool-wz5zmx67.js} +10 -10
- package/dist/packager-tool-xgh2t48c.js +255 -0
- package/dist/{packager-tool-w9bkq7zb.js → packager-tool-xgvp8a8g.js} +10 -10
- package/dist/{packager-tool-sehmpn8j.js → packager-tool-y84mhf89.js} +8 -8
- package/dist/packager-tool-ys7jsqhg.js +184 -0
- package/dist/{packager-tool-ze3fnf3s.js → packager-tool-zap4kd3z.js} +7 -2
- package/dist/{packager-tool-6g2ex7yf.js → packager-tool-zgm6kh18.js} +21 -21
- package/dist/{packager-tool-dr3zc9n9.js → packager-tool-zgzpqfcs.js} +13 -6
- package/dist/packager-tool.js +9 -9
- package/dist/{prepare-uip-ad7mfs8t.js → prepare-uip-8frvq826.js} +39 -6
- package/dist/provision-node-service-sxnfgrwy.js +86 -0
- package/dist/{pt-2RGHDOG4-gwe2ptvg.js → pt-6RCFX6CY-v42k92ey.js} +19 -4
- package/dist/{pt-BR-BIUJXIPU-2sqzhr8y.js → pt-BR-6TBBEDQS-y3f2vg5d.js} +164 -2
- package/dist/{pt-BR-PHLFHMXZ-JFPKUXIZ-1k5yafmz.js → pt-BR-7H7BP5DT-IOHWNCIU-pbdnhe7r.js} +1277 -11
- package/dist/{pt-BR-PHLFHMXZ-n6de4s7z.js → pt-BR-7H7BP5DT-z2e5aaj1.js} +1276 -10
- package/dist/{pt-BR-JWHVZE6H-gvkknv49.js → pt-BR-HDVM5NFA-azfx4gg9.js} +19 -4
- package/dist/{pt-BR-IL3KLBID-m1mvqqzb.js → pt-BR-HJIMGPPX-s18rb4vg.js} +6 -6
- package/dist/{pt-VLA5IOHO-8t4pze0h.js → pt-DZG5FPWK-6czrqk2t.js} +6 -6
- package/dist/{pt-N7ZTJT4W-GVODJMRX-jge952w8.js → pt-QWGSZKMD-RTKSQCYA-80hp6e1q.js} +1272 -11
- package/dist/{pt-N7ZTJT4W-d6c67xcn.js → pt-QWGSZKMD-m50nrkx8.js} +1271 -10
- package/dist/{pt-MHX6JGAX-xfgvdqwg.js → pt-ZSQC5RZ3-8kp3g1z5.js} +164 -2
- package/dist/{ro-FLNXVYJI-ZN43XZKK-bmkd7d2n.js → ro-B7G474JK-9k2jjk88.js} +1271 -15
- package/dist/{ro-FLNXVYJI-ytrwabg2.js → ro-B7G474JK-Q3BAA4S3-t47bbx2a.js} +1276 -10
- package/dist/{ro-53TS7SWG-hnjx1c7b.js → ro-EEKX2X3R-tz9prn67.js} +19 -4
- package/dist/{ro-3RKJNGJZ-xqj14e3d.js → ro-P33AQ6ZL-s92z1mgg.js} +164 -2
- package/dist/{ro-IRNFX376-mh84zk3n.js → ro-SX7RUXGR-kknxsr3g.js} +6 -6
- package/dist/ru-FNRVPXE5-7q150qf8.js +7 -0
- package/dist/ru-GSCT4ARZ-6OTD7JLB-cdcpfwx0.js +12 -0
- package/dist/schema-document-service-mknqzg7e.js +30 -0
- package/dist/{serialization-Z3UYT6BW-e1ks0ytt.js → serialization-7CB4AREI-6b7ng26y.js} +482 -6161
- package/dist/services/binding-id-rule.d.ts +13 -0
- package/dist/services/binding-resource-types.d.ts +12 -0
- package/dist/services/extract-schema/default-schema-document.d.ts +9 -4
- package/dist/services/extract-schema/types.d.ts +73 -0
- package/dist/services/extract-schema/validate-schema-document.d.ts +75 -0
- package/dist/services/flow-scopes.d.ts +15 -0
- package/dist/services/flow-structure-checks.d.ts +8 -0
- package/dist/services/flow-validate-service.d.ts +44 -11
- package/dist/services/idp-processor-provisioning.d.ts +35 -0
- package/dist/services/node-service.d.ts +17 -23
- package/dist/services/node-validators/extract-schema-validator.d.ts +13 -0
- package/dist/services/node-validators/index.d.ts +1 -0
- package/dist/services/node-validators/js-expression-syntax-validator.d.ts +2 -1
- package/dist/services/node-validators/types.d.ts +2 -7
- package/dist/services/vars-reference-rule.d.ts +6 -0
- package/dist/solution-auth-k0s83ake.js +11 -0
- package/dist/{solution-discovery-wwwynadq.js → solution-discovery-v5qatmz2.js} +7 -7
- package/dist/taggable-variables-nc8td6er.js +134 -0
- package/dist/{tidy-service-0xqfd039.js → tidy-service-rsxb2h63.js} +17 -18
- package/dist/tool.js +16 -14
- package/dist/{tr-PIQY7JIT-2mkwk0kq.js → tr-B6TLBSLH-1797wxkh.js} +19 -4
- package/dist/{tr-7AIZZWZ6-pdzp9n0p.js → tr-FX7WCKK2-CEOTUJRD-5ctjnkng.js} +1276 -10
- package/dist/{tr-7AIZZWZ6-ESAESUPL-0ehs3crp.js → tr-FX7WCKK2-x3rx9fjt.js} +1271 -15
- package/dist/{tr-EJKY7DVQ-rkctqva4.js → tr-OUB57VJ6-52t7zdzb.js} +6 -6
- package/dist/{tr-PQLUUHPC-qsb7btxn.js → tr-XD73CQAA-tb2nrpyp.js} +164 -2
- package/dist/utils/handle-expansion.d.ts +2 -2
- package/dist/validation.js +42 -39
- package/dist/{variable-service-wek1apw7.js → variable-service-95j5nyjh.js} +17 -12
- package/dist/{wait-for-flow-job-vy4t4tbj.js → wait-for-flow-job-k990nmdn.js} +15 -12
- package/dist/{zh-CN-NIEVKROA-q4tbbwqz.js → zh-CN-7KN22KSH-mwvgyym6.js} +19 -4
- package/dist/{zh-CN-4FH7E7UL-84y6j2ea.js → zh-CN-FKWWWLFR-n15rj6p8.js} +6 -6
- package/dist/{zh-CN-OHNTHZRI-b0hjkfqn.js → zh-CN-GWP3B2EM-c07fm17a.js} +164 -2
- package/dist/{zh-CN-CN2ARPFS-e0p2qrxf.js → zh-CN-NCMB6YM5-BB2LRIMQ-phrmedm3.js} +1276 -10
- package/dist/{zh-CN-CN2ARPFS-NIQTWDK2-zmcqpd02.js → zh-CN-NCMB6YM5-drw9bnxg.js} +1271 -15
- package/dist/{zh-TW-E4D3Y6ZR-tn3bmzz5.js → zh-TW-2DLUOA7C-9khqnr4r.js} +164 -2
- package/dist/{zh-TW-5WQA6QTM-hns69sd4.js → zh-TW-52QTEYIZ-2mj7hy3e.js} +6 -6
- package/dist/{zh-TW-MSQM7YTT-m7r7nwnj.js → zh-TW-I7BH3XP2-7rb4xpzg.js} +19 -4
- package/dist/{zh-TW-7RUFEVFK-VKVRSVTP-dg651ga1.js → zh-TW-UNIEJPLE-4vxzbphe.js} +1272 -16
- package/dist/{zh-TW-7RUFEVFK-qd7myw31.js → zh-TW-UNIEJPLE-ECGOEPYS-xxye2kfv.js} +1277 -11
- package/package.json +2 -2
- package/dist/flow-eval-node-target-xxzqch9z.js +0 -29
- package/dist/flow-eval-run-api-w4j8ra60.js +0 -9
- package/dist/flow-init-service-1zyc9wcd.js +0 -26
- package/dist/flow-io-27atafz4.js +0 -24
- package/dist/flow-validate-service-6erwscc6.js +0 -46
- package/dist/input-reader-jjq3r7ry.js +0 -11
- package/dist/localization-erchshqr.js +0 -45
- package/dist/node-service-qtpqs5en.js +0 -64
- package/dist/packager-tool-2sazb7th.js +0 -2
- package/dist/packager-tool-9217bxm2.js +0 -6
- package/dist/packager-tool-cvh1a0nm.js +0 -16
- package/dist/packager-tool-d07g2kpn.js +0 -25
- package/dist/packager-tool-f7rk9xhv.js +0 -50
- package/dist/packager-tool-f9mgybn8.js +0 -18
- package/dist/packager-tool-jd3v0bh1.js +0 -26
- package/dist/packager-tool-k3egwd1m.js +0 -15
- package/dist/packager-tool-q0f1dvtd.js +0 -40
- package/dist/packager-tool-q756a7tg.js +0 -56
- package/dist/packager-tool-wr7gn964.js +0 -2
- package/dist/packager-tool-zw2zy3h2.js +0 -461
- package/dist/ru-GSCT4ARZ-FTVCGQTL-p9re6ys2.js +0 -12
- package/dist/ru-T73OPRTC-50m7tty9.js +0 -7
- package/dist/services/connector-request-buckets.d.ts +0 -1
- package/dist/services/flow-eval-file-store.d.ts +0 -1
- package/dist/services/inline-entity-resolver.d.ts +0 -22
- package/dist/services/packaging-utils.d.ts +0 -130
- package/dist/services/preview-id-rule.d.ts +0 -33
- package/dist/services/ui-only-node-types.d.ts +0 -1
- package/dist/solution-auth-j8x84n5z.js +0 -11
- package/dist/utils/flow-io.d.ts +0 -19
- package/dist/utils/workflow-file-format.d.ts +0 -14
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import {
|
|
2
2
|
catchError2
|
|
3
|
-
} from "./packager-tool-
|
|
3
|
+
} from "./packager-tool-4hn7v05x.js";
|
|
4
|
+
import {
|
|
5
|
+
getFileSystem
|
|
6
|
+
} from "./packager-tool-1db1wqdy.js";
|
|
7
|
+
import {
|
|
8
|
+
RESULTS,
|
|
9
|
+
OutputFormatter,
|
|
10
|
+
processContext
|
|
11
|
+
} from "./packager-tool-bcsjm5hv.js";
|
|
4
12
|
|
|
5
13
|
// src/services/definitions-lookup.ts
|
|
6
14
|
function findNodeDefinition(definitions, node) {
|
|
@@ -391,7 +399,7 @@ function evaluatorCanUseSharedExpectedOutput(typeId) {
|
|
|
391
399
|
return getEvaluatorTypeDefinition(typeId)?.criteriaFields.some((field) => field.key === "expectedOutput" && field.type === "object") ?? false;
|
|
392
400
|
}
|
|
393
401
|
// ../eval-sdk/src/eval-utils.ts
|
|
394
|
-
function
|
|
402
|
+
function isRecord2(value) {
|
|
395
403
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
396
404
|
}
|
|
397
405
|
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
@@ -434,7 +442,7 @@ function buildDefaultCriteria(evaluators, expectedOutput, searchText) {
|
|
|
434
442
|
criterias[key] = { searchText };
|
|
435
443
|
} else {
|
|
436
444
|
const defaultCriteria = evaluator.evaluatorConfig.defaultEvaluationCriteria;
|
|
437
|
-
criterias[key] =
|
|
445
|
+
criterias[key] = isRecord2(defaultCriteria) ? defaultCriteria : null;
|
|
438
446
|
}
|
|
439
447
|
}
|
|
440
448
|
return criterias;
|
|
@@ -469,250 +477,6 @@ function removeEvaluatorRefFromSet(evalSet, evaluator) {
|
|
|
469
477
|
return evalSet.evaluatorRefs.length !== before;
|
|
470
478
|
}
|
|
471
479
|
|
|
472
|
-
// ../eval-sdk/src/eval-evaluation-crud.ts
|
|
473
|
-
class EvaluationCrud {
|
|
474
|
-
fs;
|
|
475
|
-
store;
|
|
476
|
-
pathResolver;
|
|
477
|
-
constructor(fs, store, pathResolver) {
|
|
478
|
-
this.fs = fs;
|
|
479
|
-
this.store = store;
|
|
480
|
-
this.pathResolver = pathResolver;
|
|
481
|
-
}
|
|
482
|
-
async add(projectPath, setIdOrName, options) {
|
|
483
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
484
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
485
|
-
const evaluators = getLinkedEvaluators(await this.listEvaluators(root), evalSet);
|
|
486
|
-
const now = new Date().toISOString();
|
|
487
|
-
const evaluation = {
|
|
488
|
-
id: crypto.randomUUID(),
|
|
489
|
-
name: options.name,
|
|
490
|
-
inputs: options.inputs,
|
|
491
|
-
expectedOutput: options.expectedOutput,
|
|
492
|
-
evaluationCriterias: options.criteria ?? buildDefaultCriteria(evaluators, options.expectedOutput, options.searchText),
|
|
493
|
-
createdAt: now,
|
|
494
|
-
updatedAt: now
|
|
495
|
-
};
|
|
496
|
-
evalSet.evaluations.push(evaluation);
|
|
497
|
-
evalSet.updatedAt = now;
|
|
498
|
-
await this.writeEvalSet(root, evalSet);
|
|
499
|
-
return evaluation;
|
|
500
|
-
}
|
|
501
|
-
async list(projectPath, setIdOrName) {
|
|
502
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
503
|
-
return evalSet.evaluations;
|
|
504
|
-
}
|
|
505
|
-
async addSimulation(projectPath, setIdOrName, evaluationIdOrName, simulation) {
|
|
506
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
507
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
508
|
-
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
509
|
-
if (!evaluation) {
|
|
510
|
-
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
511
|
-
}
|
|
512
|
-
if (!evaluation.simulations) {
|
|
513
|
-
evaluation.simulations = [];
|
|
514
|
-
}
|
|
515
|
-
const existing = evaluation.simulations.findIndex((s) => s.componentId === simulation.componentId);
|
|
516
|
-
const now = new Date().toISOString();
|
|
517
|
-
if (existing !== -1) {
|
|
518
|
-
evaluation.simulations[existing] = simulation;
|
|
519
|
-
} else {
|
|
520
|
-
evaluation.simulations.push(simulation);
|
|
521
|
-
}
|
|
522
|
-
evaluation.updatedAt = now;
|
|
523
|
-
evalSet.updatedAt = now;
|
|
524
|
-
await this.writeEvalSet(root, evalSet);
|
|
525
|
-
return simulation;
|
|
526
|
-
}
|
|
527
|
-
async listSimulations(projectPath, setIdOrName, evaluationIdOrName) {
|
|
528
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
529
|
-
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
530
|
-
if (!evaluation) {
|
|
531
|
-
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
532
|
-
}
|
|
533
|
-
return evaluation.simulations ?? [];
|
|
534
|
-
}
|
|
535
|
-
async addChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childSimulation) {
|
|
536
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
537
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
538
|
-
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
539
|
-
if (!evaluation) {
|
|
540
|
-
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
541
|
-
}
|
|
542
|
-
if (!evaluation.simulations) {
|
|
543
|
-
evaluation.simulations = [];
|
|
544
|
-
}
|
|
545
|
-
let parent = evaluation.simulations.find((s) => s.componentId === parentComponentId);
|
|
546
|
-
if (!parent) {
|
|
547
|
-
parent = {
|
|
548
|
-
componentId: parentComponentId,
|
|
549
|
-
componentType: "agent",
|
|
550
|
-
simulationStrategy: "Llm",
|
|
551
|
-
childSimulations: []
|
|
552
|
-
};
|
|
553
|
-
evaluation.simulations.push(parent);
|
|
554
|
-
}
|
|
555
|
-
if (!parent.childSimulations) {
|
|
556
|
-
parent.childSimulations = [];
|
|
557
|
-
}
|
|
558
|
-
const existing = parent.childSimulations.findIndex((s) => s.componentId === childSimulation.componentId);
|
|
559
|
-
const now = new Date().toISOString();
|
|
560
|
-
if (existing !== -1) {
|
|
561
|
-
parent.childSimulations[existing] = childSimulation;
|
|
562
|
-
} else {
|
|
563
|
-
parent.childSimulations.push(childSimulation);
|
|
564
|
-
}
|
|
565
|
-
evaluation.updatedAt = now;
|
|
566
|
-
evalSet.updatedAt = now;
|
|
567
|
-
await this.writeEvalSet(root, evalSet);
|
|
568
|
-
return childSimulation;
|
|
569
|
-
}
|
|
570
|
-
listChildSimulations(evaluation, parentComponentId) {
|
|
571
|
-
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
572
|
-
if (!parent) {
|
|
573
|
-
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
574
|
-
}
|
|
575
|
-
return parent.childSimulations ?? [];
|
|
576
|
-
}
|
|
577
|
-
async removeChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childComponentId) {
|
|
578
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
579
|
-
if (!root) {
|
|
580
|
-
return false;
|
|
581
|
-
}
|
|
582
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
583
|
-
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
584
|
-
if (!evaluation) {
|
|
585
|
-
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
586
|
-
}
|
|
587
|
-
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
588
|
-
if (!parent) {
|
|
589
|
-
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
590
|
-
}
|
|
591
|
-
const index = (parent.childSimulations ?? []).findIndex((s) => s.componentId === childComponentId);
|
|
592
|
-
if (index === -1) {
|
|
593
|
-
return false;
|
|
594
|
-
}
|
|
595
|
-
parent.childSimulations?.splice(index, 1);
|
|
596
|
-
evalSet.updatedAt = new Date().toISOString();
|
|
597
|
-
await this.writeEvalSet(root, evalSet);
|
|
598
|
-
return true;
|
|
599
|
-
}
|
|
600
|
-
async removeSimulation(projectPath, setIdOrName, evaluationIdOrName, componentId) {
|
|
601
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
602
|
-
if (!root) {
|
|
603
|
-
return false;
|
|
604
|
-
}
|
|
605
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
606
|
-
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
607
|
-
if (!evaluation) {
|
|
608
|
-
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
609
|
-
}
|
|
610
|
-
const index = (evaluation.simulations ?? []).findIndex((s) => s.componentId === componentId);
|
|
611
|
-
if (index === -1) {
|
|
612
|
-
return false;
|
|
613
|
-
}
|
|
614
|
-
evaluation.simulations?.splice(index, 1);
|
|
615
|
-
evalSet.updatedAt = new Date().toISOString();
|
|
616
|
-
await this.writeEvalSet(root, evalSet);
|
|
617
|
-
return true;
|
|
618
|
-
}
|
|
619
|
-
async remove(projectPath, setIdOrName, evaluationIdOrName) {
|
|
620
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
621
|
-
if (!root) {
|
|
622
|
-
return false;
|
|
623
|
-
}
|
|
624
|
-
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
625
|
-
const index = evalSet.evaluations.findIndex((evaluation) => evaluation.id === evaluationIdOrName || evaluation.name === evaluationIdOrName);
|
|
626
|
-
if (index === -1) {
|
|
627
|
-
return false;
|
|
628
|
-
}
|
|
629
|
-
evalSet.evaluations.splice(index, 1);
|
|
630
|
-
evalSet.updatedAt = new Date().toISOString();
|
|
631
|
-
await this.writeEvalSet(root, evalSet);
|
|
632
|
-
return true;
|
|
633
|
-
}
|
|
634
|
-
async requireEvalSet(projectPath, idOrName) {
|
|
635
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
636
|
-
const evalSet = root ? findEvalSet(await this.listEvalSets(root), idOrName) : null;
|
|
637
|
-
if (!evalSet) {
|
|
638
|
-
throw new Error(`Evaluation set "${idOrName}" not found`);
|
|
639
|
-
}
|
|
640
|
-
return evalSet;
|
|
641
|
-
}
|
|
642
|
-
async listEvalSets(root) {
|
|
643
|
-
return this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
644
|
-
}
|
|
645
|
-
async listEvaluators(root) {
|
|
646
|
-
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
647
|
-
}
|
|
648
|
-
async writeEvalSet(root, evalSet) {
|
|
649
|
-
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
|
|
653
|
-
// ../eval-sdk/src/eval-evaluator-crud.ts
|
|
654
|
-
class EvaluatorCrud {
|
|
655
|
-
fs;
|
|
656
|
-
store;
|
|
657
|
-
pathResolver;
|
|
658
|
-
constructor(fs, store, pathResolver) {
|
|
659
|
-
this.fs = fs;
|
|
660
|
-
this.store = store;
|
|
661
|
-
this.pathResolver = pathResolver;
|
|
662
|
-
}
|
|
663
|
-
async list(projectPath) {
|
|
664
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
665
|
-
if (!root) {
|
|
666
|
-
return [];
|
|
667
|
-
}
|
|
668
|
-
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
669
|
-
}
|
|
670
|
-
async add(projectPath, options) {
|
|
671
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
672
|
-
const typeId = resolveEvaluatorType(options.type);
|
|
673
|
-
const id = `${slugify(options.name)}-${crypto.randomUUID().slice(0, 8)}`;
|
|
674
|
-
const now = new Date().toISOString();
|
|
675
|
-
const fileName = `${id}.json`;
|
|
676
|
-
const evaluator = {
|
|
677
|
-
version: "1.0",
|
|
678
|
-
id,
|
|
679
|
-
name: options.name,
|
|
680
|
-
description: options.description ?? `${options.name} evaluator`,
|
|
681
|
-
evaluatorTypeId: typeId,
|
|
682
|
-
evaluatorConfig: buildEvaluatorConfig(typeId, options),
|
|
683
|
-
fileName,
|
|
684
|
-
createdAt: now,
|
|
685
|
-
updatedAt: now
|
|
686
|
-
};
|
|
687
|
-
await this.store.writeJsonFile(this.fs.path.join(root, "evaluators", fileName), evaluator);
|
|
688
|
-
return evaluator;
|
|
689
|
-
}
|
|
690
|
-
async remove(projectPath, idOrName) {
|
|
691
|
-
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
692
|
-
if (!root) {
|
|
693
|
-
return false;
|
|
694
|
-
}
|
|
695
|
-
const evaluator = findEvaluator(await this.list(projectPath), idOrName);
|
|
696
|
-
if (!evaluator?.fileName) {
|
|
697
|
-
return false;
|
|
698
|
-
}
|
|
699
|
-
await this.store.remove(this.fs.path.join(root, "evaluators", evaluator.fileName));
|
|
700
|
-
await this.removeEvaluatorRefs(root, evaluator);
|
|
701
|
-
return true;
|
|
702
|
-
}
|
|
703
|
-
async removeEvaluatorRefs(root, evaluator) {
|
|
704
|
-
const sets = await this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
705
|
-
for (const set of sets) {
|
|
706
|
-
if (removeEvaluatorRefFromSet(set, evaluator) && set.fileName) {
|
|
707
|
-
await this.writeEvalSet(root, set);
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
}
|
|
711
|
-
async writeEvalSet(root, evalSet) {
|
|
712
|
-
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
|
|
716
480
|
// ../eval-sdk/src/eval-file-store.ts
|
|
717
481
|
async function ensureDirectory(fs, path) {
|
|
718
482
|
if (await fs.exists(path)) {
|
|
@@ -770,118 +534,6 @@ class EvalFileStore {
|
|
|
770
534
|
}
|
|
771
535
|
}
|
|
772
536
|
|
|
773
|
-
// ../eval-sdk/src/eval-files.ts
|
|
774
|
-
import { randomUUID } from "node:crypto";
|
|
775
|
-
var EVAL_FILES_DIR = "eval-files";
|
|
776
|
-
var TOKEN_LENGTH = 12;
|
|
777
|
-
var EVALUATION_BINARY_PREFIX = "BINARY_FILE:";
|
|
778
|
-
var EVALUATION_BINARY_DELIMITER = `
|
|
779
|
-
---BASE64---
|
|
780
|
-
`;
|
|
781
|
-
var TEXT_EXTENSIONS = new Set([
|
|
782
|
-
".json",
|
|
783
|
-
".csv",
|
|
784
|
-
".tsv",
|
|
785
|
-
".txt",
|
|
786
|
-
".md",
|
|
787
|
-
".log",
|
|
788
|
-
".xml",
|
|
789
|
-
".html",
|
|
790
|
-
".htm",
|
|
791
|
-
".yaml",
|
|
792
|
-
".yml"
|
|
793
|
-
]);
|
|
794
|
-
var MIME_BY_EXT = {
|
|
795
|
-
".jpg": "image/jpeg",
|
|
796
|
-
".jpeg": "image/jpeg",
|
|
797
|
-
".png": "image/png",
|
|
798
|
-
".gif": "image/gif",
|
|
799
|
-
".bmp": "image/bmp",
|
|
800
|
-
".webp": "image/webp",
|
|
801
|
-
".tif": "image/tiff",
|
|
802
|
-
".tiff": "image/tiff",
|
|
803
|
-
".svg": "image/svg+xml",
|
|
804
|
-
".pdf": "application/pdf",
|
|
805
|
-
".doc": "application/msword",
|
|
806
|
-
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
807
|
-
".xls": "application/vnd.ms-excel",
|
|
808
|
-
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
809
|
-
".ppt": "application/vnd.ms-powerpoint",
|
|
810
|
-
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
811
|
-
".csv": "text/csv",
|
|
812
|
-
".tsv": "text/tab-separated-values",
|
|
813
|
-
".json": "application/json",
|
|
814
|
-
".txt": "text/plain",
|
|
815
|
-
".md": "text/markdown",
|
|
816
|
-
".log": "text/plain",
|
|
817
|
-
".xml": "application/xml",
|
|
818
|
-
".html": "text/html",
|
|
819
|
-
".htm": "text/html",
|
|
820
|
-
".yaml": "application/yaml",
|
|
821
|
-
".yml": "application/yaml"
|
|
822
|
-
};
|
|
823
|
-
|
|
824
|
-
class EvalFiles {
|
|
825
|
-
fs;
|
|
826
|
-
pathResolver;
|
|
827
|
-
constructor(fs, pathResolver) {
|
|
828
|
-
this.fs = fs;
|
|
829
|
-
this.pathResolver = pathResolver;
|
|
830
|
-
}
|
|
831
|
-
async materialize(projectPath, sourcePath) {
|
|
832
|
-
const projectRoot = await this.pathResolver.resolveProjectPath(projectPath);
|
|
833
|
-
const sourceAbsolute = this.fs.path.resolve(sourcePath);
|
|
834
|
-
if (!await this.fs.exists(sourceAbsolute)) {
|
|
835
|
-
throw new Error(`Input file not found: ${sourcePath}`);
|
|
836
|
-
}
|
|
837
|
-
const content = await this.fs.readFile(sourceAbsolute);
|
|
838
|
-
if (content === null) {
|
|
839
|
-
throw new Error(`Could not read input file: ${sourcePath}`);
|
|
840
|
-
}
|
|
841
|
-
const fullName = this.fs.path.basename(sourceAbsolute);
|
|
842
|
-
const evalRoot = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
843
|
-
const dir = this.fs.path.join(evalRoot, EVAL_FILES_DIR);
|
|
844
|
-
await this.fs.mkdir(dir);
|
|
845
|
-
const targetName = await this.resolveTargetName(dir, fullName);
|
|
846
|
-
const targetPath = this.fs.path.join(dir, targetName);
|
|
847
|
-
const mimeType = mimeFromExt(extOf(fullName));
|
|
848
|
-
if (TEXT_EXTENSIONS.has(extOf(fullName))) {
|
|
849
|
-
await this.fs.writeFile(targetPath, content);
|
|
850
|
-
} else {
|
|
851
|
-
await this.fs.writeFile(targetPath, encodeBinaryEnvelope(content, mimeType));
|
|
852
|
-
}
|
|
853
|
-
const relative = this.fs.path.relative(projectRoot, targetPath).replaceAll("\\", "/");
|
|
854
|
-
return {
|
|
855
|
-
ID: `/${relative}`,
|
|
856
|
-
FullName: fullName,
|
|
857
|
-
MimeType: mimeType
|
|
858
|
-
};
|
|
859
|
-
}
|
|
860
|
-
async resolveTargetName(dir, fullName) {
|
|
861
|
-
if (!await this.fs.exists(this.fs.path.join(dir, fullName))) {
|
|
862
|
-
return fullName;
|
|
863
|
-
}
|
|
864
|
-
const ext = extOf(fullName);
|
|
865
|
-
const stem = ext ? fullName.slice(0, -ext.length) : fullName;
|
|
866
|
-
const token = randomUUID().replaceAll("-", "").slice(0, TOKEN_LENGTH);
|
|
867
|
-
return `${stem}_${token}${ext}`;
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
function encodeBinaryEnvelope(content, mimeType) {
|
|
871
|
-
const base64Content = Buffer.from(content).toString("base64");
|
|
872
|
-
return `${EVALUATION_BINARY_PREFIX}${mimeType}:${content.byteLength}${EVALUATION_BINARY_DELIMITER}${base64Content}`;
|
|
873
|
-
}
|
|
874
|
-
function extOf(name) {
|
|
875
|
-
const dot = name.lastIndexOf(".");
|
|
876
|
-
if (dot <= 0) {
|
|
877
|
-
return "";
|
|
878
|
-
}
|
|
879
|
-
return name.slice(dot).toLowerCase();
|
|
880
|
-
}
|
|
881
|
-
function mimeFromExt(ext) {
|
|
882
|
-
return MIME_BY_EXT[ext] ?? "application/octet-stream";
|
|
883
|
-
}
|
|
884
|
-
|
|
885
537
|
// ../eval-sdk/src/eval-inject-workload-id.ts
|
|
886
538
|
var SIMULATION_STRATEGY_WIRE_VALUES = {
|
|
887
539
|
Llm: 0,
|
|
@@ -909,20 +561,20 @@ function asRecordArray(value, label) {
|
|
|
909
561
|
throw new Error(`${label} response must be an array`);
|
|
910
562
|
}
|
|
911
563
|
return value.map((item, index) => {
|
|
912
|
-
if (!
|
|
564
|
+
if (!isRecord2(item)) {
|
|
913
565
|
throw new Error(`${label} response item ${index} must be an object`);
|
|
914
566
|
}
|
|
915
567
|
return item;
|
|
916
568
|
});
|
|
917
569
|
}
|
|
918
570
|
function readRecord(value) {
|
|
919
|
-
return
|
|
571
|
+
return isRecord2(value) ? value : undefined;
|
|
920
572
|
}
|
|
921
573
|
function readRecordArray(value) {
|
|
922
574
|
if (!Array.isArray(value)) {
|
|
923
575
|
return;
|
|
924
576
|
}
|
|
925
|
-
return value.filter(
|
|
577
|
+
return value.filter(isRecord2);
|
|
926
578
|
}
|
|
927
579
|
function optionalNumber(value) {
|
|
928
580
|
return typeof value === "number" ? value : null;
|
|
@@ -1014,7 +666,7 @@ function readRemoteEvalRun(record) {
|
|
|
1014
666
|
}
|
|
1015
667
|
];
|
|
1016
668
|
}),
|
|
1017
|
-
error: typeof record.error === "string" ||
|
|
669
|
+
error: typeof record.error === "string" || isRecord2(record.error) ? record.error : null,
|
|
1018
670
|
errorMessage: typeof record.errorMessage === "string" ? record.errorMessage : null,
|
|
1019
671
|
createdAt: typeof record.createdAt === "string" ? record.createdAt : undefined,
|
|
1020
672
|
updatedAt: typeof record.updatedAt === "string" ? record.updatedAt : undefined
|
|
@@ -1042,7 +694,7 @@ class EvalRunApi {
|
|
|
1042
694
|
}
|
|
1043
695
|
async getEvalSetRun(config, workloadId, evalSetRunId) {
|
|
1044
696
|
const response = await this.requestJson(config, `/unifiedEvals/workloads/${encodeURIComponent(workloadId)}/evalSetRuns/${encodeURIComponent(evalSetRunId)}?workloadType=${encodeURIComponent(this.workloadType)}`, { method: "GET" });
|
|
1045
|
-
if (!
|
|
697
|
+
if (!isRecord2(response)) {
|
|
1046
698
|
throw new Error("Eval set run response must be an object");
|
|
1047
699
|
}
|
|
1048
700
|
return readRemoteEvalSetRun(response);
|
|
@@ -1058,7 +710,7 @@ class EvalRunApi {
|
|
|
1058
710
|
}
|
|
1059
711
|
const data = await this.requestJson(config, `/unifiedEvals/processes/${encodeURIComponent(processKey)}/simulatableComponents?${params.toString()}`, { method: "GET" }, METADATA_FETCH_TIMEOUT_MS);
|
|
1060
712
|
const components = Array.isArray(data.components) ? data.components : [];
|
|
1061
|
-
return components.filter((c) =>
|
|
713
|
+
return components.filter((c) => isRecord2(c) && typeof c.id === "string" && typeof c.name === "string" && c.isEnabled === true);
|
|
1062
714
|
}
|
|
1063
715
|
apiBase(config) {
|
|
1064
716
|
return `${config.baseUrl}/${encodeURIComponent(config.organizationId)}/${encodeURIComponent(config.tenantName)}/agentsruntime_/api`;
|
|
@@ -1107,110 +759,1108 @@ class EvalRunApi {
|
|
|
1107
759
|
}
|
|
1108
760
|
}
|
|
1109
761
|
}
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
requireOrganizationName: true
|
|
1121
|
-
});
|
|
1122
|
-
return {
|
|
1123
|
-
baseUrl: auth.baseUrl,
|
|
1124
|
-
organizationId: auth.organizationId,
|
|
1125
|
-
tenantId: auth.tenantId,
|
|
1126
|
-
tenantName: auth.tenantName,
|
|
1127
|
-
authToken: auth.accessToken,
|
|
1128
|
-
organizationName: auth.organizationName
|
|
1129
|
-
};
|
|
1130
|
-
}
|
|
1131
|
-
async function resolveEvalApiConfig() {
|
|
1132
|
-
const { getAuthContext } = await import("./index-f5kq7265.js");
|
|
1133
|
-
const auth = await getAuthContext(EVAL_AUTH_REQUIREMENTS);
|
|
1134
|
-
return {
|
|
1135
|
-
baseUrl: auth.baseUrl,
|
|
1136
|
-
organizationId: auth.organizationId,
|
|
1137
|
-
tenantName: auth.tenantName,
|
|
1138
|
-
tenantId: auth.tenantId,
|
|
1139
|
-
authToken: auth.accessToken
|
|
1140
|
-
};
|
|
1141
|
-
}
|
|
1142
|
-
// ../eval-sdk/src/eval-run-format.ts
|
|
1143
|
-
function formatDuration(durationMs) {
|
|
1144
|
-
if (durationMs === null || durationMs === undefined) {
|
|
1145
|
-
return "-";
|
|
1146
|
-
}
|
|
1147
|
-
return `${(durationMs / 1000).toFixed(1)}s`;
|
|
1148
|
-
}
|
|
1149
|
-
function formatEvaluatorScores(scores, evaluators) {
|
|
1150
|
-
if (!scores || scores.length === 0) {
|
|
1151
|
-
return "-";
|
|
762
|
+
|
|
763
|
+
// ../eval-sdk/src/eval-evaluation-crud.ts
|
|
764
|
+
class EvaluationCrud {
|
|
765
|
+
fs;
|
|
766
|
+
store;
|
|
767
|
+
pathResolver;
|
|
768
|
+
constructor(fs, store, pathResolver) {
|
|
769
|
+
this.fs = fs;
|
|
770
|
+
this.store = store;
|
|
771
|
+
this.pathResolver = pathResolver;
|
|
1152
772
|
}
|
|
1153
|
-
|
|
1154
|
-
const
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
773
|
+
async add(projectPath, setIdOrName, options) {
|
|
774
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
775
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
776
|
+
const evaluators = getLinkedEvaluators(await this.listEvaluators(root), evalSet);
|
|
777
|
+
const now = new Date().toISOString();
|
|
778
|
+
const evaluation = {
|
|
779
|
+
id: crypto.randomUUID(),
|
|
780
|
+
name: options.name,
|
|
781
|
+
inputs: options.inputs,
|
|
782
|
+
expectedOutput: options.expectedOutput,
|
|
783
|
+
evaluationCriterias: options.criteria ?? buildDefaultCriteria(evaluators, options.expectedOutput, options.searchText),
|
|
784
|
+
createdAt: now,
|
|
785
|
+
updatedAt: now
|
|
786
|
+
};
|
|
787
|
+
evalSet.evaluations.push(evaluation);
|
|
788
|
+
evalSet.updatedAt = now;
|
|
789
|
+
await this.writeEvalSet(root, evalSet);
|
|
790
|
+
return evaluation;
|
|
1161
791
|
}
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
return evaluators.find((item) => item.id === evaluatorId)?.name ?? evaluatorSnapshotName(evaluatorSnapshot) ?? evaluatorId;
|
|
1166
|
-
}
|
|
1167
|
-
function remoteRunScore(run) {
|
|
1168
|
-
if (run.result?.scores && run.result.scores.length > 0) {
|
|
1169
|
-
const sum = run.result.scores.reduce((total, score) => total + score.value, 0);
|
|
1170
|
-
return Number((sum / run.result.scores.length).toFixed(4));
|
|
792
|
+
async list(projectPath, setIdOrName) {
|
|
793
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
794
|
+
return evalSet.evaluations;
|
|
1171
795
|
}
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
if (run.evaluatorRuns && run.evaluatorRuns.length > 0) {
|
|
1179
|
-
evaluatorScores = run.evaluatorRuns.map((evaluatorRun) => {
|
|
1180
|
-
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1181
|
-
const score = evaluatorRun.result?.score;
|
|
1182
|
-
if (score?.type === "error") {
|
|
1183
|
-
return `${name}: Error`;
|
|
1184
|
-
}
|
|
1185
|
-
return `${name}: ${score?.value ?? "-"}`;
|
|
1186
|
-
}).join(", ");
|
|
1187
|
-
const justificationParts = run.evaluatorRuns.filter((evaluatorRun) => evaluatorRun.result?.score?.justification).map((evaluatorRun) => {
|
|
1188
|
-
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1189
|
-
return `[${name}] ${evaluatorRun.result?.score?.justification}`;
|
|
1190
|
-
});
|
|
1191
|
-
if (justificationParts.length > 0) {
|
|
1192
|
-
justifications = justificationParts.join(" | ");
|
|
796
|
+
async addSimulation(projectPath, setIdOrName, evaluationIdOrName, simulation) {
|
|
797
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
798
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
799
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
800
|
+
if (!evaluation) {
|
|
801
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
1193
802
|
}
|
|
1194
|
-
|
|
1195
|
-
|
|
803
|
+
if (!evaluation.simulations) {
|
|
804
|
+
evaluation.simulations = [];
|
|
805
|
+
}
|
|
806
|
+
const existing = evaluation.simulations.findIndex((s) => s.componentId === simulation.componentId);
|
|
807
|
+
const now = new Date().toISOString();
|
|
808
|
+
if (existing !== -1) {
|
|
809
|
+
evaluation.simulations[existing] = simulation;
|
|
810
|
+
} else {
|
|
811
|
+
evaluation.simulations.push(simulation);
|
|
812
|
+
}
|
|
813
|
+
evaluation.updatedAt = now;
|
|
814
|
+
evalSet.updatedAt = now;
|
|
815
|
+
await this.writeEvalSet(root, evalSet);
|
|
816
|
+
return simulation;
|
|
1196
817
|
}
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
EvaluatorScores: evaluatorScores,
|
|
1205
|
-
Tokens: typeof tokens === "number" && tokens > 0 ? tokens : "-",
|
|
1206
|
-
Duration: typeof duration === "number" ? formatDuration(duration) : "-",
|
|
1207
|
-
Error: error
|
|
1208
|
-
};
|
|
1209
|
-
if (verbose) {
|
|
1210
|
-
row.Justifications = justifications;
|
|
818
|
+
async listSimulations(projectPath, setIdOrName, evaluationIdOrName) {
|
|
819
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
820
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
821
|
+
if (!evaluation) {
|
|
822
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
823
|
+
}
|
|
824
|
+
return evaluation.simulations ?? [];
|
|
1211
825
|
}
|
|
1212
|
-
|
|
1213
|
-
|
|
826
|
+
async addChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childSimulation) {
|
|
827
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
828
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
829
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
830
|
+
if (!evaluation) {
|
|
831
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
832
|
+
}
|
|
833
|
+
if (!evaluation.simulations) {
|
|
834
|
+
evaluation.simulations = [];
|
|
835
|
+
}
|
|
836
|
+
let parent = evaluation.simulations.find((s) => s.componentId === parentComponentId);
|
|
837
|
+
if (!parent) {
|
|
838
|
+
parent = {
|
|
839
|
+
componentId: parentComponentId,
|
|
840
|
+
componentType: "agent",
|
|
841
|
+
simulationStrategy: "Llm",
|
|
842
|
+
childSimulations: []
|
|
843
|
+
};
|
|
844
|
+
evaluation.simulations.push(parent);
|
|
845
|
+
}
|
|
846
|
+
if (!parent.childSimulations) {
|
|
847
|
+
parent.childSimulations = [];
|
|
848
|
+
}
|
|
849
|
+
const existing = parent.childSimulations.findIndex((s) => s.componentId === childSimulation.componentId);
|
|
850
|
+
const now = new Date().toISOString();
|
|
851
|
+
if (existing !== -1) {
|
|
852
|
+
parent.childSimulations[existing] = childSimulation;
|
|
853
|
+
} else {
|
|
854
|
+
parent.childSimulations.push(childSimulation);
|
|
855
|
+
}
|
|
856
|
+
evaluation.updatedAt = now;
|
|
857
|
+
evalSet.updatedAt = now;
|
|
858
|
+
await this.writeEvalSet(root, evalSet);
|
|
859
|
+
return childSimulation;
|
|
860
|
+
}
|
|
861
|
+
listChildSimulations(evaluation, parentComponentId) {
|
|
862
|
+
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
863
|
+
if (!parent) {
|
|
864
|
+
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
865
|
+
}
|
|
866
|
+
return parent.childSimulations ?? [];
|
|
867
|
+
}
|
|
868
|
+
async removeChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childComponentId) {
|
|
869
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
870
|
+
if (!root) {
|
|
871
|
+
return false;
|
|
872
|
+
}
|
|
873
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
874
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
875
|
+
if (!evaluation) {
|
|
876
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
877
|
+
}
|
|
878
|
+
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
879
|
+
if (!parent) {
|
|
880
|
+
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
881
|
+
}
|
|
882
|
+
const index = (parent.childSimulations ?? []).findIndex((s) => s.componentId === childComponentId);
|
|
883
|
+
if (index === -1) {
|
|
884
|
+
return false;
|
|
885
|
+
}
|
|
886
|
+
parent.childSimulations?.splice(index, 1);
|
|
887
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
888
|
+
await this.writeEvalSet(root, evalSet);
|
|
889
|
+
return true;
|
|
890
|
+
}
|
|
891
|
+
async removeSimulation(projectPath, setIdOrName, evaluationIdOrName, componentId) {
|
|
892
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
893
|
+
if (!root) {
|
|
894
|
+
return false;
|
|
895
|
+
}
|
|
896
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
897
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
898
|
+
if (!evaluation) {
|
|
899
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
900
|
+
}
|
|
901
|
+
const index = (evaluation.simulations ?? []).findIndex((s) => s.componentId === componentId);
|
|
902
|
+
if (index === -1) {
|
|
903
|
+
return false;
|
|
904
|
+
}
|
|
905
|
+
evaluation.simulations?.splice(index, 1);
|
|
906
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
907
|
+
await this.writeEvalSet(root, evalSet);
|
|
908
|
+
return true;
|
|
909
|
+
}
|
|
910
|
+
async remove(projectPath, setIdOrName, evaluationIdOrName) {
|
|
911
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
912
|
+
if (!root) {
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
916
|
+
const index = evalSet.evaluations.findIndex((evaluation) => evaluation.id === evaluationIdOrName || evaluation.name === evaluationIdOrName);
|
|
917
|
+
if (index === -1) {
|
|
918
|
+
return false;
|
|
919
|
+
}
|
|
920
|
+
evalSet.evaluations.splice(index, 1);
|
|
921
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
922
|
+
await this.writeEvalSet(root, evalSet);
|
|
923
|
+
return true;
|
|
924
|
+
}
|
|
925
|
+
async requireEvalSet(projectPath, idOrName) {
|
|
926
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
927
|
+
const evalSet = root ? findEvalSet(await this.listEvalSets(root), idOrName) : null;
|
|
928
|
+
if (!evalSet) {
|
|
929
|
+
throw new Error(`Evaluation set "${idOrName}" not found`);
|
|
930
|
+
}
|
|
931
|
+
return evalSet;
|
|
932
|
+
}
|
|
933
|
+
async listEvalSets(root) {
|
|
934
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
935
|
+
}
|
|
936
|
+
async listEvaluators(root) {
|
|
937
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
938
|
+
}
|
|
939
|
+
async writeEvalSet(root, evalSet) {
|
|
940
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// ../eval-sdk/src/eval-evaluator-crud.ts
|
|
945
|
+
class EvaluatorCrud {
|
|
946
|
+
fs;
|
|
947
|
+
store;
|
|
948
|
+
pathResolver;
|
|
949
|
+
constructor(fs, store, pathResolver) {
|
|
950
|
+
this.fs = fs;
|
|
951
|
+
this.store = store;
|
|
952
|
+
this.pathResolver = pathResolver;
|
|
953
|
+
}
|
|
954
|
+
async list(projectPath) {
|
|
955
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
956
|
+
if (!root) {
|
|
957
|
+
return [];
|
|
958
|
+
}
|
|
959
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
960
|
+
}
|
|
961
|
+
async add(projectPath, options) {
|
|
962
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
963
|
+
const typeId = resolveEvaluatorType(options.type);
|
|
964
|
+
const id = `${slugify(options.name)}-${crypto.randomUUID().slice(0, 8)}`;
|
|
965
|
+
const now = new Date().toISOString();
|
|
966
|
+
const fileName = `${id}.json`;
|
|
967
|
+
const evaluator = {
|
|
968
|
+
version: "1.0",
|
|
969
|
+
id,
|
|
970
|
+
name: options.name,
|
|
971
|
+
description: options.description ?? `${options.name} evaluator`,
|
|
972
|
+
evaluatorTypeId: typeId,
|
|
973
|
+
evaluatorConfig: buildEvaluatorConfig(typeId, options),
|
|
974
|
+
fileName,
|
|
975
|
+
createdAt: now,
|
|
976
|
+
updatedAt: now
|
|
977
|
+
};
|
|
978
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "evaluators", fileName), evaluator);
|
|
979
|
+
return evaluator;
|
|
980
|
+
}
|
|
981
|
+
async remove(projectPath, idOrName) {
|
|
982
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
983
|
+
if (!root) {
|
|
984
|
+
return false;
|
|
985
|
+
}
|
|
986
|
+
const evaluator = findEvaluator(await this.list(projectPath), idOrName);
|
|
987
|
+
if (!evaluator?.fileName) {
|
|
988
|
+
return false;
|
|
989
|
+
}
|
|
990
|
+
await this.store.remove(this.fs.path.join(root, "evaluators", evaluator.fileName));
|
|
991
|
+
await this.removeEvaluatorRefs(root, evaluator);
|
|
992
|
+
return true;
|
|
993
|
+
}
|
|
994
|
+
async removeEvaluatorRefs(root, evaluator) {
|
|
995
|
+
const sets = await this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
996
|
+
for (const set of sets) {
|
|
997
|
+
if (removeEvaluatorRefFromSet(set, evaluator) && set.fileName) {
|
|
998
|
+
await this.writeEvalSet(root, set);
|
|
999
|
+
}
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
async writeEvalSet(root, evalSet) {
|
|
1003
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
// ../eval-sdk/src/eval-files.ts
|
|
1008
|
+
import { randomUUID } from "node:crypto";
|
|
1009
|
+
var EVAL_FILES_DIR = "eval-files";
|
|
1010
|
+
var TOKEN_LENGTH = 12;
|
|
1011
|
+
var EVALUATION_BINARY_PREFIX = "BINARY_FILE:";
|
|
1012
|
+
var EVALUATION_BINARY_DELIMITER = `
|
|
1013
|
+
---BASE64---
|
|
1014
|
+
`;
|
|
1015
|
+
var TEXT_EXTENSIONS = new Set([
|
|
1016
|
+
".json",
|
|
1017
|
+
".csv",
|
|
1018
|
+
".tsv",
|
|
1019
|
+
".txt",
|
|
1020
|
+
".md",
|
|
1021
|
+
".log",
|
|
1022
|
+
".xml",
|
|
1023
|
+
".html",
|
|
1024
|
+
".htm",
|
|
1025
|
+
".yaml",
|
|
1026
|
+
".yml"
|
|
1027
|
+
]);
|
|
1028
|
+
var MIME_BY_EXT = {
|
|
1029
|
+
".jpg": "image/jpeg",
|
|
1030
|
+
".jpeg": "image/jpeg",
|
|
1031
|
+
".png": "image/png",
|
|
1032
|
+
".gif": "image/gif",
|
|
1033
|
+
".bmp": "image/bmp",
|
|
1034
|
+
".webp": "image/webp",
|
|
1035
|
+
".tif": "image/tiff",
|
|
1036
|
+
".tiff": "image/tiff",
|
|
1037
|
+
".svg": "image/svg+xml",
|
|
1038
|
+
".pdf": "application/pdf",
|
|
1039
|
+
".doc": "application/msword",
|
|
1040
|
+
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
1041
|
+
".xls": "application/vnd.ms-excel",
|
|
1042
|
+
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
1043
|
+
".ppt": "application/vnd.ms-powerpoint",
|
|
1044
|
+
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
1045
|
+
".csv": "text/csv",
|
|
1046
|
+
".tsv": "text/tab-separated-values",
|
|
1047
|
+
".json": "application/json",
|
|
1048
|
+
".txt": "text/plain",
|
|
1049
|
+
".md": "text/markdown",
|
|
1050
|
+
".log": "text/plain",
|
|
1051
|
+
".xml": "application/xml",
|
|
1052
|
+
".html": "text/html",
|
|
1053
|
+
".htm": "text/html",
|
|
1054
|
+
".yaml": "application/yaml",
|
|
1055
|
+
".yml": "application/yaml"
|
|
1056
|
+
};
|
|
1057
|
+
|
|
1058
|
+
class EvalFiles {
|
|
1059
|
+
fs;
|
|
1060
|
+
pathResolver;
|
|
1061
|
+
constructor(fs, pathResolver) {
|
|
1062
|
+
this.fs = fs;
|
|
1063
|
+
this.pathResolver = pathResolver;
|
|
1064
|
+
}
|
|
1065
|
+
async materialize(projectPath, sourcePath) {
|
|
1066
|
+
const projectRoot = await this.pathResolver.resolveProjectPath(projectPath);
|
|
1067
|
+
const sourceAbsolute = this.fs.path.resolve(sourcePath);
|
|
1068
|
+
if (!await this.fs.exists(sourceAbsolute)) {
|
|
1069
|
+
throw new Error(`Input file not found: ${sourcePath}`);
|
|
1070
|
+
}
|
|
1071
|
+
const content = await this.fs.readFile(sourceAbsolute);
|
|
1072
|
+
if (content === null) {
|
|
1073
|
+
throw new Error(`Could not read input file: ${sourcePath}`);
|
|
1074
|
+
}
|
|
1075
|
+
const fullName = this.fs.path.basename(sourceAbsolute);
|
|
1076
|
+
const evalRoot = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
1077
|
+
const dir = this.fs.path.join(evalRoot, EVAL_FILES_DIR);
|
|
1078
|
+
await this.fs.mkdir(dir);
|
|
1079
|
+
const targetName = await this.resolveTargetName(dir, fullName);
|
|
1080
|
+
const targetPath = this.fs.path.join(dir, targetName);
|
|
1081
|
+
const mimeType = mimeFromExt(extOf(fullName));
|
|
1082
|
+
if (TEXT_EXTENSIONS.has(extOf(fullName))) {
|
|
1083
|
+
await this.fs.writeFile(targetPath, content);
|
|
1084
|
+
} else {
|
|
1085
|
+
await this.fs.writeFile(targetPath, encodeBinaryEnvelope(content, mimeType));
|
|
1086
|
+
}
|
|
1087
|
+
const relative = this.fs.path.relative(projectRoot, targetPath).replaceAll("\\", "/");
|
|
1088
|
+
return {
|
|
1089
|
+
ID: `/${relative}`,
|
|
1090
|
+
FullName: fullName,
|
|
1091
|
+
MimeType: mimeType
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
async resolveTargetName(dir, fullName) {
|
|
1095
|
+
if (!await this.fs.exists(this.fs.path.join(dir, fullName))) {
|
|
1096
|
+
return fullName;
|
|
1097
|
+
}
|
|
1098
|
+
const ext = extOf(fullName);
|
|
1099
|
+
const stem = ext ? fullName.slice(0, -ext.length) : fullName;
|
|
1100
|
+
const token = randomUUID().replaceAll("-", "").slice(0, TOKEN_LENGTH);
|
|
1101
|
+
return `${stem}_${token}${ext}`;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
function encodeBinaryEnvelope(content, mimeType) {
|
|
1105
|
+
const base64Content = Buffer.from(content).toString("base64");
|
|
1106
|
+
return `${EVALUATION_BINARY_PREFIX}${mimeType}:${content.byteLength}${EVALUATION_BINARY_DELIMITER}${base64Content}`;
|
|
1107
|
+
}
|
|
1108
|
+
function extOf(name) {
|
|
1109
|
+
const dot = name.lastIndexOf(".");
|
|
1110
|
+
if (dot <= 0) {
|
|
1111
|
+
return "";
|
|
1112
|
+
}
|
|
1113
|
+
return name.slice(dot).toLowerCase();
|
|
1114
|
+
}
|
|
1115
|
+
function mimeFromExt(ext) {
|
|
1116
|
+
return MIME_BY_EXT[ext] ?? "application/octet-stream";
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
// ../eval-sdk/src/eval-set-crud.ts
|
|
1120
|
+
var DATASET_SCHEMA_VERSION = 4;
|
|
1121
|
+
|
|
1122
|
+
class EvalSetCrud {
|
|
1123
|
+
fs;
|
|
1124
|
+
store;
|
|
1125
|
+
adapter;
|
|
1126
|
+
constructor(fs, store, adapter) {
|
|
1127
|
+
this.fs = fs;
|
|
1128
|
+
this.store = store;
|
|
1129
|
+
this.adapter = adapter;
|
|
1130
|
+
}
|
|
1131
|
+
async list(projectPath) {
|
|
1132
|
+
const root = await this.adapter.paths.resolveEvalRoot(projectPath, false);
|
|
1133
|
+
if (!root) {
|
|
1134
|
+
return [];
|
|
1135
|
+
}
|
|
1136
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
1137
|
+
}
|
|
1138
|
+
async add(projectPath, options) {
|
|
1139
|
+
const resolvedProjectPath = await this.adapter.paths.resolveProjectPath(projectPath);
|
|
1140
|
+
const root = await this.adapter.paths.resolveEvalRoot(projectPath, true);
|
|
1141
|
+
const id = crypto.randomUUID();
|
|
1142
|
+
const fileName = `evaluation-set-${slugify(options.name)}-${id.slice(0, 8)}.json`;
|
|
1143
|
+
const evaluators = await this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
1144
|
+
const now = new Date().toISOString();
|
|
1145
|
+
const agentNodeId = await this.adapter.classifyEntryPoint(resolvedProjectPath, options.entryPoint);
|
|
1146
|
+
const selectedEntrypoint = options.entryPoint ?? await this.adapter.paths.readDefaultEntryPointNodeId(resolvedProjectPath);
|
|
1147
|
+
const evaluatorRefs = this.adapter.resolveEvaluatorRefs(evaluators, options.evaluatorRefs);
|
|
1148
|
+
const evalSet = {
|
|
1149
|
+
version: "1.0",
|
|
1150
|
+
...agentNodeId !== undefined ? { schemaVersion: DATASET_SCHEMA_VERSION } : {},
|
|
1151
|
+
id,
|
|
1152
|
+
name: options.name,
|
|
1153
|
+
evaluatorRefs,
|
|
1154
|
+
evaluations: [],
|
|
1155
|
+
...agentNodeId !== undefined ? {
|
|
1156
|
+
target: {
|
|
1157
|
+
kind: "agentNode",
|
|
1158
|
+
nodeId: agentNodeId
|
|
1159
|
+
},
|
|
1160
|
+
nodeEval: true
|
|
1161
|
+
} : {},
|
|
1162
|
+
...selectedEntrypoint !== undefined ? { selectedEntrypoint } : {},
|
|
1163
|
+
fileName,
|
|
1164
|
+
createdAt: now,
|
|
1165
|
+
updatedAt: now
|
|
1166
|
+
};
|
|
1167
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", fileName), evalSet);
|
|
1168
|
+
return evalSet;
|
|
1169
|
+
}
|
|
1170
|
+
async remove(projectPath, idOrName) {
|
|
1171
|
+
const root = await this.adapter.paths.resolveEvalRoot(projectPath, false);
|
|
1172
|
+
if (!root) {
|
|
1173
|
+
return false;
|
|
1174
|
+
}
|
|
1175
|
+
const evalSet = findEvalSet(await this.list(projectPath), idOrName);
|
|
1176
|
+
if (!evalSet?.fileName) {
|
|
1177
|
+
return false;
|
|
1178
|
+
}
|
|
1179
|
+
await this.store.remove(this.fs.path.join(root, "eval-sets", evalSet.fileName));
|
|
1180
|
+
return true;
|
|
1181
|
+
}
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// ../eval-sdk/src/eval-service.ts
|
|
1185
|
+
class EvalService {
|
|
1186
|
+
evaluators;
|
|
1187
|
+
evalSets;
|
|
1188
|
+
evaluations;
|
|
1189
|
+
files;
|
|
1190
|
+
adapter;
|
|
1191
|
+
constructor(fs = getFileSystem(), adapter) {
|
|
1192
|
+
this.adapter = adapter;
|
|
1193
|
+
const store = new EvalFileStore(fs);
|
|
1194
|
+
const paths = adapter.paths;
|
|
1195
|
+
this.evaluators = new EvaluatorCrud(fs, store, paths);
|
|
1196
|
+
this.evalSets = new EvalSetCrud(fs, store, adapter);
|
|
1197
|
+
this.evaluations = new EvaluationCrud(fs, store, paths);
|
|
1198
|
+
this.files = new EvalFiles(fs, paths);
|
|
1199
|
+
}
|
|
1200
|
+
listEvalSets(projectPath) {
|
|
1201
|
+
return this.evalSets.list(projectPath);
|
|
1202
|
+
}
|
|
1203
|
+
listEvaluators(projectPath) {
|
|
1204
|
+
return this.evaluators.list(projectPath);
|
|
1205
|
+
}
|
|
1206
|
+
addEvaluator(projectPath, options) {
|
|
1207
|
+
return this.evaluators.add(projectPath, options);
|
|
1208
|
+
}
|
|
1209
|
+
removeEvaluator(projectPath, idOrName) {
|
|
1210
|
+
return this.evaluators.remove(projectPath, idOrName);
|
|
1211
|
+
}
|
|
1212
|
+
addEvalSet(projectPath, options) {
|
|
1213
|
+
return this.evalSets.add(projectPath, options);
|
|
1214
|
+
}
|
|
1215
|
+
removeEvalSet(projectPath, idOrName) {
|
|
1216
|
+
return this.evalSets.remove(projectPath, idOrName);
|
|
1217
|
+
}
|
|
1218
|
+
async addEvaluation(projectPath, setIdOrName, options) {
|
|
1219
|
+
const inputFiles = options.inputFiles ?? [];
|
|
1220
|
+
for (const file of inputFiles) {
|
|
1221
|
+
if (file.key in options.inputs) {
|
|
1222
|
+
throw new Error(`Input "${file.key}" was provided in both --inputs and --input-file`);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
const evalSet = findEvalSet(await this.evalSets.list(projectPath), setIdOrName);
|
|
1226
|
+
const nodeTargetId = evalSet ? this.adapter.readNodeTargetId(evalSet) : undefined;
|
|
1227
|
+
if (nodeTargetId !== undefined) {
|
|
1228
|
+
if (inputFiles.length > 0) {
|
|
1229
|
+
throw new Error(`File inputs (--input-file) are not supported for node-scoped eval sets yet. Pass the value inline via --inputs, or use a flow-scoped eval set.`);
|
|
1230
|
+
}
|
|
1231
|
+
await this.adapter.validateAgentInputs(projectPath, nodeTargetId, options.inputs);
|
|
1232
|
+
if (options.expectedOutput) {
|
|
1233
|
+
await this.adapter.validateAgentOutputs(projectPath, nodeTargetId, options.expectedOutput);
|
|
1234
|
+
}
|
|
1235
|
+
} else {
|
|
1236
|
+
await this.adapter.validateFlowInputs(projectPath, options.inputs, inputFiles, evalSet?.selectedEntrypoint);
|
|
1237
|
+
if (options.expectedOutput) {
|
|
1238
|
+
await this.adapter.validateFlowOutputs(projectPath, options.expectedOutput, evalSet?.selectedEntrypoint);
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
const inputs = { ...options.inputs };
|
|
1242
|
+
for (const file of inputFiles) {
|
|
1243
|
+
inputs[file.key] = await this.files.materialize(projectPath, file.sourcePath);
|
|
1244
|
+
}
|
|
1245
|
+
return this.evaluations.add(projectPath, setIdOrName, {
|
|
1246
|
+
...options,
|
|
1247
|
+
inputs
|
|
1248
|
+
});
|
|
1249
|
+
}
|
|
1250
|
+
listEvaluations(projectPath, setIdOrName) {
|
|
1251
|
+
return this.evaluations.list(projectPath, setIdOrName);
|
|
1252
|
+
}
|
|
1253
|
+
removeEvaluation(projectPath, setIdOrName, evaluationIdOrName) {
|
|
1254
|
+
return this.evaluations.remove(projectPath, setIdOrName, evaluationIdOrName);
|
|
1255
|
+
}
|
|
1256
|
+
async addSimulation(projectPath, setIdOrName, evaluationIdOrName, simulation) {
|
|
1257
|
+
await this.adapter.locateComponent?.(projectPath, simulation.componentId);
|
|
1258
|
+
if (simulation.simulationStrategy === "Llm" && simulation.outputSchema === undefined) {
|
|
1259
|
+
const resolved = await this.adapter.readSimulationOutputSchema(projectPath, simulation.componentId);
|
|
1260
|
+
if (!resolved.nodeFound) {
|
|
1261
|
+
throw new Error(`Component "${simulation.componentId}" not found in the flow. Verify the component ID.`);
|
|
1262
|
+
}
|
|
1263
|
+
if (!resolved.schema) {
|
|
1264
|
+
throw new Error(`Could not auto-resolve output schema for component "${simulation.componentId}". ` + `The node has no declared outputs.`);
|
|
1265
|
+
}
|
|
1266
|
+
simulation = { ...simulation, outputSchema: resolved.schema };
|
|
1267
|
+
}
|
|
1268
|
+
return this.evaluations.addSimulation(projectPath, setIdOrName, evaluationIdOrName, simulation);
|
|
1269
|
+
}
|
|
1270
|
+
listSimulations(projectPath, setIdOrName, evaluationIdOrName) {
|
|
1271
|
+
return this.evaluations.listSimulations(projectPath, setIdOrName, evaluationIdOrName);
|
|
1272
|
+
}
|
|
1273
|
+
removeSimulation(projectPath, setIdOrName, evaluationIdOrName, componentId) {
|
|
1274
|
+
return this.evaluations.removeSimulation(projectPath, setIdOrName, evaluationIdOrName, componentId);
|
|
1275
|
+
}
|
|
1276
|
+
async addChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childSimulation) {
|
|
1277
|
+
let warning;
|
|
1278
|
+
if (childSimulation.outputSchema === undefined) {
|
|
1279
|
+
const resolved = await this.adapter.readChildSimulationSchema(projectPath, parentComponentId, childSimulation.componentId);
|
|
1280
|
+
if (resolved.schema) {
|
|
1281
|
+
childSimulation = {
|
|
1282
|
+
...childSimulation,
|
|
1283
|
+
outputSchema: resolved.schema
|
|
1284
|
+
};
|
|
1285
|
+
} else if (childSimulation.simulationStrategy === "Llm") {
|
|
1286
|
+
if (!resolved.parentFound) {
|
|
1287
|
+
const detail = resolved.reason ? ` ${resolved.reason}` : "";
|
|
1288
|
+
throw new Error(`Parent agent "${parentComponentId}" not found in the flow.${detail} Verify the parent component ID.`);
|
|
1289
|
+
}
|
|
1290
|
+
if (resolved.errorDetail) {
|
|
1291
|
+
throw new Error(`Could not auto-resolve output schema for "${childSimulation.componentId}" inside agent "${parentComponentId}": ${resolved.errorDetail}`);
|
|
1292
|
+
}
|
|
1293
|
+
throw new Error(`Could not auto-resolve output schema for "${childSimulation.componentId}" inside agent "${parentComponentId}". ` + `The tool has no declared outputs.`);
|
|
1294
|
+
} else if (childSimulation.simulationStrategy === "Static") {
|
|
1295
|
+
if (!resolved.parentFound) {
|
|
1296
|
+
const detail = resolved.reason ? ` ${resolved.reason}` : "";
|
|
1297
|
+
warning = `Parent agent "${parentComponentId}" not found in the flow.${detail} Mock value key validation was skipped.`;
|
|
1298
|
+
} else if (resolved.errorDetail) {
|
|
1299
|
+
warning = `Could not resolve output schema for "${childSimulation.componentId}": ${resolved.errorDetail}. ` + `Mock value key validation was skipped.`;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
if (childSimulation.simulationStrategy === "Static" && childSimulation.outputSchema !== undefined && childSimulation.mockValue !== undefined) {
|
|
1304
|
+
validateMockValueKeys(childSimulation.mockValue, childSimulation.outputSchema, childSimulation.componentId);
|
|
1305
|
+
}
|
|
1306
|
+
const simulation = await this.evaluations.addChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childSimulation);
|
|
1307
|
+
return { simulation, warning };
|
|
1308
|
+
}
|
|
1309
|
+
async listChildSimulations(projectPath, setIdOrName, evaluationIdOrName, parentComponentId) {
|
|
1310
|
+
const evalSet = await this.evaluations.list(projectPath, setIdOrName);
|
|
1311
|
+
const evaluation = evalSet.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
1312
|
+
if (!evaluation) {
|
|
1313
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
1314
|
+
}
|
|
1315
|
+
return this.evaluations.listChildSimulations(evaluation, parentComponentId);
|
|
1316
|
+
}
|
|
1317
|
+
removeChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childComponentId) {
|
|
1318
|
+
return this.evaluations.removeChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childComponentId);
|
|
1319
|
+
}
|
|
1320
|
+
static evaluatorTypeAliases() {
|
|
1321
|
+
return evaluatorTypeAliases();
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
function validateMockValueKeys(mockValue, outputSchema, componentId) {
|
|
1325
|
+
if (!isRecord2(mockValue) || !isRecord2(outputSchema))
|
|
1326
|
+
return;
|
|
1327
|
+
if (outputSchema.additionalProperties === true || typeof outputSchema.additionalProperties === "object" && outputSchema.additionalProperties !== null) {
|
|
1328
|
+
return;
|
|
1329
|
+
}
|
|
1330
|
+
const properties = isRecord2(outputSchema.properties) ? outputSchema.properties : undefined;
|
|
1331
|
+
if (!properties)
|
|
1332
|
+
return;
|
|
1333
|
+
const schemaKeys = new Set(Object.keys(properties));
|
|
1334
|
+
const mockKeys = Object.keys(mockValue);
|
|
1335
|
+
const unexpected = mockKeys.filter((k) => !schemaKeys.has(k));
|
|
1336
|
+
if (unexpected.length > 0) {
|
|
1337
|
+
throw new Error(`Mock value for "${componentId}" has keys not in the output schema: ${unexpected.join(", ")}. ` + `Expected keys: ${[...schemaKeys].join(", ")}`);
|
|
1338
|
+
}
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
// ../eval-sdk/src/eval-command-helpers.ts
|
|
1342
|
+
function parseJsonValue(raw, label) {
|
|
1343
|
+
try {
|
|
1344
|
+
return JSON.parse(raw);
|
|
1345
|
+
} catch (err) {
|
|
1346
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1347
|
+
throw new Error(`Invalid JSON for ${label}: ${message}`);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
function parseJsonObject(raw, label) {
|
|
1351
|
+
const parsed = JSON.parse(raw);
|
|
1352
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1353
|
+
throw new Error(`${label} must be a JSON object`);
|
|
1354
|
+
}
|
|
1355
|
+
return parsed;
|
|
1356
|
+
}
|
|
1357
|
+
function parseCriteriaOption(raw) {
|
|
1358
|
+
const parsed = parseJsonObject(raw, "--criteria");
|
|
1359
|
+
const criteria = {};
|
|
1360
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
1361
|
+
if (value === null || value === undefined) {
|
|
1362
|
+
criteria[key] = value;
|
|
1363
|
+
} else if (typeof value === "object" && !Array.isArray(value)) {
|
|
1364
|
+
criteria[key] = value;
|
|
1365
|
+
} else {
|
|
1366
|
+
throw new TypeError(`--criteria value for "${key}" must be a JSON object or null`);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
return criteria;
|
|
1370
|
+
}
|
|
1371
|
+
function parseCommaSeparated(value) {
|
|
1372
|
+
if (value === undefined) {
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
return value.split(",").map((item) => item.trim()).filter(Boolean);
|
|
1376
|
+
}
|
|
1377
|
+
function collectInputFile(value, previous) {
|
|
1378
|
+
return [...previous, value];
|
|
1379
|
+
}
|
|
1380
|
+
function parseInputFiles(raw) {
|
|
1381
|
+
return raw.map((entry) => {
|
|
1382
|
+
const eq = entry.indexOf("=");
|
|
1383
|
+
if (eq <= 0 || eq === entry.length - 1) {
|
|
1384
|
+
throw new Error(`--input-file must be "<key>=<path>" (got "${entry}")`);
|
|
1385
|
+
}
|
|
1386
|
+
const key = entry.slice(0, eq).trim();
|
|
1387
|
+
const sourcePath = entry.slice(eq + 1).trim();
|
|
1388
|
+
if (!key || !sourcePath) {
|
|
1389
|
+
throw new Error(`--input-file must be "<key>=<path>" (got "${entry}")`);
|
|
1390
|
+
}
|
|
1391
|
+
return { key, sourcePath };
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
1394
|
+
function outputError(message, error) {
|
|
1395
|
+
OutputFormatter.error({
|
|
1396
|
+
Result: RESULTS.Failure,
|
|
1397
|
+
Message: message,
|
|
1398
|
+
Instructions: error.message
|
|
1399
|
+
});
|
|
1400
|
+
processContext.exit(1);
|
|
1401
|
+
}
|
|
1402
|
+
// ../eval-sdk/src/eval-command-tree.ts
|
|
1403
|
+
function registerEvalCommands(program, config) {
|
|
1404
|
+
const evalCmd = program.command("eval").description(config.evalDescription);
|
|
1405
|
+
registerEvaluatorCommands(evalCmd, config);
|
|
1406
|
+
registerEvalSetCommands(evalCmd, config);
|
|
1407
|
+
registerEvaluationCommands(evalCmd, config);
|
|
1408
|
+
registerSimulationCommands(evalCmd, config);
|
|
1409
|
+
config.registerRunCommands(evalCmd);
|
|
1410
|
+
}
|
|
1411
|
+
function registerEvaluatorCommands(evalCmd, config) {
|
|
1412
|
+
const evaluator = evalCmd.command("evaluators").alias("evaluator").description(config.evaluatorsDescription);
|
|
1413
|
+
evaluator.command("create").alias("add").description(config.evaluatorAddDescription).argument("<name>", "Evaluator name").requiredOption("--type <type>", `Evaluator type: ${config.evaluatorTypeAliases.join(", ")}`).option("--description <text>", "Evaluator description").option("--target-key <key>", "Target output key", "*").option("--model <model>", "LLM model for LLM judge evaluators").option("--prompt <prompt>", "Custom LLM judge prompt").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (name, options) => {
|
|
1414
|
+
const [error] = await catchError2((async () => {
|
|
1415
|
+
const service = await config.loadService();
|
|
1416
|
+
const created = await service.addEvaluator(options.path, {
|
|
1417
|
+
name,
|
|
1418
|
+
type: options.type,
|
|
1419
|
+
description: options.description,
|
|
1420
|
+
targetOutputKey: options.targetKey,
|
|
1421
|
+
model: options.model,
|
|
1422
|
+
prompt: options.prompt
|
|
1423
|
+
});
|
|
1424
|
+
OutputFormatter.success({
|
|
1425
|
+
Result: RESULTS.Success,
|
|
1426
|
+
Code: `${config.codePrefix}EvaluatorAdd`,
|
|
1427
|
+
Data: {
|
|
1428
|
+
Name: created.name,
|
|
1429
|
+
Id: created.id,
|
|
1430
|
+
Type: created.evaluatorTypeId,
|
|
1431
|
+
File: created.fileName
|
|
1432
|
+
}
|
|
1433
|
+
});
|
|
1434
|
+
})());
|
|
1435
|
+
if (error) {
|
|
1436
|
+
outputError("Failed to add evaluator", error);
|
|
1437
|
+
}
|
|
1438
|
+
});
|
|
1439
|
+
config.addRowWindowOptions(evaluator.command("list")).description(config.evaluatorListDescription).option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (options) => {
|
|
1440
|
+
const [error] = await catchError2((async () => {
|
|
1441
|
+
const service = await config.loadService();
|
|
1442
|
+
const evaluators = await service.listEvaluators(options.path);
|
|
1443
|
+
const window = config.rowWindow(evaluators, options);
|
|
1444
|
+
OutputFormatter.success({
|
|
1445
|
+
Result: RESULTS.Success,
|
|
1446
|
+
Code: `${config.codePrefix}EvaluatorList`,
|
|
1447
|
+
Pagination: window.pagination,
|
|
1448
|
+
Data: window.rows.map((item) => ({
|
|
1449
|
+
Name: item.name ?? (typeof item.evaluatorConfig.name === "string" ? item.evaluatorConfig.name : ""),
|
|
1450
|
+
Id: item.id,
|
|
1451
|
+
Type: item.evaluatorTypeId,
|
|
1452
|
+
TargetKey: item.evaluatorConfig.targetOutputKey ?? "*",
|
|
1453
|
+
File: item.fileName
|
|
1454
|
+
}))
|
|
1455
|
+
});
|
|
1456
|
+
})());
|
|
1457
|
+
if (error) {
|
|
1458
|
+
outputError("Failed to list evaluators", error);
|
|
1459
|
+
}
|
|
1460
|
+
});
|
|
1461
|
+
evaluator.command("delete").alias("remove").description(config.evaluatorRemoveDescription).argument("<id>", "Evaluator ID, name, or file base name").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (id, options) => {
|
|
1462
|
+
const [error] = await catchError2((async () => {
|
|
1463
|
+
const service = await config.loadService();
|
|
1464
|
+
const removed = await service.removeEvaluator(options.path, id);
|
|
1465
|
+
if (!removed) {
|
|
1466
|
+
throw new Error(`Evaluator "${id}" not found`);
|
|
1467
|
+
}
|
|
1468
|
+
OutputFormatter.success({
|
|
1469
|
+
Result: RESULTS.Success,
|
|
1470
|
+
Code: `${config.codePrefix}EvaluatorRemove`,
|
|
1471
|
+
Data: { Status: "Evaluator removed", Id: id }
|
|
1472
|
+
});
|
|
1473
|
+
})());
|
|
1474
|
+
if (error) {
|
|
1475
|
+
outputError("Failed to remove evaluator", error);
|
|
1476
|
+
}
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
function registerEvalSetCommands(evalCmd, config) {
|
|
1480
|
+
const set = evalCmd.command("sets").alias("set").description("Manage evaluation sets");
|
|
1481
|
+
set.command("create").alias("add").description("Create an evaluation set").argument("<name>", "Evaluation set name").option("--evaluators <refs>", "Comma-separated evaluator IDs or file base names (default: all)").option("--entry-point <id>", config.setEntryPointDescription).option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (name, options) => {
|
|
1482
|
+
const [error] = await catchError2((async () => {
|
|
1483
|
+
const service = await config.loadService();
|
|
1484
|
+
const { readEvalSetScope, readNodeTargetId } = await config.loadNodeTarget();
|
|
1485
|
+
const created = await service.addEvalSet(options.path, {
|
|
1486
|
+
name,
|
|
1487
|
+
evaluatorRefs: parseCommaSeparated(options.evaluators),
|
|
1488
|
+
entryPoint: options.entryPoint
|
|
1489
|
+
});
|
|
1490
|
+
const targetNode = readNodeTargetId(created);
|
|
1491
|
+
OutputFormatter.success({
|
|
1492
|
+
Result: RESULTS.Success,
|
|
1493
|
+
Code: `${config.codePrefix}SetAdd`,
|
|
1494
|
+
Data: {
|
|
1495
|
+
Name: created.name,
|
|
1496
|
+
Id: created.id,
|
|
1497
|
+
Evaluators: created.evaluatorRefs.length,
|
|
1498
|
+
File: created.fileName,
|
|
1499
|
+
Scope: readEvalSetScope(created),
|
|
1500
|
+
...targetNode !== undefined ? { TargetNode: targetNode } : {}
|
|
1501
|
+
}
|
|
1502
|
+
});
|
|
1503
|
+
})());
|
|
1504
|
+
if (error) {
|
|
1505
|
+
outputError("Failed to create evaluation set", error);
|
|
1506
|
+
}
|
|
1507
|
+
});
|
|
1508
|
+
config.addRowWindowOptions(set.command("list")).description("List evaluation sets").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (options) => {
|
|
1509
|
+
const [error] = await catchError2((async () => {
|
|
1510
|
+
const service = await config.loadService();
|
|
1511
|
+
const { readEvalSetScope, readNodeTargetId } = await config.loadNodeTarget();
|
|
1512
|
+
const sets = await service.listEvalSets(options.path);
|
|
1513
|
+
const window = config.rowWindow(sets, options);
|
|
1514
|
+
OutputFormatter.success({
|
|
1515
|
+
Result: RESULTS.Success,
|
|
1516
|
+
Code: `${config.codePrefix}SetList`,
|
|
1517
|
+
Pagination: window.pagination,
|
|
1518
|
+
Data: window.rows.map((item) => {
|
|
1519
|
+
const targetNode = readNodeTargetId(item);
|
|
1520
|
+
return {
|
|
1521
|
+
Name: item.name,
|
|
1522
|
+
Id: item.id,
|
|
1523
|
+
DataPoints: item.evaluations.length,
|
|
1524
|
+
Evaluators: item.evaluatorRefs.length,
|
|
1525
|
+
File: item.fileName,
|
|
1526
|
+
Scope: readEvalSetScope(item),
|
|
1527
|
+
...targetNode !== undefined ? { TargetNode: targetNode } : {}
|
|
1528
|
+
};
|
|
1529
|
+
})
|
|
1530
|
+
});
|
|
1531
|
+
})());
|
|
1532
|
+
if (error) {
|
|
1533
|
+
outputError("Failed to list evaluation sets", error);
|
|
1534
|
+
}
|
|
1535
|
+
});
|
|
1536
|
+
set.command("delete").alias("remove").description("Remove an evaluation set").argument("<id>", "Evaluation set ID, name, or file base name").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (id, options) => {
|
|
1537
|
+
const [error] = await catchError2((async () => {
|
|
1538
|
+
const service = await config.loadService();
|
|
1539
|
+
const removed = await service.removeEvalSet(options.path, id);
|
|
1540
|
+
if (!removed) {
|
|
1541
|
+
throw new Error(`Evaluation set "${id}" not found`);
|
|
1542
|
+
}
|
|
1543
|
+
OutputFormatter.success({
|
|
1544
|
+
Result: RESULTS.Success,
|
|
1545
|
+
Code: `${config.codePrefix}SetRemove`,
|
|
1546
|
+
Data: { Status: "Evaluation set removed", Id: id }
|
|
1547
|
+
});
|
|
1548
|
+
})());
|
|
1549
|
+
if (error) {
|
|
1550
|
+
outputError("Failed to remove evaluation set", error);
|
|
1551
|
+
}
|
|
1552
|
+
});
|
|
1553
|
+
}
|
|
1554
|
+
function registerEvaluationCommands(evalCmd, config) {
|
|
1555
|
+
evalCmd.command("create").alias("add").description("Add a data point to an evaluation set").argument("<name>", "Data point name").requiredOption("--set <name>", "Evaluation set name or ID").option("--inputs <json>", "Input values as a JSON object").option("--input-file <key=path>", "Attach a file as input <key>; repeatable", collectInputFile, []).option("--expected <json>", "Expected output as a JSON object").option("--criteria <json>", "Per-evaluator criteria JSON object keyed by evaluator id").option("--search-text <text>", "Search text for contains evaluators").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (name, options) => {
|
|
1556
|
+
const [error] = await catchError2((async () => {
|
|
1557
|
+
const inputs = options.inputs === undefined ? {} : parseJsonObject(options.inputs, "--inputs");
|
|
1558
|
+
const inputFiles = parseInputFiles(options.inputFile);
|
|
1559
|
+
const expectedOutput = options.expected !== undefined ? parseJsonObject(options.expected, "--expected") : undefined;
|
|
1560
|
+
const criteria = options.criteria !== undefined ? parseCriteriaOption(options.criteria) : undefined;
|
|
1561
|
+
const service = await config.loadService();
|
|
1562
|
+
const evaluation = await service.addEvaluation(options.path, options.set, {
|
|
1563
|
+
name,
|
|
1564
|
+
inputs,
|
|
1565
|
+
inputFiles,
|
|
1566
|
+
expectedOutput,
|
|
1567
|
+
criteria,
|
|
1568
|
+
searchText: options.searchText
|
|
1569
|
+
});
|
|
1570
|
+
OutputFormatter.success({
|
|
1571
|
+
Result: RESULTS.Success,
|
|
1572
|
+
Code: `${config.codePrefix}Add`,
|
|
1573
|
+
Data: {
|
|
1574
|
+
Status: "Data point added",
|
|
1575
|
+
Name: evaluation.name,
|
|
1576
|
+
Id: evaluation.id,
|
|
1577
|
+
Set: options.set
|
|
1578
|
+
}
|
|
1579
|
+
});
|
|
1580
|
+
})());
|
|
1581
|
+
if (error) {
|
|
1582
|
+
outputError("Failed to add evaluation data point", error);
|
|
1583
|
+
}
|
|
1584
|
+
});
|
|
1585
|
+
config.addRowWindowOptions(evalCmd.command("list")).description("List data points in an evaluation set").requiredOption("--set <name>", "Evaluation set name or ID").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (options) => {
|
|
1586
|
+
const [error] = await catchError2((async () => {
|
|
1587
|
+
const service = await config.loadService();
|
|
1588
|
+
const evaluations = await service.listEvaluations(options.path, options.set);
|
|
1589
|
+
const window = config.rowWindow(evaluations, options);
|
|
1590
|
+
OutputFormatter.success({
|
|
1591
|
+
Result: RESULTS.Success,
|
|
1592
|
+
Code: `${config.codePrefix}List`,
|
|
1593
|
+
Pagination: window.pagination,
|
|
1594
|
+
Data: window.rows.map((item) => ({
|
|
1595
|
+
Name: item.name,
|
|
1596
|
+
Id: item.id,
|
|
1597
|
+
Inputs: JSON.stringify(item.inputs),
|
|
1598
|
+
Expected: item.expectedOutput ? JSON.stringify(item.expectedOutput) : "-",
|
|
1599
|
+
Evaluators: Object.keys(item.evaluationCriterias ?? {}).length
|
|
1600
|
+
}))
|
|
1601
|
+
});
|
|
1602
|
+
})());
|
|
1603
|
+
if (error) {
|
|
1604
|
+
outputError("Failed to list evaluation data points", error);
|
|
1605
|
+
}
|
|
1606
|
+
});
|
|
1607
|
+
evalCmd.command("delete").alias("remove").description("Remove a data point from an evaluation set").argument("<id>", "Data point ID or name").requiredOption("--set <name>", "Evaluation set name or ID").option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (id, options) => {
|
|
1608
|
+
const [error] = await catchError2((async () => {
|
|
1609
|
+
const service = await config.loadService();
|
|
1610
|
+
const removed = await service.removeEvaluation(options.path, options.set, id);
|
|
1611
|
+
if (!removed) {
|
|
1612
|
+
throw new Error(`Data point "${id}" not found`);
|
|
1613
|
+
}
|
|
1614
|
+
OutputFormatter.success({
|
|
1615
|
+
Result: RESULTS.Success,
|
|
1616
|
+
Code: `${config.codePrefix}Remove`,
|
|
1617
|
+
Data: { Status: "Data point removed", Id: id }
|
|
1618
|
+
});
|
|
1619
|
+
})());
|
|
1620
|
+
if (error) {
|
|
1621
|
+
outputError("Failed to remove evaluation data point", error);
|
|
1622
|
+
}
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
function registerSimulationCommands(evalCmd, config) {
|
|
1626
|
+
const sim = config.simulation;
|
|
1627
|
+
const rejectParent = (parent) => {
|
|
1628
|
+
if (!sim.childSimulations && parent !== undefined) {
|
|
1629
|
+
throw new Error(sim.parentNotSupportedError ?? "--parent is not supported for this tool.");
|
|
1630
|
+
}
|
|
1631
|
+
};
|
|
1632
|
+
const simulation = evalCmd.command("simulations").alias("simulation").description("Manage simulations on eval data points");
|
|
1633
|
+
const createCmd = simulation.command("create").alias("add").description("Add or update a simulation on a data point").argument("<component-id>", "Component ID to simulate").requiredOption("--set <name>", "Evaluation set name or ID").requiredOption("--data-point <id>", "Data point name or ID").requiredOption("--strategy <strategy>", "Simulation strategy: Static or Llm").addOption(simulation.createOption("--component-type <type>", `${sim.componentTypeDescription}. One of: ${sim.componentTypes.join(", ")}`).choices(sim.componentTypes)).option("--component-description <text>", "Component description").option("--simulation-instructions <text>", "Simulation instructions (for Llm strategy)").option("--mock-value <json>", "Mock output value as JSON (for Static strategy)");
|
|
1634
|
+
if (sim.outputSchemaOption) {
|
|
1635
|
+
createCmd.option("--output-schema <json>", "Output JSON schema for the simulated component (for Llm strategy; required for BPMN, which cannot auto-resolve it)");
|
|
1636
|
+
}
|
|
1637
|
+
createCmd.option("--parent <component-id>", sim.parentDescription).option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (componentId, options) => {
|
|
1638
|
+
const [error] = await catchError2((async () => {
|
|
1639
|
+
rejectParent(options.parent);
|
|
1640
|
+
const { strategy, componentType } = resolveSimulationKind(options, sim);
|
|
1641
|
+
const mockValue = options.mockValue !== undefined ? parseJsonValue(options.mockValue, "--mock-value") : undefined;
|
|
1642
|
+
const outputSchema = sim.outputSchemaOption && options.outputSchema !== undefined ? parseJsonObject(options.outputSchema, "--output-schema") : undefined;
|
|
1643
|
+
const simulationInput = {
|
|
1644
|
+
componentId,
|
|
1645
|
+
componentType,
|
|
1646
|
+
componentDescription: options.componentDescription,
|
|
1647
|
+
simulationStrategy: strategy,
|
|
1648
|
+
simulationInstruction: options.simulationInstructions,
|
|
1649
|
+
mockValue,
|
|
1650
|
+
...sim.outputSchemaOption ? { outputSchema } : {}
|
|
1651
|
+
};
|
|
1652
|
+
const service = await config.loadService();
|
|
1653
|
+
if (sim.childSimulations && options.parent !== undefined) {
|
|
1654
|
+
const { simulation: created, warning } = await service.addChildSimulation(options.path, options.set, options.dataPoint, options.parent, simulationInput);
|
|
1655
|
+
const data = {
|
|
1656
|
+
ComponentId: created.componentId,
|
|
1657
|
+
ComponentType: created.componentType,
|
|
1658
|
+
Strategy: created.simulationStrategy,
|
|
1659
|
+
Parent: options.parent,
|
|
1660
|
+
DataPoint: options.dataPoint,
|
|
1661
|
+
Set: options.set
|
|
1662
|
+
};
|
|
1663
|
+
if (warning) {
|
|
1664
|
+
data.Warning = warning;
|
|
1665
|
+
OutputFormatter.log({ Warning: warning });
|
|
1666
|
+
}
|
|
1667
|
+
OutputFormatter.success({
|
|
1668
|
+
Result: RESULTS.Success,
|
|
1669
|
+
Code: `${config.codePrefix}ChildSimulationAdd`,
|
|
1670
|
+
Data: data
|
|
1671
|
+
});
|
|
1672
|
+
} else {
|
|
1673
|
+
const created = await service.addSimulation(options.path, options.set, options.dataPoint, simulationInput);
|
|
1674
|
+
OutputFormatter.success({
|
|
1675
|
+
Result: RESULTS.Success,
|
|
1676
|
+
Code: `${config.codePrefix}SimulationAdd`,
|
|
1677
|
+
Data: {
|
|
1678
|
+
ComponentId: created.componentId,
|
|
1679
|
+
ComponentType: created.componentType,
|
|
1680
|
+
Strategy: created.simulationStrategy,
|
|
1681
|
+
DataPoint: options.dataPoint,
|
|
1682
|
+
Set: options.set
|
|
1683
|
+
}
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1686
|
+
})());
|
|
1687
|
+
if (error) {
|
|
1688
|
+
outputError("Failed to add simulation", error);
|
|
1689
|
+
}
|
|
1690
|
+
});
|
|
1691
|
+
config.addRowWindowOptions(simulation.command("list")).description("List simulations on a data point").requiredOption("--set <name>", "Evaluation set name or ID").requiredOption("--data-point <id>", "Data point name or ID").option("--parent <component-id>", sim.parentListDescription).option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (options) => {
|
|
1692
|
+
const [error] = await catchError2((async () => {
|
|
1693
|
+
rejectParent(options.parent);
|
|
1694
|
+
const service = await config.loadService();
|
|
1695
|
+
const childParent = sim.childSimulations && options.parent !== undefined ? options.parent : undefined;
|
|
1696
|
+
const simulations = childParent !== undefined ? await service.listChildSimulations(options.path, options.set, options.dataPoint, childParent) : await service.listSimulations(options.path, options.set, options.dataPoint);
|
|
1697
|
+
const window = config.rowWindow(simulations, options);
|
|
1698
|
+
OutputFormatter.success({
|
|
1699
|
+
Result: RESULTS.Success,
|
|
1700
|
+
Code: childParent !== undefined ? `${config.codePrefix}ChildSimulationList` : `${config.codePrefix}SimulationList`,
|
|
1701
|
+
Pagination: window.pagination,
|
|
1702
|
+
Data: window.rows.map((s) => ({
|
|
1703
|
+
ComponentId: s.componentId,
|
|
1704
|
+
ComponentType: s.componentType,
|
|
1705
|
+
ComponentDescription: s.componentDescription ?? "-",
|
|
1706
|
+
Strategy: s.simulationStrategy,
|
|
1707
|
+
Instruction: s.simulationInstruction ?? "-",
|
|
1708
|
+
...childParent !== undefined ? { Parent: childParent } : {}
|
|
1709
|
+
}))
|
|
1710
|
+
});
|
|
1711
|
+
})());
|
|
1712
|
+
if (error) {
|
|
1713
|
+
outputError("Failed to list simulations", error);
|
|
1714
|
+
}
|
|
1715
|
+
});
|
|
1716
|
+
simulation.command("delete").alias("remove").description("Remove a simulation from a data point").argument("<component-id>", "Component ID to remove").requiredOption("--set <name>", "Evaluation set name or ID").requiredOption("--data-point <id>", "Data point name or ID").option("--parent <component-id>", sim.parentDeleteDescription).option("--path <path>", config.pathDescription, ".").trackedAction(processContext, async (componentId, options) => {
|
|
1717
|
+
const [error] = await catchError2((async () => {
|
|
1718
|
+
rejectParent(options.parent);
|
|
1719
|
+
const service = await config.loadService();
|
|
1720
|
+
const childParent = sim.childSimulations && options.parent !== undefined ? options.parent : undefined;
|
|
1721
|
+
const removed = childParent !== undefined ? await service.removeChildSimulation(options.path, options.set, options.dataPoint, childParent, componentId) : await service.removeSimulation(options.path, options.set, options.dataPoint, componentId);
|
|
1722
|
+
if (!removed) {
|
|
1723
|
+
throw new Error(childParent !== undefined ? `Child simulation for component "${componentId}" not found on parent "${childParent}"` : `Simulation for component "${componentId}" not found`);
|
|
1724
|
+
}
|
|
1725
|
+
OutputFormatter.success({
|
|
1726
|
+
Result: RESULTS.Success,
|
|
1727
|
+
Code: childParent !== undefined ? `${config.codePrefix}ChildSimulationRemove` : `${config.codePrefix}SimulationRemove`,
|
|
1728
|
+
Data: {
|
|
1729
|
+
Status: childParent !== undefined ? "Child simulation removed" : "Simulation removed",
|
|
1730
|
+
ComponentId: componentId,
|
|
1731
|
+
...childParent !== undefined ? { Parent: childParent } : {}
|
|
1732
|
+
}
|
|
1733
|
+
});
|
|
1734
|
+
})());
|
|
1735
|
+
if (error) {
|
|
1736
|
+
outputError("Failed to remove simulation", error);
|
|
1737
|
+
}
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
function resolveSimulationKind(options, sim) {
|
|
1741
|
+
const strategy = options.strategy;
|
|
1742
|
+
if (strategy !== "Static" && strategy !== "Llm") {
|
|
1743
|
+
throw new Error(`--strategy must be Static or Llm (got "${strategy}")`);
|
|
1744
|
+
}
|
|
1745
|
+
const componentType = options.componentType ?? (sim.childSimulations && options.parent !== undefined ? "Node" : undefined);
|
|
1746
|
+
if (componentType === undefined) {
|
|
1747
|
+
throw new Error(sim.childSimulations ? "--component-type is required (or use --parent to default to Node)" : "--component-type is required");
|
|
1748
|
+
}
|
|
1749
|
+
if (strategy === "Static" && options.mockValue === undefined) {
|
|
1750
|
+
throw new Error("--mock-value is required for the Static strategy");
|
|
1751
|
+
}
|
|
1752
|
+
if (strategy === "Llm" && options.simulationInstructions === undefined) {
|
|
1753
|
+
throw new Error("--simulation-instructions is required for the Llm strategy");
|
|
1754
|
+
}
|
|
1755
|
+
if (sim.outputSchemaOption && strategy === "Llm" && options.outputSchema === undefined) {
|
|
1756
|
+
throw new Error("--output-schema is required for the Llm strategy (BPMN cannot auto-resolve the component's output schema)");
|
|
1757
|
+
}
|
|
1758
|
+
return { strategy, componentType };
|
|
1759
|
+
}
|
|
1760
|
+
// ../eval-sdk/src/eval-login.ts
|
|
1761
|
+
var EVAL_AUTH_REQUIREMENTS = {
|
|
1762
|
+
requireOrganizationId: true,
|
|
1763
|
+
requireTenantId: true,
|
|
1764
|
+
requireTenantName: true
|
|
1765
|
+
};
|
|
1766
|
+
async function resolveLoginConfig() {
|
|
1767
|
+
const { getAuthContext } = await import("./index-r6crd7tx.js");
|
|
1768
|
+
const auth = await getAuthContext({
|
|
1769
|
+
...EVAL_AUTH_REQUIREMENTS,
|
|
1770
|
+
requireOrganizationName: true
|
|
1771
|
+
});
|
|
1772
|
+
return {
|
|
1773
|
+
baseUrl: auth.baseUrl,
|
|
1774
|
+
organizationId: auth.organizationId,
|
|
1775
|
+
tenantId: auth.tenantId,
|
|
1776
|
+
tenantName: auth.tenantName,
|
|
1777
|
+
authToken: auth.accessToken,
|
|
1778
|
+
organizationName: auth.organizationName
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
async function resolveEvalApiConfig() {
|
|
1782
|
+
const { getAuthContext } = await import("./index-r6crd7tx.js");
|
|
1783
|
+
const auth = await getAuthContext(EVAL_AUTH_REQUIREMENTS);
|
|
1784
|
+
return {
|
|
1785
|
+
baseUrl: auth.baseUrl,
|
|
1786
|
+
organizationId: auth.organizationId,
|
|
1787
|
+
tenantName: auth.tenantName,
|
|
1788
|
+
tenantId: auth.tenantId,
|
|
1789
|
+
authToken: auth.accessToken
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
// ../eval-sdk/src/eval-run-format.ts
|
|
1793
|
+
function formatDuration(durationMs) {
|
|
1794
|
+
if (durationMs === null || durationMs === undefined) {
|
|
1795
|
+
return "-";
|
|
1796
|
+
}
|
|
1797
|
+
return `${(durationMs / 1000).toFixed(1)}s`;
|
|
1798
|
+
}
|
|
1799
|
+
function formatEvaluatorScores(scores, evaluators) {
|
|
1800
|
+
if (!scores || scores.length === 0) {
|
|
1801
|
+
return "-";
|
|
1802
|
+
}
|
|
1803
|
+
return scores.map((score) => {
|
|
1804
|
+
const evaluator = evaluators.find((item) => item.id === score.evaluatorId);
|
|
1805
|
+
return `${evaluator?.name ?? score.evaluatorId}: ${score.value}`;
|
|
1806
|
+
}).join(", ");
|
|
1807
|
+
}
|
|
1808
|
+
function evaluatorSnapshotName(snapshot) {
|
|
1809
|
+
if (typeof snapshot === "object" && snapshot !== null && !Array.isArray(snapshot) && typeof snapshot.name === "string") {
|
|
1810
|
+
return snapshot.name;
|
|
1811
|
+
}
|
|
1812
|
+
return;
|
|
1813
|
+
}
|
|
1814
|
+
function remoteEvaluatorName(evaluatorId, evaluatorSnapshot, evaluators) {
|
|
1815
|
+
return evaluators.find((item) => item.id === evaluatorId)?.name ?? evaluatorSnapshotName(evaluatorSnapshot) ?? evaluatorId;
|
|
1816
|
+
}
|
|
1817
|
+
function remoteRunScore(run) {
|
|
1818
|
+
if (run.result?.scores && run.result.scores.length > 0) {
|
|
1819
|
+
const sum = run.result.scores.reduce((total, score) => total + score.value, 0);
|
|
1820
|
+
return Number((sum / run.result.scores.length).toFixed(4));
|
|
1821
|
+
}
|
|
1822
|
+
const value = run.result?.score?.value;
|
|
1823
|
+
return typeof value === "number" || typeof value === "boolean" ? value : "-";
|
|
1824
|
+
}
|
|
1825
|
+
function formatRemoteEvalRunRow(run, evaluators, verbose = false) {
|
|
1826
|
+
let evaluatorScores = "-";
|
|
1827
|
+
let justifications = "-";
|
|
1828
|
+
if (run.evaluatorRuns && run.evaluatorRuns.length > 0) {
|
|
1829
|
+
evaluatorScores = run.evaluatorRuns.map((evaluatorRun) => {
|
|
1830
|
+
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1831
|
+
const score = evaluatorRun.result?.score;
|
|
1832
|
+
if (score?.type === "error") {
|
|
1833
|
+
return `${name}: Error`;
|
|
1834
|
+
}
|
|
1835
|
+
return `${name}: ${score?.value ?? "-"}`;
|
|
1836
|
+
}).join(", ");
|
|
1837
|
+
const justificationParts = run.evaluatorRuns.filter((evaluatorRun) => evaluatorRun.result?.score?.justification).map((evaluatorRun) => {
|
|
1838
|
+
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1839
|
+
return `[${name}] ${evaluatorRun.result?.score?.justification}`;
|
|
1840
|
+
});
|
|
1841
|
+
if (justificationParts.length > 0) {
|
|
1842
|
+
justifications = justificationParts.join(" | ");
|
|
1843
|
+
}
|
|
1844
|
+
} else if (run.result?.scores && run.result.scores.length > 0) {
|
|
1845
|
+
evaluatorScores = formatEvaluatorScores(run.result.scores, evaluators);
|
|
1846
|
+
}
|
|
1847
|
+
const tokens = run.completionMetrics?.tokens;
|
|
1848
|
+
const duration = run.completionMetrics?.duration;
|
|
1849
|
+
const error = run.errorMessage ?? (typeof run.error === "string" ? run.error : run.error?.message) ?? "-";
|
|
1850
|
+
const row = {
|
|
1851
|
+
DataPoint: run.evalSnapshot?.name ?? run.id,
|
|
1852
|
+
Status: run.status,
|
|
1853
|
+
Score: remoteRunScore(run),
|
|
1854
|
+
EvaluatorScores: evaluatorScores,
|
|
1855
|
+
Tokens: typeof tokens === "number" && tokens > 0 ? tokens : "-",
|
|
1856
|
+
Duration: typeof duration === "number" ? formatDuration(duration) : "-",
|
|
1857
|
+
Error: error
|
|
1858
|
+
};
|
|
1859
|
+
if (verbose) {
|
|
1860
|
+
row.Justifications = justifications;
|
|
1861
|
+
}
|
|
1862
|
+
return row;
|
|
1863
|
+
}
|
|
1214
1864
|
function isFailedRemoteRun(run) {
|
|
1215
1865
|
const status = run.status.toLowerCase();
|
|
1216
1866
|
if (status === "failed" || status === "workloadfailed" || run.errorMessage || run.error) {
|
|
@@ -1222,7 +1872,30 @@ function isFailedRemoteRun(run) {
|
|
|
1222
1872
|
return run.evaluatorRuns?.some((evaluatorRun) => evaluatorRun.result?.score?.type === "error" || evaluatorRun.result?.score?.value === false) ?? false;
|
|
1223
1873
|
}
|
|
1224
1874
|
function isTerminalRemoteStatus(status) {
|
|
1225
|
-
return [
|
|
1875
|
+
return [
|
|
1876
|
+
"completed",
|
|
1877
|
+
"partiallycompleted",
|
|
1878
|
+
"failed",
|
|
1879
|
+
"workloadfailed"
|
|
1880
|
+
].includes(status.toLowerCase());
|
|
1881
|
+
}
|
|
1882
|
+
function csvField(value) {
|
|
1883
|
+
if (value === null || value === undefined) {
|
|
1884
|
+
return "";
|
|
1885
|
+
}
|
|
1886
|
+
let text = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
1887
|
+
if (typeof value === "string" && text !== "-" && /^[=+\-@\t\r]/.test(text)) {
|
|
1888
|
+
text = `'${text}`;
|
|
1889
|
+
}
|
|
1890
|
+
return /["\r\n,]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
|
1891
|
+
}
|
|
1892
|
+
function rowsToCsv(rows) {
|
|
1893
|
+
const headers = Object.keys(rows[0] ?? {});
|
|
1894
|
+
return [
|
|
1895
|
+
headers.join(","),
|
|
1896
|
+
...rows.map((row) => headers.map((header) => csvField(row[header])).join(","))
|
|
1897
|
+
].join(`
|
|
1898
|
+
`);
|
|
1226
1899
|
}
|
|
1227
1900
|
function formatEvalSetRunRow(run, evaluators) {
|
|
1228
1901
|
return {
|
|
@@ -1246,6 +1919,6 @@ function findMatchingDefinition(nodeType, typeVersion, definitions) {
|
|
|
1246
1919
|
return definitions.find((d) => d.nodeType === nodeType);
|
|
1247
1920
|
}
|
|
1248
1921
|
|
|
1249
|
-
export { findNodeDefinition, evaluatorTypeAliases,
|
|
1922
|
+
export { findNodeDefinition, registerEvalCommands, evaluatorTypeAliases, isRecord2, isUUID, findEvalSet, getLinkedEvaluators, ensureDirectory, EvalFileStore, injectWorkloadId, resolveLoginConfig, resolveEvalApiConfig, EvalRunApi, formatDuration, formatEvaluatorScores, remoteRunScore, formatRemoteEvalRunRow, isFailedRemoteRun, isTerminalRemoteStatus, rowsToCsv, formatEvalSetRunRow, EvalService, findMatchingDefinition };
|
|
1250
1923
|
|
|
1251
|
-
//# debugId=
|
|
1924
|
+
//# debugId=3669B1E8AE369E2064756E2164756E21
|