@anvia/core 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -0
- package/dist/agent/agent-stream.js +7 -0
- package/dist/agent/agent-stream.js.map +1 -0
- package/dist/agent/agent-tool.js +24 -0
- package/dist/agent/agent-tool.js.map +1 -0
- package/dist/agent/agent.js +89 -0
- package/dist/agent/agent.js.map +1 -0
- package/dist/agent/errors.js +17 -0
- package/dist/agent/errors.js.map +1 -0
- package/dist/agent/ids.js +7 -0
- package/dist/agent/ids.js.map +1 -0
- package/dist/agent/index.d.ts +127 -9
- package/dist/agent/index.js +103 -21
- package/dist/agent/interactions/index.js +2 -2
- package/dist/agent/interactions.js +22 -0
- package/dist/agent/interactions.js.map +1 -0
- package/dist/agent/lifecycle.js +9 -0
- package/dist/agent/lifecycle.js.map +1 -0
- package/dist/agent/output-schema.js +10 -0
- package/dist/agent/output-schema.js.map +1 -0
- package/dist/agent/resolve-options.js +46 -0
- package/dist/agent/resolve-options.js.map +1 -0
- package/dist/agent/resolved-agent.js +92 -0
- package/dist/agent/resolved-agent.js.map +1 -0
- package/dist/agent/run-types.js +1 -0
- package/dist/agent/run-types.js.map +1 -0
- package/dist/agent/snapshot.js +17 -0
- package/dist/agent/snapshot.js.map +1 -0
- package/dist/agent/team/agent-team.js +98 -0
- package/dist/agent/team/agent-team.js.map +1 -0
- package/dist/agent/team/errors.js +9 -0
- package/dist/agent/team/errors.js.map +1 -0
- package/dist/agent/team/index.js +104 -0
- package/dist/agent/team/index.js.map +1 -0
- package/dist/agent/team/types.js +1 -0
- package/dist/agent/team/types.js.map +1 -0
- package/dist/agent/tool-catalog.js +37 -0
- package/dist/agent/tool-catalog.js.map +1 -0
- package/dist/agent/tool-state.js +11 -0
- package/dist/agent/tool-state.js.map +1 -0
- package/dist/agent/types.js +1 -0
- package/dist/agent/types.js.map +1 -0
- package/dist/agent/vector-context.js +10 -0
- package/dist/agent/vector-context.js.map +1 -0
- package/dist/{agent-DTmzs1Qr.d.ts → agent-C9VhENdj.d.ts} +2 -3
- package/dist/{chunk-2QZJTG2E.js → chunk-23GEYG2X.js} +11 -71
- package/dist/chunk-23GEYG2X.js.map +1 -0
- package/dist/{chunk-I3Q7XSBB.js → chunk-23LRXKQX.js} +6 -4
- package/dist/{chunk-I3Q7XSBB.js.map → chunk-23LRXKQX.js.map} +1 -1
- package/dist/{chunk-YK4WAAS4.js → chunk-2NLINXRF.js} +1 -1
- package/dist/{chunk-SPD4XVR7.js → chunk-2XSMGVBG.js} +1 -1
- package/dist/chunk-3J2CSVPR.js +1 -0
- package/dist/chunk-3J2CSVPR.js.map +1 -0
- package/dist/chunk-3P3YVJ7V.js +1 -0
- package/dist/chunk-3P3YVJ7V.js.map +1 -0
- package/dist/chunk-3TAQXJBC.js +107 -0
- package/dist/chunk-3TAQXJBC.js.map +1 -0
- package/dist/chunk-47NTHIVB.js +76 -0
- package/dist/chunk-47NTHIVB.js.map +1 -0
- package/dist/chunk-4AW2633B.js +9 -0
- package/dist/chunk-4AW2633B.js.map +1 -0
- package/dist/{chunk-3XQGVDU5.js → chunk-4C7AXYOM.js} +1 -1
- package/dist/{chunk-IZNOP6JG.js → chunk-4FIVU53H.js} +1 -1
- package/dist/chunk-4ITAV7YA.js +1 -0
- package/dist/chunk-4ITAV7YA.js.map +1 -0
- package/dist/{chunk-7JLAIN6E.js → chunk-4ONUNWGC.js} +2 -2
- package/dist/chunk-4UUIGAIX.js +1 -0
- package/dist/chunk-4UUIGAIX.js.map +1 -0
- package/dist/chunk-53TPAXYR.js +1 -0
- package/dist/chunk-53TPAXYR.js.map +1 -0
- package/dist/chunk-5AETKC4L.js +107 -0
- package/dist/chunk-5AETKC4L.js.map +1 -0
- package/dist/chunk-5I2NVRHC.js +146 -0
- package/dist/chunk-5I2NVRHC.js.map +1 -0
- package/dist/chunk-5IURMWFN.js +53 -0
- package/dist/chunk-5IURMWFN.js.map +1 -0
- package/dist/chunk-5M6GTA3T.js +1 -0
- package/dist/chunk-5M6GTA3T.js.map +1 -0
- package/dist/chunk-5N5W7CX7.js +18 -0
- package/dist/chunk-5N5W7CX7.js.map +1 -0
- package/dist/chunk-5P2YOQSA.js +355 -0
- package/dist/chunk-5P2YOQSA.js.map +1 -0
- package/dist/chunk-5S7T2URX.js +1 -0
- package/dist/chunk-5S7T2URX.js.map +1 -0
- package/dist/chunk-5V533R7F.js +19 -0
- package/dist/chunk-5V533R7F.js.map +1 -0
- package/dist/chunk-5WLIHNNJ.js +56 -0
- package/dist/chunk-5WLIHNNJ.js.map +1 -0
- package/dist/chunk-6BTJC3ES.js +37 -0
- package/dist/chunk-6BTJC3ES.js.map +1 -0
- package/dist/chunk-6N4ENZO4.js +24 -0
- package/dist/chunk-6N4ENZO4.js.map +1 -0
- package/dist/{chunk-4UQALF5Y.js → chunk-6RQYWZI3.js} +2 -36
- package/dist/chunk-6RQYWZI3.js.map +1 -0
- package/dist/chunk-6TAWRKFK.js +86 -0
- package/dist/chunk-6TAWRKFK.js.map +1 -0
- package/dist/chunk-7F53DOML.js +53 -0
- package/dist/chunk-7F53DOML.js.map +1 -0
- package/dist/chunk-7KAGWJ5M.js +112 -0
- package/dist/chunk-7KAGWJ5M.js.map +1 -0
- package/dist/chunk-7LBPRXZP.js +1 -0
- package/dist/chunk-7LBPRXZP.js.map +1 -0
- package/dist/chunk-A7BXVZXZ.js +25 -0
- package/dist/chunk-A7BXVZXZ.js.map +1 -0
- package/dist/chunk-B47EXUHL.js +67 -0
- package/dist/chunk-B47EXUHL.js.map +1 -0
- package/dist/chunk-B5JQDR33.js +14 -0
- package/dist/chunk-B5JQDR33.js.map +1 -0
- package/dist/chunk-B5YTCKUJ.js +158 -0
- package/dist/chunk-B5YTCKUJ.js.map +1 -0
- package/dist/chunk-BDDC72GF.js +189 -0
- package/dist/chunk-BDDC72GF.js.map +1 -0
- package/dist/chunk-BDKFFFVX.js +342 -0
- package/dist/chunk-BDKFFFVX.js.map +1 -0
- package/dist/{chunk-QGX73TSQ.js → chunk-BONQTXRQ.js} +3 -12
- package/dist/chunk-BONQTXRQ.js.map +1 -0
- package/dist/chunk-BUPC72Y2.js +1007 -0
- package/dist/chunk-BUPC72Y2.js.map +1 -0
- package/dist/chunk-CKKNO4XB.js +70 -0
- package/dist/chunk-CKKNO4XB.js.map +1 -0
- package/dist/chunk-CX7ZVTO5.js +22 -0
- package/dist/chunk-CX7ZVTO5.js.map +1 -0
- package/dist/chunk-DU4WA6LQ.js +1 -0
- package/dist/chunk-DU4WA6LQ.js.map +1 -0
- package/dist/chunk-EJR6VVB3.js +63 -0
- package/dist/chunk-EJR6VVB3.js.map +1 -0
- package/dist/chunk-EKZRAWBP.js +110 -0
- package/dist/chunk-EKZRAWBP.js.map +1 -0
- package/dist/chunk-ENZ2XX27.js +26 -0
- package/dist/chunk-ENZ2XX27.js.map +1 -0
- package/dist/chunk-EOAJBBII.js +1 -0
- package/dist/chunk-EOAJBBII.js.map +1 -0
- package/dist/chunk-ERN7P6Q4.js +1 -0
- package/dist/chunk-ERN7P6Q4.js.map +1 -0
- package/dist/chunk-ESOFEAYK.js +57 -0
- package/dist/chunk-ESOFEAYK.js.map +1 -0
- package/dist/chunk-ETYB5JO6.js +175 -0
- package/dist/chunk-ETYB5JO6.js.map +1 -0
- package/dist/chunk-EZR5AV3J.js +74 -0
- package/dist/chunk-EZR5AV3J.js.map +1 -0
- package/dist/chunk-F2OKLHGB.js +9 -0
- package/dist/chunk-F2OKLHGB.js.map +1 -0
- package/dist/chunk-F3E2RLC7.js +342 -0
- package/dist/chunk-F3E2RLC7.js.map +1 -0
- package/dist/chunk-FAPIZJS2.js +54 -0
- package/dist/chunk-FAPIZJS2.js.map +1 -0
- package/dist/chunk-GDJNOZ47.js +1 -0
- package/dist/chunk-GDJNOZ47.js.map +1 -0
- package/dist/chunk-GDY2Z2SZ.js +83 -0
- package/dist/chunk-GDY2Z2SZ.js.map +1 -0
- package/dist/chunk-GLMYZ6OA.js +20 -0
- package/dist/chunk-GLMYZ6OA.js.map +1 -0
- package/dist/{chunk-XEVOC433.js → chunk-GPVXCY5I.js} +2 -73
- package/dist/chunk-GPVXCY5I.js.map +1 -0
- package/dist/{chunk-KSKST3KP.js → chunk-H3H74RWJ.js} +1 -1
- package/dist/chunk-HAAGSIZH.js +89 -0
- package/dist/chunk-HAAGSIZH.js.map +1 -0
- package/dist/{chunk-K5L7R7XM.js → chunk-HHMLE3TZ.js} +6 -54
- package/dist/chunk-HHMLE3TZ.js.map +1 -0
- package/dist/{chunk-JLJMVRRE.js → chunk-HP34GRQX.js} +10 -83
- package/dist/chunk-HP34GRQX.js.map +1 -0
- package/dist/chunk-HTI343NP.js +1 -0
- package/dist/chunk-HTI343NP.js.map +1 -0
- package/dist/{chunk-T2C3CGDQ.js → chunk-HTI7DZJM.js} +6 -4
- package/dist/{chunk-T2C3CGDQ.js.map → chunk-HTI7DZJM.js.map} +1 -1
- package/dist/chunk-IDNL6IH4.js +294 -0
- package/dist/chunk-IDNL6IH4.js.map +1 -0
- package/dist/chunk-IV6YQTY2.js +47 -0
- package/dist/chunk-IV6YQTY2.js.map +1 -0
- package/dist/chunk-J4SINPUA.js +939 -0
- package/dist/chunk-J4SINPUA.js.map +1 -0
- package/dist/chunk-J6S6MT7O.js +32 -0
- package/dist/chunk-J6S6MT7O.js.map +1 -0
- package/dist/chunk-JB3RFK6K.js +168 -0
- package/dist/chunk-JB3RFK6K.js.map +1 -0
- package/dist/chunk-JBQNCUXA.js +34 -0
- package/dist/chunk-JBQNCUXA.js.map +1 -0
- package/dist/chunk-JMJFK3A4.js +1 -0
- package/dist/chunk-JMJFK3A4.js.map +1 -0
- package/dist/chunk-JMMZICU7.js +74 -0
- package/dist/chunk-JMMZICU7.js.map +1 -0
- package/dist/chunk-JOQ2T3QP.js +663 -0
- package/dist/chunk-JOQ2T3QP.js.map +1 -0
- package/dist/chunk-K4IBS7VB.js +55 -0
- package/dist/chunk-K4IBS7VB.js.map +1 -0
- package/dist/chunk-KBBQSP4M.js +16 -0
- package/dist/chunk-KBBQSP4M.js.map +1 -0
- package/dist/chunk-KFAVEL5F.js +68 -0
- package/dist/chunk-KFAVEL5F.js.map +1 -0
- package/dist/chunk-KITX3HDN.js +17 -0
- package/dist/chunk-KITX3HDN.js.map +1 -0
- package/dist/chunk-KO5NMTAM.js +1 -0
- package/dist/chunk-KO5NMTAM.js.map +1 -0
- package/dist/chunk-KY4VXIN2.js +118 -0
- package/dist/chunk-KY4VXIN2.js.map +1 -0
- package/dist/chunk-KZGSXJKU.js +29 -0
- package/dist/chunk-KZGSXJKU.js.map +1 -0
- package/dist/chunk-LAIATEUB.js +24 -0
- package/dist/chunk-LAIATEUB.js.map +1 -0
- package/dist/{chunk-OJBFDBLG.js → chunk-LBYJPMHC.js} +2 -2
- package/dist/chunk-LKIABZNR.js +49 -0
- package/dist/chunk-LKIABZNR.js.map +1 -0
- package/dist/chunk-LQJ4KHMY.js +23 -0
- package/dist/chunk-LQJ4KHMY.js.map +1 -0
- package/dist/chunk-MEN4AFDZ.js +1 -0
- package/dist/chunk-MEN4AFDZ.js.map +1 -0
- package/dist/chunk-MHOJJYDK.js +93 -0
- package/dist/chunk-MHOJJYDK.js.map +1 -0
- package/dist/chunk-MJCMKZFP.js +30 -0
- package/dist/chunk-MJCMKZFP.js.map +1 -0
- package/dist/chunk-MT3BLVYN.js +31 -0
- package/dist/chunk-MT3BLVYN.js.map +1 -0
- package/dist/chunk-NF6WBPFG.js +37 -0
- package/dist/chunk-NF6WBPFG.js.map +1 -0
- package/dist/chunk-NNEKBZSM.js +66 -0
- package/dist/chunk-NNEKBZSM.js.map +1 -0
- package/dist/chunk-NPJ2T3L5.js +19 -0
- package/dist/chunk-NPJ2T3L5.js.map +1 -0
- package/dist/chunk-NVALGVQG.js +150 -0
- package/dist/chunk-NVALGVQG.js.map +1 -0
- package/dist/chunk-OJF67RNM.js +1 -0
- package/dist/chunk-OJF67RNM.js.map +1 -0
- package/dist/chunk-PHLGGRHC.js +89 -0
- package/dist/chunk-PHLGGRHC.js.map +1 -0
- package/dist/chunk-PIMD4DYG.js +104 -0
- package/dist/chunk-PIMD4DYG.js.map +1 -0
- package/dist/chunk-PJNE75WC.js +1 -0
- package/dist/chunk-PJNE75WC.js.map +1 -0
- package/dist/{chunk-J6LVLV6P.js → chunk-PLDJCCVL.js} +1 -1
- package/dist/{chunk-3RWESPUG.js → chunk-POXHJF3H.js} +6 -51
- package/dist/chunk-POXHJF3H.js.map +1 -0
- package/dist/chunk-PPJ7SYQL.js +81 -0
- package/dist/chunk-PPJ7SYQL.js.map +1 -0
- package/dist/chunk-PT777EQ3.js +48 -0
- package/dist/chunk-PT777EQ3.js.map +1 -0
- package/dist/chunk-PYLPLFJZ.js +318 -0
- package/dist/chunk-PYLPLFJZ.js.map +1 -0
- package/dist/chunk-QEPBHHAP.js +41 -0
- package/dist/chunk-QEPBHHAP.js.map +1 -0
- package/dist/chunk-QHGYCYV6.js +91 -0
- package/dist/chunk-QHGYCYV6.js.map +1 -0
- package/dist/chunk-R6722CIU.js +28 -0
- package/dist/chunk-R6722CIU.js.map +1 -0
- package/dist/chunk-RA4YVN43.js +1 -0
- package/dist/chunk-RA4YVN43.js.map +1 -0
- package/dist/chunk-RT5LUEO3.js +2339 -0
- package/dist/chunk-RT5LUEO3.js.map +1 -0
- package/dist/chunk-RXKUIJ77.js +236 -0
- package/dist/chunk-RXKUIJ77.js.map +1 -0
- package/dist/chunk-SC4SUIEY.js +1 -0
- package/dist/chunk-SC4SUIEY.js.map +1 -0
- package/dist/chunk-SQAAVYJG.js +12 -0
- package/dist/chunk-SQAAVYJG.js.map +1 -0
- package/dist/chunk-SXE4J43E.js +59 -0
- package/dist/chunk-SXE4J43E.js.map +1 -0
- package/dist/{chunk-EFLT7XZD.js → chunk-SYULMZGL.js} +6 -4
- package/dist/{chunk-EFLT7XZD.js.map → chunk-SYULMZGL.js.map} +1 -1
- package/dist/chunk-T4KS577P.js +58 -0
- package/dist/chunk-T4KS577P.js.map +1 -0
- package/dist/chunk-TFIOU6UR.js +78 -0
- package/dist/chunk-TFIOU6UR.js.map +1 -0
- package/dist/chunk-TIZGADU4.js +375 -0
- package/dist/chunk-TIZGADU4.js.map +1 -0
- package/dist/chunk-TP3MCUXS.js +1 -0
- package/dist/chunk-TP3MCUXS.js.map +1 -0
- package/dist/chunk-TSPQD5HW.js +613 -0
- package/dist/chunk-TSPQD5HW.js.map +1 -0
- package/dist/{chunk-JTJU56ZV.js → chunk-UOGM62JL.js} +11 -7
- package/dist/{chunk-JTJU56ZV.js.map → chunk-UOGM62JL.js.map} +1 -1
- package/dist/chunk-USRKPEQN.js +55 -0
- package/dist/chunk-USRKPEQN.js.map +1 -0
- package/dist/chunk-UTAB3XQI.js +667 -0
- package/dist/chunk-UTAB3XQI.js.map +1 -0
- package/dist/chunk-V3FFZWF3.js +182 -0
- package/dist/chunk-V3FFZWF3.js.map +1 -0
- package/dist/chunk-V4OZ7ISA.js +9 -0
- package/dist/chunk-V4OZ7ISA.js.map +1 -0
- package/dist/chunk-VZ5ZZBNX.js +139 -0
- package/dist/chunk-VZ5ZZBNX.js.map +1 -0
- package/dist/{chunk-AHLKV6KP.js → chunk-WN6AVBO4.js} +1 -1
- package/dist/chunk-WXR5CWSC.js +1 -0
- package/dist/chunk-WXR5CWSC.js.map +1 -0
- package/dist/chunk-X6WS3XWL.js +36 -0
- package/dist/chunk-X6WS3XWL.js.map +1 -0
- package/dist/{chunk-Q5BCNYED.js → chunk-XJRNBTIV.js} +7 -4
- package/dist/chunk-XQV3XNVT.js +225 -0
- package/dist/chunk-XQV3XNVT.js.map +1 -0
- package/dist/chunk-XV3DH5GC.js +58 -0
- package/dist/chunk-XV3DH5GC.js.map +1 -0
- package/dist/chunk-YWRXOPHC.js +43 -0
- package/dist/chunk-YWRXOPHC.js.map +1 -0
- package/dist/chunk-YZVFIW5D.js +27 -0
- package/dist/chunk-YZVFIW5D.js.map +1 -0
- package/dist/chunk-ZHC5CD6J.js +33 -0
- package/dist/chunk-ZHC5CD6J.js.map +1 -0
- package/dist/chunk-ZPJFXRQZ.js +23 -0
- package/dist/chunk-ZPJFXRQZ.js.map +1 -0
- package/dist/chunk-ZPXOBQDS.js +57 -0
- package/dist/chunk-ZPXOBQDS.js.map +1 -0
- package/dist/chunk-ZUFS6R7L.js +155 -0
- package/dist/chunk-ZUFS6R7L.js.map +1 -0
- package/dist/chunk-ZXLCTQUX.js +62 -0
- package/dist/chunk-ZXLCTQUX.js.map +1 -0
- package/dist/chunk-ZYXMWEL4.js +59 -0
- package/dist/chunk-ZYXMWEL4.js.map +1 -0
- package/dist/completion/controls.js +11 -0
- package/dist/completion/controls.js.map +1 -0
- package/dist/completion/documents.js +9 -0
- package/dist/completion/documents.js.map +1 -0
- package/dist/completion/generate-completion.js +23 -0
- package/dist/completion/generate-completion.js.map +1 -0
- package/dist/completion/index.js +23 -12
- package/dist/completion/json.js +7 -0
- package/dist/completion/json.js.map +1 -0
- package/dist/completion/message-schema.js +18 -0
- package/dist/completion/message-schema.js.map +1 -0
- package/dist/completion/provider-output-error.js +12 -0
- package/dist/completion/provider-output-error.js.map +1 -0
- package/dist/completion/stream-accumulator.js +10 -0
- package/dist/completion/stream-accumulator.js.map +1 -0
- package/dist/completion/types.js +28 -0
- package/dist/completion/types.js.map +1 -0
- package/dist/documents/chunk-text.js +7 -0
- package/dist/documents/chunk-text.js.map +1 -0
- package/dist/documents/index.js +5 -2
- package/dist/documents/text-document.js +8 -0
- package/dist/documents/text-document.js.map +1 -0
- package/dist/embeddings/distance.js +17 -0
- package/dist/embeddings/distance.js.map +1 -0
- package/dist/embeddings/embed.js +18 -0
- package/dist/embeddings/embed.js.map +1 -0
- package/dist/embeddings/index.js +11 -6
- package/dist/embeddings/types.js +2 -0
- package/dist/embeddings/types.js.map +1 -0
- package/dist/evals/advanced-metrics.js +46 -0
- package/dist/evals/advanced-metrics.js.map +1 -0
- package/dist/evals/agent-target.js +10 -0
- package/dist/evals/agent-target.js.map +1 -0
- package/dist/evals/cli.js +40 -0
- package/dist/evals/cli.js.map +1 -0
- package/dist/evals/execution.js +15 -0
- package/dist/evals/execution.js.map +1 -0
- package/dist/evals/format.js +13 -0
- package/dist/evals/format.js.map +1 -0
- package/dist/evals/index.d.ts +4 -4
- package/dist/evals/index.js +87 -2660
- package/dist/evals/index.js.map +1 -1
- package/dist/evals/judge.js +27 -0
- package/dist/evals/judge.js.map +1 -0
- package/dist/evals/metric.js +7 -0
- package/dist/evals/metric.js.map +1 -0
- package/dist/evals/metrics.js +53 -0
- package/dist/evals/metrics.js.map +1 -0
- package/dist/evals/outcome.js +7 -0
- package/dist/evals/outcome.js.map +1 -0
- package/dist/evals/reporting.js +11 -0
- package/dist/evals/reporting.js.map +1 -0
- package/dist/evals/runner.js +29 -0
- package/dist/evals/runner.js.map +1 -0
- package/dist/evals/selectors.js +18 -0
- package/dist/evals/selectors.js.map +1 -0
- package/dist/evals/suite.js +11 -0
- package/dist/evals/suite.js.map +1 -0
- package/dist/evals/types.js +1 -0
- package/dist/evals/types.js.map +1 -0
- package/dist/extractor/extractor.js +23 -0
- package/dist/extractor/extractor.js.map +1 -0
- package/dist/extractor/index.js +16 -7
- package/dist/guardrails/actions.js +13 -0
- package/dist/guardrails/actions.js.map +1 -0
- package/dist/guardrails/index.d.ts +3 -4
- package/dist/guardrails/index.js +15 -7
- package/dist/guardrails/message.js +9 -0
- package/dist/guardrails/message.js.map +1 -0
- package/dist/guardrails/policy.js +17 -0
- package/dist/guardrails/policy.js.map +1 -0
- package/dist/guardrails/runtime.js +11 -0
- package/dist/guardrails/runtime.js.map +1 -0
- package/dist/guardrails/text.js +9 -0
- package/dist/guardrails/text.js.map +1 -0
- package/dist/guardrails/types.js +1 -0
- package/dist/guardrails/types.js.map +1 -0
- package/dist/hooks/control.js +17 -0
- package/dist/hooks/control.js.map +1 -0
- package/dist/hooks/index.js +19 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/types.js +2 -0
- package/dist/hooks/types.js.map +1 -0
- package/dist/image-generation/generate-image.js +11 -0
- package/dist/image-generation/generate-image.js.map +1 -0
- package/dist/image-generation/index.js +7 -4
- package/dist/image-generation/types.js +2 -0
- package/dist/image-generation/types.js.map +1 -0
- package/dist/index.d.ts +5 -7
- package/dist/index.js +163 -57
- package/dist/internal/abort.js +13 -0
- package/dist/internal/abort.js.map +1 -0
- package/dist/internal/agent-runtime/agent-run.js +81 -0
- package/dist/internal/agent-runtime/agent-run.js.map +1 -0
- package/dist/internal/agent-runtime/approval-request.js +1 -0
- package/dist/internal/agent-runtime/approval-request.js.map +1 -0
- package/dist/internal/agent-runtime/approval-requirement.js +9 -0
- package/dist/internal/agent-runtime/approval-requirement.js.map +1 -0
- package/dist/internal/agent-runtime/continuation-state.js +26 -0
- package/dist/internal/agent-runtime/continuation-state.js.map +1 -0
- package/dist/internal/agent-runtime/interaction-suspension.js +25 -0
- package/dist/internal/agent-runtime/interaction-suspension.js.map +1 -0
- package/dist/internal/agent-runtime/memory-scope.js +8 -0
- package/dist/internal/agent-runtime/memory-scope.js.map +1 -0
- package/dist/internal/agent-runtime/memory.js +24 -0
- package/dist/internal/agent-runtime/memory.js.map +1 -0
- package/dist/internal/agent-runtime/prepared-tool-call.js +19 -0
- package/dist/internal/agent-runtime/prepared-tool-call.js.map +1 -0
- package/dist/internal/agent-runtime/retrieval.js +35 -0
- package/dist/internal/agent-runtime/retrieval.js.map +1 -0
- package/dist/internal/agent-runtime/run-options.js +9 -0
- package/dist/internal/agent-runtime/run-options.js.map +1 -0
- package/dist/internal/agent-runtime/run-validation.js +9 -0
- package/dist/internal/agent-runtime/run-validation.js.map +1 -0
- package/dist/internal/agent-runtime/stream-events.js +11 -0
- package/dist/internal/agent-runtime/stream-events.js.map +1 -0
- package/dist/internal/agent-runtime/structured-output.js +13 -0
- package/dist/internal/agent-runtime/structured-output.js.map +1 -0
- package/dist/internal/agent-runtime/tool-execution.js +55 -0
- package/dist/internal/agent-runtime/tool-execution.js.map +1 -0
- package/dist/internal/agent.d.ts +5 -5
- package/dist/internal/agent.js +93 -57
- package/dist/internal/agent.js.map +1 -1
- package/dist/internal/async-queue.js +7 -0
- package/dist/internal/async-queue.js.map +1 -0
- package/dist/internal/completion-request.js +11 -0
- package/dist/internal/completion-request.js.map +1 -0
- package/dist/internal/concurrency.js +7 -0
- package/dist/internal/concurrency.js.map +1 -0
- package/dist/internal/json-object.js +8 -0
- package/dist/internal/json-object.js.map +1 -0
- package/dist/internal/rag-text.js +7 -0
- package/dist/internal/rag-text.js.map +1 -0
- package/dist/internal/record.js +7 -0
- package/dist/internal/record.js.map +1 -0
- package/dist/internal/team-runtime/coordination.js +12 -0
- package/dist/internal/team-runtime/coordination.js.map +1 -0
- package/dist/internal/team-runtime/member.js +26 -0
- package/dist/internal/team-runtime/member.js.map +1 -0
- package/dist/internal/team-runtime/policy.js +7 -0
- package/dist/internal/team-runtime/policy.js.map +1 -0
- package/dist/internal/team-runtime/run.js +95 -0
- package/dist/internal/team-runtime/run.js.map +1 -0
- package/dist/internal/team-runtime/stream.js +9 -0
- package/dist/internal/team-runtime/stream.js.map +1 -0
- package/dist/internal/team-runtime/tools.js +17 -0
- package/dist/internal/team-runtime/tools.js.map +1 -0
- package/dist/internal/type-utils.js +1 -0
- package/dist/internal/type-utils.js.map +1 -0
- package/dist/internal/vector-search-options.js +9 -0
- package/dist/internal/vector-search-options.js.map +1 -0
- package/dist/mcp/index.js +3 -1
- package/dist/mcp/tool.js +7 -0
- package/dist/mcp/tool.js.map +1 -0
- package/dist/mcp/types.js +2 -0
- package/dist/mcp/types.js.map +1 -0
- package/dist/memory/assert.js +11 -0
- package/dist/memory/assert.js.map +1 -0
- package/dist/memory/compaction.js +26 -0
- package/dist/memory/compaction.js.map +1 -0
- package/dist/memory/errors.js +9 -0
- package/dist/memory/errors.js.map +1 -0
- package/dist/memory/index.js +23 -10
- package/dist/memory/options.js +10 -0
- package/dist/memory/options.js.map +1 -0
- package/dist/memory/scope-key.js +7 -0
- package/dist/memory/scope-key.js.map +1 -0
- package/dist/memory/types.js +2 -0
- package/dist/memory/types.js.map +1 -0
- package/dist/{types-CRda8x7p.d.ts → middleware-CWjnMbiH.d.ts} +64 -5
- package/dist/model-call-options.js +1 -0
- package/dist/model-call-options.js.map +1 -0
- package/dist/model-listing/errors.js +7 -0
- package/dist/model-listing/errors.js.map +1 -0
- package/dist/model-listing/index.js +4 -13
- package/dist/model-listing/index.js.map +1 -1
- package/dist/model-listing/types.js +2 -0
- package/dist/model-listing/types.js.map +1 -0
- package/dist/observability/group.js +30 -0
- package/dist/observability/group.js.map +1 -0
- package/dist/observability/index.d.ts +4 -6
- package/dist/observability/index.js +16 -8
- package/dist/observability/snapshot.js +7 -0
- package/dist/observability/snapshot.js.map +1 -0
- package/dist/observability/types.js +1 -0
- package/dist/observability/types.js.map +1 -0
- package/dist/pipeline/errors.js +9 -0
- package/dist/pipeline/errors.js.map +1 -0
- package/dist/pipeline/graph.js +21 -0
- package/dist/pipeline/graph.js.map +1 -0
- package/dist/pipeline/index.d.ts +4 -6
- package/dist/pipeline/index.js +27 -805
- package/dist/pipeline/index.js.map +1 -1
- package/dist/pipeline/observability.js +15 -0
- package/dist/pipeline/observability.js.map +1 -0
- package/dist/pipeline/pipeline.js +30 -0
- package/dist/pipeline/pipeline.js.map +1 -0
- package/dist/pipeline/runtime.js +17 -0
- package/dist/pipeline/runtime.js.map +1 -0
- package/dist/pipeline/types.js +1 -0
- package/dist/pipeline/types.js.map +1 -0
- package/dist/retry.js +20 -0
- package/dist/retry.js.map +1 -0
- package/dist/schema/index.js +1 -0
- package/dist/schema/index.js.map +1 -0
- package/dist/schema/zod-schema.js +7 -0
- package/dist/schema/zod-schema.js.map +1 -0
- package/dist/skills/index.js +41 -12
- package/dist/skills/instructions.js +7 -0
- package/dist/skills/instructions.js.map +1 -0
- package/dist/skills/load.js +39 -0
- package/dist/skills/load.js.map +1 -0
- package/dist/skills/local.js +9 -0
- package/dist/skills/local.js.map +1 -0
- package/dist/skills/tools.js +37 -0
- package/dist/skills/tools.js.map +1 -0
- package/dist/skills/types.js +7 -0
- package/dist/skills/types.js.map +1 -0
- package/dist/speech-generation/generate-speech.js +11 -0
- package/dist/speech-generation/generate-speech.js.map +1 -0
- package/dist/speech-generation/index.js +7 -4
- package/dist/speech-generation/types.js +2 -0
- package/dist/speech-generation/types.js.map +1 -0
- package/dist/streaming/index.js +3 -37
- package/dist/streaming/index.js.map +1 -1
- package/dist/streaming/readable-stream.js +7 -0
- package/dist/streaming/readable-stream.js.map +1 -0
- package/dist/{text-CmjXxakS.d.ts → text-GbW4WYMg.d.ts} +1 -1
- package/dist/tool/create-tool.js +14 -0
- package/dist/tool/create-tool.js.map +1 -0
- package/dist/tool/dynamic-tools.js +35 -0
- package/dist/tool/dynamic-tools.js.map +1 -0
- package/dist/tool/errors.js +11 -0
- package/dist/tool/errors.js.map +1 -0
- package/dist/tool/index.d.ts +3 -2
- package/dist/tool/index.js +41 -15
- package/dist/tool/middleware.js +7 -0
- package/dist/tool/middleware.js.map +1 -0
- package/dist/tool/question-tool.js +17 -0
- package/dist/tool/question-tool.js.map +1 -0
- package/dist/tool/skill-tool-marker.js +9 -0
- package/dist/tool/skill-tool-marker.js.map +1 -0
- package/dist/tool/think-tool.js +15 -0
- package/dist/tool/think-tool.js.map +1 -0
- package/dist/tool/tool.js +17 -0
- package/dist/tool/tool.js.map +1 -0
- package/dist/transcription/index.js +7 -4
- package/dist/transcription/transcribe.js +11 -0
- package/dist/transcription/transcribe.js.map +1 -0
- package/dist/transcription/types.js +2 -0
- package/dist/transcription/types.js.map +1 -0
- package/dist/{types-h0EL6Xn0.d.ts → types-Bqw0AuFl.d.ts} +6 -2
- package/dist/vector-store/filter.js +9 -0
- package/dist/vector-store/filter.js.map +1 -0
- package/dist/vector-store/index.js +31 -11
- package/dist/vector-store/ingest.js +19 -0
- package/dist/vector-store/ingest.js.map +1 -0
- package/dist/vector-store/lsh.js +7 -0
- package/dist/vector-store/lsh.js.map +1 -0
- package/dist/vector-store/retrieve.js +15 -0
- package/dist/vector-store/retrieve.js.map +1 -0
- package/dist/vector-store/types.js +1 -0
- package/dist/vector-store/types.js.map +1 -0
- package/package.json +8 -5
- package/dist/chunk-2QZJTG2E.js.map +0 -1
- package/dist/chunk-3RWESPUG.js.map +0 -1
- package/dist/chunk-4UQALF5Y.js.map +0 -1
- package/dist/chunk-AZB6N7P4.js +0 -320
- package/dist/chunk-AZB6N7P4.js.map +0 -1
- package/dist/chunk-JLJMVRRE.js.map +0 -1
- package/dist/chunk-K5L7R7XM.js.map +0 -1
- package/dist/chunk-OI3LSMJG.js +0 -1545
- package/dist/chunk-OI3LSMJG.js.map +0 -1
- package/dist/chunk-QGX73TSQ.js.map +0 -1
- package/dist/chunk-XC3LVC4K.js +0 -753
- package/dist/chunk-XC3LVC4K.js.map +0 -1
- package/dist/chunk-XEVOC433.js.map +0 -1
- package/dist/chunk-XSV5C4R5.js +0 -400
- package/dist/chunk-XSV5C4R5.js.map +0 -1
- package/dist/chunk-XU54PLSD.js +0 -4786
- package/dist/chunk-XU54PLSD.js.map +0 -1
- package/dist/middleware-mY_lpjDp.d.ts +0 -59
- package/dist/type-utils-CtHVDRn_.d.ts +0 -6
- /package/dist/{chunk-YK4WAAS4.js.map → chunk-2NLINXRF.js.map} +0 -0
- /package/dist/{chunk-SPD4XVR7.js.map → chunk-2XSMGVBG.js.map} +0 -0
- /package/dist/{chunk-3XQGVDU5.js.map → chunk-4C7AXYOM.js.map} +0 -0
- /package/dist/{chunk-IZNOP6JG.js.map → chunk-4FIVU53H.js.map} +0 -0
- /package/dist/{chunk-7JLAIN6E.js.map → chunk-4ONUNWGC.js.map} +0 -0
- /package/dist/{chunk-KSKST3KP.js.map → chunk-H3H74RWJ.js.map} +0 -0
- /package/dist/{chunk-OJBFDBLG.js.map → chunk-LBYJPMHC.js.map} +0 -0
- /package/dist/{chunk-J6LVLV6P.js.map → chunk-PLDJCCVL.js.map} +0 -0
- /package/dist/{chunk-AHLKV6KP.js.map → chunk-WN6AVBO4.js.map} +0 -0
- /package/dist/{chunk-Q5BCNYED.js.map → chunk-XJRNBTIV.js.map} +0 -0
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
import {
|
|
2
|
+
resolveActual,
|
|
3
|
+
resolveActualText,
|
|
4
|
+
resolveExpected,
|
|
5
|
+
resolveJudgePrompt
|
|
6
|
+
} from "./chunk-ZPXOBQDS.js";
|
|
7
|
+
import {
|
|
8
|
+
evalValuesEqual,
|
|
9
|
+
formatValue
|
|
10
|
+
} from "./chunk-ZYXMWEL4.js";
|
|
11
|
+
import {
|
|
12
|
+
EvalOutcome
|
|
13
|
+
} from "./chunk-EJR6VVB3.js";
|
|
14
|
+
import {
|
|
15
|
+
extract
|
|
16
|
+
} from "./chunk-UOGM62JL.js";
|
|
17
|
+
import {
|
|
18
|
+
cosineSimilarity
|
|
19
|
+
} from "./chunk-7F53DOML.js";
|
|
20
|
+
import {
|
|
21
|
+
embedText
|
|
22
|
+
} from "./chunk-HHMLE3TZ.js";
|
|
23
|
+
|
|
24
|
+
// src/evals/metrics.ts
|
|
25
|
+
import { z } from "zod";
|
|
26
|
+
function exactMatch(options = {}) {
|
|
27
|
+
return {
|
|
28
|
+
name: options.name ?? "exact_match",
|
|
29
|
+
required: options.required ?? true,
|
|
30
|
+
dataType: "BOOLEAN",
|
|
31
|
+
direction: "higher_is_better",
|
|
32
|
+
threshold: 1,
|
|
33
|
+
async evaluate(args) {
|
|
34
|
+
const actual = await resolveActual(options.actual, args);
|
|
35
|
+
const expected = await resolveExpected(options.expected, args);
|
|
36
|
+
if (expected === void 0) {
|
|
37
|
+
return EvalOutcome.invalid("No expected value provided for exact match.");
|
|
38
|
+
}
|
|
39
|
+
const passed = evalValuesEqual(actual, expected);
|
|
40
|
+
return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function contains(options = {}) {
|
|
45
|
+
return {
|
|
46
|
+
name: options.name ?? "contains",
|
|
47
|
+
required: options.required ?? true,
|
|
48
|
+
dataType: "BOOLEAN",
|
|
49
|
+
direction: "higher_is_better",
|
|
50
|
+
threshold: 1,
|
|
51
|
+
async evaluate(args) {
|
|
52
|
+
const actual = await resolveActualText(options.actual, args);
|
|
53
|
+
const expected = await resolveExpected(options.expected, args);
|
|
54
|
+
if (expected === void 0) {
|
|
55
|
+
return EvalOutcome.invalid("No expected value provided for contains.");
|
|
56
|
+
}
|
|
57
|
+
if (typeof expected !== "string" && !(expected instanceof RegExp)) {
|
|
58
|
+
return EvalOutcome.invalid("Contains expected value must be a string or RegExp.");
|
|
59
|
+
}
|
|
60
|
+
const passed = expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);
|
|
61
|
+
return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function notContains(options = {}) {
|
|
66
|
+
return {
|
|
67
|
+
name: options.name ?? "not_contains",
|
|
68
|
+
required: options.required ?? true,
|
|
69
|
+
dataType: "BOOLEAN",
|
|
70
|
+
direction: "higher_is_better",
|
|
71
|
+
threshold: 1,
|
|
72
|
+
async evaluate(args) {
|
|
73
|
+
const actual = await resolveActualText(options.actual, args);
|
|
74
|
+
const expected = await resolveExpected(options.expected, args);
|
|
75
|
+
if (expected === void 0) {
|
|
76
|
+
return EvalOutcome.invalid("No expected value provided for notContains.");
|
|
77
|
+
}
|
|
78
|
+
if (typeof expected !== "string" && !(expected instanceof RegExp)) {
|
|
79
|
+
return EvalOutcome.invalid("notContains expected value must be a string or RegExp.");
|
|
80
|
+
}
|
|
81
|
+
const found = textExpectationMatches(expected, actual);
|
|
82
|
+
return found ? EvalOutcome.fail(false, {
|
|
83
|
+
comment: `Output contained forbidden value ${String(expected)}.`
|
|
84
|
+
}) : EvalOutcome.pass(true);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
function containsAll(options) {
|
|
89
|
+
return containsListMetric("contains_all", "all", options);
|
|
90
|
+
}
|
|
91
|
+
function containsAny(options) {
|
|
92
|
+
return containsListMetric("contains_any", "any", options);
|
|
93
|
+
}
|
|
94
|
+
function containsListMetric(defaultName, mode, options) {
|
|
95
|
+
return {
|
|
96
|
+
name: options.name ?? defaultName,
|
|
97
|
+
required: options.required ?? true,
|
|
98
|
+
dataType: "BOOLEAN",
|
|
99
|
+
direction: "higher_is_better",
|
|
100
|
+
threshold: 1,
|
|
101
|
+
async evaluate(args) {
|
|
102
|
+
const actual = await resolveActualText(options.actual, args);
|
|
103
|
+
const expected = await resolveExpected(options.expected, args);
|
|
104
|
+
if (!Array.isArray(expected) || expected.length === 0) {
|
|
105
|
+
return EvalOutcome.invalid(`${defaultName} expected value must be a non-empty array.`);
|
|
106
|
+
}
|
|
107
|
+
if (expected.some((value) => typeof value !== "string" && !(value instanceof RegExp))) {
|
|
108
|
+
return EvalOutcome.invalid(`${defaultName} expected values must be strings or RegExp.`);
|
|
109
|
+
}
|
|
110
|
+
const matches2 = expected.map((value) => textExpectationMatches(value, actual));
|
|
111
|
+
const passed = mode === "all" ? matches2.every(Boolean) : matches2.some(Boolean);
|
|
112
|
+
if (passed) return EvalOutcome.pass(true);
|
|
113
|
+
const missing = expected.filter((_, index) => !matches2[index]).map(String);
|
|
114
|
+
const comment = mode === "all" ? `Output was missing: ${missing.join(", ")}.` : `Output matched none of: ${expected.map(String).join(", ")}.`;
|
|
115
|
+
return EvalOutcome.fail(false, { comment });
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function matches(options) {
|
|
120
|
+
return regexMetric("matches", false, options);
|
|
121
|
+
}
|
|
122
|
+
function doesNotMatch(options) {
|
|
123
|
+
return regexMetric("does_not_match", true, options);
|
|
124
|
+
}
|
|
125
|
+
function regexMetric(defaultName, negate, options) {
|
|
126
|
+
return {
|
|
127
|
+
name: options.name ?? defaultName,
|
|
128
|
+
required: options.required ?? true,
|
|
129
|
+
dataType: "BOOLEAN",
|
|
130
|
+
direction: "higher_is_better",
|
|
131
|
+
threshold: 1,
|
|
132
|
+
async evaluate(args) {
|
|
133
|
+
const actual = await resolveActualText(options.actual, args);
|
|
134
|
+
const expected = await resolveExpected(options.expected, args);
|
|
135
|
+
if (!(expected instanceof RegExp)) {
|
|
136
|
+
return EvalOutcome.invalid(`${defaultName} expected value must be a RegExp.`);
|
|
137
|
+
}
|
|
138
|
+
const matched = regexMatches(expected, actual);
|
|
139
|
+
const passed = negate ? !matched : matched;
|
|
140
|
+
return passed ? EvalOutcome.pass(true) : EvalOutcome.fail(false, {
|
|
141
|
+
comment: negate ? `Output matched forbidden pattern ${String(expected)}.` : `Output did not match ${String(expected)}.`
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
function maxLength(options) {
|
|
147
|
+
return {
|
|
148
|
+
name: options.name ?? "max_length",
|
|
149
|
+
required: options.required ?? true,
|
|
150
|
+
dataType: "BOOLEAN",
|
|
151
|
+
direction: "higher_is_better",
|
|
152
|
+
threshold: 1,
|
|
153
|
+
async evaluate(args) {
|
|
154
|
+
const actual = await resolveActualText(options.actual, args);
|
|
155
|
+
const max = await resolveOption(options.max, args);
|
|
156
|
+
if (!Number.isInteger(max) || max < 0) {
|
|
157
|
+
return EvalOutcome.invalid("maxLength max must be a non-negative integer.");
|
|
158
|
+
}
|
|
159
|
+
const length = Array.from(actual).length;
|
|
160
|
+
return length <= max ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Output length ${length} exceeded maximum ${max}.` });
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
function requiredFields(options) {
|
|
165
|
+
return {
|
|
166
|
+
name: options.name ?? "required_fields",
|
|
167
|
+
required: options.required ?? true,
|
|
168
|
+
dataType: "BOOLEAN",
|
|
169
|
+
direction: "higher_is_better",
|
|
170
|
+
threshold: 1,
|
|
171
|
+
async evaluate(args) {
|
|
172
|
+
const actual = await resolveActual(options.actual, args);
|
|
173
|
+
const expected = await resolveOption(options.expected, args);
|
|
174
|
+
if (!Array.isArray(expected) || expected.length === 0 || expected.some((field) => typeof field !== "string" || field.length === 0)) {
|
|
175
|
+
return EvalOutcome.invalid(
|
|
176
|
+
"requiredFields expected value must be a non-empty string array."
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
if (typeof actual !== "object" || actual === null || Array.isArray(actual)) {
|
|
180
|
+
return EvalOutcome.invalid("requiredFields actual value must be an object.");
|
|
181
|
+
}
|
|
182
|
+
const missing = expected.filter((field) => !Object.hasOwn(actual, field));
|
|
183
|
+
return missing.length === 0 ? EvalOutcome.pass(true) : EvalOutcome.fail(false, { comment: `Missing required fields: ${missing.join(", ")}.` });
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
function regexMatches(pattern, text) {
|
|
188
|
+
pattern.lastIndex = 0;
|
|
189
|
+
const matched = pattern.test(text);
|
|
190
|
+
pattern.lastIndex = 0;
|
|
191
|
+
return matched;
|
|
192
|
+
}
|
|
193
|
+
function textExpectationMatches(expected, actual) {
|
|
194
|
+
return expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);
|
|
195
|
+
}
|
|
196
|
+
async function resolveOption(value, args) {
|
|
197
|
+
return typeof value === "function" ? value(args) : value;
|
|
198
|
+
}
|
|
199
|
+
function semanticSimilarity(options) {
|
|
200
|
+
const threshold = unitInterval(options.threshold, "semanticSimilarity threshold");
|
|
201
|
+
return {
|
|
202
|
+
name: options.name ?? "semantic_similarity",
|
|
203
|
+
required: options.required ?? true,
|
|
204
|
+
dataType: "NUMERIC",
|
|
205
|
+
direction: "higher_is_better",
|
|
206
|
+
threshold,
|
|
207
|
+
async evaluate(args) {
|
|
208
|
+
const actual = await resolveActualText(options.actual, args);
|
|
209
|
+
const expected = await resolveExpected(options.expected, args);
|
|
210
|
+
if (expected === void 0) {
|
|
211
|
+
return EvalOutcome.invalid("No expected value provided for semantic similarity.");
|
|
212
|
+
}
|
|
213
|
+
if (typeof expected !== "string") {
|
|
214
|
+
return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
|
|
215
|
+
}
|
|
216
|
+
const [{ embedding: actualEmbedding }, { embedding: expectedEmbedding }] = await Promise.all([
|
|
217
|
+
embedText({ model: options.model, text: actual }),
|
|
218
|
+
embedText({ model: options.model, text: expected })
|
|
219
|
+
]);
|
|
220
|
+
const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
|
|
221
|
+
return score >= threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${threshold}.` });
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
function llmJudge(options) {
|
|
226
|
+
const retries = extractionRetries(options.retries);
|
|
227
|
+
return {
|
|
228
|
+
name: options.name ?? "llm_judge",
|
|
229
|
+
required: options.required ?? true,
|
|
230
|
+
async evaluate(args) {
|
|
231
|
+
try {
|
|
232
|
+
const result = await extract({
|
|
233
|
+
model: options.model,
|
|
234
|
+
outputSchema: options.schema,
|
|
235
|
+
instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema.",
|
|
236
|
+
text: await resolveJudgePrompt(options.prompt, args),
|
|
237
|
+
retries
|
|
238
|
+
});
|
|
239
|
+
return options.passes(result.output) ? EvalOutcome.pass(result.output, { usage: result.usage }) : EvalOutcome.fail(result.output, { usage: result.usage });
|
|
240
|
+
} catch (error) {
|
|
241
|
+
return EvalOutcome.fromError(error);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function llmScore(options) {
|
|
247
|
+
const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
|
|
248
|
+
if (criteria.trim().length === 0) throw new TypeError("llmScore criteria must not be empty.");
|
|
249
|
+
const threshold = unitInterval(options.threshold, "llmScore threshold");
|
|
250
|
+
const retries = extractionRetries(options.retries);
|
|
251
|
+
return {
|
|
252
|
+
name: options.name ?? "llm_score",
|
|
253
|
+
required: options.required ?? true,
|
|
254
|
+
dataType: "NUMERIC",
|
|
255
|
+
projectScore: (score) => score.score,
|
|
256
|
+
direction: "higher_is_better",
|
|
257
|
+
threshold,
|
|
258
|
+
async evaluate(args) {
|
|
259
|
+
try {
|
|
260
|
+
const result = await extract({
|
|
261
|
+
model: options.model,
|
|
262
|
+
outputSchema: z.object({
|
|
263
|
+
score: z.number(),
|
|
264
|
+
feedback: z.string()
|
|
265
|
+
}),
|
|
266
|
+
instructions: options.instructions ?? `Score the eval case against these criteria:
|
|
267
|
+
${criteria}
|
|
268
|
+
|
|
269
|
+
Return a score between 0 and 1 and brief feedback.`,
|
|
270
|
+
text: await resolveJudgePrompt(options.prompt, args),
|
|
271
|
+
retries
|
|
272
|
+
});
|
|
273
|
+
const score = result.output;
|
|
274
|
+
if (score.score < 0 || score.score > 1) {
|
|
275
|
+
return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
|
|
276
|
+
score,
|
|
277
|
+
usage: result.usage
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
return score.score >= threshold ? EvalOutcome.pass(score, { comment: score.feedback, usage: result.usage }) : EvalOutcome.fail(score, { comment: score.feedback, usage: result.usage });
|
|
281
|
+
} catch (error) {
|
|
282
|
+
return EvalOutcome.fromError(error);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
function extractionRetries(retries) {
|
|
288
|
+
const retryCount = retries ?? 0;
|
|
289
|
+
if (!Number.isInteger(retryCount) || retryCount < 0) {
|
|
290
|
+
throw new RangeError("Eval metric retries must be a non-negative integer.");
|
|
291
|
+
}
|
|
292
|
+
if (retryCount === 0) {
|
|
293
|
+
return void 0;
|
|
294
|
+
}
|
|
295
|
+
return { maxAttempts: retryCount + 1 };
|
|
296
|
+
}
|
|
297
|
+
function unitInterval(value, label) {
|
|
298
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) {
|
|
299
|
+
throw new RangeError(`${label} must be between 0 and 1.`);
|
|
300
|
+
}
|
|
301
|
+
return value;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export {
|
|
305
|
+
exactMatch,
|
|
306
|
+
contains,
|
|
307
|
+
notContains,
|
|
308
|
+
containsAll,
|
|
309
|
+
containsAny,
|
|
310
|
+
matches,
|
|
311
|
+
doesNotMatch,
|
|
312
|
+
maxLength,
|
|
313
|
+
requiredFields,
|
|
314
|
+
semanticSimilarity,
|
|
315
|
+
llmJudge,
|
|
316
|
+
llmScore
|
|
317
|
+
};
|
|
318
|
+
//# sourceMappingURL=chunk-PYLPLFJZ.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/evals/metrics.ts"],"sourcesContent":["import { z } from \"zod\";\nimport type { CompletionModel } from \"../completion\";\nimport { cosineSimilarity, type EmbeddingModel, embedText } from \"../embeddings\";\nimport { extract } from \"../extractor\";\nimport type { ZodSchema } from \"../schema\";\nimport { evalValuesEqual, formatValue } from \"./format\";\nimport { EvalOutcome } from \"./outcome\";\nimport { resolveActual, resolveActualText, resolveExpected, resolveJudgePrompt } from \"./selectors\";\nimport type { EvalCaseRequirements, EvalMetric, SelectorOrValue, ValueSelector } from \"./types\";\n\nexport type ExactMatchOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n expected?: SelectorOrValue<Input, Output, Expected, unknown> | undefined;\n};\n\nexport function exactMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ExactMatchOptions<Input, Output, Expected> & {\n expected: Exclude<ExactMatchOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function exactMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options?: Omit<ExactMatchOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: unknown }>;\nexport function exactMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ExactMatchOptions<Input, Output, Expected> & { name?: Name | undefined } = {},\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return {\n name: (options.name ?? \"exact_match\") as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActual(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (expected === undefined) {\n return EvalOutcome.invalid(\"No expected value provided for exact match.\");\n }\n const passed = evalValuesEqual(actual, expected);\n return passed\n ? EvalOutcome.pass(true)\n : EvalOutcome.fail(false, { comment: `Expected ${formatValue(expected)}.` });\n },\n };\n}\n\nexport type ContainsOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n expected?: SelectorOrValue<Input, Output, Expected, string | RegExp> | undefined;\n};\n\nexport function contains<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsOptions<Input, Output, Expected> & {\n expected: Exclude<ContainsOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function contains<Input, Output, Expected = unknown, const Name extends string = string>(\n options?: Omit<ContainsOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: string | RegExp }>;\nexport function contains<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsOptions<Input, Output, Expected> & { name?: Name | undefined } = {},\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return {\n name: (options.name ?? \"contains\") as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (expected === undefined) {\n return EvalOutcome.invalid(\"No expected value provided for contains.\");\n }\n if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n return EvalOutcome.invalid(\"Contains expected value must be a string or RegExp.\");\n }\n const passed =\n expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);\n return passed\n ? EvalOutcome.pass(true)\n : EvalOutcome.fail(false, { comment: `Output did not contain ${String(expected)}.` });\n },\n };\n}\n\nexport type NotContainsOptions<Input, Output, Expected = unknown> = ContainsOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function notContains<Input, Output, Expected = unknown, const Name extends string = string>(\n options: NotContainsOptions<Input, Output, Expected> & {\n expected: Exclude<NotContainsOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function notContains<Input, Output, Expected = unknown, const Name extends string = string>(\n options?: Omit<NotContainsOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: string | RegExp }>;\nexport function notContains<Input, Output, Expected = unknown, const Name extends string = string>(\n options: NotContainsOptions<Input, Output, Expected> & { name?: Name | undefined } = {},\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return {\n name: (options.name ?? \"not_contains\") as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (expected === undefined) {\n return EvalOutcome.invalid(\"No expected value provided for notContains.\");\n }\n if (typeof expected !== \"string\" && !(expected instanceof RegExp)) {\n return EvalOutcome.invalid(\"notContains expected value must be a string or RegExp.\");\n }\n const found = textExpectationMatches(expected, actual);\n return found\n ? EvalOutcome.fail(false, {\n comment: `Output contained forbidden value ${String(expected)}.`,\n })\n : EvalOutcome.pass(true);\n },\n };\n}\n\nexport type ContainsListOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n expected?: SelectorOrValue<Input, Output, Expected, ReadonlyArray<string | RegExp>> | undefined;\n};\n\nexport type ContainsAllOptions<Input, Output, Expected = unknown> = ContainsListOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsAllOptions<Input, Output, Expected> & {\n expected: Exclude<ContainsAllOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(\n options: Omit<ContainsAllOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: ReadonlyArray<string | RegExp> }>;\nexport function containsAll<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsAllOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return containsListMetric(\"contains_all\", \"all\", options);\n}\n\nexport type ContainsAnyOptions<Input, Output, Expected = unknown> = ContainsListOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsAnyOptions<Input, Output, Expected> & {\n expected: Exclude<ContainsAnyOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(\n options: Omit<ContainsAnyOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: ReadonlyArray<string | RegExp> }>;\nexport function containsAny<Input, Output, Expected = unknown, const Name extends string = string>(\n options: ContainsAnyOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return containsListMetric(\"contains_any\", \"any\", options);\n}\n\nfunction containsListMetric<Input, Output, Expected, const Name extends string>(\n defaultName: string,\n mode: \"all\" | \"any\",\n options: ContainsListOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name> {\n return {\n name: (options.name ?? defaultName) as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (!Array.isArray(expected) || expected.length === 0) {\n return EvalOutcome.invalid(`${defaultName} expected value must be a non-empty array.`);\n }\n if (expected.some((value) => typeof value !== \"string\" && !(value instanceof RegExp))) {\n return EvalOutcome.invalid(`${defaultName} expected values must be strings or RegExp.`);\n }\n const matches = expected.map((value) => textExpectationMatches(value, actual));\n const passed = mode === \"all\" ? matches.every(Boolean) : matches.some(Boolean);\n if (passed) return EvalOutcome.pass(true);\n const missing = expected.filter((_, index) => !matches[index]).map(String);\n const comment =\n mode === \"all\"\n ? `Output was missing: ${missing.join(\", \")}.`\n : `Output matched none of: ${expected.map(String).join(\", \")}.`;\n return EvalOutcome.fail(false, { comment });\n },\n };\n}\n\nexport type MatchesOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n expected?: SelectorOrValue<Input, Output, Expected, RegExp> | undefined;\n};\n\nexport function matches<Input, Output, Expected = unknown, const Name extends string = string>(\n options: MatchesOptions<Input, Output, Expected> & {\n expected: Exclude<MatchesOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function matches<Input, Output, Expected = unknown, const Name extends string = string>(\n options: Omit<MatchesOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: RegExp }>;\nexport function matches<Input, Output, Expected = unknown, const Name extends string = string>(\n options: MatchesOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return regexMetric(\"matches\", false, options);\n}\n\nexport type DoesNotMatchOptions<Input, Output, Expected = unknown> = MatchesOptions<\n Input,\n Output,\n Expected\n>;\n\nexport function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options: DoesNotMatchOptions<Input, Output, Expected> & {\n expected: Exclude<DoesNotMatchOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name>;\nexport function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options: Omit<DoesNotMatchOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, boolean, Expected, Name, { expected: RegExp }>;\nexport function doesNotMatch<Input, Output, Expected = unknown, const Name extends string = string>(\n options: DoesNotMatchOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name, EvalCaseRequirements> {\n return regexMetric(\"does_not_match\", true, options);\n}\n\nfunction regexMetric<Input, Output, Expected, const Name extends string>(\n defaultName: string,\n negate: boolean,\n options: MatchesOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name> {\n return {\n name: (options.name ?? defaultName) as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (!(expected instanceof RegExp)) {\n return EvalOutcome.invalid(`${defaultName} expected value must be a RegExp.`);\n }\n const matched = regexMatches(expected, actual);\n const passed = negate ? !matched : matched;\n return passed\n ? EvalOutcome.pass(true)\n : EvalOutcome.fail(false, {\n comment: negate\n ? `Output matched forbidden pattern ${String(expected)}.`\n : `Output did not match ${String(expected)}.`,\n });\n },\n };\n}\n\nexport type MaxLengthOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n max: SelectorOrValue<Input, Output, Expected, number>;\n};\n\nexport function maxLength<Input, Output, Expected = unknown, const Name extends string = string>(\n options: MaxLengthOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name> {\n return {\n name: (options.name ?? \"max_length\") as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const max = await resolveOption(options.max, args);\n if (!Number.isInteger(max) || max < 0) {\n return EvalOutcome.invalid(\"maxLength max must be a non-negative integer.\");\n }\n const length = Array.from(actual).length;\n return length <= max\n ? EvalOutcome.pass(true)\n : EvalOutcome.fail(false, { comment: `Output length ${length} exceeded maximum ${max}.` });\n },\n };\n}\n\nexport type RequiredFieldsOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n actual?: ValueSelector<Input, Output, Expected, unknown> | undefined;\n expected: SelectorOrValue<Input, Output, Expected, readonly string[]>;\n};\n\nexport function requiredFields<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: RequiredFieldsOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, boolean, Expected, Name> {\n return {\n name: (options.name ?? \"required_fields\") as Name,\n required: options.required ?? true,\n dataType: \"BOOLEAN\",\n direction: \"higher_is_better\",\n threshold: 1,\n async evaluate(args) {\n const actual = await resolveActual(options.actual, args);\n const expected = await resolveOption(options.expected, args);\n if (\n !Array.isArray(expected) ||\n expected.length === 0 ||\n expected.some((field) => typeof field !== \"string\" || field.length === 0)\n ) {\n return EvalOutcome.invalid(\n \"requiredFields expected value must be a non-empty string array.\",\n );\n }\n if (typeof actual !== \"object\" || actual === null || Array.isArray(actual)) {\n return EvalOutcome.invalid(\"requiredFields actual value must be an object.\");\n }\n const missing = expected.filter((field) => !Object.hasOwn(actual, field));\n return missing.length === 0\n ? EvalOutcome.pass(true)\n : EvalOutcome.fail(false, { comment: `Missing required fields: ${missing.join(\", \")}.` });\n },\n };\n}\n\nfunction regexMatches(pattern: RegExp, text: string): boolean {\n pattern.lastIndex = 0;\n const matched = pattern.test(text);\n pattern.lastIndex = 0;\n return matched;\n}\n\nfunction textExpectationMatches(expected: string | RegExp, actual: string): boolean {\n return expected instanceof RegExp ? regexMatches(expected, actual) : actual.includes(expected);\n}\n\nasync function resolveOption<Input, Output, Expected, Value>(\n value: SelectorOrValue<Input, Output, Expected, Value>,\n args: Parameters<ValueSelector<Input, Output, Expected, Value>>[0],\n): Promise<Value> {\n return typeof value === \"function\"\n ? (value as ValueSelector<Input, Output, Expected, Value>)(args)\n : value;\n}\n\nexport type SemanticSimilarityOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: EmbeddingModel;\n threshold: number;\n actual?: ValueSelector<Input, Output, Expected, string> | undefined;\n expected?: SelectorOrValue<Input, Output, Expected, string> | undefined;\n};\n\nexport function semanticSimilarity<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: SemanticSimilarityOptions<Input, Output, Expected> & {\n expected: Exclude<SemanticSimilarityOptions<Input, Output, Expected>[\"expected\"], undefined>;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name>;\nexport function semanticSimilarity<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: Omit<SemanticSimilarityOptions<Input, Output, Expected>, \"expected\"> & {\n expected?: undefined;\n name?: Name | undefined;\n },\n): EvalMetric<Input, Output, number, Expected, Name, { expected: string }>;\nexport function semanticSimilarity<\n Input,\n Output,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: SemanticSimilarityOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, number, Expected, Name, EvalCaseRequirements> {\n const threshold = unitInterval(options.threshold, \"semanticSimilarity threshold\");\n return {\n name: (options.name ?? \"semantic_similarity\") as Name,\n required: options.required ?? true,\n dataType: \"NUMERIC\",\n direction: \"higher_is_better\",\n threshold,\n async evaluate(args) {\n const actual = await resolveActualText(options.actual, args);\n const expected = await resolveExpected(options.expected, args);\n if (expected === undefined) {\n return EvalOutcome.invalid(\"No expected value provided for semantic similarity.\");\n }\n if (typeof expected !== \"string\") {\n return EvalOutcome.invalid(\"Semantic similarity expected value must be a string.\");\n }\n const [{ embedding: actualEmbedding }, { embedding: expectedEmbedding }] = await Promise.all([\n embedText({ model: options.model, text: actual }),\n embedText({ model: options.model, text: expected }),\n ]);\n const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);\n return score >= threshold\n ? EvalOutcome.pass(score)\n : EvalOutcome.fail(score, { comment: `Similarity below threshold ${threshold}.` });\n },\n };\n}\n\nexport type LlmJudgeOptions<Input, Output, SchemaOutput, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: CompletionModel;\n schema: ZodSchema<SchemaOutput>;\n passes(value: SchemaOutput): boolean;\n instructions?: string | undefined;\n retries?: number | undefined;\n prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmJudge<\n Input,\n Output,\n SchemaOutput,\n Expected = unknown,\n const Name extends string = string,\n>(\n options: LlmJudgeOptions<Input, Output, SchemaOutput, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, SchemaOutput, Expected, Name> {\n const retries = extractionRetries(options.retries);\n return {\n name: (options.name ?? \"llm_judge\") as Name,\n required: options.required ?? true,\n async evaluate(args) {\n try {\n const result = await extract({\n model: options.model,\n outputSchema: options.schema,\n instructions:\n options.instructions ??\n \"Judge the eval case by the requested schema. Submit the judgment using the schema.\",\n text: await resolveJudgePrompt(options.prompt, args),\n retries,\n });\n return options.passes(result.output)\n ? EvalOutcome.pass(result.output, { usage: result.usage })\n : EvalOutcome.fail(result.output, { usage: result.usage });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n },\n };\n}\n\nexport type LlmScoreMetricScore = {\n score: number;\n feedback: string;\n};\n\nexport type LlmScoreOptions<Input, Output, Expected = unknown> = {\n name?: string | undefined;\n required?: boolean | undefined;\n model: CompletionModel;\n threshold: number;\n criteria: string | string[];\n instructions?: string | undefined;\n retries?: number | undefined;\n prompt?: ValueSelector<Input, Output, Expected, string> | undefined;\n};\n\nexport function llmScore<Input, Output, Expected = unknown, const Name extends string = string>(\n options: LlmScoreOptions<Input, Output, Expected> & { name?: Name | undefined },\n): EvalMetric<Input, Output, LlmScoreMetricScore, Expected, Name> {\n const criteria = Array.isArray(options.criteria) ? options.criteria.join(\"\\n\") : options.criteria;\n if (criteria.trim().length === 0) throw new TypeError(\"llmScore criteria must not be empty.\");\n const threshold = unitInterval(options.threshold, \"llmScore threshold\");\n const retries = extractionRetries(options.retries);\n return {\n name: (options.name ?? \"llm_score\") as Name,\n required: options.required ?? true,\n dataType: \"NUMERIC\",\n projectScore: (score) => score.score,\n direction: \"higher_is_better\",\n threshold,\n async evaluate(args) {\n try {\n const result = await extract({\n model: options.model,\n outputSchema: z.object({\n score: z.number(),\n feedback: z.string(),\n }),\n instructions:\n options.instructions ??\n `Score the eval case against these criteria:\\n${criteria}\\n\\nReturn a score between 0 and 1 and brief feedback.`,\n text: await resolveJudgePrompt(options.prompt, args),\n retries,\n });\n const score = result.output;\n if (score.score < 0 || score.score > 1) {\n return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {\n score,\n usage: result.usage,\n });\n }\n return score.score >= threshold\n ? EvalOutcome.pass(score, { comment: score.feedback, usage: result.usage })\n : EvalOutcome.fail(score, { comment: score.feedback, usage: result.usage });\n } catch (error) {\n return EvalOutcome.fromError(error);\n }\n },\n };\n}\n\nfunction extractionRetries(retries: number | undefined): { maxAttempts: number } | undefined {\n const retryCount = retries ?? 0;\n if (!Number.isInteger(retryCount) || retryCount < 0) {\n throw new RangeError(\"Eval metric retries must be a non-negative integer.\");\n }\n if (retryCount === 0) {\n return undefined;\n }\n return { maxAttempts: retryCount + 1 };\n}\n\nfunction unitInterval(value: number, label: string): number {\n if (!Number.isFinite(value) || value < 0 || value > 1) {\n throw new RangeError(`${label} must be between 0 and 1.`);\n }\n return value;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,SAAS;AA6BX,SAAS,WACd,UAAoF,CAAC,GACX;AAC1E,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,YAAM,SAAS,gBAAgB,QAAQ,QAAQ;AAC/C,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,YAAY,YAAY,QAAQ,CAAC,IAAI,CAAC;AAAA,IAC/E;AAAA,EACF;AACF;AAqBO,SAAS,SACd,UAAkF,CAAC,GACT;AAC1E,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,0CAA0C;AAAA,MACvE;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,YAAM,SACJ,oBAAoB,SAAS,aAAa,UAAU,MAAM,IAAI,OAAO,SAAS,QAAQ;AACxF,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,0BAA0B,OAAO,QAAQ,CAAC,IAAI,CAAC;AAAA,IACxF;AAAA,EACF;AACF;AAoBO,SAAS,YACd,UAAqF,CAAC,GACZ;AAC1E,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,6CAA6C;AAAA,MAC1E;AACA,UAAI,OAAO,aAAa,YAAY,EAAE,oBAAoB,SAAS;AACjE,eAAO,YAAY,QAAQ,wDAAwD;AAAA,MACrF;AACA,YAAM,QAAQ,uBAAuB,UAAU,MAAM;AACrD,aAAO,QACH,YAAY,KAAK,OAAO;AAAA,QACtB,SAAS,oCAAoC,OAAO,QAAQ,CAAC;AAAA,MAC/D,CAAC,IACD,YAAY,KAAK,IAAI;AAAA,IAC3B;AAAA,EACF;AACF;AA2BO,SAAS,YACd,SAC0E;AAC1E,SAAO,mBAAmB,gBAAgB,OAAO,OAAO;AAC1D;AAoBO,SAAS,YACd,SAC0E;AAC1E,SAAO,mBAAmB,gBAAgB,OAAO,OAAO;AAC1D;AAEA,SAAS,mBACP,aACA,MACA,SACoD;AACpD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,CAAC,MAAM,QAAQ,QAAQ,KAAK,SAAS,WAAW,GAAG;AACrD,eAAO,YAAY,QAAQ,GAAG,WAAW,4CAA4C;AAAA,MACvF;AACA,UAAI,SAAS,KAAK,CAAC,UAAU,OAAO,UAAU,YAAY,EAAE,iBAAiB,OAAO,GAAG;AACrF,eAAO,YAAY,QAAQ,GAAG,WAAW,6CAA6C;AAAA,MACxF;AACA,YAAMA,WAAU,SAAS,IAAI,CAAC,UAAU,uBAAuB,OAAO,MAAM,CAAC;AAC7E,YAAM,SAAS,SAAS,QAAQA,SAAQ,MAAM,OAAO,IAAIA,SAAQ,KAAK,OAAO;AAC7E,UAAI,OAAQ,QAAO,YAAY,KAAK,IAAI;AACxC,YAAM,UAAU,SAAS,OAAO,CAAC,GAAG,UAAU,CAACA,SAAQ,KAAK,CAAC,EAAE,IAAI,MAAM;AACzE,YAAM,UACJ,SAAS,QACL,uBAAuB,QAAQ,KAAK,IAAI,CAAC,MACzC,2BAA2B,SAAS,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC;AAChE,aAAO,YAAY,KAAK,OAAO,EAAE,QAAQ,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;AAqBO,SAAS,QACd,SAC0E;AAC1E,SAAO,YAAY,WAAW,OAAO,OAAO;AAC9C;AAoBO,SAAS,aACd,SAC0E;AAC1E,SAAO,YAAY,kBAAkB,MAAM,OAAO;AACpD;AAEA,SAAS,YACP,aACA,QACA,SACoD;AACpD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,EAAE,oBAAoB,SAAS;AACjC,eAAO,YAAY,QAAQ,GAAG,WAAW,mCAAmC;AAAA,MAC9E;AACA,YAAM,UAAU,aAAa,UAAU,MAAM;AAC7C,YAAM,SAAS,SAAS,CAAC,UAAU;AACnC,aAAO,SACH,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO;AAAA,QACtB,SAAS,SACL,oCAAoC,OAAO,QAAQ,CAAC,MACpD,wBAAwB,OAAO,QAAQ,CAAC;AAAA,MAC9C,CAAC;AAAA,IACP;AAAA,EACF;AACF;AASO,SAAS,UACd,SACoD;AACpD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,MAAM,MAAM,cAAc,QAAQ,KAAK,IAAI;AACjD,UAAI,CAAC,OAAO,UAAU,GAAG,KAAK,MAAM,GAAG;AACrC,eAAO,YAAY,QAAQ,+CAA+C;AAAA,MAC5E;AACA,YAAM,SAAS,MAAM,KAAK,MAAM,EAAE;AAClC,aAAO,UAAU,MACb,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,iBAAiB,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAAA,IAC7F;AAAA,EACF;AACF;AASO,SAAS,eAMd,SACoD;AACpD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX,WAAW;AAAA,IACX,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,cAAc,QAAQ,QAAQ,IAAI;AACvD,YAAM,WAAW,MAAM,cAAc,QAAQ,UAAU,IAAI;AAC3D,UACE,CAAC,MAAM,QAAQ,QAAQ,KACvB,SAAS,WAAW,KACpB,SAAS,KAAK,CAAC,UAAU,OAAO,UAAU,YAAY,MAAM,WAAW,CAAC,GACxE;AACA,eAAO,YAAY;AAAA,UACjB;AAAA,QACF;AAAA,MACF;AACA,UAAI,OAAO,WAAW,YAAY,WAAW,QAAQ,MAAM,QAAQ,MAAM,GAAG;AAC1E,eAAO,YAAY,QAAQ,gDAAgD;AAAA,MAC7E;AACA,YAAM,UAAU,SAAS,OAAO,CAAC,UAAU,CAAC,OAAO,OAAO,QAAQ,KAAK,CAAC;AACxE,aAAO,QAAQ,WAAW,IACtB,YAAY,KAAK,IAAI,IACrB,YAAY,KAAK,OAAO,EAAE,SAAS,4BAA4B,QAAQ,KAAK,IAAI,CAAC,IAAI,CAAC;AAAA,IAC5F;AAAA,EACF;AACF;AAEA,SAAS,aAAa,SAAiB,MAAuB;AAC5D,UAAQ,YAAY;AACpB,QAAM,UAAU,QAAQ,KAAK,IAAI;AACjC,UAAQ,YAAY;AACpB,SAAO;AACT;AAEA,SAAS,uBAAuB,UAA2B,QAAyB;AAClF,SAAO,oBAAoB,SAAS,aAAa,UAAU,MAAM,IAAI,OAAO,SAAS,QAAQ;AAC/F;AAEA,eAAe,cACb,OACA,MACgB;AAChB,SAAO,OAAO,UAAU,aACnB,MAAwD,IAAI,IAC7D;AACN;AAiCO,SAAS,mBAMd,SACyE;AACzE,QAAM,YAAY,aAAa,QAAQ,WAAW,8BAA8B;AAChF,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,WAAW;AAAA,IACX;AAAA,IACA,MAAM,SAAS,MAAM;AACnB,YAAM,SAAS,MAAM,kBAAkB,QAAQ,QAAQ,IAAI;AAC3D,YAAM,WAAW,MAAM,gBAAgB,QAAQ,UAAU,IAAI;AAC7D,UAAI,aAAa,QAAW;AAC1B,eAAO,YAAY,QAAQ,qDAAqD;AAAA,MAClF;AACA,UAAI,OAAO,aAAa,UAAU;AAChC,eAAO,YAAY,QAAQ,sDAAsD;AAAA,MACnF;AACA,YAAM,CAAC,EAAE,WAAW,gBAAgB,GAAG,EAAE,WAAW,kBAAkB,CAAC,IAAI,MAAM,QAAQ,IAAI;AAAA,QAC3F,UAAU,EAAE,OAAO,QAAQ,OAAO,MAAM,OAAO,CAAC;AAAA,QAChD,UAAU,EAAE,OAAO,QAAQ,OAAO,MAAM,SAAS,CAAC;AAAA,MACpD,CAAC;AACD,YAAM,QAAQ,iBAAiB,gBAAgB,QAAQ,kBAAkB,MAAM;AAC/E,aAAO,SAAS,YACZ,YAAY,KAAK,KAAK,IACtB,YAAY,KAAK,OAAO,EAAE,SAAS,8BAA8B,SAAS,IAAI,CAAC;AAAA,IACrF;AAAA,EACF;AACF;AAaO,SAAS,SAOd,SACyD;AACzD,QAAM,UAAU,kBAAkB,QAAQ,OAAO;AACjD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,SAAS,MAAM,QAAQ;AAAA,UAC3B,OAAO,QAAQ;AAAA,UACf,cAAc,QAAQ;AAAA,UACtB,cACE,QAAQ,gBACR;AAAA,UACF,MAAM,MAAM,mBAAmB,QAAQ,QAAQ,IAAI;AAAA,UACnD;AAAA,QACF,CAAC;AACD,eAAO,QAAQ,OAAO,OAAO,MAAM,IAC/B,YAAY,KAAK,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,CAAC,IACvD,YAAY,KAAK,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,CAAC;AAAA,MAC7D,SAAS,OAAO;AACd,eAAO,YAAY,UAAU,KAAK;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACF;AAkBO,SAAS,SACd,SACgE;AAChE,QAAM,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAAI,QAAQ,SAAS,KAAK,IAAI,IAAI,QAAQ;AACzF,MAAI,SAAS,KAAK,EAAE,WAAW,EAAG,OAAM,IAAI,UAAU,sCAAsC;AAC5F,QAAM,YAAY,aAAa,QAAQ,WAAW,oBAAoB;AACtE,QAAM,UAAU,kBAAkB,QAAQ,OAAO;AACjD,SAAO;AAAA,IACL,MAAO,QAAQ,QAAQ;AAAA,IACvB,UAAU,QAAQ,YAAY;AAAA,IAC9B,UAAU;AAAA,IACV,cAAc,CAAC,UAAU,MAAM;AAAA,IAC/B,WAAW;AAAA,IACX;AAAA,IACA,MAAM,SAAS,MAAM;AACnB,UAAI;AACF,cAAM,SAAS,MAAM,QAAQ;AAAA,UAC3B,OAAO,QAAQ;AAAA,UACf,cAAc,EAAE,OAAO;AAAA,YACrB,OAAO,EAAE,OAAO;AAAA,YAChB,UAAU,EAAE,OAAO;AAAA,UACrB,CAAC;AAAA,UACD,cACE,QAAQ,gBACR;AAAA,EAAgD,QAAQ;AAAA;AAAA;AAAA,UAC1D,MAAM,MAAM,mBAAmB,QAAQ,QAAQ,IAAI;AAAA,UACnD;AAAA,QACF,CAAC;AACD,cAAM,QAAQ,OAAO;AACrB,YAAI,MAAM,QAAQ,KAAK,MAAM,QAAQ,GAAG;AACtC,iBAAO,YAAY,QAAQ,SAAS,MAAM,KAAK,gCAAgC;AAAA,YAC7E;AAAA,YACA,OAAO,OAAO;AAAA,UAChB,CAAC;AAAA,QACH;AACA,eAAO,MAAM,SAAS,YAClB,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,UAAU,OAAO,OAAO,MAAM,CAAC,IACxE,YAAY,KAAK,OAAO,EAAE,SAAS,MAAM,UAAU,OAAO,OAAO,MAAM,CAAC;AAAA,MAC9E,SAAS,OAAO;AACd,eAAO,YAAY,UAAU,KAAK;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,SAAkE;AAC3F,QAAM,aAAa,WAAW;AAC9B,MAAI,CAAC,OAAO,UAAU,UAAU,KAAK,aAAa,GAAG;AACnD,UAAM,IAAI,WAAW,qDAAqD;AAAA,EAC5E;AACA,MAAI,eAAe,GAAG;AACpB,WAAO;AAAA,EACT;AACA,SAAO,EAAE,aAAa,aAAa,EAAE;AACvC;AAEA,SAAS,aAAa,OAAe,OAAuB;AAC1D,MAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,UAAM,IAAI,WAAW,GAAG,KAAK,2BAA2B;AAAA,EAC1D;AACA,SAAO;AACT;","names":["matches"]}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// src/streaming/readable-stream.ts
|
|
2
|
+
function toReadableStream(events, _options = {}) {
|
|
3
|
+
const encoder = new TextEncoder();
|
|
4
|
+
const iterator = events[Symbol.asyncIterator]();
|
|
5
|
+
return new ReadableStream({
|
|
6
|
+
async pull(controller) {
|
|
7
|
+
try {
|
|
8
|
+
const next = await iterator.next();
|
|
9
|
+
if (next.done === true) {
|
|
10
|
+
controller.close();
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
controller.enqueue(encoder.encode(`${JSON.stringify(next.value)}
|
|
14
|
+
`));
|
|
15
|
+
} catch (error) {
|
|
16
|
+
controller.enqueue(
|
|
17
|
+
encoder.encode(`${JSON.stringify({ type: "error", error: serializeError(error) })}
|
|
18
|
+
`)
|
|
19
|
+
);
|
|
20
|
+
controller.close();
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
async cancel() {
|
|
24
|
+
await iterator.return?.();
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
function serializeError(error) {
|
|
29
|
+
if (error instanceof Error) {
|
|
30
|
+
return {
|
|
31
|
+
name: error.name,
|
|
32
|
+
message: error.message
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
return error;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export {
|
|
39
|
+
toReadableStream
|
|
40
|
+
};
|
|
41
|
+
//# sourceMappingURL=chunk-QEPBHHAP.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/streaming/readable-stream.ts"],"sourcesContent":["export type ReadableStreamOptions = {\n format?: \"jsonl\";\n};\n\nexport function toReadableStream<T>(\n events: AsyncIterable<T>,\n _options: ReadableStreamOptions = {},\n): ReadableStream<Uint8Array> {\n const encoder = new TextEncoder();\n const iterator = events[Symbol.asyncIterator]();\n\n return new ReadableStream<Uint8Array>({\n async pull(controller) {\n try {\n const next = await iterator.next();\n if (next.done === true) {\n controller.close();\n return;\n }\n\n controller.enqueue(encoder.encode(`${JSON.stringify(next.value)}\\n`));\n } catch (error) {\n controller.enqueue(\n encoder.encode(`${JSON.stringify({ type: \"error\", error: serializeError(error) })}\\n`),\n );\n controller.close();\n }\n },\n async cancel() {\n await iterator.return?.();\n },\n });\n}\n\nfunction serializeError(error: unknown): unknown {\n if (error instanceof Error) {\n return {\n name: error.name,\n message: error.message,\n };\n }\n\n return error;\n}\n"],"mappings":";AAIO,SAAS,iBACd,QACA,WAAkC,CAAC,GACP;AAC5B,QAAM,UAAU,IAAI,YAAY;AAChC,QAAM,WAAW,OAAO,OAAO,aAAa,EAAE;AAE9C,SAAO,IAAI,eAA2B;AAAA,IACpC,MAAM,KAAK,YAAY;AACrB,UAAI;AACF,cAAM,OAAO,MAAM,SAAS,KAAK;AACjC,YAAI,KAAK,SAAS,MAAM;AACtB,qBAAW,MAAM;AACjB;AAAA,QACF;AAEA,mBAAW,QAAQ,QAAQ,OAAO,GAAG,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,CAAI,CAAC;AAAA,MACtE,SAAS,OAAO;AACd,mBAAW;AAAA,UACT,QAAQ,OAAO,GAAG,KAAK,UAAU,EAAE,MAAM,SAAS,OAAO,eAAe,KAAK,EAAE,CAAC,CAAC;AAAA,CAAI;AAAA,QACvF;AACA,mBAAW,MAAM;AAAA,MACnB;AAAA,IACF;AAAA,IACA,MAAM,SAAS;AACb,YAAM,SAAS,SAAS;AAAA,IAC1B;AAAA,EACF,CAAC;AACH;AAEA,SAAS,eAAe,OAAyB;AAC/C,MAAI,iBAAiB,OAAO;AAC1B,WAAO;AAAA,MACL,MAAM,MAAM;AAAA,MACZ,SAAS,MAAM;AAAA,IACjB;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import {
|
|
2
|
+
embedDocuments
|
|
3
|
+
} from "./chunk-HHMLE3TZ.js";
|
|
4
|
+
import {
|
|
5
|
+
chunkTextDocuments
|
|
6
|
+
} from "./chunk-TFIOU6UR.js";
|
|
7
|
+
|
|
8
|
+
// src/vector-store/ingest.ts
|
|
9
|
+
async function ingestVectorText(options) {
|
|
10
|
+
return ingestVectorDocuments({
|
|
11
|
+
store: options.store,
|
|
12
|
+
documents: [options.document],
|
|
13
|
+
embeddingModel: options.embeddingModel,
|
|
14
|
+
chunking: options.chunking,
|
|
15
|
+
retries: options.retries,
|
|
16
|
+
concurrency: options.concurrency,
|
|
17
|
+
providerOptions: options.providerOptions,
|
|
18
|
+
abortSignal: options.abortSignal
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
async function ingestVectorDocuments(options) {
|
|
22
|
+
for (const document of options.documents) {
|
|
23
|
+
validateVectorMetadata(document.metadata, document.id);
|
|
24
|
+
}
|
|
25
|
+
const chunks = chunkTextDocuments({
|
|
26
|
+
documents: options.documents,
|
|
27
|
+
chunking: options.chunking
|
|
28
|
+
});
|
|
29
|
+
const chunksByDocument = groupChunksByDocument(chunks);
|
|
30
|
+
const embedded = await embedDocuments({
|
|
31
|
+
model: options.embeddingModel,
|
|
32
|
+
documents: [...options.documents],
|
|
33
|
+
id: (document) => document.id,
|
|
34
|
+
content: (document) => {
|
|
35
|
+
const documentChunks = chunksByDocument.get(document.id);
|
|
36
|
+
if (documentChunks === void 0) {
|
|
37
|
+
throw new TypeError(`Text document ${document.id} has no chunks`);
|
|
38
|
+
}
|
|
39
|
+
return documentChunks.map((chunk) => chunk.text);
|
|
40
|
+
},
|
|
41
|
+
metadata: (document) => document.metadata,
|
|
42
|
+
retries: options.retries,
|
|
43
|
+
concurrency: options.concurrency,
|
|
44
|
+
abortSignal: options.abortSignal
|
|
45
|
+
});
|
|
46
|
+
await options.store.upsert({
|
|
47
|
+
documents: embedded.documents,
|
|
48
|
+
providerOptions: options.providerOptions
|
|
49
|
+
});
|
|
50
|
+
return { documents: embedded.documents };
|
|
51
|
+
}
|
|
52
|
+
function validateVectorMetadata(metadata, documentId) {
|
|
53
|
+
if (metadata === void 0) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (typeof metadata !== "object" || metadata === null || Array.isArray(metadata)) {
|
|
57
|
+
throw new TypeError(`Text document ${documentId} metadata must be an object`);
|
|
58
|
+
}
|
|
59
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
60
|
+
validateVectorMetadataValue(value, `Text document ${documentId} metadata.${key}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function validateVectorMetadataValue(value, label) {
|
|
64
|
+
if (value === null || typeof value === "string" || typeof value === "boolean") {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
68
|
+
throw new TypeError(`${label} must be a finite vector metadata value`);
|
|
69
|
+
}
|
|
70
|
+
if (Number.isInteger(value) && !Number.isSafeInteger(value)) {
|
|
71
|
+
throw new TypeError(`${label} must be a safe integer`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function groupChunksByDocument(chunks) {
|
|
75
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
76
|
+
for (const chunk of chunks) {
|
|
77
|
+
const documentChunks = grouped.get(chunk.documentId);
|
|
78
|
+
if (documentChunks === void 0) {
|
|
79
|
+
grouped.set(chunk.documentId, [chunk]);
|
|
80
|
+
} else {
|
|
81
|
+
documentChunks.push(chunk);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return grouped;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export {
|
|
88
|
+
ingestVectorText,
|
|
89
|
+
ingestVectorDocuments
|
|
90
|
+
};
|
|
91
|
+
//# sourceMappingURL=chunk-QHGYCYV6.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/vector-store/ingest.ts"],"sourcesContent":["import type { JsonObject } from \"../completion\";\nimport {\n chunkTextDocuments,\n type TextDocument,\n type TextDocumentChunk,\n type TextDocumentChunkingOptions,\n} from \"../documents\";\nimport {\n embedDocuments,\n type EmbeddedDocument,\n type EmbeddingModel,\n type VectorMetadata,\n type VectorMetadataValue,\n} from \"../embeddings\";\nimport type { RetrySetting } from \"../retry\";\nimport type { VectorStore } from \"./types\";\n\nexport type IngestVectorDocumentsOptions<Metadata extends VectorMetadata = VectorMetadata> =\n Readonly<{\n store: VectorStore<TextDocument<Metadata>, Metadata>;\n documents: readonly TextDocument<Metadata>[];\n embeddingModel: EmbeddingModel;\n chunking?: TextDocumentChunkingOptions | undefined;\n retries?: RetrySetting | undefined;\n concurrency?: number | undefined;\n providerOptions?: JsonObject | undefined;\n abortSignal?: AbortSignal | undefined;\n }>;\n\nexport type IngestVectorTextOptions<Metadata extends VectorMetadata = VectorMetadata> = Omit<\n IngestVectorDocumentsOptions<Metadata>,\n \"documents\"\n> &\n Readonly<{ document: TextDocument<Metadata> }>;\n\nexport type IngestVectorDocumentsResult<Metadata extends VectorMetadata = VectorMetadata> =\n Readonly<{\n documents: readonly EmbeddedDocument<TextDocument<Metadata>, Metadata>[];\n }>;\n\nexport async function ingestVectorText<Metadata extends VectorMetadata = VectorMetadata>(\n options: IngestVectorTextOptions<Metadata>,\n): Promise<IngestVectorDocumentsResult<Metadata>> {\n return ingestVectorDocuments({\n store: options.store,\n documents: [options.document],\n embeddingModel: options.embeddingModel,\n chunking: options.chunking,\n retries: options.retries,\n concurrency: options.concurrency,\n providerOptions: options.providerOptions,\n abortSignal: options.abortSignal,\n });\n}\n\nexport async function ingestVectorDocuments<Metadata extends VectorMetadata = VectorMetadata>(\n options: IngestVectorDocumentsOptions<Metadata>,\n): Promise<IngestVectorDocumentsResult<Metadata>> {\n for (const document of options.documents) {\n validateVectorMetadata(document.metadata, document.id);\n }\n const chunks = chunkTextDocuments({\n documents: options.documents,\n chunking: options.chunking,\n });\n const chunksByDocument = groupChunksByDocument(chunks);\n const embedded = await embedDocuments({\n model: options.embeddingModel,\n documents: [...options.documents],\n id: (document) => document.id,\n content: (document) => {\n const documentChunks = chunksByDocument.get(document.id);\n if (documentChunks === undefined) {\n throw new TypeError(`Text document ${document.id} has no chunks`);\n }\n return documentChunks.map((chunk) => chunk.text);\n },\n metadata: (document) => document.metadata,\n retries: options.retries,\n concurrency: options.concurrency,\n abortSignal: options.abortSignal,\n });\n await options.store.upsert({\n documents: embedded.documents,\n providerOptions: options.providerOptions,\n });\n return { documents: embedded.documents };\n}\n\nfunction validateVectorMetadata(metadata: VectorMetadata | undefined, documentId: string): void {\n if (metadata === undefined) {\n return;\n }\n if (typeof metadata !== \"object\" || metadata === null || Array.isArray(metadata)) {\n throw new TypeError(`Text document ${documentId} metadata must be an object`);\n }\n for (const [key, value] of Object.entries(metadata)) {\n validateVectorMetadataValue(value, `Text document ${documentId} metadata.${key}`);\n }\n}\n\nfunction validateVectorMetadataValue(value: VectorMetadataValue, label: string): void {\n if (value === null || typeof value === \"string\" || typeof value === \"boolean\") {\n return;\n }\n if (typeof value !== \"number\" || !Number.isFinite(value)) {\n throw new TypeError(`${label} must be a finite vector metadata value`);\n }\n if (Number.isInteger(value) && !Number.isSafeInteger(value)) {\n throw new TypeError(`${label} must be a safe integer`);\n }\n}\n\nfunction groupChunksByDocument<Metadata>(\n chunks: readonly TextDocumentChunk<Metadata>[],\n): ReadonlyMap<string, readonly TextDocumentChunk<Metadata>[]> {\n const grouped = new Map<string, TextDocumentChunk<Metadata>[]>();\n for (const chunk of chunks) {\n const documentChunks = grouped.get(chunk.documentId);\n if (documentChunks === undefined) {\n grouped.set(chunk.documentId, [chunk]);\n } else {\n documentChunks.push(chunk);\n }\n }\n return grouped;\n}\n"],"mappings":";;;;;;;;AAwCA,eAAsB,iBACpB,SACgD;AAChD,SAAO,sBAAsB;AAAA,IAC3B,OAAO,QAAQ;AAAA,IACf,WAAW,CAAC,QAAQ,QAAQ;AAAA,IAC5B,gBAAgB,QAAQ;AAAA,IACxB,UAAU,QAAQ;AAAA,IAClB,SAAS,QAAQ;AAAA,IACjB,aAAa,QAAQ;AAAA,IACrB,iBAAiB,QAAQ;AAAA,IACzB,aAAa,QAAQ;AAAA,EACvB,CAAC;AACH;AAEA,eAAsB,sBACpB,SACgD;AAChD,aAAW,YAAY,QAAQ,WAAW;AACxC,2BAAuB,SAAS,UAAU,SAAS,EAAE;AAAA,EACvD;AACA,QAAM,SAAS,mBAAmB;AAAA,IAChC,WAAW,QAAQ;AAAA,IACnB,UAAU,QAAQ;AAAA,EACpB,CAAC;AACD,QAAM,mBAAmB,sBAAsB,MAAM;AACrD,QAAM,WAAW,MAAM,eAAe;AAAA,IACpC,OAAO,QAAQ;AAAA,IACf,WAAW,CAAC,GAAG,QAAQ,SAAS;AAAA,IAChC,IAAI,CAAC,aAAa,SAAS;AAAA,IAC3B,SAAS,CAAC,aAAa;AACrB,YAAM,iBAAiB,iBAAiB,IAAI,SAAS,EAAE;AACvD,UAAI,mBAAmB,QAAW;AAChC,cAAM,IAAI,UAAU,iBAAiB,SAAS,EAAE,gBAAgB;AAAA,MAClE;AACA,aAAO,eAAe,IAAI,CAAC,UAAU,MAAM,IAAI;AAAA,IACjD;AAAA,IACA,UAAU,CAAC,aAAa,SAAS;AAAA,IACjC,SAAS,QAAQ;AAAA,IACjB,aAAa,QAAQ;AAAA,IACrB,aAAa,QAAQ;AAAA,EACvB,CAAC;AACD,QAAM,QAAQ,MAAM,OAAO;AAAA,IACzB,WAAW,SAAS;AAAA,IACpB,iBAAiB,QAAQ;AAAA,EAC3B,CAAC;AACD,SAAO,EAAE,WAAW,SAAS,UAAU;AACzC;AAEA,SAAS,uBAAuB,UAAsC,YAA0B;AAC9F,MAAI,aAAa,QAAW;AAC1B;AAAA,EACF;AACA,MAAI,OAAO,aAAa,YAAY,aAAa,QAAQ,MAAM,QAAQ,QAAQ,GAAG;AAChF,UAAM,IAAI,UAAU,iBAAiB,UAAU,6BAA6B;AAAA,EAC9E;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,QAAQ,GAAG;AACnD,gCAA4B,OAAO,iBAAiB,UAAU,aAAa,GAAG,EAAE;AAAA,EAClF;AACF;AAEA,SAAS,4BAA4B,OAA4B,OAAqB;AACpF,MAAI,UAAU,QAAQ,OAAO,UAAU,YAAY,OAAO,UAAU,WAAW;AAC7E;AAAA,EACF;AACA,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,GAAG;AACxD,UAAM,IAAI,UAAU,GAAG,KAAK,yCAAyC;AAAA,EACvE;AACA,MAAI,OAAO,UAAU,KAAK,KAAK,CAAC,OAAO,cAAc,KAAK,GAAG;AAC3D,UAAM,IAAI,UAAU,GAAG,KAAK,yBAAyB;AAAA,EACvD;AACF;AAEA,SAAS,sBACP,QAC6D;AAC7D,QAAM,UAAU,oBAAI,IAA2C;AAC/D,aAAW,SAAS,QAAQ;AAC1B,UAAM,iBAAiB,QAAQ,IAAI,MAAM,UAAU;AACnD,QAAI,mBAAmB,QAAW;AAChC,cAAQ,IAAI,MAAM,YAAY,CAAC,KAAK,CAAC;AAAA,IACvC,OAAO;AACL,qBAAe,KAAK,KAAK;AAAA,IAC3B;AAAA,EACF;AACA,SAAO;AACT;","names":[]}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
// src/pipeline/errors.ts
|
|
2
|
+
var PipelineAgentSuspensionError = class extends Error {
|
|
3
|
+
constructor(result) {
|
|
4
|
+
super("Pipeline agent stage suspended for human interaction.");
|
|
5
|
+
this.result = result;
|
|
6
|
+
this.name = "PipelineAgentSuspensionError";
|
|
7
|
+
}
|
|
8
|
+
result;
|
|
9
|
+
};
|
|
10
|
+
var PipelineObserverDispatchError = class extends AggregateError {
|
|
11
|
+
phase;
|
|
12
|
+
failures;
|
|
13
|
+
constructor(phase, failures) {
|
|
14
|
+
super(
|
|
15
|
+
failures.map((failure) => failure.error),
|
|
16
|
+
`Pipeline observer ${phase} failed for ${failures.map((failure) => failure.observer).join(", ")}.`
|
|
17
|
+
);
|
|
18
|
+
this.name = "PipelineObserverDispatchError";
|
|
19
|
+
this.phase = phase;
|
|
20
|
+
this.failures = Object.freeze([...failures]);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export {
|
|
25
|
+
PipelineAgentSuspensionError,
|
|
26
|
+
PipelineObserverDispatchError
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=chunk-R6722CIU.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/pipeline/errors.ts"],"sourcesContent":["import type { AgentInteractionOutcome } from \"../agent/run-types\";\n\nexport class PipelineAgentSuspensionError extends Error {\n constructor(readonly result: AgentInteractionOutcome) {\n super(\"Pipeline agent stage suspended for human interaction.\");\n this.name = \"PipelineAgentSuspensionError\";\n }\n}\n\nexport type PipelineObserverFailure = {\n readonly observer: string;\n readonly error: unknown;\n};\n\nexport class PipelineObserverDispatchError extends AggregateError {\n readonly phase: string;\n readonly failures: readonly PipelineObserverFailure[];\n\n constructor(phase: string, failures: readonly PipelineObserverFailure[]) {\n super(\n failures.map((failure) => failure.error),\n `Pipeline observer ${phase} failed for ${failures\n .map((failure) => failure.observer)\n .join(\", \")}.`,\n );\n this.name = \"PipelineObserverDispatchError\";\n this.phase = phase;\n this.failures = Object.freeze([...failures]);\n }\n}\n"],"mappings":";AAEO,IAAM,+BAAN,cAA2C,MAAM;AAAA,EACtD,YAAqB,QAAiC;AACpD,UAAM,uDAAuD;AAD1C;AAEnB,SAAK,OAAO;AAAA,EACd;AAAA,EAHqB;AAIvB;AAOO,IAAM,gCAAN,cAA4C,eAAe;AAAA,EACvD;AAAA,EACA;AAAA,EAET,YAAY,OAAe,UAA8C;AACvE;AAAA,MACE,SAAS,IAAI,CAAC,YAAY,QAAQ,KAAK;AAAA,MACvC,qBAAqB,KAAK,eAAe,SACtC,IAAI,CAAC,YAAY,QAAQ,QAAQ,EACjC,KAAK,IAAI,CAAC;AAAA,IACf;AACA,SAAK,OAAO;AACZ,SAAK,QAAQ;AACb,SAAK,WAAW,OAAO,OAAO,CAAC,GAAG,QAAQ,CAAC;AAAA,EAC7C;AACF;","names":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
//# sourceMappingURL=chunk-RA4YVN43.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|