@uipath/flow-tool 1.204.0-preview.255 → 1.205.0-preview.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/THIRD-PARTY-NOTICES.md +36 -12
- package/dist/{authoring-bppe07p0.js → authoring-e1k40hyw.js} +6 -5
- package/dist/{binding-service-fw3rzae4.js → binding-service-yw91615m.js} +38 -30
- package/dist/{connector-service-2z7s7qtd.js → connector-service-ttm75a4n.js} +224 -47
- package/dist/de-UZI7FJDO-awjpqfz0.js +200 -0
- package/dist/{de-MUJPA4EB-dpg7t2rw.js → de-UZI7FJDO-mwvc4cxe.js} +29 -2
- package/dist/{de-FGHEJRCI-rv513mfh.js → de-WLW6KWOJ-2qnsqdtw.js} +238 -31
- package/dist/debug-conversation-trigger-xsm3ycqx.js +30 -0
- package/dist/{debug-http-client-p4kf38yt.js → debug-http-client-yqjzsnhw.js} +5 -4
- package/dist/{debug-service-893erynq.js → debug-service-nra3n02y.js} +6 -5
- package/dist/{edge-service-6zr5f09c.js → edge-service-8fv0m681.js} +16 -12
- package/dist/{embedded-file-reader-v8c2yj2a.js → embedded-file-reader-r66242ey.js} +2 -2
- package/dist/en-EJCBA3LK-0wb3b1xy.js +146 -0
- package/dist/{en-FBPVNKFN-dqt573xn.js → en-EJCBA3LK-dtpdghgj.js} +12 -4
- package/dist/{en-BWGKA5OE-1qkvb9rn.js → en-M7GQYBV2-5nfmpnfz.js} +88 -6
- package/dist/{es-7SE7HX2X-8yh1aj2y.js → es-4UBUTCQA-m1ytpyx2.js} +29 -2
- package/dist/es-4UBUTCQA-t7hsackg.js +200 -0
- package/dist/{es-KR3ZSNMP-84zphx0x.js → es-5THCNRFN-tqwyrdgy.js} +235 -28
- package/dist/{es-MX-2MYFZU7X-3rek4r73.js → es-MX-5RELIXKI-9jhqtddc.js} +29 -2
- package/dist/es-MX-5RELIXKI-jtjbcxbb.js +200 -0
- package/dist/{es-MX-XXZJGC4R-2mr8jtgm.js → es-MX-TWZBRAXK-gqd9swzw.js} +237 -30
- package/dist/filter-engine-7ep90g7f.js +11 -0
- package/dist/{flow-api-kjvpsxhw.js → flow-api-sat010z1.js} +16 -10
- package/dist/{flow-debug-service-9q9k756w.js → flow-debug-service-cgawm96k.js} +96 -58
- package/dist/{flow-eval-node-target-zvk7fcb2.js → flow-eval-node-target-dah1gmmh.js} +17 -17
- package/dist/flow-eval-run-api-xdygjzfv.js +13 -0
- package/dist/{flow-eval-run-service-2hfqyadh.js → flow-eval-run-service-03j1szf6.js} +73 -80
- package/dist/{flow-eval-service-m5n2svke.js → flow-eval-service-9j46ta9k.js} +167 -415
- package/dist/{flow-init-sdk-source-z9qx1zkt.js → flow-init-sdk-source-xvdvkq7d.js} +6 -5
- package/dist/{flow-init-service-8y8b76zm.js → flow-init-service-0a4y3a46.js} +19 -15
- package/dist/{flow-io-w11yspjx.js → flow-io-1wabd32s.js} +15 -11
- package/dist/{flow-migrate-service-v7ch4t3h.js → flow-migrate-service-mkqpbwtv.js} +16 -12
- package/dist/{flow-pack-service-xs8v5mba.js → flow-pack-service-7dj2hhyz.js} +47 -38
- package/dist/flow-validate-service-1ktjayjv.js +70 -0
- package/dist/{fr-Y6NPVODW-8vr1ja4b.js → fr-CERONSRC-edpxhbmb.js} +237 -30
- package/dist/{fr-MDFKNZ4C-h4a122k9.js → fr-VBM5WRC2-63z7w4mh.js} +29 -2
- package/dist/fr-VBM5WRC2-707ha692.js +200 -0
- package/dist/{governance-fetcher-18pk0mgv.js → governance-fetcher-76hxssjm.js} +7 -6
- package/dist/{hitl-service-ehnfkwhp.js → hitl-service-vcbhd4cb.js} +15 -11
- package/dist/{index-aaq9rwfv.js → index-1b2je9n2.js} +1088 -233
- package/dist/index-d090f35d.js +66 -0
- package/dist/{index-yj4xk2aw.js → index-dgdnbb2g.js} +7 -5
- package/dist/{index-ng2f77my.js → index-fyc0k5dd.js} +985 -617
- package/dist/{index-9hqj0ee4.js → index-hz008fep.js} +6 -5
- package/dist/{index-vrhcbxnk.js → index-nsn889vf.js} +6 -5
- package/dist/{index-2s8rmmw6.js → index-sqhbn7ew.js} +4 -4
- package/dist/{index-tnz5ks4t.js → index-wycgq0ff.js} +65 -2230
- package/dist/index.js +17 -14
- package/dist/init.js +19 -15
- package/dist/inline-entity-resolver-64pqwk86.js +45 -0
- package/dist/input-reader-n7kzdrev.js +12 -0
- package/dist/{ja-UTQEAP34-8e0q6swp.js → ja-43QL7CGO-c3q8gre9.js} +29 -2
- package/dist/ja-43QL7CGO-h6qfxfay.js +200 -0
- package/dist/{ja-Q5PXKJ2V-p4msxzjd.js → ja-U2MCNHGT-gz0ap1jk.js} +235 -28
- package/dist/{known-types-q6fcyq2e.js → known-types-3025scz1.js} +5 -5
- package/dist/ko-TLL6Z6CD-ag6gjynr.js +200 -0
- package/dist/{ko-AXCHW2OK-askct7f6.js → ko-TLL6Z6CD-wrt3n8a9.js} +29 -2
- package/dist/{ko-AYBDSRZG-41mdmsg1.js → ko-XAUYQWDR-zp81gjp4.js} +237 -30
- package/dist/{maestro-automate-entry-qak46eg0.js → maestro-automate-entry-6f9act3w.js} +2 -2
- package/dist/{node-cvwqsgg0.js → node-hz6cvp9d.js} +29 -15
- package/dist/{node-service-544fv7as.js → node-service-7wz8gx7h.js} +41 -30
- package/dist/{node-strategy-fdnkvq4a.js → node-strategy-1dmmhvf3.js} +1 -1
- package/dist/node-sync-service-yvyf7chw.js +61 -0
- package/dist/packager-tool-10vc8pq4.js +2 -0
- package/dist/{packager-tool-29b6r6eg.js → packager-tool-2h15x2kb.js} +6 -6
- package/dist/{packager-tool-h7cjs093.js → packager-tool-2z0r1f32.js} +668 -57
- package/dist/packager-tool-4kr6g0zw.js +16 -0
- package/dist/packager-tool-4x7tzvv0.js +1176 -0
- package/dist/{packager-tool-d88rqwk8.js → packager-tool-6z1hsm2h.js} +13 -5
- package/dist/packager-tool-7hw344bt.js +107 -0
- package/dist/{packager-tool-pb7wzsn4.js → packager-tool-7wnxyvf9.js} +519 -437
- package/dist/{packager-tool-m8xtf5ej.js → packager-tool-8g5931fz.js} +7 -4
- package/dist/{packager-tool-h9tzz0qd.js → packager-tool-961vsbsc.js} +15 -10
- package/dist/packager-tool-9era57c0.js +28 -0
- package/dist/packager-tool-9gprvfcy.js +23 -0
- package/dist/packager-tool-9jxzgsxc.js +14 -0
- package/dist/{packager-tool-zxp4hya4.js → packager-tool-a5afh1dc.js} +34 -13
- package/dist/{packager-tool-02f63a08.js → packager-tool-apawc11t.js} +88 -762
- package/dist/{packager-tool-tf3d2y8z.js → packager-tool-aw9v0raw.js} +2 -2
- package/dist/packager-tool-cqtsj6yh.js +2 -0
- package/dist/{packager-tool-5nqhxb9e.js → packager-tool-dwd6cc01.js} +16110 -13031
- package/dist/{packager-tool-t605qewm.js → packager-tool-dxgrhfq5.js} +2 -2
- package/dist/{packager-tool-awkq26v0.js → packager-tool-e7kvq8hf.js} +5 -3
- package/dist/{packager-tool-2y2mmdnq.js → packager-tool-fjqpf1y6.js} +6 -24
- package/dist/{packager-tool-3dc9gz35.js → packager-tool-frjt46rb.js} +1 -1
- package/dist/{packager-tool-y8egb0fa.js → packager-tool-gqq3vkgh.js} +2573 -535
- package/dist/{packager-tool-w5a1hhyj.js → packager-tool-hxyjn885.js} +13 -2
- package/dist/packager-tool-j3228zs9.js +35 -0
- package/dist/{packager-tool-hhy1gr2k.js → packager-tool-jgfzfqjm.js} +1 -1
- package/dist/{packager-tool-5csvkk54.js → packager-tool-kb520rm5.js} +172 -71
- package/dist/{packager-tool-k04k2rnq.js → packager-tool-kmv2p8z4.js} +20 -8
- package/dist/{packager-tool-6h69pd2a.js → packager-tool-kn86394c.js} +127 -61
- package/dist/packager-tool-mgcz986r.js +143 -0
- package/dist/{packager-tool-3h2ktz6q.js → packager-tool-myhbp761.js} +11 -1
- package/dist/{packager-tool-5tk1eg46.js → packager-tool-ns9xn7zq.js} +13 -13
- package/dist/{packager-tool-vbfz8k7w.js → packager-tool-p8gab24q.js} +52 -32
- package/dist/{packager-tool-stq8800d.js → packager-tool-pacrqxv2.js} +17 -4
- package/dist/{packager-tool-7q6z6npd.js → packager-tool-pcvhcn2k.js} +93 -11
- package/dist/packager-tool-phw8ydjx.js +331 -0
- package/dist/{packager-tool-mx0gg6yh.js → packager-tool-q8akejne.js} +6 -4
- package/dist/packager-tool-qcqb9hf8.js +26 -0
- package/dist/{packager-tool-n89vrpex.js → packager-tool-skcjythd.js} +10907 -25608
- package/dist/{packager-tool-v37kemg5.js → packager-tool-sphnextf.js} +5 -6
- package/dist/{packager-tool-1ca047de.js → packager-tool-sqpesv2e.js} +12 -56
- package/dist/packager-tool-t7ahhaer.js +877 -0
- package/dist/{packager-tool-zgm9mqrh.js → packager-tool-thnkpqwc.js} +1802 -378
- package/dist/{packager-tool-peawy22c.js → packager-tool-tqjfdpda.js} +53 -8
- package/dist/{packager-tool-gb4sm499.js → packager-tool-vd3rn4fj.js} +463 -327
- package/dist/{packager-tool-5tjzg56t.js → packager-tool-wh660fny.js} +1 -1
- package/dist/{packager-tool-tyg399d6.js → packager-tool-wze74wyk.js} +74 -47
- package/dist/{packager-tool-zbhqrk3p.js → packager-tool-x20684x5.js} +1 -1
- package/dist/packager-tool-xj0agqpk.js +11 -0
- package/dist/packager-tool-xk78j7dz.js +1256 -0
- package/dist/{packager-tool-py93409h.js → packager-tool-ymma2s85.js} +2 -2
- package/dist/packager-tool-z0yxfhdj.js +40 -0
- package/dist/packager-tool.js +13 -10
- package/dist/{pt-BR-TKHNHMET-sn91xyq0.js → pt-BR-GNQNAAZ2-xadharv4.js} +237 -30
- package/dist/{pt-BR-Y6K6XHXK-mp3hwhw1.js → pt-BR-USP2T7GD-pedn77m1.js} +29 -2
- package/dist/pt-BR-USP2T7GD-rppy42ym.js +200 -0
- package/dist/{pt-3DIO5QWG-59b3f4bc.js → pt-RMIAKZB4-67enhx3m.js} +29 -2
- package/dist/pt-RMIAKZB4-t80exmp7.js +200 -0
- package/dist/{pt-ROXU257Y-2f6bxev4.js → pt-TO2DARY2-ftcce1bv.js} +237 -30
- package/dist/ro-GHOBYW2S-4zevc9dy.js +200 -0
- package/dist/{ro-BWR3ONRR-vzx3hy8a.js → ro-GHOBYW2S-mhnqsgaa.js} +29 -2
- package/dist/{ro-AVAWPKOO-b89pw79r.js → ro-WUYJV6OW-a8d9qy1n.js} +235 -28
- package/dist/ru-WBLQ5B2U-dpqvtfwr.js +10 -0
- package/dist/services/connector-request-buckets.d.ts +1 -0
- package/dist/services/flow-eval-file-store.d.ts +1 -11
- package/dist/services/flow-eval-utils.d.ts +1 -32
- package/dist/services/flow-validate-service.d.ts +51 -0
- package/dist/services/inline-entity-resolver.d.ts +22 -0
- package/dist/services/node-service.d.ts +21 -1
- package/dist/services/packaging-utils.d.ts +15 -2
- package/dist/{shared-utils-pnbvyyb2.js → shared-utils-tqnx96mg.js} +4 -3
- package/dist/solution-auth-8qz5pw7z.js +14 -0
- package/dist/solution-discovery-vqravtkh.js +44 -0
- package/dist/{tidy-service-j70ccg27.js → tidy-service-tqjea6g2.js} +143 -39
- package/dist/tool.js +17 -14
- package/dist/{tr-5IIHSFW5-hw4cewew.js → tr-GDTLECMM-k5fhmq13.js} +29 -2
- package/dist/tr-GDTLECMM-kngksm3r.js +200 -0
- package/dist/{tr-JMIP7ZSL-c39b6pe4.js → tr-M3F5A6AU-bb0tg39m.js} +235 -28
- package/dist/utils/flow-io.d.ts +2 -1
- package/dist/validation.js +39 -33
- package/dist/{variable-service-86t2r5m9.js → variable-service-kqnqxmx2.js} +16 -12
- package/dist/{wait-for-flow-job-d8m6mqb4.js → wait-for-flow-job-b0vj8828.js} +15 -12
- package/dist/{zh-CN-IFFXSEJS-9wm9gmrv.js → zh-CN-C5G4PC7C-ds9gkptg.js} +235 -28
- package/dist/zh-CN-I5D7WHBD-ge92tbx9.js +200 -0
- package/dist/{zh-CN-R2KU4U5L-tc1hat59.js → zh-CN-I5D7WHBD-sr5h0a98.js} +29 -2
- package/dist/{zh-TW-GNKMELBF-j674515r.js → zh-TW-2AVLM723-288pkc1z.js} +29 -2
- package/dist/zh-TW-2AVLM723-xsmap3gk.js +200 -0
- package/dist/{zh-TW-IYBNYAMG-8727c7v2.js → zh-TW-62HZIG6O-787kdwex.js} +234 -27
- package/package.json +2 -2
- package/dist/debug-conversation-trigger-hgszjdhx.js +0 -44
- package/dist/flow-eval-run-api-byx3cacw.js +0 -13
- package/dist/flow-validate-service-12v3bz1b.js +0 -64
- package/dist/input-reader-vhar4g0k.js +0 -11
- package/dist/node-sync-service-ccwgtyte.js +0 -40
- package/dist/packager-tool-4g5hk66k.js +0 -454
- package/dist/packager-tool-kg35b9s6.js +0 -15
- package/dist/packager-tool-pz5nqg0s.js +0 -64
- package/dist/packager-tool-t3ysatjd.js +0 -521
- package/dist/packager-tool-vd6jt4n8.js +0 -208
- package/dist/packager-tool-vscvjrhn.js +0 -265
- package/dist/services/flow-debug-service.d.ts +0 -320
- package/dist/services/flow-eval-schema/deterministic-evaluators.d.ts +0 -110
- package/dist/services/flow-eval-schema/evaluator-fields.d.ts +0 -66
- package/dist/services/flow-eval-schema/helpers.d.ts +0 -9
- package/dist/services/flow-eval-schema/index.d.ts +0 -4
- package/dist/services/flow-eval-schema/llm-judge-evaluators.d.ts +0 -174
- package/dist/services/flow-eval-schema/prompts.d.ts +0 -2
- package/dist/services/flow-eval-schema/registry.d.ts +0 -375
- package/dist/services/flow-eval-schema/tool-call-evaluators.d.ts +0 -93
- package/dist/services/flow-eval-schema/types.d.ts +0 -95
- package/dist/services/flow-eval-types.d.ts +0 -6
- package/dist/utils/write-resolved-flow.d.ts +0 -7
- /package/dist/{headless-45dypanh.js → headless-fw8pwyzm.js} +0 -0
- /package/dist/{localization-xecwjpe2.js → localization-d6gjmrsw.js} +0 -0
|
@@ -0,0 +1,1256 @@
|
|
|
1
|
+
import {
|
|
2
|
+
catchError
|
|
3
|
+
} from "./packager-tool-t7ahhaer.js";
|
|
4
|
+
import {
|
|
5
|
+
__require
|
|
6
|
+
} from "./packager-tool-1yh8g550.js";
|
|
7
|
+
|
|
8
|
+
// ../eval-sdk/src/eval-schema/prompts.ts
|
|
9
|
+
var DEFAULT_EVALUATOR_PROMPT = `As an expert evaluator, analyze the semantic similarity of these JSON contents to determine a score from 0-100. Focus on comparing the meaning and contextual equivalence of corresponding fields, accounting for alternative valid expressions, synonyms, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.
|
|
10
|
+
----
|
|
11
|
+
ExpectedOutput:
|
|
12
|
+
{{ExpectedOutput}}
|
|
13
|
+
----
|
|
14
|
+
ActualOutput:
|
|
15
|
+
{{ActualOutput}}
|
|
16
|
+
`;
|
|
17
|
+
var DEFAULT_TRAJECTORY_EVALUATOR_PROMPT = `As an expert evaluator, determine how well the agent did on a scale of 0-100. Focus on if the simulation was successful and if the agent behaved according to the expected output accounting for alternative valid expressions, and reasonable variations in language while maintaining high standards for accuracy and completeness. Provide your score with a justification, explaining briefly and concisely why you gave that score.
|
|
18
|
+
----
|
|
19
|
+
UserOrSyntheticInputGivenToAgent:
|
|
20
|
+
{{UserOrSyntheticInput}}
|
|
21
|
+
----
|
|
22
|
+
SimulationInstructions:
|
|
23
|
+
{{SimulationInstructions}}
|
|
24
|
+
----
|
|
25
|
+
ExpectedAgentBehavior:
|
|
26
|
+
{{ExpectedAgentBehavior}}
|
|
27
|
+
----
|
|
28
|
+
AgentRunHistory:
|
|
29
|
+
{{AgentRunHistory}}
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
// ../eval-sdk/src/eval-schema/evaluator-fields.ts
|
|
33
|
+
var targetOutputKeyConfigField = {
|
|
34
|
+
key: "targetOutputKey",
|
|
35
|
+
label: "Target Output Key",
|
|
36
|
+
type: "targetOutputKey",
|
|
37
|
+
required: false,
|
|
38
|
+
description: "The key in the output to evaluate"
|
|
39
|
+
};
|
|
40
|
+
var caseSensitiveConfigField = {
|
|
41
|
+
key: "caseSensitive",
|
|
42
|
+
label: "Case Sensitive",
|
|
43
|
+
type: "boolean",
|
|
44
|
+
required: false,
|
|
45
|
+
default: false,
|
|
46
|
+
description: "If true, the comparison is case-sensitive"
|
|
47
|
+
};
|
|
48
|
+
var negatedConfigField = {
|
|
49
|
+
key: "negated",
|
|
50
|
+
label: "Negated",
|
|
51
|
+
type: "boolean",
|
|
52
|
+
required: false,
|
|
53
|
+
default: false,
|
|
54
|
+
description: "If true, the evaluation passes when the match is not found"
|
|
55
|
+
};
|
|
56
|
+
function strictModeConfigField(description) {
|
|
57
|
+
return {
|
|
58
|
+
key: "strict",
|
|
59
|
+
label: "Strict Mode",
|
|
60
|
+
type: "boolean",
|
|
61
|
+
required: false,
|
|
62
|
+
default: false,
|
|
63
|
+
description
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
var expectedOutputCriteriaField = {
|
|
67
|
+
key: "expectedOutput",
|
|
68
|
+
label: "Expected Output",
|
|
69
|
+
type: "object",
|
|
70
|
+
required: false,
|
|
71
|
+
description: "The expected output value for this evaluator"
|
|
72
|
+
};
|
|
73
|
+
var llmBaseConfigFields = [
|
|
74
|
+
{
|
|
75
|
+
key: "model",
|
|
76
|
+
label: "Model",
|
|
77
|
+
type: "llmModel",
|
|
78
|
+
required: true,
|
|
79
|
+
description: "The LLM model to use for judging"
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
key: "prompt",
|
|
83
|
+
label: "Evaluation Prompt",
|
|
84
|
+
type: "string",
|
|
85
|
+
required: true,
|
|
86
|
+
default: DEFAULT_EVALUATOR_PROMPT,
|
|
87
|
+
description: "Custom prompt template for the LLM judge"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
key: "temperature",
|
|
91
|
+
label: "Temperature",
|
|
92
|
+
type: "number",
|
|
93
|
+
required: true,
|
|
94
|
+
default: 0,
|
|
95
|
+
min: 0,
|
|
96
|
+
max: 2,
|
|
97
|
+
step: 0.1,
|
|
98
|
+
description: "Temperature for LLM generation"
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
key: "maxTokens",
|
|
102
|
+
label: "Max Tokens",
|
|
103
|
+
type: "number",
|
|
104
|
+
required: true,
|
|
105
|
+
default: 8192,
|
|
106
|
+
min: 0,
|
|
107
|
+
max: 16384,
|
|
108
|
+
step: 256,
|
|
109
|
+
description: "Maximum tokens for LLM response"
|
|
110
|
+
}
|
|
111
|
+
];
|
|
112
|
+
var llmTrajectoryBaseConfigFields = llmBaseConfigFields.map((field) => field.key === "prompt" ? { ...field, default: DEFAULT_TRAJECTORY_EVALUATOR_PROMPT } : field);
|
|
113
|
+
|
|
114
|
+
// ../eval-sdk/src/eval-schema/deterministic-evaluators.ts
|
|
115
|
+
var DETERMINISTIC_EVALUATOR_TYPES = {
|
|
116
|
+
"uipath-contains": {
|
|
117
|
+
name: "Contains Text",
|
|
118
|
+
description: "Check if output contains specific text",
|
|
119
|
+
category: "deterministic",
|
|
120
|
+
shortDescription: "Passes when output contains given text",
|
|
121
|
+
configFields: [
|
|
122
|
+
targetOutputKeyConfigField,
|
|
123
|
+
caseSensitiveConfigField,
|
|
124
|
+
{
|
|
125
|
+
...negatedConfigField,
|
|
126
|
+
label: "Negated (should NOT contain)"
|
|
127
|
+
}
|
|
128
|
+
],
|
|
129
|
+
criteriaFields: [
|
|
130
|
+
{
|
|
131
|
+
key: "searchText",
|
|
132
|
+
label: "Search Text",
|
|
133
|
+
type: "string",
|
|
134
|
+
required: false,
|
|
135
|
+
minLength: 1,
|
|
136
|
+
description: "The text to search for in the output"
|
|
137
|
+
}
|
|
138
|
+
],
|
|
139
|
+
examples: {
|
|
140
|
+
criteria: { searchText: "loan approved" }
|
|
141
|
+
}
|
|
142
|
+
},
|
|
143
|
+
"uipath-exact-match": {
|
|
144
|
+
name: "Exact Match",
|
|
145
|
+
description: "Check if output exactly matches expected value",
|
|
146
|
+
category: "deterministic",
|
|
147
|
+
shortDescription: "Passes when output is an exact match",
|
|
148
|
+
configFields: [
|
|
149
|
+
targetOutputKeyConfigField,
|
|
150
|
+
caseSensitiveConfigField,
|
|
151
|
+
negatedConfigField
|
|
152
|
+
],
|
|
153
|
+
criteriaFields: [expectedOutputCriteriaField],
|
|
154
|
+
examples: {
|
|
155
|
+
criteria: { expectedOutput: { value: "Application approved" } }
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
"uipath-json-similarity": {
|
|
159
|
+
name: "JSON Similarity",
|
|
160
|
+
description: "Compare JSON output similarity",
|
|
161
|
+
category: "deterministic",
|
|
162
|
+
shortDescription: "Passes when JSON structures are similar",
|
|
163
|
+
configFields: [targetOutputKeyConfigField],
|
|
164
|
+
criteriaFields: [expectedOutputCriteriaField],
|
|
165
|
+
examples: {
|
|
166
|
+
criteria: { expectedOutput: { status: "ok" } }
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
// ../eval-sdk/src/eval-schema/llm-judge-evaluators.ts
|
|
172
|
+
var LLM_JUDGE_EVALUATOR_TYPES = {
|
|
173
|
+
"uipath-llm-judge-output-semantic-similarity": {
|
|
174
|
+
name: "LLM Judge Output",
|
|
175
|
+
description: "Use LLM to judge semantic similarity of outputs",
|
|
176
|
+
category: "llm-judge",
|
|
177
|
+
shortDescription: "AI judges if output semantically matches expected",
|
|
178
|
+
configFields: [...llmBaseConfigFields, targetOutputKeyConfigField],
|
|
179
|
+
criteriaFields: [expectedOutputCriteriaField],
|
|
180
|
+
examples: {
|
|
181
|
+
config: { model: "gpt-4o", temperature: 0 },
|
|
182
|
+
criteria: {
|
|
183
|
+
expectedOutput: {
|
|
184
|
+
text: "The loan has been approved for the requested amount."
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
},
|
|
189
|
+
"uipath-llm-judge-output-strict-json-similarity": {
|
|
190
|
+
name: "LLM Judge Strict JSON",
|
|
191
|
+
description: "Use LLM to judge strict JSON similarity",
|
|
192
|
+
category: "llm-judge",
|
|
193
|
+
shortDescription: "AI judges strict JSON structural match",
|
|
194
|
+
configFields: [...llmBaseConfigFields, targetOutputKeyConfigField],
|
|
195
|
+
criteriaFields: [expectedOutputCriteriaField],
|
|
196
|
+
examples: {
|
|
197
|
+
config: { model: "gpt-4o", temperature: 0 },
|
|
198
|
+
criteria: { expectedOutput: { result: "success" } }
|
|
199
|
+
}
|
|
200
|
+
},
|
|
201
|
+
"uipath-llm-judge-trajectory-similarity": {
|
|
202
|
+
name: "LLM Judge Trajectory",
|
|
203
|
+
description: "Use LLM to judge agent trajectory and behavior",
|
|
204
|
+
category: "llm-judge",
|
|
205
|
+
shortDescription: "AI judges if agent followed expected behavior",
|
|
206
|
+
configFields: llmTrajectoryBaseConfigFields,
|
|
207
|
+
criteriaFields: [
|
|
208
|
+
{
|
|
209
|
+
key: "expectedAgentBehavior",
|
|
210
|
+
label: "Expected Agent Behavior",
|
|
211
|
+
type: "string",
|
|
212
|
+
required: false,
|
|
213
|
+
description: "Description of expected agent behavior and actions"
|
|
214
|
+
}
|
|
215
|
+
],
|
|
216
|
+
examples: {
|
|
217
|
+
config: { model: "gpt-4o" },
|
|
218
|
+
criteria: {
|
|
219
|
+
expectedAgentBehavior: "Agent checks eligibility, then calls approval API"
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
},
|
|
223
|
+
"uipath-llm-judge-trajectory-simulation": {
|
|
224
|
+
name: "LLM Judge Simulation",
|
|
225
|
+
description: "Use LLM to judge simulation trajectory",
|
|
226
|
+
category: "llm-judge",
|
|
227
|
+
shortDescription: "AI judges simulated agent execution path",
|
|
228
|
+
configFields: llmTrajectoryBaseConfigFields,
|
|
229
|
+
criteriaFields: [
|
|
230
|
+
{
|
|
231
|
+
key: "expectedAgentBehavior",
|
|
232
|
+
label: "Expected Agent Behavior",
|
|
233
|
+
type: "string",
|
|
234
|
+
required: false,
|
|
235
|
+
description: "Description of expected agent behavior during simulation"
|
|
236
|
+
}
|
|
237
|
+
],
|
|
238
|
+
examples: {
|
|
239
|
+
config: { model: "gpt-4o" },
|
|
240
|
+
criteria: {
|
|
241
|
+
expectedAgentBehavior: "Agent simulates the user journey through checkout flow"
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
// ../eval-sdk/src/eval-schema/tool-call-evaluators.ts
|
|
248
|
+
var TOOL_CALL_EVALUATOR_TYPES = {
|
|
249
|
+
"uipath-tool-call-args": {
|
|
250
|
+
name: "Tool Call Arguments",
|
|
251
|
+
description: "Verify tool call arguments match expected values",
|
|
252
|
+
category: "tool-call",
|
|
253
|
+
shortDescription: "Checks that tools were called with correct arguments",
|
|
254
|
+
configFields: [
|
|
255
|
+
strictModeConfigField("If true, requires exact match of all tool call arguments"),
|
|
256
|
+
{
|
|
257
|
+
key: "subset",
|
|
258
|
+
label: "Subset Mode",
|
|
259
|
+
type: "boolean",
|
|
260
|
+
required: false,
|
|
261
|
+
default: false,
|
|
262
|
+
description: "If true, allows actual tool calls to be a subset of expected"
|
|
263
|
+
}
|
|
264
|
+
],
|
|
265
|
+
criteriaFields: [
|
|
266
|
+
{
|
|
267
|
+
key: "toolCalls",
|
|
268
|
+
label: "Tool Calls",
|
|
269
|
+
type: "toolCallArgs",
|
|
270
|
+
required: false,
|
|
271
|
+
description: "Expected tool calls with names and arguments"
|
|
272
|
+
}
|
|
273
|
+
]
|
|
274
|
+
},
|
|
275
|
+
"uipath-tool-call-count": {
|
|
276
|
+
name: "Tool Call Count",
|
|
277
|
+
description: "Verify tool call counts match expected values",
|
|
278
|
+
category: "tool-call",
|
|
279
|
+
shortDescription: "Checks number of times each tool was called",
|
|
280
|
+
configFields: [
|
|
281
|
+
strictModeConfigField("If true, requires exact count match")
|
|
282
|
+
],
|
|
283
|
+
criteriaFields: [
|
|
284
|
+
{
|
|
285
|
+
key: "toolCallsCount",
|
|
286
|
+
label: "Tool Calls Count",
|
|
287
|
+
type: "toolCallCount",
|
|
288
|
+
required: false,
|
|
289
|
+
description: "Expected tool call counts with comparison operators"
|
|
290
|
+
}
|
|
291
|
+
]
|
|
292
|
+
},
|
|
293
|
+
"uipath-tool-call-order": {
|
|
294
|
+
name: "Tool Call Order",
|
|
295
|
+
description: "Verify tool calls occur in expected order",
|
|
296
|
+
category: "tool-call",
|
|
297
|
+
shortDescription: "Checks that tools were called in the right sequence",
|
|
298
|
+
configFields: [
|
|
299
|
+
strictModeConfigField("If true, requires exact order match")
|
|
300
|
+
],
|
|
301
|
+
criteriaFields: [
|
|
302
|
+
{
|
|
303
|
+
key: "toolCallsOrder",
|
|
304
|
+
label: "Tool Calls Order",
|
|
305
|
+
type: "toolCallOrder",
|
|
306
|
+
required: false,
|
|
307
|
+
description: "Tool names in expected execution order"
|
|
308
|
+
}
|
|
309
|
+
]
|
|
310
|
+
},
|
|
311
|
+
"uipath-tool-call-output": {
|
|
312
|
+
name: "Tool Call Output",
|
|
313
|
+
description: "Verify tool call outputs match expected values",
|
|
314
|
+
category: "tool-call",
|
|
315
|
+
shortDescription: "Checks that tool outputs match expected values",
|
|
316
|
+
configFields: [
|
|
317
|
+
strictModeConfigField("If true, requires exact output match")
|
|
318
|
+
],
|
|
319
|
+
criteriaFields: [
|
|
320
|
+
{
|
|
321
|
+
key: "toolOutputs",
|
|
322
|
+
label: "Tool Outputs",
|
|
323
|
+
type: "toolCallOutput",
|
|
324
|
+
required: false,
|
|
325
|
+
description: "Expected tool outputs with names and output values"
|
|
326
|
+
}
|
|
327
|
+
]
|
|
328
|
+
}
|
|
329
|
+
};
|
|
330
|
+
|
|
331
|
+
// ../eval-sdk/src/eval-schema/registry.ts
|
|
332
|
+
var EVALUATOR_TYPE_REGISTRY = {
|
|
333
|
+
...DETERMINISTIC_EVALUATOR_TYPES,
|
|
334
|
+
...LLM_JUDGE_EVALUATOR_TYPES,
|
|
335
|
+
...TOOL_CALL_EVALUATOR_TYPES
|
|
336
|
+
};
|
|
337
|
+
var EVALUATOR_TYPE_ALIASES = {
|
|
338
|
+
"exact-match": "uipath-exact-match",
|
|
339
|
+
"json-similarity": "uipath-json-similarity",
|
|
340
|
+
contains: "uipath-contains",
|
|
341
|
+
"llm-judge-output": "uipath-llm-judge-output-semantic-similarity",
|
|
342
|
+
"llm-judge-strict-json": "uipath-llm-judge-output-strict-json-similarity",
|
|
343
|
+
"llm-judge-trajectory": "uipath-llm-judge-trajectory-similarity",
|
|
344
|
+
"llm-judge-trajectory-simulation": "uipath-llm-judge-trajectory-simulation"
|
|
345
|
+
};
|
|
346
|
+
|
|
347
|
+
// ../eval-sdk/src/eval-schema/helpers.ts
|
|
348
|
+
function evaluatorTypeAliases() {
|
|
349
|
+
return Object.keys(EVALUATOR_TYPE_ALIASES);
|
|
350
|
+
}
|
|
351
|
+
function isEvaluatorTypeId(value) {
|
|
352
|
+
return Object.hasOwn(EVALUATOR_TYPE_REGISTRY, value);
|
|
353
|
+
}
|
|
354
|
+
function getEvaluatorTypeDefinition(typeId) {
|
|
355
|
+
if (!isEvaluatorTypeId(typeId)) {
|
|
356
|
+
return null;
|
|
357
|
+
}
|
|
358
|
+
return EVALUATOR_TYPE_REGISTRY[typeId];
|
|
359
|
+
}
|
|
360
|
+
function resolveEvaluatorType(type) {
|
|
361
|
+
const alias = Object.hasOwn(EVALUATOR_TYPE_ALIASES, type) ? EVALUATOR_TYPE_ALIASES[type] : undefined;
|
|
362
|
+
const resolved = alias ?? type;
|
|
363
|
+
if (!isEvaluatorTypeId(resolved)) {
|
|
364
|
+
throw new Error(`Unknown evaluator type "${type}". Valid types: ${evaluatorTypeAliases().join(", ")}`);
|
|
365
|
+
}
|
|
366
|
+
return resolved;
|
|
367
|
+
}
|
|
368
|
+
function getDefaultConfigForEvaluatorType(typeId) {
|
|
369
|
+
return collectFieldDefaults(EVALUATOR_TYPE_REGISTRY[typeId].configFields);
|
|
370
|
+
}
|
|
371
|
+
function collectFieldDefaults(fields) {
|
|
372
|
+
const defaults = {};
|
|
373
|
+
for (const field of fields) {
|
|
374
|
+
if (field.default !== undefined) {
|
|
375
|
+
defaults[field.key] = field.default;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
return defaults;
|
|
379
|
+
}
|
|
380
|
+
function evaluatorTypeHasConfigField(typeId, fieldKey) {
|
|
381
|
+
return getEvaluatorTypeDefinition(typeId)?.configFields.some((field) => field.key === fieldKey) ?? false;
|
|
382
|
+
}
|
|
383
|
+
function evaluatorCanUseSharedExpectedOutput(typeId) {
|
|
384
|
+
return getEvaluatorTypeDefinition(typeId)?.criteriaFields.some((field) => field.key === "expectedOutput" && field.type === "object") ?? false;
|
|
385
|
+
}
|
|
386
|
+
// ../eval-sdk/src/eval-utils.ts
|
|
387
|
+
function isRecord(value) {
|
|
388
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
389
|
+
}
|
|
390
|
+
var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
391
|
+
function isUUID(value) {
|
|
392
|
+
return UUID_RE.test(value);
|
|
393
|
+
}
|
|
394
|
+
function stripJsonExt(name) {
|
|
395
|
+
return name.endsWith(".json") ? name.slice(0, -".json".length) : name;
|
|
396
|
+
}
|
|
397
|
+
function slugify(text) {
|
|
398
|
+
const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
399
|
+
return slug || "untitled";
|
|
400
|
+
}
|
|
401
|
+
function getEvaluatorRef(evaluator) {
|
|
402
|
+
return stripJsonExt(evaluator.fileName ?? `${evaluator.id}.json`);
|
|
403
|
+
}
|
|
404
|
+
function buildEvaluatorConfig(typeId, options) {
|
|
405
|
+
const evaluatorConfig = {
|
|
406
|
+
name: options.name,
|
|
407
|
+
...getDefaultConfigForEvaluatorType(typeId)
|
|
408
|
+
};
|
|
409
|
+
if (evaluatorTypeHasConfigField(typeId, "targetOutputKey")) {
|
|
410
|
+
evaluatorConfig.targetOutputKey = options.targetOutputKey ?? "*";
|
|
411
|
+
}
|
|
412
|
+
if (options.model !== undefined) {
|
|
413
|
+
evaluatorConfig.model = options.model;
|
|
414
|
+
}
|
|
415
|
+
if (options.prompt !== undefined) {
|
|
416
|
+
evaluatorConfig.prompt = options.prompt;
|
|
417
|
+
}
|
|
418
|
+
return evaluatorConfig;
|
|
419
|
+
}
|
|
420
|
+
function buildDefaultCriteria(evaluators, expectedOutput, searchText) {
|
|
421
|
+
const criterias = {};
|
|
422
|
+
for (const evaluator of evaluators) {
|
|
423
|
+
const key = evaluator.id;
|
|
424
|
+
if (expectedOutput !== undefined && evaluatorCanUseSharedExpectedOutput(evaluator.evaluatorTypeId)) {
|
|
425
|
+
criterias[key] = { expectedOutput };
|
|
426
|
+
} else if (searchText !== undefined && evaluator.evaluatorTypeId === "uipath-contains") {
|
|
427
|
+
criterias[key] = { searchText };
|
|
428
|
+
} else {
|
|
429
|
+
const defaultCriteria = evaluator.evaluatorConfig.defaultEvaluationCriteria;
|
|
430
|
+
criterias[key] = isRecord(defaultCriteria) ? defaultCriteria : null;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
return criterias;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
// ../eval-sdk/src/eval-selectors.ts
|
|
437
|
+
function findEvaluator(evaluators, idOrName) {
|
|
438
|
+
return evaluators.find((ev) => ev.id === idOrName || ev.name === idOrName || getEvaluatorRef(ev) === idOrName) ?? null;
|
|
439
|
+
}
|
|
440
|
+
function findEvalSet(sets, idOrName) {
|
|
441
|
+
return sets.find((set) => set.id === idOrName || set.name === idOrName || stripJsonExt(set.fileName ?? "") === idOrName) ?? null;
|
|
442
|
+
}
|
|
443
|
+
function getLinkedEvaluators(allEvaluators, evalSet) {
|
|
444
|
+
const refs = new Set(evalSet.evaluatorRefs.map(stripJsonExt));
|
|
445
|
+
return allEvaluators.filter((evaluator) => {
|
|
446
|
+
const candidates = [
|
|
447
|
+
evaluator.id,
|
|
448
|
+
evaluator.fileName,
|
|
449
|
+
getEvaluatorRef(evaluator)
|
|
450
|
+
].filter((value) => value !== undefined);
|
|
451
|
+
return candidates.some((candidate) => refs.has(stripJsonExt(candidate)));
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
function removeEvaluatorRefFromSet(evalSet, evaluator) {
|
|
455
|
+
const removedRefs = new Set([
|
|
456
|
+
evaluator.id,
|
|
457
|
+
getEvaluatorRef(evaluator),
|
|
458
|
+
evaluator.fileName
|
|
459
|
+
]);
|
|
460
|
+
const before = evalSet.evaluatorRefs.length;
|
|
461
|
+
evalSet.evaluatorRefs = evalSet.evaluatorRefs.filter((ref) => !removedRefs.has(ref));
|
|
462
|
+
return evalSet.evaluatorRefs.length !== before;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
// ../eval-sdk/src/eval-evaluation-crud.ts
|
|
466
|
+
class EvaluationCrud {
|
|
467
|
+
fs;
|
|
468
|
+
store;
|
|
469
|
+
pathResolver;
|
|
470
|
+
constructor(fs, store, pathResolver) {
|
|
471
|
+
this.fs = fs;
|
|
472
|
+
this.store = store;
|
|
473
|
+
this.pathResolver = pathResolver;
|
|
474
|
+
}
|
|
475
|
+
async add(projectPath, setIdOrName, options) {
|
|
476
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
477
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
478
|
+
const evaluators = getLinkedEvaluators(await this.listEvaluators(root), evalSet);
|
|
479
|
+
const now = new Date().toISOString();
|
|
480
|
+
const evaluation = {
|
|
481
|
+
id: crypto.randomUUID(),
|
|
482
|
+
name: options.name,
|
|
483
|
+
inputs: options.inputs,
|
|
484
|
+
expectedOutput: options.expectedOutput,
|
|
485
|
+
evaluationCriterias: options.criteria ?? buildDefaultCriteria(evaluators, options.expectedOutput, options.searchText),
|
|
486
|
+
createdAt: now,
|
|
487
|
+
updatedAt: now
|
|
488
|
+
};
|
|
489
|
+
evalSet.evaluations.push(evaluation);
|
|
490
|
+
evalSet.updatedAt = now;
|
|
491
|
+
await this.writeEvalSet(root, evalSet);
|
|
492
|
+
return evaluation;
|
|
493
|
+
}
|
|
494
|
+
async list(projectPath, setIdOrName) {
|
|
495
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
496
|
+
return evalSet.evaluations;
|
|
497
|
+
}
|
|
498
|
+
async addSimulation(projectPath, setIdOrName, evaluationIdOrName, simulation) {
|
|
499
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
500
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
501
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
502
|
+
if (!evaluation) {
|
|
503
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
504
|
+
}
|
|
505
|
+
if (!evaluation.simulations) {
|
|
506
|
+
evaluation.simulations = [];
|
|
507
|
+
}
|
|
508
|
+
const existing = evaluation.simulations.findIndex((s) => s.componentId === simulation.componentId);
|
|
509
|
+
const now = new Date().toISOString();
|
|
510
|
+
if (existing !== -1) {
|
|
511
|
+
evaluation.simulations[existing] = simulation;
|
|
512
|
+
} else {
|
|
513
|
+
evaluation.simulations.push(simulation);
|
|
514
|
+
}
|
|
515
|
+
evaluation.updatedAt = now;
|
|
516
|
+
evalSet.updatedAt = now;
|
|
517
|
+
await this.writeEvalSet(root, evalSet);
|
|
518
|
+
return simulation;
|
|
519
|
+
}
|
|
520
|
+
async listSimulations(projectPath, setIdOrName, evaluationIdOrName) {
|
|
521
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
522
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
523
|
+
if (!evaluation) {
|
|
524
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
525
|
+
}
|
|
526
|
+
return evaluation.simulations ?? [];
|
|
527
|
+
}
|
|
528
|
+
async addChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childSimulation) {
|
|
529
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
530
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
531
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
532
|
+
if (!evaluation) {
|
|
533
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
534
|
+
}
|
|
535
|
+
if (!evaluation.simulations) {
|
|
536
|
+
evaluation.simulations = [];
|
|
537
|
+
}
|
|
538
|
+
let parent = evaluation.simulations.find((s) => s.componentId === parentComponentId);
|
|
539
|
+
if (!parent) {
|
|
540
|
+
parent = {
|
|
541
|
+
componentId: parentComponentId,
|
|
542
|
+
componentType: "agent",
|
|
543
|
+
simulationStrategy: "Llm",
|
|
544
|
+
childSimulations: []
|
|
545
|
+
};
|
|
546
|
+
evaluation.simulations.push(parent);
|
|
547
|
+
}
|
|
548
|
+
if (!parent.childSimulations) {
|
|
549
|
+
parent.childSimulations = [];
|
|
550
|
+
}
|
|
551
|
+
const existing = parent.childSimulations.findIndex((s) => s.componentId === childSimulation.componentId);
|
|
552
|
+
const now = new Date().toISOString();
|
|
553
|
+
if (existing !== -1) {
|
|
554
|
+
parent.childSimulations[existing] = childSimulation;
|
|
555
|
+
} else {
|
|
556
|
+
parent.childSimulations.push(childSimulation);
|
|
557
|
+
}
|
|
558
|
+
evaluation.updatedAt = now;
|
|
559
|
+
evalSet.updatedAt = now;
|
|
560
|
+
await this.writeEvalSet(root, evalSet);
|
|
561
|
+
return childSimulation;
|
|
562
|
+
}
|
|
563
|
+
listChildSimulations(evaluation, parentComponentId) {
|
|
564
|
+
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
565
|
+
if (!parent) {
|
|
566
|
+
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
567
|
+
}
|
|
568
|
+
return parent.childSimulations ?? [];
|
|
569
|
+
}
|
|
570
|
+
async removeChildSimulation(projectPath, setIdOrName, evaluationIdOrName, parentComponentId, childComponentId) {
|
|
571
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
572
|
+
if (!root) {
|
|
573
|
+
return false;
|
|
574
|
+
}
|
|
575
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
576
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
577
|
+
if (!evaluation) {
|
|
578
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
579
|
+
}
|
|
580
|
+
const parent = (evaluation.simulations ?? []).find((s) => s.componentId === parentComponentId);
|
|
581
|
+
if (!parent) {
|
|
582
|
+
throw new Error(`Simulation for component "${parentComponentId}" not found`);
|
|
583
|
+
}
|
|
584
|
+
const index = (parent.childSimulations ?? []).findIndex((s) => s.componentId === childComponentId);
|
|
585
|
+
if (index === -1) {
|
|
586
|
+
return false;
|
|
587
|
+
}
|
|
588
|
+
parent.childSimulations?.splice(index, 1);
|
|
589
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
590
|
+
await this.writeEvalSet(root, evalSet);
|
|
591
|
+
return true;
|
|
592
|
+
}
|
|
593
|
+
async removeSimulation(projectPath, setIdOrName, evaluationIdOrName, componentId) {
|
|
594
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
595
|
+
if (!root) {
|
|
596
|
+
return false;
|
|
597
|
+
}
|
|
598
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
599
|
+
const evaluation = evalSet.evaluations.find((e) => e.id === evaluationIdOrName || e.name === evaluationIdOrName);
|
|
600
|
+
if (!evaluation) {
|
|
601
|
+
throw new Error(`Data point "${evaluationIdOrName}" not found`);
|
|
602
|
+
}
|
|
603
|
+
const index = (evaluation.simulations ?? []).findIndex((s) => s.componentId === componentId);
|
|
604
|
+
if (index === -1) {
|
|
605
|
+
return false;
|
|
606
|
+
}
|
|
607
|
+
evaluation.simulations?.splice(index, 1);
|
|
608
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
609
|
+
await this.writeEvalSet(root, evalSet);
|
|
610
|
+
return true;
|
|
611
|
+
}
|
|
612
|
+
async remove(projectPath, setIdOrName, evaluationIdOrName) {
|
|
613
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
614
|
+
if (!root) {
|
|
615
|
+
return false;
|
|
616
|
+
}
|
|
617
|
+
const evalSet = await this.requireEvalSet(projectPath, setIdOrName);
|
|
618
|
+
const index = evalSet.evaluations.findIndex((evaluation) => evaluation.id === evaluationIdOrName || evaluation.name === evaluationIdOrName);
|
|
619
|
+
if (index === -1) {
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
evalSet.evaluations.splice(index, 1);
|
|
623
|
+
evalSet.updatedAt = new Date().toISOString();
|
|
624
|
+
await this.writeEvalSet(root, evalSet);
|
|
625
|
+
return true;
|
|
626
|
+
}
|
|
627
|
+
async requireEvalSet(projectPath, idOrName) {
|
|
628
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
629
|
+
const evalSet = root ? findEvalSet(await this.listEvalSets(root), idOrName) : null;
|
|
630
|
+
if (!evalSet) {
|
|
631
|
+
throw new Error(`Evaluation set "${idOrName}" not found`);
|
|
632
|
+
}
|
|
633
|
+
return evalSet;
|
|
634
|
+
}
|
|
635
|
+
async listEvalSets(root) {
|
|
636
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
637
|
+
}
|
|
638
|
+
async listEvaluators(root) {
|
|
639
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
640
|
+
}
|
|
641
|
+
async writeEvalSet(root, evalSet) {
|
|
642
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// ../eval-sdk/src/eval-evaluator-crud.ts
|
|
647
|
+
class EvaluatorCrud {
|
|
648
|
+
fs;
|
|
649
|
+
store;
|
|
650
|
+
pathResolver;
|
|
651
|
+
constructor(fs, store, pathResolver) {
|
|
652
|
+
this.fs = fs;
|
|
653
|
+
this.store = store;
|
|
654
|
+
this.pathResolver = pathResolver;
|
|
655
|
+
}
|
|
656
|
+
async list(projectPath) {
|
|
657
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
658
|
+
if (!root) {
|
|
659
|
+
return [];
|
|
660
|
+
}
|
|
661
|
+
return this.store.readJsonFiles(this.fs.path.join(root, "evaluators"));
|
|
662
|
+
}
|
|
663
|
+
async add(projectPath, options) {
|
|
664
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
665
|
+
const typeId = resolveEvaluatorType(options.type);
|
|
666
|
+
const id = `${slugify(options.name)}-${crypto.randomUUID().slice(0, 8)}`;
|
|
667
|
+
const now = new Date().toISOString();
|
|
668
|
+
const fileName = `${id}.json`;
|
|
669
|
+
const evaluator = {
|
|
670
|
+
version: "1.0",
|
|
671
|
+
id,
|
|
672
|
+
name: options.name,
|
|
673
|
+
description: options.description ?? `${options.name} evaluator`,
|
|
674
|
+
evaluatorTypeId: typeId,
|
|
675
|
+
evaluatorConfig: buildEvaluatorConfig(typeId, options),
|
|
676
|
+
fileName,
|
|
677
|
+
createdAt: now,
|
|
678
|
+
updatedAt: now
|
|
679
|
+
};
|
|
680
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "evaluators", fileName), evaluator);
|
|
681
|
+
return evaluator;
|
|
682
|
+
}
|
|
683
|
+
async remove(projectPath, idOrName) {
|
|
684
|
+
const root = await this.pathResolver.resolveEvalRoot(projectPath, false);
|
|
685
|
+
if (!root) {
|
|
686
|
+
return false;
|
|
687
|
+
}
|
|
688
|
+
const evaluator = findEvaluator(await this.list(projectPath), idOrName);
|
|
689
|
+
if (!evaluator?.fileName) {
|
|
690
|
+
return false;
|
|
691
|
+
}
|
|
692
|
+
await this.store.remove(this.fs.path.join(root, "evaluators", evaluator.fileName));
|
|
693
|
+
await this.removeEvaluatorRefs(root, evaluator);
|
|
694
|
+
return true;
|
|
695
|
+
}
|
|
696
|
+
async removeEvaluatorRefs(root, evaluator) {
|
|
697
|
+
const sets = await this.store.readJsonFiles(this.fs.path.join(root, "eval-sets"));
|
|
698
|
+
for (const set of sets) {
|
|
699
|
+
if (removeEvaluatorRefFromSet(set, evaluator) && set.fileName) {
|
|
700
|
+
await this.writeEvalSet(root, set);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
async writeEvalSet(root, evalSet) {
|
|
705
|
+
await this.store.writeJsonFile(this.fs.path.join(root, "eval-sets", evalSet.fileName ?? `${evalSet.id}.json`), evalSet);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
// ../eval-sdk/src/eval-file-store.ts
|
|
710
|
+
async function ensureDirectory(fs, path) {
|
|
711
|
+
if (await fs.exists(path)) {
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
const parent = fs.path.dirname(path);
|
|
715
|
+
if (parent && parent !== path && !await fs.exists(parent)) {
|
|
716
|
+
await ensureDirectory(fs, parent);
|
|
717
|
+
}
|
|
718
|
+
const [error] = await catchError(fs.mkdir(path));
|
|
719
|
+
if (error && !await fs.exists(path)) {
|
|
720
|
+
throw error;
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
class EvalFileStore {
|
|
725
|
+
fs;
|
|
726
|
+
constructor(fs) {
|
|
727
|
+
this.fs = fs;
|
|
728
|
+
}
|
|
729
|
+
async readJsonFiles(dir) {
|
|
730
|
+
const [readDirError, files] = await catchError(this.fs.readdir(dir));
|
|
731
|
+
if (readDirError) {
|
|
732
|
+
return [];
|
|
733
|
+
}
|
|
734
|
+
const items = [];
|
|
735
|
+
for (const file of files) {
|
|
736
|
+
if (!file.endsWith(".json")) {
|
|
737
|
+
continue;
|
|
738
|
+
}
|
|
739
|
+
const filePath = this.fs.path.join(dir, file);
|
|
740
|
+
const [readError, raw] = await catchError(this.fs.readFile(filePath, "utf-8"));
|
|
741
|
+
if (readError) {
|
|
742
|
+
throw new Error(`Failed to read JSON file "${filePath}": ${readError.message}`);
|
|
743
|
+
}
|
|
744
|
+
if (raw === null) {
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
const [parseError, data] = catchError(() => JSON.parse(String(raw)));
|
|
748
|
+
if (parseError) {
|
|
749
|
+
throw new Error(`Invalid JSON in "${filePath}": ${parseError.message}`);
|
|
750
|
+
}
|
|
751
|
+
data.fileName = file;
|
|
752
|
+
items.push(data);
|
|
753
|
+
}
|
|
754
|
+
return items;
|
|
755
|
+
}
|
|
756
|
+
async writeJsonFile(path, value) {
|
|
757
|
+
await ensureDirectory(this.fs, this.fs.path.dirname(path));
|
|
758
|
+
await this.fs.writeFile(path, `${JSON.stringify(value, null, 2)}
|
|
759
|
+
`);
|
|
760
|
+
}
|
|
761
|
+
async remove(path) {
|
|
762
|
+
await this.fs.rm(path);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
// ../eval-sdk/src/eval-files.ts
|
|
767
|
+
import { randomUUID } from "node:crypto";
|
|
768
|
+
var EVAL_FILES_DIR = "eval-files";
|
|
769
|
+
var TOKEN_LENGTH = 12;
|
|
770
|
+
var EVALUATION_BINARY_PREFIX = "BINARY_FILE:";
|
|
771
|
+
var EVALUATION_BINARY_DELIMITER = `
|
|
772
|
+
---BASE64---
|
|
773
|
+
`;
|
|
774
|
+
var TEXT_EXTENSIONS = new Set([
|
|
775
|
+
".json",
|
|
776
|
+
".csv",
|
|
777
|
+
".tsv",
|
|
778
|
+
".txt",
|
|
779
|
+
".md",
|
|
780
|
+
".log",
|
|
781
|
+
".xml",
|
|
782
|
+
".html",
|
|
783
|
+
".htm",
|
|
784
|
+
".yaml",
|
|
785
|
+
".yml"
|
|
786
|
+
]);
|
|
787
|
+
var MIME_BY_EXT = {
|
|
788
|
+
".jpg": "image/jpeg",
|
|
789
|
+
".jpeg": "image/jpeg",
|
|
790
|
+
".png": "image/png",
|
|
791
|
+
".gif": "image/gif",
|
|
792
|
+
".bmp": "image/bmp",
|
|
793
|
+
".webp": "image/webp",
|
|
794
|
+
".tif": "image/tiff",
|
|
795
|
+
".tiff": "image/tiff",
|
|
796
|
+
".svg": "image/svg+xml",
|
|
797
|
+
".pdf": "application/pdf",
|
|
798
|
+
".doc": "application/msword",
|
|
799
|
+
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
800
|
+
".xls": "application/vnd.ms-excel",
|
|
801
|
+
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
802
|
+
".ppt": "application/vnd.ms-powerpoint",
|
|
803
|
+
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
804
|
+
".csv": "text/csv",
|
|
805
|
+
".tsv": "text/tab-separated-values",
|
|
806
|
+
".json": "application/json",
|
|
807
|
+
".txt": "text/plain",
|
|
808
|
+
".md": "text/markdown",
|
|
809
|
+
".log": "text/plain",
|
|
810
|
+
".xml": "application/xml",
|
|
811
|
+
".html": "text/html",
|
|
812
|
+
".htm": "text/html",
|
|
813
|
+
".yaml": "application/yaml",
|
|
814
|
+
".yml": "application/yaml"
|
|
815
|
+
};
|
|
816
|
+
|
|
817
|
+
class EvalFiles {
|
|
818
|
+
fs;
|
|
819
|
+
pathResolver;
|
|
820
|
+
constructor(fs, pathResolver) {
|
|
821
|
+
this.fs = fs;
|
|
822
|
+
this.pathResolver = pathResolver;
|
|
823
|
+
}
|
|
824
|
+
async materialize(projectPath, sourcePath) {
|
|
825
|
+
const projectRoot = await this.pathResolver.resolveProjectPath(projectPath);
|
|
826
|
+
const sourceAbsolute = this.fs.path.resolve(sourcePath);
|
|
827
|
+
if (!await this.fs.exists(sourceAbsolute)) {
|
|
828
|
+
throw new Error(`Input file not found: ${sourcePath}`);
|
|
829
|
+
}
|
|
830
|
+
const content = await this.fs.readFile(sourceAbsolute);
|
|
831
|
+
if (content === null) {
|
|
832
|
+
throw new Error(`Could not read input file: ${sourcePath}`);
|
|
833
|
+
}
|
|
834
|
+
const fullName = this.fs.path.basename(sourceAbsolute);
|
|
835
|
+
const evalRoot = await this.pathResolver.resolveEvalRoot(projectPath, true);
|
|
836
|
+
const dir = this.fs.path.join(evalRoot, EVAL_FILES_DIR);
|
|
837
|
+
await this.fs.mkdir(dir);
|
|
838
|
+
const targetName = await this.resolveTargetName(dir, fullName);
|
|
839
|
+
const targetPath = this.fs.path.join(dir, targetName);
|
|
840
|
+
const mimeType = mimeFromExt(extOf(fullName));
|
|
841
|
+
if (TEXT_EXTENSIONS.has(extOf(fullName))) {
|
|
842
|
+
await this.fs.writeFile(targetPath, content);
|
|
843
|
+
} else {
|
|
844
|
+
await this.fs.writeFile(targetPath, encodeBinaryEnvelope(content, mimeType));
|
|
845
|
+
}
|
|
846
|
+
const relative = this.fs.path.relative(projectRoot, targetPath).replaceAll("\\", "/");
|
|
847
|
+
return {
|
|
848
|
+
ID: `/${relative}`,
|
|
849
|
+
FullName: fullName,
|
|
850
|
+
MimeType: mimeType
|
|
851
|
+
};
|
|
852
|
+
}
|
|
853
|
+
async resolveTargetName(dir, fullName) {
|
|
854
|
+
if (!await this.fs.exists(this.fs.path.join(dir, fullName))) {
|
|
855
|
+
return fullName;
|
|
856
|
+
}
|
|
857
|
+
const ext = extOf(fullName);
|
|
858
|
+
const stem = ext ? fullName.slice(0, -ext.length) : fullName;
|
|
859
|
+
const token = randomUUID().replaceAll("-", "").slice(0, TOKEN_LENGTH);
|
|
860
|
+
return `${stem}_${token}${ext}`;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
function encodeBinaryEnvelope(content, mimeType) {
|
|
864
|
+
const base64Content = Buffer.from(content).toString("base64");
|
|
865
|
+
return `${EVALUATION_BINARY_PREFIX}${mimeType}:${content.byteLength}${EVALUATION_BINARY_DELIMITER}${base64Content}`;
|
|
866
|
+
}
|
|
867
|
+
function extOf(name) {
|
|
868
|
+
const dot = name.lastIndexOf(".");
|
|
869
|
+
if (dot <= 0) {
|
|
870
|
+
return "";
|
|
871
|
+
}
|
|
872
|
+
return name.slice(dot).toLowerCase();
|
|
873
|
+
}
|
|
874
|
+
function mimeFromExt(ext) {
|
|
875
|
+
return MIME_BY_EXT[ext] ?? "application/octet-stream";
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
// ../eval-sdk/src/eval-inject-workload-id.ts
|
|
879
|
+
var SIMULATION_STRATEGY_WIRE_VALUES = {
|
|
880
|
+
Llm: 0,
|
|
881
|
+
Static: 2
|
|
882
|
+
};
|
|
883
|
+
function injectWorkloadId(simulation, defaultWorkloadId) {
|
|
884
|
+
const { childSimulations: rawChildSimulations, ...rest } = simulation;
|
|
885
|
+
const workloadId = typeof simulation.workloadId === "string" ? simulation.workloadId : defaultWorkloadId;
|
|
886
|
+
const childSimulations = Array.isArray(rawChildSimulations) ? rawChildSimulations.map((child) => injectWorkloadId(child, workloadId)) : undefined;
|
|
887
|
+
const wireStrategy = SIMULATION_STRATEGY_WIRE_VALUES[simulation.simulationStrategy];
|
|
888
|
+
if (wireStrategy === undefined) {
|
|
889
|
+
throw new Error(`Unknown simulationStrategy "${String(simulation.simulationStrategy)}" on component "${simulation.componentId}". Supported values: ${Object.keys(SIMULATION_STRATEGY_WIRE_VALUES).join(", ")}.`);
|
|
890
|
+
}
|
|
891
|
+
return {
|
|
892
|
+
...rest,
|
|
893
|
+
workloadId,
|
|
894
|
+
simulationStrategy: wireStrategy,
|
|
895
|
+
...childSimulations ? { childSimulations } : {}
|
|
896
|
+
};
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// ../eval-sdk/src/eval-run-readers.ts
|
|
900
|
+
function asRecordArray(value, label) {
|
|
901
|
+
if (!Array.isArray(value)) {
|
|
902
|
+
throw new Error(`${label} response must be an array`);
|
|
903
|
+
}
|
|
904
|
+
return value.map((item, index) => {
|
|
905
|
+
if (!isRecord(item)) {
|
|
906
|
+
throw new Error(`${label} response item ${index} must be an object`);
|
|
907
|
+
}
|
|
908
|
+
return item;
|
|
909
|
+
});
|
|
910
|
+
}
|
|
911
|
+
function readRecord(value) {
|
|
912
|
+
return isRecord(value) ? value : undefined;
|
|
913
|
+
}
|
|
914
|
+
function readRecordArray(value) {
|
|
915
|
+
if (!Array.isArray(value)) {
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
return value.filter(isRecord);
|
|
919
|
+
}
|
|
920
|
+
function optionalNumber(value) {
|
|
921
|
+
return typeof value === "number" ? value : null;
|
|
922
|
+
}
|
|
923
|
+
function optionalString(value) {
|
|
924
|
+
return typeof value === "string" ? value : "";
|
|
925
|
+
}
|
|
926
|
+
function statusString(value) {
|
|
927
|
+
if (typeof value === "string") {
|
|
928
|
+
return value;
|
|
929
|
+
}
|
|
930
|
+
if (typeof value === "number") {
|
|
931
|
+
const statusByCode = {
|
|
932
|
+
0: "Pending",
|
|
933
|
+
1: "Running",
|
|
934
|
+
2: "Completed",
|
|
935
|
+
3: "PartiallyCompleted",
|
|
936
|
+
4: "Failed"
|
|
937
|
+
};
|
|
938
|
+
return statusByCode[value] ?? String(value);
|
|
939
|
+
}
|
|
940
|
+
return "";
|
|
941
|
+
}
|
|
942
|
+
function readEvaluatorScores(value) {
|
|
943
|
+
const records = readRecordArray(value);
|
|
944
|
+
if (!records) {
|
|
945
|
+
return null;
|
|
946
|
+
}
|
|
947
|
+
return records.flatMap((score) => {
|
|
948
|
+
const evaluatorId = score.evaluatorId;
|
|
949
|
+
const scoreValue = score.value;
|
|
950
|
+
if (typeof evaluatorId !== "string" || typeof scoreValue !== "number") {
|
|
951
|
+
return [];
|
|
952
|
+
}
|
|
953
|
+
return [{ evaluatorId, value: scoreValue }];
|
|
954
|
+
});
|
|
955
|
+
}
|
|
956
|
+
function readRemoteEvalSetRun(record) {
|
|
957
|
+
return {
|
|
958
|
+
id: optionalString(record.id),
|
|
959
|
+
workloadId: optionalString(record.workloadId),
|
|
960
|
+
evalSetId: typeof record.evalSetId === "string" ? record.evalSetId : null,
|
|
961
|
+
status: statusString(record.status),
|
|
962
|
+
score: optionalNumber(record.score),
|
|
963
|
+
evaluatorScores: readEvaluatorScores(record.evaluatorScores),
|
|
964
|
+
config: readRecord(record.config) ?? {},
|
|
965
|
+
numberOfEvalsExecuted: optionalNumber(record.numberOfEvalsExecuted),
|
|
966
|
+
durationMilliseconds: optionalNumber(record.durationMilliseconds),
|
|
967
|
+
createdAt: optionalString(record.createdAt),
|
|
968
|
+
updatedAt: optionalString(record.updatedAt)
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
function readRemoteEvalRun(record) {
|
|
972
|
+
const snapshot = readRecord(record.evalSnapshot);
|
|
973
|
+
const result = readRecord(record.result);
|
|
974
|
+
return {
|
|
975
|
+
id: optionalString(record.id),
|
|
976
|
+
evalSetRunId: optionalString(record.evalSetRunId),
|
|
977
|
+
evalSnapshot: snapshot ? {
|
|
978
|
+
id: typeof snapshot.id === "string" ? snapshot.id : undefined,
|
|
979
|
+
name: typeof snapshot.name === "string" ? snapshot.name : undefined,
|
|
980
|
+
inputs: readRecord(snapshot.inputs),
|
|
981
|
+
evaluationCriterias: readRecord(snapshot.evaluationCriterias),
|
|
982
|
+
expectedOutput: readRecord(snapshot.expectedOutput) ?? (snapshot.expectedOutput === null ? null : undefined)
|
|
983
|
+
} : undefined,
|
|
984
|
+
status: statusString(record.status),
|
|
985
|
+
result: result ? {
|
|
986
|
+
output: readRecord(result.output),
|
|
987
|
+
scores: readEvaluatorScores(result.scores) ?? undefined,
|
|
988
|
+
score: readRecord(result.score)
|
|
989
|
+
} : null,
|
|
990
|
+
completionMetrics: readRecord(record.completionMetrics),
|
|
991
|
+
evaluatorRuns: readRecordArray(record.evaluatorRuns)?.flatMap((evaluatorRun) => {
|
|
992
|
+
const evaluatorId = evaluatorRun.evaluatorId;
|
|
993
|
+
if (typeof evaluatorId !== "string") {
|
|
994
|
+
return [];
|
|
995
|
+
}
|
|
996
|
+
const evaluatorResult = readRecord(evaluatorRun.result);
|
|
997
|
+
return [
|
|
998
|
+
{
|
|
999
|
+
evaluatorId,
|
|
1000
|
+
evaluatorSnapshot: evaluatorRun.evaluatorSnapshot,
|
|
1001
|
+
status: typeof evaluatorRun.status === "string" ? evaluatorRun.status : undefined,
|
|
1002
|
+
result: evaluatorResult ? {
|
|
1003
|
+
output: readRecord(evaluatorResult.output),
|
|
1004
|
+
score: readRecord(evaluatorResult.score)
|
|
1005
|
+
} : null,
|
|
1006
|
+
completionMetrics: readRecord(evaluatorRun.completionMetrics)
|
|
1007
|
+
}
|
|
1008
|
+
];
|
|
1009
|
+
}),
|
|
1010
|
+
error: typeof record.error === "string" || isRecord(record.error) ? record.error : null,
|
|
1011
|
+
errorMessage: typeof record.errorMessage === "string" ? record.errorMessage : null,
|
|
1012
|
+
createdAt: typeof record.createdAt === "string" ? record.createdAt : undefined,
|
|
1013
|
+
updatedAt: typeof record.updatedAt === "string" ? record.updatedAt : undefined
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
// ../eval-sdk/src/eval-run-api.ts
|
|
1018
|
+
var METADATA_FETCH_TIMEOUT_MS = 30000;
|
|
1019
|
+
var MAX_ERROR_BODY_LENGTH = 400;
|
|
1020
|
+
|
|
1021
|
+
class EvalRunApi {
|
|
1022
|
+
workloadType;
|
|
1023
|
+
constructor(workloadType) {
|
|
1024
|
+
this.workloadType = workloadType;
|
|
1025
|
+
}
|
|
1026
|
+
async executeAndEvaluate(config, payload) {
|
|
1027
|
+
return this.requestJson(config, "/unifiedEvals/workloads/executeAndEvaluate", {
|
|
1028
|
+
method: "POST",
|
|
1029
|
+
body: JSON.stringify(payload)
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
async listEvalSetRuns(config, workloadId) {
|
|
1033
|
+
const response = await this.requestJson(config, `/unifiedEvals/workloads/${encodeURIComponent(workloadId)}/evalSetRuns?workloadType=${encodeURIComponent(this.workloadType)}`, { method: "GET" });
|
|
1034
|
+
return asRecordArray(response, "Eval set runs").map(readRemoteEvalSetRun);
|
|
1035
|
+
}
|
|
1036
|
+
async getEvalSetRun(config, workloadId, evalSetRunId) {
|
|
1037
|
+
const response = await this.requestJson(config, `/unifiedEvals/workloads/${encodeURIComponent(workloadId)}/evalSetRuns/${encodeURIComponent(evalSetRunId)}?workloadType=${encodeURIComponent(this.workloadType)}`, { method: "GET" });
|
|
1038
|
+
if (!isRecord(response)) {
|
|
1039
|
+
throw new Error("Eval set run response must be an object");
|
|
1040
|
+
}
|
|
1041
|
+
return readRemoteEvalSetRun(response);
|
|
1042
|
+
}
|
|
1043
|
+
async getEvalRuns(config, workloadId, evalSetRunId) {
|
|
1044
|
+
const response = await this.requestJson(config, `/unifiedEvals/workloads/${encodeURIComponent(workloadId)}/evalSetRuns/${encodeURIComponent(evalSetRunId)}/evalRuns?workloadType=${encodeURIComponent(this.workloadType)}`, { method: "GET" });
|
|
1045
|
+
return asRecordArray(response, "Eval runs").map(readRemoteEvalRun);
|
|
1046
|
+
}
|
|
1047
|
+
async getSimulatableComponents(config, processKey, processVersion) {
|
|
1048
|
+
const params = new URLSearchParams({ workloadType: "Agents" });
|
|
1049
|
+
if (processVersion) {
|
|
1050
|
+
params.set("processVersion", processVersion);
|
|
1051
|
+
}
|
|
1052
|
+
const data = await this.requestJson(config, `/unifiedEvals/processes/${encodeURIComponent(processKey)}/simulatableComponents?${params.toString()}`, { method: "GET" }, METADATA_FETCH_TIMEOUT_MS);
|
|
1053
|
+
const components = Array.isArray(data.components) ? data.components : [];
|
|
1054
|
+
return components.filter((c) => isRecord(c) && typeof c.id === "string" && typeof c.name === "string" && c.isEnabled === true);
|
|
1055
|
+
}
|
|
1056
|
+
apiBase(config) {
|
|
1057
|
+
return `${config.baseUrl}/${encodeURIComponent(config.organizationId)}/${encodeURIComponent(config.tenantName)}/agentsruntime_/api`;
|
|
1058
|
+
}
|
|
1059
|
+
headers(config) {
|
|
1060
|
+
return {
|
|
1061
|
+
Authorization: `Bearer ${config.authToken}`,
|
|
1062
|
+
Accept: "application/json",
|
|
1063
|
+
"Content-Type": "application/json",
|
|
1064
|
+
"X-UiPath-Internal-TenantId": config.tenantId,
|
|
1065
|
+
"X-UiPath-Internal-AccountId": config.organizationId
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
async requestJson(config, path, request, timeoutMs) {
|
|
1069
|
+
let response;
|
|
1070
|
+
let text;
|
|
1071
|
+
try {
|
|
1072
|
+
response = await fetch(`${this.apiBase(config)}${path}`, {
|
|
1073
|
+
method: request.method,
|
|
1074
|
+
headers: {
|
|
1075
|
+
...this.headers(config),
|
|
1076
|
+
...request.headers
|
|
1077
|
+
},
|
|
1078
|
+
body: request.body,
|
|
1079
|
+
...timeoutMs !== undefined ? { signal: AbortSignal.timeout(timeoutMs) } : {}
|
|
1080
|
+
});
|
|
1081
|
+
text = await response.text();
|
|
1082
|
+
} catch (err) {
|
|
1083
|
+
if (err instanceof DOMException && err.name === "TimeoutError") {
|
|
1084
|
+
throw new Error(`Unified eval API request timed out after ${timeoutMs}ms (${request.method} ${path})`);
|
|
1085
|
+
}
|
|
1086
|
+
throw new Error(`Unified eval API request failed (${request.method} ${path}): ${err instanceof Error ? err.message : String(err)}`);
|
|
1087
|
+
}
|
|
1088
|
+
if (!response.ok) {
|
|
1089
|
+
const body = text.length > MAX_ERROR_BODY_LENGTH ? `${text.slice(0, MAX_ERROR_BODY_LENGTH)}… (truncated)` : text;
|
|
1090
|
+
throw new Error(`Unified eval API returned ${response.status}: ${body}`);
|
|
1091
|
+
}
|
|
1092
|
+
if (!text) {
|
|
1093
|
+
throw new Error(`Unified eval API returned ${response.status} with empty body`);
|
|
1094
|
+
}
|
|
1095
|
+
try {
|
|
1096
|
+
return JSON.parse(text);
|
|
1097
|
+
} catch {
|
|
1098
|
+
const preview = text.length > MAX_ERROR_BODY_LENGTH ? `${text.slice(0, MAX_ERROR_BODY_LENGTH)}… (truncated)` : text;
|
|
1099
|
+
throw new Error(`Unified eval API returned ${response.status} with non-JSON body (${request.method} ${path}): ${preview}`);
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
}
|
|
1103
|
+
// ../eval-sdk/src/eval-login.ts
|
|
1104
|
+
async function validateLoginFields() {
|
|
1105
|
+
const { getLoginStatusAsync } = await import("./index-sqhbn7ew.js");
|
|
1106
|
+
const [loginError, login] = await catchError(getLoginStatusAsync());
|
|
1107
|
+
if (loginError) {
|
|
1108
|
+
throw new Error(loginError.message);
|
|
1109
|
+
}
|
|
1110
|
+
if (login.loginStatus !== "Logged in" || !login.baseUrl) {
|
|
1111
|
+
throw new Error("Not logged in. Run 'uip login' first.");
|
|
1112
|
+
}
|
|
1113
|
+
if (!login.organizationId) {
|
|
1114
|
+
throw new Error("Organization ID not available from login.");
|
|
1115
|
+
}
|
|
1116
|
+
if (!login.tenantId) {
|
|
1117
|
+
throw new Error("Tenant ID not available from login.");
|
|
1118
|
+
}
|
|
1119
|
+
if (!login.tenantName) {
|
|
1120
|
+
throw new Error("Tenant name not available from login.");
|
|
1121
|
+
}
|
|
1122
|
+
if (!login.accessToken) {
|
|
1123
|
+
throw new Error("Access token not available from login.");
|
|
1124
|
+
}
|
|
1125
|
+
return {
|
|
1126
|
+
baseUrl: login.baseUrl,
|
|
1127
|
+
organizationId: login.organizationId,
|
|
1128
|
+
tenantId: login.tenantId,
|
|
1129
|
+
tenantName: login.tenantName,
|
|
1130
|
+
accessToken: login.accessToken,
|
|
1131
|
+
organizationName: login.organizationName || undefined
|
|
1132
|
+
};
|
|
1133
|
+
}
|
|
1134
|
+
async function resolveLoginConfig() {
|
|
1135
|
+
const fields = await validateLoginFields();
|
|
1136
|
+
if (!fields.organizationName) {
|
|
1137
|
+
throw new Error("Organization name not available from login. Required for Studio Web evaluation.");
|
|
1138
|
+
}
|
|
1139
|
+
return {
|
|
1140
|
+
baseUrl: fields.baseUrl,
|
|
1141
|
+
organizationId: fields.organizationId,
|
|
1142
|
+
tenantId: fields.tenantId,
|
|
1143
|
+
tenantName: fields.tenantName,
|
|
1144
|
+
authToken: fields.accessToken,
|
|
1145
|
+
organizationName: fields.organizationName
|
|
1146
|
+
};
|
|
1147
|
+
}
|
|
1148
|
+
async function resolveEvalApiConfig() {
|
|
1149
|
+
const fields = await validateLoginFields();
|
|
1150
|
+
return {
|
|
1151
|
+
baseUrl: fields.baseUrl,
|
|
1152
|
+
organizationId: fields.organizationId,
|
|
1153
|
+
tenantName: fields.tenantName,
|
|
1154
|
+
tenantId: fields.tenantId,
|
|
1155
|
+
authToken: fields.accessToken
|
|
1156
|
+
};
|
|
1157
|
+
}
|
|
1158
|
+
// ../eval-sdk/src/eval-run-format.ts
|
|
1159
|
+
function formatDuration(durationMs) {
|
|
1160
|
+
if (durationMs === null || durationMs === undefined) {
|
|
1161
|
+
return "-";
|
|
1162
|
+
}
|
|
1163
|
+
return `${(durationMs / 1000).toFixed(1)}s`;
|
|
1164
|
+
}
|
|
1165
|
+
function formatEvaluatorScores(scores, evaluators) {
|
|
1166
|
+
if (!scores || scores.length === 0) {
|
|
1167
|
+
return "-";
|
|
1168
|
+
}
|
|
1169
|
+
return scores.map((score) => {
|
|
1170
|
+
const evaluator = evaluators.find((item) => item.id === score.evaluatorId);
|
|
1171
|
+
return `${evaluator?.name ?? score.evaluatorId}: ${score.value}`;
|
|
1172
|
+
}).join(", ");
|
|
1173
|
+
}
|
|
1174
|
+
function evaluatorSnapshotName(snapshot) {
|
|
1175
|
+
if (typeof snapshot === "object" && snapshot !== null && !Array.isArray(snapshot) && typeof snapshot.name === "string") {
|
|
1176
|
+
return snapshot.name;
|
|
1177
|
+
}
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
function remoteEvaluatorName(evaluatorId, evaluatorSnapshot, evaluators) {
|
|
1181
|
+
return evaluators.find((item) => item.id === evaluatorId)?.name ?? evaluatorSnapshotName(evaluatorSnapshot) ?? evaluatorId;
|
|
1182
|
+
}
|
|
1183
|
+
function remoteRunScore(run) {
|
|
1184
|
+
if (run.result?.scores && run.result.scores.length > 0) {
|
|
1185
|
+
const sum = run.result.scores.reduce((total, score) => total + score.value, 0);
|
|
1186
|
+
return Number((sum / run.result.scores.length).toFixed(4));
|
|
1187
|
+
}
|
|
1188
|
+
const value = run.result?.score?.value;
|
|
1189
|
+
return typeof value === "number" || typeof value === "boolean" ? value : "-";
|
|
1190
|
+
}
|
|
1191
|
+
function formatRemoteEvalRunRow(run, evaluators, verbose = false) {
|
|
1192
|
+
let evaluatorScores = "-";
|
|
1193
|
+
let justifications = "-";
|
|
1194
|
+
if (run.evaluatorRuns && run.evaluatorRuns.length > 0) {
|
|
1195
|
+
evaluatorScores = run.evaluatorRuns.map((evaluatorRun) => {
|
|
1196
|
+
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1197
|
+
const score = evaluatorRun.result?.score;
|
|
1198
|
+
if (score?.type === "error") {
|
|
1199
|
+
return `${name}: Error`;
|
|
1200
|
+
}
|
|
1201
|
+
return `${name}: ${score?.value ?? "-"}`;
|
|
1202
|
+
}).join(", ");
|
|
1203
|
+
const justificationParts = run.evaluatorRuns.filter((evaluatorRun) => evaluatorRun.result?.score?.justification).map((evaluatorRun) => {
|
|
1204
|
+
const name = remoteEvaluatorName(evaluatorRun.evaluatorId, evaluatorRun.evaluatorSnapshot, evaluators);
|
|
1205
|
+
return `[${name}] ${evaluatorRun.result?.score?.justification}`;
|
|
1206
|
+
});
|
|
1207
|
+
if (justificationParts.length > 0) {
|
|
1208
|
+
justifications = justificationParts.join(" | ");
|
|
1209
|
+
}
|
|
1210
|
+
} else if (run.result?.scores && run.result.scores.length > 0) {
|
|
1211
|
+
evaluatorScores = formatEvaluatorScores(run.result.scores, evaluators);
|
|
1212
|
+
}
|
|
1213
|
+
const tokens = run.completionMetrics?.tokens;
|
|
1214
|
+
const duration = run.completionMetrics?.duration;
|
|
1215
|
+
const error = run.errorMessage ?? (typeof run.error === "string" ? run.error : run.error?.message) ?? "-";
|
|
1216
|
+
const row = {
|
|
1217
|
+
DataPoint: run.evalSnapshot?.name ?? run.id,
|
|
1218
|
+
Status: run.status,
|
|
1219
|
+
Score: remoteRunScore(run),
|
|
1220
|
+
EvaluatorScores: evaluatorScores,
|
|
1221
|
+
Tokens: typeof tokens === "number" && tokens > 0 ? tokens : "-",
|
|
1222
|
+
Duration: typeof duration === "number" ? formatDuration(duration) : "-",
|
|
1223
|
+
Error: error
|
|
1224
|
+
};
|
|
1225
|
+
if (verbose) {
|
|
1226
|
+
row.Justifications = justifications;
|
|
1227
|
+
}
|
|
1228
|
+
return row;
|
|
1229
|
+
}
|
|
1230
|
+
function isFailedRemoteRun(run) {
|
|
1231
|
+
const status = run.status.toLowerCase();
|
|
1232
|
+
if (status === "failed" || status === "workloadfailed" || run.errorMessage || run.error) {
|
|
1233
|
+
return true;
|
|
1234
|
+
}
|
|
1235
|
+
if (run.result?.score?.type === "error") {
|
|
1236
|
+
return true;
|
|
1237
|
+
}
|
|
1238
|
+
return run.evaluatorRuns?.some((evaluatorRun) => evaluatorRun.result?.score?.type === "error" || evaluatorRun.result?.score?.value === false) ?? false;
|
|
1239
|
+
}
|
|
1240
|
+
function isTerminalRemoteStatus(status) {
|
|
1241
|
+
return ["completed", "partiallycompleted", "failed"].includes(status.toLowerCase());
|
|
1242
|
+
}
|
|
1243
|
+
function formatEvalSetRunRow(run, evaluators) {
|
|
1244
|
+
return {
|
|
1245
|
+
EvalSetRunId: run.id,
|
|
1246
|
+
Status: run.status,
|
|
1247
|
+
Score: run.score ?? "-",
|
|
1248
|
+
DataPoints: run.numberOfEvalsExecuted ?? "-",
|
|
1249
|
+
Duration: formatDuration(run.durationMilliseconds),
|
|
1250
|
+
EvaluatorScores: formatEvaluatorScores(run.evaluatorScores, evaluators),
|
|
1251
|
+
CreatedAt: run.createdAt
|
|
1252
|
+
};
|
|
1253
|
+
}
|
|
1254
|
+
export { evaluatorTypeAliases, isRecord, isUUID, slugify, findEvalSet, getLinkedEvaluators, EvaluationCrud, EvaluatorCrud, ensureDirectory, EvalFileStore, EvalFiles, injectWorkloadId, resolveLoginConfig, resolveEvalApiConfig, EvalRunApi, formatDuration, formatEvaluatorScores, remoteRunScore, formatRemoteEvalRunRow, isFailedRemoteRun, isTerminalRemoteStatus, formatEvalSetRunRow };
|
|
1255
|
+
|
|
1256
|
+
//# debugId=BBF99570DB2BC04964756E2164756E21
|