@salesforce/plugin-agent 1.31.3 → 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/README.md +23 -19
  2. package/lib/commands/agent/test/run-eval.d.ts +1 -0
  3. package/lib/commands/agent/test/run-eval.js +1 -0
  4. package/lib/commands/agent/test/run-eval.js.map +1 -1
  5. package/lib/evalNormalizer.js +11 -1
  6. package/lib/evalNormalizer.js.map +1 -1
  7. package/lib/yamlSpecTranslator.js +12 -2
  8. package/lib/yamlSpecTranslator.js.map +1 -1
  9. package/messages/agent.generate.test-spec.md +2 -0
  10. package/messages/agent.test.run-eval.md +9 -5
  11. package/oclif.manifest.json +6 -5
  12. package/package.json +4 -4
  13. package/schemas/agent-test-run__eval.json +5 -1
package/README.md CHANGED
@@ -116,7 +116,7 @@ EXAMPLES
116
116
  $ sf agent activate --api-name Resort_Manager --target-org my-org
117
117
  ```
118
118
 
119
- _See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/activate.ts)_
119
+ _See code: [src/commands/agent/activate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/activate.ts)_
120
120
 
121
121
  ## `sf agent create`
122
122
 
@@ -183,7 +183,7 @@ EXAMPLES
183
183
  $ sf agent create --name "Resort Manager" --spec specs/resortManagerAgent.yaml --preview
184
184
  ```
185
185
 
186
- _See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/create.ts)_
186
+ _See code: [src/commands/agent/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/create.ts)_
187
187
 
188
188
  ## `sf agent deactivate`
189
189
 
@@ -223,7 +223,7 @@ EXAMPLES
223
223
  $ sf agent deactivate --api-name Resort_Manager --target-org my-org
224
224
  ```
225
225
 
226
- _See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/deactivate.ts)_
226
+ _See code: [src/commands/agent/deactivate.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/deactivate.ts)_
227
227
 
228
228
  ## `sf agent generate agent-spec`
229
229
 
@@ -330,7 +330,7 @@ EXAMPLES
330
330
  $ sf agent generate agent-spec --tone formal --agent-user resortmanager@myorg.com
331
331
  ```
332
332
 
333
- _See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/agent-spec.ts)_
333
+ _See code: [src/commands/agent/generate/agent-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/generate/agent-spec.ts)_
334
334
 
335
335
  ## `sf agent generate authoring-bundle`
336
336
 
@@ -407,7 +407,7 @@ EXAMPLES
407
407
  other-package-dir/main/default --target-org my-dev-org
408
408
  ```
409
409
 
410
- _See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/authoring-bundle.ts)_
410
+ _See code: [src/commands/agent/generate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/generate/authoring-bundle.ts)_
411
411
 
412
412
  ## `sf agent generate template`
413
413
 
@@ -455,7 +455,7 @@ EXAMPLES
455
455
  force-app/main/default/bots/My_Awesome_Agent/My_Awesome_Agent.bot-meta.xml --agent-version 1
456
456
  ```
457
457
 
458
- _See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/template.ts)_
458
+ _See code: [src/commands/agent/generate/template.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/generate/template.ts)_
459
459
 
460
460
  ## `sf agent generate test-spec`
461
461
 
@@ -494,6 +494,10 @@ DESCRIPTION
494
494
  - (Optional) Conversation history: Boilerplate for additional context you can add to the test in the form of a
495
495
  conversation history.
496
496
 
497
+ You can manually add contextVariables to test cases in the generated YAML file to inject contextual data (such as
498
+ CaseId or RoutableId) into agent sessions. This is useful for testing agent behavior with different contextual
499
+ information.
500
+
497
501
  When your test spec is ready, you then run the "agent test create" command to actually create the test in your org and
498
502
  synchronize the metadata with your DX project. The metadata type for an agent test is AiEvaluationDefinition.
499
503
 
@@ -516,7 +520,7 @@ EXAMPLES
516
520
  force-app//main/default/aiEvaluationDefinitions/Resort_Manager_Tests.aiEvaluationDefinition-meta.xml
517
521
  ```
518
522
 
519
- _See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/generate/test-spec.ts)_
523
+ _See code: [src/commands/agent/generate/test-spec.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/generate/test-spec.ts)_
520
524
 
521
525
  ## `sf agent preview`
522
526
 
@@ -589,7 +593,7 @@ EXAMPLES
589
593
  $ sf agent preview --use-live-actions --apex-debug --output-dir transcripts/my-preview
590
594
  ```
591
595
 
592
- _See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview.ts)_
596
+ _See code: [src/commands/agent/preview.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/preview.ts)_
593
597
 
594
598
  ## `sf agent preview end`
595
599
 
@@ -644,7 +648,7 @@ EXAMPLES
644
648
  $ sf agent preview end --authoring-bundle My_Local_Agent
645
649
  ```
646
650
 
647
- _See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/end.ts)_
651
+ _See code: [src/commands/agent/preview/end.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/preview/end.ts)_
648
652
 
649
653
  ## `sf agent preview send`
650
654
 
@@ -702,7 +706,7 @@ EXAMPLES
702
706
  $ sf agent preview send --utterance "what can you help me with?" --authoring-bundle My_Local_Agent
703
707
  ```
704
708
 
705
- _See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/send.ts)_
709
+ _See code: [src/commands/agent/preview/send.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/preview/send.ts)_
706
710
 
707
711
  ## `sf agent preview sessions`
708
712
 
@@ -735,7 +739,7 @@ EXAMPLES
735
739
  $ sf agent preview sessions
736
740
  ```
737
741
 
738
- _See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/sessions.ts)_
742
+ _See code: [src/commands/agent/preview/sessions.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/preview/sessions.ts)_
739
743
 
740
744
  ## `sf agent preview start`
741
745
 
@@ -792,7 +796,7 @@ EXAMPLES
792
796
  $ sf agent preview start --api-name My_Published_Agent
793
797
  ```
794
798
 
795
- _See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/preview/start.ts)_
799
+ _See code: [src/commands/agent/preview/start.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/preview/start.ts)_
796
800
 
797
801
  ## `sf agent publish authoring-bundle`
798
802
 
@@ -841,7 +845,7 @@ EXAMPLES
841
845
  $ sf agent publish authoring-bundle --api-name MyAuthoringbundle --target-org my-dev-org
842
846
  ```
843
847
 
844
- _See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/publish/authoring-bundle.ts)_
848
+ _See code: [src/commands/agent/publish/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/publish/authoring-bundle.ts)_
845
849
 
846
850
  ## `sf agent test create`
847
851
 
@@ -896,7 +900,7 @@ EXAMPLES
896
900
  $ sf agent test create --spec specs/Resort_Manager-testSpec.yaml --api-name Resort_Manager_Test --preview
897
901
  ```
898
902
 
899
- _See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/create.ts)_
903
+ _See code: [src/commands/agent/test/create.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/test/create.ts)_
900
904
 
901
905
  ## `sf agent test list`
902
906
 
@@ -931,7 +935,7 @@ EXAMPLES
931
935
  $ sf agent test list --target-org my-org
932
936
  ```
933
937
 
934
- _See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/list.ts)_
938
+ _See code: [src/commands/agent/test/list.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/test/list.ts)_
935
939
 
936
940
  ## `sf agent test results`
937
941
 
@@ -997,7 +1001,7 @@ FLAG DESCRIPTIONS
997
1001
  expression when using custom evaluations.
998
1002
  ```
999
1003
 
1000
- _See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/results.ts)_
1004
+ _See code: [src/commands/agent/test/results.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/test/results.ts)_
1001
1005
 
1002
1006
  ## `sf agent test resume`
1003
1007
 
@@ -1070,7 +1074,7 @@ FLAG DESCRIPTIONS
1070
1074
  expression when using custom evaluations.
1071
1075
  ```
1072
1076
 
1073
- _See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/resume.ts)_
1077
+ _See code: [src/commands/agent/test/resume.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/test/resume.ts)_
1074
1078
 
1075
1079
  ## `sf agent test run`
1076
1080
 
@@ -1144,7 +1148,7 @@ FLAG DESCRIPTIONS
1144
1148
  expression when using custom evaluations.
1145
1149
  ```
1146
1150
 
1147
- _See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/test/run.ts)_
1151
+ _See code: [src/commands/agent/test/run.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/test/run.ts)_
1148
1152
 
1149
1153
  ## `sf agent validate authoring-bundle`
1150
1154
 
@@ -1191,6 +1195,6 @@ EXAMPLES
1191
1195
  $ sf agent validate authoring-bundle --api-name MyAuthoringBundle --target-org my-dev-org
1192
1196
  ```
1193
1197
 
1194
- _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.31.3/src/commands/agent/validate/authoring-bundle.ts)_
1198
+ _See code: [src/commands/agent/validate/authoring-bundle.ts](https://github.com/salesforcecli/plugin-agent/blob/1.32.0/src/commands/agent/validate/authoring-bundle.ts)_
1195
1199
 
1196
1200
  <!-- commandsstop -->
package/lib/commands/agent/test/run-eval.d.ts CHANGED
@@ -5,6 +5,7 @@ export type RunEvalResult = {
5
5
  id: string;
6
6
  status: string;
7
7
  evaluations: unknown[];
8
+ outputs: unknown[];
8
9
  }>;
9
10
  summary: {
10
11
  passed: number;
package/lib/commands/agent/test/run-eval.js CHANGED
@@ -99,6 +99,7 @@ function buildResultSummary(mergedResponse) {
99
99
  id: testId,
100
100
  status: failed > 0 || testErrors.length > 0 ? 'failed' : 'passed',
101
101
  evaluations: evalResults,
102
+ outputs: tr.outputs ?? [],
102
103
  });
103
104
  }
104
105
  return { summary, testSummaries };
package/lib/commands/agent/test/run-eval.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"run-eval.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAO,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAoB,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAwB,aAAa,EAAqB,MAAM,2BAA2B,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAElG,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;AAe1F,KAAK,UAAU,aAAa,CAAC,GAAQ;IACnC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAsB,GAAG,IAAI,CAAC,WAAW,2BAA2B,CAAC,CAAC;IAEzG,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE;QACrB,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,OAAoB,EAAE,OAAmB;IAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,OAAO,IAAI,CAAC,OAAO,CAA0B;QAC3C,GAAG,EAAE,yDAAyD;QAC9D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,uBAAuB,EAAE,aAAa,OAAO,CAAC,KAAK,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,0BAA0B,EAAE,OAAO,CAAC,WAAW;YAC/C,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,qBAAqB,EAAE,sBAAsB;YAC7C,oBAAoB,EAAE,aAAa;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAAe;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,iDAAiD;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,uDAAuD,cAAc,GAAG,CACzE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,2CAA2C;IAC3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,sDAAsD,OAAO,wCAAwC,CACtG,CAAC;IACF,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,QAAQ,CAAC,WAAW,CAAC,4BAA4B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAQ,EACR,OAAoC,EACpC,GAA0B;IAE1B,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAEzC,yDAAyD;IACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,YAAY,GAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,kBAAkB,CAAC,cAA+B;IAIzD,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAkE,EAAE,CAAC;IAExF,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,UAAqC,CAAC;QACjD,MAAM,MAAM,GAAI,EAAE,CAAC,EAAa,IAAI,SAAS,CAAC;QAC9C,MAAM,WAAW,GAAI,EAAE,CAAC,kBAAqD,IAAI,EAAE,CAAC;QACpF,MAAM,UAAU,GAAI,EAAE,CAAC,MAAoB,IAAI,EAAE,CAAC;QAElD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QAEtF,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpC,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,WAAW,EAAE,WAAW;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,SAAwB;IAC7D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,MAAM,CAAU,MAAM,GAAG,IAAI,CAAC;IAE9B,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;YACvB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SACvD,CAAC;QACF,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;YAClB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;SACnD,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC;YAC5B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,4BAA4B,CAAC;SAC3D,CAAC;KACH,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAEhC,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAE5B,oFAAoF;QACpF,IAAI,CAAC;YACH,sDAAsD;YACtD,wEAAwE;YACxE,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YACD,qCAAqC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;YACnD,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAoB,CAAC;QACzB,IAAI,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,oDAAoD;YACpD,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAElC,kEAAkE;YAClE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAgB,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClF,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,wEAAwE;QACxE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACrE,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;wBACzC,qCAAqC;wBACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;wBACxB,qCAAqC;wBACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEhE,wBAAwB;QACxB,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3D,qBAAqB;QACrB,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAE9E,MAAM,cAAc,GAAoB,EAAE,OAAO,EAAE,UAAwC,EAAE,CAAC;QAE9F,mBAAmB;QACnB,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,OAAO,CAAiB,CAAC;QACzE,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEpB,yCAAyC;QACzC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAEtE,yCAAyC;QACzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC"}
1
+ {"version":3,"file":"run-eval.js","sourceRoot":"","sources":["../../../../src/commands/agent/test/run-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC/D,OAAO,EAAE,QAAQ,EAAO,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAoB,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAClG,OAAO,EAAwB,aAAa,EAAqB,MAAM,2BAA2B,CAAC;AACnG,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAElG,QAAQ,CAAC,kCAAkC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC7D,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,CAAC,0BAA0B,EAAE,qBAAqB,CAAC,CAAC;AAe1F,KAAK,UAAU,aAAa,CAAC,GAAQ;IACnC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAsB,GAAG,IAAI,CAAC,WAAW,2BAA2B,CAAC,CAAC;IAEzG,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,QAAQ,EAAE;QACrB,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,GAAQ,EAAE,OAAoB,EAAE,OAAmB;IAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,OAAO,IAAI,CAAC,OAAO,CAA0B;QAC3C,GAAG,EAAE,yDAAyD;QAC9D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,uBAAuB,EAAE,aAAa,OAAO,CAAC,KAAK,EAAE;YACrD,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,0BAA0B,EAAE,OAAO,CAAC,WAAW;YAC/C,gBAAgB,EAAE,OAAO,CAAC,MAAM;YAChC,qBAAqB,EAAE,sBAAsB;YAC7C,oBAAoB,EAAE,aAAa;SACpC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAQ,EAAE,OAAe;IACnD,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;IAEjC,iDAAiD;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAChC,uDAAuD,cAAc,GAAG,CACzE,CAAC;IACF,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,MAAM,QAAQ,CAAC,WAAW,CAAC,qBAAqB,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAExC,2CAA2C;IAC3C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CACpC,sDAAsD,OAAO,wCAAwC,CACtG,CAAC;IACF,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,QAAQ,CAAC,WAAW,CAAC,4BAA4B,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAQ,EACR,OAAoC,EACpC,GAA0B;IAE1B,0DAA0D;IAC1D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;IAEzC,yDAAyD;IACzD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,oBAAoB,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5F,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChD,MAAM,YAAY,GAAgB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAChE,OAAO,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACtD,OAAO,YAAY,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,SAAS,kBAAkB,CAAC,cAA+B;IAIzD,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,aAAa,GAAsF,EAAE,CAAC;IAE5G,KAAK,MAAM,UAAU,IAAI,cAAc,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACtD,MAAM,EAAE,GAAG,UAAqC,CAAC;QACjD,MAAM,MAAM,GAAI,EAAE,CAAC,EAAa,IAAI,SAAS,CAAC;QAC9C,MAAM,WAAW,GAAI,EAAE,CAAC,kBAAqD,IAAI,EAAE,CAAC;QACpF,MAAM,UAAU,GAAI,EAAE,CAAC,MAAoB,IAAI,EAAE,CAAC;QAElD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC;QACrE,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC;QAEtF,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC;QACzB,OAAO,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;QAEpC,aAAa,CAAC,IAAI,CAAC;YACjB,EAAE,EAAE,MAAM;YACV,MAAM,EAAE,MAAM,GAAG,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,WAAW,EAAE,WAAW;YACxB,OAAO,EAAG,EAAE,CAAC,OAAqB,IAAI,EAAE;SACzC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,SAAwB;IAC7D,MAAM,CAAU,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,CAAU,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACjE,MAAM,CAAU,QAAQ,GAAG,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;IACtB,MAAM,CAAU,MAAM,GAAG,IAAI,CAAC;IAE9B,MAAM,CAAU,KAAK,GAAG;QAC7B,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE;QACjC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE;QACpC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACjB,IAAI,EAAE,GAAG;YACT,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;YAClD,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC;YACvB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,wBAAwB,CAAC;SACvD,CAAC;QACF,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC;YAClB,IAAI,EAAE,GAAG;YACT,OAAO,EAAE,EAAE;YACX,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,oBAAoB,CAAC;SACnD,CAAC;QACF,eAAe,EAAE,gBAAgB,EAAE;QACnC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC;YAC1B,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,0BAA0B,CAAC;SACzD,CAAC;QACF,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC;YAC5B,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,4BAA4B,CAAC;SAC3D,CAAC;KACH,CAAC;IAEK,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;QAEhC,0DAA0D;QAC1D,IAAI,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC;QAE5B,oFAAoF;QACpF,IAAI,CAAC;YACH,sDAAsD;YACtD,wEAAwE;YACxE,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACzB,CAAC;YACD,qCAAqC;QACvC,CAAC;QAAC,MAAM,CAAC;YACP,mDAAmD;YACnD,UAAU,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,6BAA6B;QAC7B,IAAI,OAAoB,CAAC;QACzB,IAAI,YAAY,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;QAErC,IAAI,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/B,oDAAoD;YACpD,MAAM,IAAI,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACvC,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAElC,kEAAkE;YAClE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,IAAI,CAAC,WAAW,CAAC;gBAChC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,mBAAmB,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAgB,CAAC;YAClD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClF,MAAM,QAAQ,CAAC,WAAW,CAAC,sBAAsB,EAAE,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACzF,CAAC;QAED,wEAAwE;QACxE,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;YACrE,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;wBACzC,qCAAqC;wBACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;wBACxB,qCAAqC;wBACrC,IAAI,CAAC,gBAAgB,GAAG,SAAS,CAAC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3B,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QAED,sBAAsB;QACtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAEhE,wBAAwB;QACxB,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAE3D,qBAAqB;QACrB,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAE9E,MAAM,cAAc,GAAoB,EAAE,OAAO,EAAE,UAAwC,EAAE,CAAC;QAE9F,mBAAmB;QACnB,MAAM,YAAY,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,OAAO,CAAiB,CAAC;QACzE,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEpB,yCAAyC;QACzC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAEtE,yCAAyC;QACzC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7C,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IAC3C,CAAC"}
package/lib/evalNormalizer.js CHANGED
@@ -47,7 +47,17 @@ const ASSERTION_VALID_FIELDS = new Set([
47
47
  'metric_name',
48
48
  ]);
49
49
  const VALID_AGENT_FIELDS = {
50
- 'agent.create_session': new Set(['type', 'id', 'agent_id', 'agent_version_id', 'use_agent_api', 'planner_id']),
50
+ 'agent.create_session': new Set([
51
+ 'type',
52
+ 'id',
53
+ 'agent_id',
54
+ 'agent_version_id',
55
+ 'use_agent_api',
56
+ 'planner_id',
57
+ 'state',
58
+ 'setupSessionContext',
59
+ 'context_variables',
60
+ ]),
51
61
  'agent.send_message': new Set(['type', 'id', 'session_id', 'utterance']),
52
62
  'agent.get_state': new Set(['type', 'id', 'session_id']),
53
63
  };
package/lib/evalNormalizer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"evalNormalizer.js","sourceRoot":"","sources":["../src/evalNormalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAqBH,mCAAmC;AAEnC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,0BAA0B;IAC1B,mCAAmC;IACnC,2BAA2B;IAC3B,+BAA+B;CAChC,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAC,CAAC;AAEjG,MAAM,oBAAoB,GAA2B;IACnD,0BAA0B,EAAE,wBAAwB;IACpD,mCAAmC,EAAE,yBAAyB;IAC9D,2BAA2B,EAAE,iBAAiB;IAC9C,+BAA+B,EAAE,qBAAqB;CACvD,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,IAAI;IACJ,kBAAkB;IAClB,kBAAkB;IAClB,aAAa;IACb,WAAW;CACZ,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,MAAM;IACN,IAAI;IACJ,QAAQ;IACR,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,aAAa;IACb,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAgC;IACtD,sBAAsB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,kBAAkB,EAAE,eAAe,EAAE,YAAY,CAAC,CAAC;IAC9G,oBAAoB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IACxE,iBAAiB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;CACzD,CAAC;AAEF,+BAA+B;AAE/B,MAAM,iBAAiB,GAA2B;IAChD,OAAO,EAAE,UAAU;IACnB,cAAc,EAAE,kBAAkB;IAClC,SAAS,EAAE,YAAY;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,WAAW;IAClB,MAAM,EAAE,WAAW;IACnB,YAAY,EAAE,WAAW;IACzB,WAAW,EAAE,WAAW;CACzB,CAAC;AAEF,MAAM,qBAAqB,GAA2B;IACpD,OAAO,EAAE,QAAQ;IACjB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,WAAW,EAAE,QAAQ;IACrB,YAAY,EAAE,QAAQ;IACtB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,UAAU,EAAE,UAAU;CACvB,CAAC;AAEF,wDAAwD;AAExD,MAAM,mBAAmB,GAA2B;IAClD,WAAW,EAAE,eAAe;IAC5B,SAAS,EAAE,YAAY;IACvB,mBAAmB,EAAE,YAAY;IACjC,qBAAqB,EAAE,YAAY;IACnC,kBAAkB,EAAE,YAAY;IAChC,gBAAgB,EAAE,YAAY;CAC/B,CAAC;AAEF,0CAA0C;AAE1C,MAAM,qBAAqB,GAA2B;IACpD,MAAM,EAAE,kBAAkB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;IAChC,cAAc,EAAE,kBAAkB;IAClC,aAAa,EAAE,kBAAkB;IACjC,eAAe,EAAE,kBAAkB;IACnC,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;CACjC,CAAC;AAEF,4CAA4C;AAE5C,MAAM,uBAAuB,GAA2B;IACtD,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,UAAU;IAC1B,gBAAgB,EAAE,QAAQ;IAC1B,gBAAgB,EAAE,UAAU;IAC5B,aAAa,EAAE,QAAQ;IACvB,eAAe,EAAE,UAAU;IAC3B,QAAQ,EAAE,QAAQ;IAClB,YAAY,EAAE,UAAU;CACzB,CAAC;AAEF,sCAAsC;AAEtC,+FAA+F;AAC/F,MAAM,aAAa,GAA2B;IAC5C,qBAAqB,EAAE,+CAA+C;IACtE,8BAA8B,EAAE,wDAAwD;IACxF,+BAA+B,EAAE,wDAAwD;IACzF,QAAQ,EAAE,UAAU;IACpB,mBAAmB,EAAE,UAAU;CAChC,CAAC;AAEF,2BAA2B;AAE3B;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAoB;IACnD,MAAM,UAAU,GAAgB;QAC9B,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,IAAI,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACrC,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAClC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;YACxC,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;YACpC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAC9B,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC,CAAC;KACH,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,0CAA0C;AAE1C;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAiB;IACrD,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAoC,CAAC;QAEjE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAE9D,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,0EAA0E;QAC1E,UAAU,CAAC,IAAI,GAAG,aAAa,cAAc,EAAE,CAAC;QAChD,OAAO,UAAU,CAAC,cAAc,CAAC;QAEjC,+DAA+D;QAC/D,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,QAAQ,IAAI,UAAU,CAAC,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAe,CAAC;gBAE9C,kFAAkF;gBAClF,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACnD,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;oBACzD,UAAU,CAAC,MAAM,GAAG,IAAI,MAAM,IAAI,UAAU,GAAG,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC;gBACjC,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC,KAAK,CAAC;QAC1B,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,UAAU,CAAC,EAAE,GAAG,QAAQ,WAAW,EAAE,CAAC;QACxC,CAAC;QACD,WAAW,EAAE,CAAC;QAEd,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;QAEtC,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACjE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACrE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAqB,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAiB;IAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB;YAAE,OAAO,IAAI,CAAC;QAEtD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACrE,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;gBACxB,IAAI,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,EAAE,CAAC;oBAC/B,UAAU,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAc,EAAE,OAA+B;IACxE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,UAAoB,EAAE,QAAgB;IACvE,iBAAiB,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAC;IAErD,qCAAqC;IACrC,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;SAAM,IAAI,UAAU,CAAC,WAAW,KAAK,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,UAAoB,EAAE,QAAgB;IACzE,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,UAAU,IAAI,UAAU,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxE,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC1D,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,UAAU,CAAC,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAiB;IACxD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,yBAAyB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,2BAA2B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,CAAC;QACD,sFAAsF;QACtF,gGAAgG;QAEhG,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAiB;IACpD,8CAA8C;IAC9C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,WAAW,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YAC7B,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,cAAc,CAAC;IAElC,SAAS,YAAY,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5C,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7B,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEtC,IAAI,CAAC,CAAC,GAAG,IAAI,WAAW,CAAC;gBAAE,OAAO,KAAK,CAAC;YAExC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAE7B,yDAAyD;YACzD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBAC1C,KAAK,GAAG,UAAU,CAAC;YACrB,CAAC;YAED,OAAO,aAAa,GAAG,KAAK,KAAK,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,OAAO,GAA4B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;iBAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1E,MAAM,MAAM,GAA4B,EAAE,CAAC;gBAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;oBACpE,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,GAAI,GAAiB,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CACtD,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CACrD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,OAAmB,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,IAAI,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC1D,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAiB;IACvD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAEjC,cAAc;QACd,IAAI,QAAQ,IAAI,kBAAkB,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,qBAAqB;QACrB,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,uBAAuB;QACvB,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,wEAAwE;QACxE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0BAA0B;AAE1B;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAiB,EAAE,SAAiB;IACnE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
1
+ {"version":3,"file":"evalNormalizer.js","sourceRoot":"","sources":["../src/evalNormalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAqBH,mCAAmC;AAEnC,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC;IACjC,0BAA0B;IAC1B,mCAAmC;IACnC,2BAA2B;IAC3B,+BAA+B;CAChC,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,4BAA4B,EAAE,0BAA0B,CAAC,CAAC,CAAC;AAEjG,MAAM,oBAAoB,GAA2B;IACnD,0BAA0B,EAAE,wBAAwB;IACpD,mCAAmC,EAAE,yBAAyB;IAC9D,2BAA2B,EAAE,iBAAiB;IAC9C,+BAA+B,EAAE,qBAAqB;CACvD,CAAC;AAEF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,MAAM;IACN,IAAI;IACJ,kBAAkB;IAClB,kBAAkB;IAClB,aAAa;IACb,WAAW;CACZ,CAAC,CAAC;AAEH,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,MAAM;IACN,IAAI;IACJ,QAAQ;IACR,UAAU;IACV,UAAU;IACV,WAAW;IACX,WAAW;IACX,aAAa;IACb,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAgC;IACtD,sBAAsB,EAAE,IAAI,GAAG,CAAC;QAC9B,MAAM;QACN,IAAI;QACJ,UAAU;QACV,kBAAkB;QAClB,eAAe;QACf,YAAY;QACZ,OAAO;QACP,qBAAqB;QACrB,mBAAmB;KACpB,CAAC;IACF,oBAAoB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;IACxE,iBAAiB,EAAE,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;CACzD,CAAC;AAEF,+BAA+B;AAE/B,MAAM,iBAAiB,GAA2B;IAChD,OAAO,EAAE,UAAU;IACnB,cAAc,EAAE,kBAAkB;IAClC,SAAS,EAAE,YAAY;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,WAAW;IACpB,KAAK,EAAE,WAAW;IAClB,MAAM,EAAE,WAAW;IACnB,YAAY,EAAE,WAAW;IACzB,WAAW,EAAE,WAAW;CACzB,CAAC;AAEF,MAAM,qBAAqB,GAA2B;IACpD,OAAO,EAAE,QAAQ;IACjB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,WAAW,EAAE,QAAQ;IACrB,YAAY,EAAE,QAAQ;IACtB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,UAAU;IAC1B,UAAU,EAAE,UAAU;CACvB,CAAC;AAEF,wDAAwD;AAExD,MAAM,mBAAmB,GAA2B;IAClD,WAAW,EAAE,eAAe;IAC5B,SAAS,EAAE,YAAY;IACvB,mBAAmB,EAAE,YAAY;IACjC,qBAAqB,EAAE,YAAY;IACnC,kBAAkB,EAAE,YAAY;IAChC,gBAAgB,EAAE,YAAY;CAC/B,CAAC;AAEF,0CAA0C;AAE1C,MAAM,qBAAqB,GAA2B;IACpD,MAAM,EAAE,kBAAkB;IAC1B,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;IAChC,cAAc,EAAE,kBAAkB;IAClC,aAAa,EAAE,kBAAkB;IACjC,eAAe,EAAE,kBAAkB;IACnC,QAAQ,EAAE,kBAAkB;IAC5B,YAAY,EAAE,kBAAkB;CACjC,CAAC;AAEF,4CAA4C;AAE5C,MAAM,uBAAuB,GAA2B;IACtD,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,UAAU;IAC1B,gBAAgB,EAAE,QAAQ;IAC1B,gBAAgB,EAAE,UAAU;IAC5B,aAAa,EAAE,QAAQ;IACvB,eAAe,EAAE,UAAU;IAC3B,QAAQ,EAAE,QAAQ;IAClB,YAAY,EAAE,UAAU;CACzB,CAAC;AAEF,sCAAsC;AAEtC,+FAA+F;AAC/F,MAAM,aAAa,GAA2B;IAC5C,qBAAqB,EAAE,+CAA+C;IACtE,8BAA8B,EAAE,wDAAwD;IACxF,+BAA+B,EAAE,wDAAwD;IACzF,QAAQ,EAAE,UAAU;IACpB,mBAAmB,EAAE,UAAU;CAChC,CAAC;AAEF,2BAA2B;AAE3B;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAoB;IACnD,MAAM,UAAU,GAAgB;QAC9B,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAChC,IAAI,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5B,KAAK,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;YACrC,KAAK,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACjC,KAAK,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAClC,KAAK,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;YACxC,KAAK,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;YACpC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YAC9B,KAAK,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;YACvC,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,CAAC;QAC5B,CAAC,CAAC;KACH,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,0CAA0C;AAE1C;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAiB;IACrD,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,cAAc,GAAG,IAAI,CAAC,cAAoC,CAAC;QAEjE,4EAA4E;QAC5E,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAE9D,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,0EAA0E;QAC1E,UAAU,CAAC,IAAI,GAAG,aAAa,cAAc,EAAE,CAAC;QAChD,OAAO,UAAU,CAAC,cAAc,CAAC;QAEjC,+DAA+D;QAC/D,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,QAAQ,IAAI,UAAU,CAAC,EAAE,CAAC;gBAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,KAAe,CAAC;gBAE9C,kFAAkF;gBAClF,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACvC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;oBACf,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;oBAC/C,MAAM,SAAS,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACnD,MAAM,UAAU,GAAG,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;oBACzD,UAAU,CAAC,MAAM,GAAG,IAAI,MAAM,IAAI,UAAU,GAAG,CAAC;gBAClD,CAAC;qBAAM,CAAC;oBACN,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC;gBACjC,CAAC;YACH,CAAC;YACD,OAAO,UAAU,CAAC,KAAK,CAAC;QAC1B,CAAC;QAED,8BAA8B;QAC9B,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAC3C,UAAU,CAAC,EAAE,GAAG,QAAQ,WAAW,EAAE,CAAC;QACxC,CAAC;QACD,WAAW,EAAE,CAAC;QAEd,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,SAAS,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,IAAI,EAAE,CAAC;QAEtC,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBACjE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACrE,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,OAAO,IAAI,SAAS,CAAC,EAAE,CAAC;oBAClD,SAAS,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;oBACtC,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAqB,CAAC;IAC/B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAiB;IAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB;YAAE,OAAO,IAAI,CAAC;QAEtD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;YACrE,IAAI,KAAK,IAAI,UAAU,EAAE,CAAC;gBACxB,IAAI,CAAC,CAAC,SAAS,IAAI,UAAU,CAAC,EAAE,CAAC;oBAC/B,UAAU,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC5C,CAAC;gBACD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,IAAc,EAAE,OAA+B;IACxE,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,IAAI,IAAI,IAAI,SAAS,IAAI,IAAI,EAAE,CAAC;YAC9C,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAAC,UAAoB,EAAE,QAAgB;IACvE,iBAAiB,CAAC,UAAU,EAAE,qBAAqB,CAAC,CAAC;IAErD,qCAAqC;IACrC,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;SAAM,IAAI,UAAU,CAAC,WAAW,KAAK,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,UAAU,CAAC,WAAW,GAAG,aAAa,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,2BAA2B,CAAC,UAAoB,EAAE,QAAgB;IACzE,iBAAiB,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IAEvD,0BAA0B;IAC1B,IAAI,UAAU,IAAI,UAAU,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACxE,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC1D,CAAC;IAED,mDAAmD;IACnD,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,EAAE,CAAC;QACnC,UAAU,CAAC,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAiB;IACxD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QAEpD,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;QAE/B,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,yBAAyB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QAClD,CAAC;aAAM,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9C,2BAA2B,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpD,CAAC;QACD,sFAAsF;QACtF,gGAAgG;QAEhG,OAAO,UAAsB,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAiB;IACpD,8CAA8C;IAC9C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3C,WAAW,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YAC7B,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,cAAc,CAAC;IAElC,SAAS,YAAY,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE5C,OAAO,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,EAAE;YACtD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,MAAM,GAAG,CAAC;gBAAE,OAAO,KAAK,CAAC;YAE7B,MAAM,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACrC,IAAI,KAAK,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAEtC,IAAI,CAAC,CAAC,GAAG,IAAI,WAAW,CAAC;gBAAE,OAAO,KAAK,CAAC;YAExC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;YAE7B,yDAAyD;YACzD,IAAI,KAAK,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBAC1C,KAAK,GAAG,UAAU,CAAC;YACrB,CAAC;YAED,OAAO,aAAa,GAAG,KAAK,KAAK,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,OAAO,GAA4B,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;iBAAM,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1E,MAAM,MAAM,GAA4B,EAAE,CAAC;gBAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAA8B,CAAC,EAAE,CAAC;oBACpE,MAAM,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;YACxB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,GAAI,GAAiB,CAAC,GAAG,CAAC,CAAC,IAAa,EAAE,EAAE,CACtD,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CACrD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;YACrB,CAAC;QACH,CAAC;QACD,OAAO,OAAmB,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,IAAI,IAAI,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;YACzC,IAAI,CAAC,CAAC,eAAe,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,EAAE,CAAC;gBAC1D,OAAO,EAAE,GAAG,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;YAC1C,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAiB;IACvD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAEjC,cAAc;QACd,IAAI,QAAQ,IAAI,kBAAkB,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzB,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,qBAAqB;QACrB,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBAClC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,uBAAuB;QACvB,IAAI,oBAAoB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,MAAM,QAAQ,GAA4B,EAAE,CAAC;YAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9C,IAAI,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACpC,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;gBACtB,CAAC;YACH,CAAC;YACD,OAAO,QAAoB,CAAC;QAC9B,CAAC;QAED,wEAAwE;QACxE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0BAA0B;AAE1B;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAiB,EAAE,SAAiB;IACnE,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
package/lib/yamlSpecTranslator.js CHANGED
@@ -87,11 +87,21 @@ export function translateTestCase(testCase, index, specName) {
87
87
  const id = specName ? `${specName}_case_${index}` : `test_case_${index}`;
88
88
  const steps = [];
89
89
  // 1. agent.create_session
90
- steps.push({
90
+ const createSessionStep = {
91
91
  type: 'agent.create_session',
92
92
  id: 'cs',
93
93
  use_agent_api: true,
94
- });
94
+ };
95
+ if (testCase.contextVariables && testCase.contextVariables.length > 0) {
96
+ // Validate for duplicate names
97
+ const names = testCase.contextVariables.map((cv) => cv.name);
98
+ const duplicates = names.filter((name, idx) => names.indexOf(name) !== idx);
99
+ if (duplicates.length > 0) {
100
+ throw new Error(`Duplicate contextVariable names found in test case ${index}: ${[...new Set(duplicates)].join(', ')}. Each contextVariable name must be unique.`);
101
+ }
102
+ createSessionStep.context_variables = Object.fromEntries(testCase.contextVariables.map((cv) => [cv.name, cv.value]));
103
+ }
104
+ steps.push(createSessionStep);
95
105
  // 2. Conversation history — only user messages become send_message steps
96
106
  let historyIdx = 0;
97
107
  if (testCase.conversationHistory) {
package/lib/yamlSpecTranslator.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"yamlSpecTranslator.js","sourceRoot":"","sources":["../src/yamlSpecTranslator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8BAA8B;AAE9B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAI1C,4DAA4D;AAE5D,MAAM,eAAe,GAA2B;IAC9C,yBAAyB,EAAE,eAAe;IAC1C,uBAAuB,EAAE,oDAAoD;IAC7E,gCAAgC,EAAE,6DAA6D;IAC/F,iCAAiC,EAAE,6DAA6D;CACjG,CAAC;AAEF,2DAA2D;AAE3D,MAAM,oBAAoB,GAA2B;IACnD,iBAAiB,EAAE,4BAA4B;IAC/C,kBAAkB,EAAE,6BAA6B;CAClD,CAAC;AAEF,4CAA4C;AAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,gCAAgC;IAChC,iCAAiC;CAClC,CAAC,CAAC;AAEH,qBAAqB;AAErB;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB,EAAE,KAAa,EAAE,QAAiB;IACpF,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;IACzE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,sBAAsB;QAC5B,EAAE,EAAE,IAAI;QACR,aAAa,EAAE,IAAI;KACpB,CAAC,CAAC;IAEH,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,oBAAoB;oBAC1B,EAAE,EAAE,WAAW,UAAU,EAAE;oBAC3B,UAAU,EAAE,iBAAiB;oBAC7B,SAAS,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,oBAAoB;QAC1B,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,iBAAiB;QAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,iBAAiB;YACvB,EAAE,EAAE,IAAI;YACR,UAAU,EAAE,iBAAiB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mCAAmC;YACzC,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,QAAQ,CAAC,aAAa;YAChC,MAAM,EAAE,oDAAoD;YAC5D,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,qCAAqC;YAC3C,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,6DAA6D;YACrE,QAAQ,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,+BAA+B;YACrC,EAAE,EAAE,eAAe;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,2BAA2B;AAE3B;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtD,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/F,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,UAA8D,EAC9D,KAAa;IAEb,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzF,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,EAAE,EAAE,UAAU,KAAK,EAAE;QACrB,QAAQ;QACR,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC"}
1
+ {"version":3,"file":"yamlSpecTranslator.js","sourceRoot":"","sources":["../src/yamlSpecTranslator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,8BAA8B;AAE9B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAI1C,4DAA4D;AAE5D,MAAM,eAAe,GAA2B;IAC9C,yBAAyB,EAAE,eAAe;IAC1C,uBAAuB,EAAE,oDAAoD;IAC7E,gCAAgC,EAAE,6DAA6D;IAC/F,iCAAiC,EAAE,6DAA6D;CACjG,CAAC;AAEF,2DAA2D;AAE3D,MAAM,oBAAoB,GAA2B;IACnD,iBAAiB,EAAE,4BAA4B;IAC/C,kBAAkB,EAAE,6BAA6B;CAClD,CAAC;AAEF,4CAA4C;AAC5C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,uBAAuB;IACvB,gCAAgC;IAChC,iCAAiC;CAClC,CAAC,CAAC;AAEH,qBAAqB;AAErB;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3E,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,GAAG,GAAG,MAAiC,CAAC;QAC9C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,MAAM,GAAY,SAAS,CAAC,OAAO,CAAC,CAAC;IAC3C,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IACD,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjC,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,MAAkB,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAc;IAC9C,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9E,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAkB,EAAE,KAAa,EAAE,QAAiB;IACpF,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,SAAS,KAAK,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,EAAE,CAAC;IACzE,MAAM,KAAK,GAAe,EAAE,CAAC;IAE7B,0BAA0B;IAC1B,MAAM,iBAAiB,GAAa;QAClC,IAAI,EAAE,sBAAsB;QAC5B,EAAE,EAAE,IAAI;QACR,aAAa,EAAE,IAAI;KACpB,CAAC;IAEF,IAAI,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtE,+BAA+B;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QAC5E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACb,sDAAsD,KAAK,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAC3F,IAAI,CACL,6CAA6C,CAC/C,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,iBAAiB,GAAG,MAAM,CAAC,WAAW,CACtD,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAE9B,yEAAyE;IACzE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,mBAAmB,EAAE,CAAC;YACjD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC1B,KAAK,CAAC,IAAI,CAAC;oBACT,IAAI,EAAE,oBAAoB;oBAC1B,EAAE,EAAE,WAAW,UAAU,EAAE;oBAC3B,UAAU,EAAE,iBAAiB;oBAC7B,SAAS,EAAE,KAAK,CAAC,OAAO;iBACzB,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,KAAK,CAAC,IAAI,CAAC;QACT,IAAI,EAAE,oBAAoB;QAC1B,EAAE,EAAE,IAAI;QACR,UAAU,EAAE,iBAAiB;QAC7B,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,iBAAiB;YACvB,EAAE,EAAE,IAAI;YACR,UAAU,EAAE,iBAAiB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,gBAAgB;IAChB,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,mCAAmC;YACzC,EAAE,EAAE,aAAa;YACjB,QAAQ,EAAE,QAAQ,CAAC,aAAa;YAChC,MAAM,EAAE,oDAAoD;YAC5D,QAAQ,EAAE,UAAU;SACrB,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,qCAAqC;YAC3C,EAAE,EAAE,eAAe;YACnB,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,6DAA6D;YACrE,QAAQ,EAAE,gBAAgB;SAC3B,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,+BAA+B;YACrC,EAAE,EAAE,eAAe;YACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,QAAQ,EAAE,QAAQ,CAAC,eAAe;YAClC,MAAM,EAAE,eAAe;YACvB,SAAS,EAAE,GAAG;SACf,CAAC,CAAC;IACL,CAAC;IAED,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,QAAQ,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE;YAC3D,MAAM,IAAI,GAAG,yBAAyB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;AACvB,CAAC;AAED,2BAA2B;AAE3B;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,aAAa,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtD,IAAI,QAAQ,CAAC,eAAe,KAAK,SAAS,IAAI,QAAQ,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/F,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,KAAK,MAAM,UAAU,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9D,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,yBAAyB,CAChC,UAA8D,EAC9D,KAAa;IAEb,MAAM,QAAQ,GAAG,oBAAoB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,aAAa,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzF,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;IAElB,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC1C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACrC,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,EAAE,EAAE,UAAU,KAAK,EAAE;QACrB,QAAQ;QACR,MAAM;QACN,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;AACvC,CAAC"}
package/messages/agent.generate.test-spec.md CHANGED
@@ -15,6 +15,8 @@ To generate a specific agent test case, this command prompts you for this inform
15
15
  - (Optional) Custom evaluation: Test an agent's response for specific strings or numbers.
16
16
  - (Optional) Conversation history: Boilerplate for additional context you can add to the test in the form of a conversation history.
17
17
 
18
+ You can manually add contextVariables to test cases in the generated YAML file to inject contextual data (such as CaseId or RoutableId) into agent sessions. This is useful for testing agent behavior with different contextual information.
19
+
18
20
  When your test spec is ready, you then run the "agent test create" command to actually create the test in your org and synchronize the metadata with your DX project. The metadata type for an agent test is AiEvaluationDefinition.
19
21
 
20
22
  If you have an existing AiEvaluationDefinition metadata XML file in your DX project, you can generate its equivalent YAML test spec file with the --from-definition flag.
package/messages/agent.test.run-eval.md CHANGED
@@ -6,15 +6,15 @@ Run evaluation tests against an Agentforce agent.
6
6
 
7
7
  Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.
8
8
 
9
- When you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`.
9
+ When you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`. YAML test specs also support contextVariables, which allow you to inject contextual data (such as CaseId or RoutableId) into agent sessions for testing with different contexts.
10
10
 
11
- When you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization.
11
+ When you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization. JSON payloads can also include context_variables on agent.create_session steps for the same contextual testing capabilities.
12
12
 
13
13
  Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.
14
14
 
15
15
  # flags.spec.summary
16
16
 
17
- Path to test spec file (YAML or JSON). Use `-` for stdin.
17
+ Path to test spec file (YAML or JSON). Supports reading from stdin when piping content.
18
18
 
19
19
  # flags.api-name.summary
20
20
 
@@ -54,9 +54,13 @@ Disable auto-normalization of field names and shorthand references.
54
54
 
55
55
  <%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit
56
56
 
57
- - Pipe JSON payload from stdin:
57
+ - Run tests with contextVariables to inject contextual data into agent sessions (add contextVariables to test cases in your YAML spec):
58
58
 
59
- $ echo '{"tests":[...]}' | <%= config.bin %> <%= command.id %> --spec - --target-org my-org
59
+ <%= config.bin %> <%= command.id %> --spec tests/agent-with-context.yaml --target-org my-org
60
+
61
+ - Pipe JSON payload from stdin (--spec flag is automatically populated from stdin):
62
+
63
+ $ echo '{"tests":[...]}' | <%= config.bin %> <%= command.id %> --spec --target-org my-org
60
64
 
61
65
  # info.batchProgress
62
66
 
package/oclif.manifest.json CHANGED
@@ -726,7 +726,7 @@
726
726
  "agent:generate:test-spec": {
727
727
  "aliases": [],
728
728
  "args": {},
729
- "description": "The first step when using Salesforce CLI to create an agent test in your org is to use this interactive command to generate a local YAML-formatted test spec file. The test spec YAML file contains information about the agent being tested, such as its API name, and then one or more test cases. This command uses the metadata components in your DX project when prompting for information, such as the agent API name; it doesn't look in your org.\n\nTo generate a specific agent test case, this command prompts you for this information; when possible, the command provides a list of options for you to choose from:\n\n- Utterance: Natural language statement, question, or command used to test the agent.\n- Expected topic: API name of the topic you expect the agent to use when responding to the utterance.\n- Expected actions: One or more API names of the expection actions the agent takes.\n- Expected outcome: Natural language description of the outcome you expect.\n- (Optional) Custom evaluation: Test an agent's response for specific strings or numbers.\n- (Optional) Conversation history: Boilerplate for additional context you can add to the test in the form of a conversation history.\n\nWhen your test spec is ready, you then run the \"agent test create\" command to actually create the test in your org and synchronize the metadata with your DX project. The metadata type for an agent test is AiEvaluationDefinition.\n\nIf you have an existing AiEvaluationDefinition metadata XML file in your DX project, you can generate its equivalent YAML test spec file with the --from-definition flag.",
729
+ "description": "The first step when using Salesforce CLI to create an agent test in your org is to use this interactive command to generate a local YAML-formatted test spec file. The test spec YAML file contains information about the agent being tested, such as its API name, and then one or more test cases. This command uses the metadata components in your DX project when prompting for information, such as the agent API name; it doesn't look in your org.\n\nTo generate a specific agent test case, this command prompts you for this information; when possible, the command provides a list of options for you to choose from:\n\n- Utterance: Natural language statement, question, or command used to test the agent.\n- Expected topic: API name of the topic you expect the agent to use when responding to the utterance.\n- Expected actions: One or more API names of the expection actions the agent takes.\n- Expected outcome: Natural language description of the outcome you expect.\n- (Optional) Custom evaluation: Test an agent's response for specific strings or numbers.\n- (Optional) Conversation history: Boilerplate for additional context you can add to the test in the form of a conversation history.\n\nYou can manually add contextVariables to test cases in the generated YAML file to inject contextual data (such as CaseId or RoutableId) into agent sessions. This is useful for testing agent behavior with different contextual information.\n\nWhen your test spec is ready, you then run the \"agent test create\" command to actually create the test in your org and synchronize the metadata with your DX project. The metadata type for an agent test is AiEvaluationDefinition.\n\nIf you have an existing AiEvaluationDefinition metadata XML file in your DX project, you can generate its equivalent YAML test spec file with the --from-definition flag.",
730
730
  "examples": [
731
731
  "Generate an agent test spec YAML file interactively:\n<%= config.bin %> <%= command.id %>",
732
732
  "Generate an agent test spec YAML file and specify a name for the new file; if the file exists, overwrite it without confirmation:\n<%= config.bin %> <%= command.id %> --output-file specs/Resort_Manager-new-version-testSpec.yaml --force-overwrite",
@@ -1627,13 +1627,14 @@
1627
1627
  "agent:test:run-eval": {
1628
1628
  "aliases": [],
1629
1629
  "args": {},
1630
- "description": "Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.\n\nWhen you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`.\n\nWhen you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization.\n\nSupports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.",
1630
+ "description": "Execute rich evaluation tests against an Agentforce agent using the Einstein Evaluation API. Supports both YAML test specs (same format as `sf agent generate test-spec`) and JSON payloads.\n\nWhen you provide a YAML test spec, the command automatically translates test cases into Evaluation API calls and infers the agent name from the spec's `subjectName` field. This means you can use the same test spec with both `sf agent test run` and `sf agent test run-eval`. YAML test specs also support contextVariables, which allow you to inject contextual data (such as CaseId or RoutableId) into agent sessions for testing with different contexts.\n\nWhen you provide a JSON payload, it's sent directly to the API with optional normalization. The normalizer auto-corrects common field name mistakes, converts shorthand references to JSONPath, and injects defaults. Use `--no-normalize` to disable this auto-normalization. JSON payloads can also include context_variables on agent.create_session steps for the same contextual testing capabilities.\n\nSupports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings.",
1631
1631
  "examples": [
1632
1632
  "Run tests using a YAML test spec on the org with alias \"my-org\":\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org",
1633
1633
  "Run tests using a YAML spec with explicit agent name override; use your default org:\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --api-name My_Agent --target-org my-org",
1634
1634
  "Run tests using a JSON payload:\n<%= config.bin %> <%= command.id %> --spec tests/eval-payload.json --target-org my-org",
1635
1635
  "Run tests and output results in JUnit format; useful for continuous integration and deployment (CI/CD):\n<%= config.bin %> <%= command.id %> --spec tests/my-agent-testSpec.yaml --target-org my-org --result-format junit",
1636
- "Pipe JSON payload from stdin:\n$ echo '{\"tests\":[...]}' | <%= config.bin %> <%= command.id %> --spec - --target-org my-org"
1636
+ "Run tests with contextVariables to inject contextual data into agent sessions (add contextVariables to test cases in your YAML spec):\n<%= config.bin %> <%= command.id %> --spec tests/agent-with-context.yaml --target-org my-org",
1637
+ "Pipe JSON payload from stdin (--spec flag is automatically populated from stdin):\n$ echo '{\"tests\":[...]}' | <%= config.bin %> <%= command.id %> --spec --target-org my-org"
1637
1638
  ],
1638
1639
  "flags": {
1639
1640
  "json": {
@@ -1672,7 +1673,7 @@
1672
1673
  "char": "s",
1673
1674
  "name": "spec",
1674
1675
  "required": true,
1675
- "summary": "Path to test spec file (YAML or JSON). Use `-` for stdin.",
1676
+ "summary": "Path to test spec file (YAML or JSON). Supports reading from stdin when piping content.",
1676
1677
  "hasDynamicHelp": false,
1677
1678
  "multiple": false,
1678
1679
  "type": "option"
@@ -1953,5 +1954,5 @@
1953
1954
  ]
1954
1955
  }
1955
1956
  },
1956
- "version": "1.31.3"
1957
+ "version": "1.32.0"
1957
1958
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@salesforce/plugin-agent",
3
3
  "description": "Commands to interact with Salesforce agents",
4
- "version": "1.31.3",
4
+ "version": "1.32.0",
5
5
  "author": "Salesforce",
6
6
  "bugs": "https://github.com/forcedotcom/cli/issues",
7
7
  "enableO11y": true,
@@ -28,7 +28,7 @@
28
28
  "yaml": "^2.8.2"
29
29
  },
30
30
  "devDependencies": {
31
- "@oclif/plugin-command-snapshot": "^5.3.10",
31
+ "@oclif/plugin-command-snapshot": "^5.3.11",
32
32
  "@oclif/test": "^4.1.15",
33
33
  "@salesforce/cli-plugins-testkit": "^5.3.41",
34
34
  "@salesforce/dev-scripts": "^11.0.4",
@@ -240,7 +240,7 @@
240
240
  "exports": "./lib/index.js",
241
241
  "type": "module",
242
242
  "sfdx": {
243
- "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.31.3.crt",
244
- "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.31.3.sig"
243
+ "publicKeyUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.32.0.crt",
244
+ "signatureUrl": "https://developer.salesforce.com/media/salesforce-cli/security/@salesforce/plugin-agent/1.32.0.sig"
245
245
  }
246
246
  }
package/schemas/agent-test-run__eval.json CHANGED
@@ -19,9 +19,13 @@
19
19
  "evaluations": {
20
20
  "type": "array",
21
21
  "items": {}
22
+ },
23
+ "outputs": {
24
+ "type": "array",
25
+ "items": {}
22
26
  }
23
27
  },
24
- "required": ["id", "status", "evaluations"],
28
+ "required": ["id", "status", "evaluations", "outputs"],
25
29
  "additionalProperties": false
26
30
  }
27
31
  },