@memberjunction/testing-engine 2.133.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/variable-resolver.test.d.ts +5 -0
- package/dist/__tests__/variable-resolver.test.d.ts.map +1 -0
- package/dist/__tests__/variable-resolver.test.js +412 -0
- package/dist/__tests__/variable-resolver.test.js.map +1 -0
- package/dist/drivers/AgentEvalDriver.d.ts +10 -0
- package/dist/drivers/AgentEvalDriver.d.ts.map +1 -1
- package/dist/drivers/AgentEvalDriver.js +47 -5
- package/dist/drivers/AgentEvalDriver.js.map +1 -1
- package/dist/engine/TestEngine.d.ts +12 -0
- package/dist/engine/TestEngine.d.ts.map +1 -1
- package/dist/engine/TestEngine.js +73 -6
- package/dist/engine/TestEngine.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +8 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/variable-resolver.d.ts +82 -0
- package/dist/utils/variable-resolver.d.ts.map +1 -0
- package/dist/utils/variable-resolver.js +298 -0
- package/dist/utils/variable-resolver.js.map +1 -0
- package/package.json +17 -12
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"variable-resolver.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/variable-resolver.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Unit tests for VariableResolver
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const variable_resolver_1 = require("../utils/variable-resolver");
|
|
7
|
+
describe('VariableResolver', () => {
|
|
8
|
+
let resolver;
|
|
9
|
+
beforeEach(() => {
|
|
10
|
+
resolver = new variable_resolver_1.VariableResolver();
|
|
11
|
+
});
|
|
12
|
+
// ============================================================================
|
|
13
|
+
// Schema Parsing Tests
|
|
14
|
+
// ============================================================================
|
|
15
|
+
describe('parseTypeSchema', () => {
|
|
16
|
+
it('should return null for null input', () => {
|
|
17
|
+
const result = resolver.parseTypeSchema(null);
|
|
18
|
+
expect(result).toBeNull();
|
|
19
|
+
});
|
|
20
|
+
it('should return null for empty string', () => {
|
|
21
|
+
const result = resolver.parseTypeSchema('');
|
|
22
|
+
expect(result).toBeNull();
|
|
23
|
+
});
|
|
24
|
+
it('should return null for invalid JSON', () => {
|
|
25
|
+
const result = resolver.parseTypeSchema('not valid json');
|
|
26
|
+
expect(result).toBeNull();
|
|
27
|
+
});
|
|
28
|
+
it('should parse valid schema', () => {
|
|
29
|
+
const schema = {
|
|
30
|
+
schemaVersion: '1.0',
|
|
31
|
+
variables: [
|
|
32
|
+
{
|
|
33
|
+
name: 'TestVar',
|
|
34
|
+
displayName: 'Test Variable',
|
|
35
|
+
dataType: 'string',
|
|
36
|
+
valueSource: 'freeform',
|
|
37
|
+
required: false
|
|
38
|
+
}
|
|
39
|
+
]
|
|
40
|
+
};
|
|
41
|
+
const result = resolver.parseTypeSchema(JSON.stringify(schema));
|
|
42
|
+
expect(result).toEqual(schema);
|
|
43
|
+
});
|
|
44
|
+
it('should throw for unsupported schema version', () => {
|
|
45
|
+
const schema = {
|
|
46
|
+
schemaVersion: '2.0',
|
|
47
|
+
variables: []
|
|
48
|
+
};
|
|
49
|
+
expect(() => resolver.parseTypeSchema(JSON.stringify(schema)))
|
|
50
|
+
.toThrow(variable_resolver_1.VariableResolutionError);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
describe('parseTestConfig', () => {
|
|
54
|
+
it('should return null for null input', () => {
|
|
55
|
+
const result = resolver.parseTestConfig(null);
|
|
56
|
+
expect(result).toBeNull();
|
|
57
|
+
});
|
|
58
|
+
it('should parse valid config', () => {
|
|
59
|
+
const config = {
|
|
60
|
+
variables: {
|
|
61
|
+
TestVar: { exposed: true, defaultValue: 'test' }
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
const result = resolver.parseTestConfig(JSON.stringify(config));
|
|
65
|
+
expect(result).toEqual(config);
|
|
66
|
+
});
|
|
67
|
+
});
|
|
68
|
+
describe('parseSuiteConfig', () => {
|
|
69
|
+
it('should return null for null input', () => {
|
|
70
|
+
const result = resolver.parseSuiteConfig(null);
|
|
71
|
+
expect(result).toBeNull();
|
|
72
|
+
});
|
|
73
|
+
it('should parse valid config', () => {
|
|
74
|
+
const config = {
|
|
75
|
+
variables: {
|
|
76
|
+
TestVar: 'suite-value'
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
const result = resolver.parseSuiteConfig(JSON.stringify(config));
|
|
80
|
+
expect(result).toEqual(config);
|
|
81
|
+
});
|
|
82
|
+
});
|
|
83
|
+
// ============================================================================
|
|
84
|
+
// Variable Resolution Tests
|
|
85
|
+
// ============================================================================
|
|
86
|
+
describe('resolveVariables', () => {
|
|
87
|
+
const baseTypeSchema = {
|
|
88
|
+
schemaVersion: '1.0',
|
|
89
|
+
variables: [
|
|
90
|
+
{
|
|
91
|
+
name: 'AIConfiguration',
|
|
92
|
+
displayName: 'AI Configuration',
|
|
93
|
+
dataType: 'string',
|
|
94
|
+
valueSource: 'static',
|
|
95
|
+
possibleValues: [
|
|
96
|
+
{ value: 'claude', label: 'Claude' },
|
|
97
|
+
{ value: 'gpt4', label: 'GPT-4' },
|
|
98
|
+
{ value: 'gemini', label: 'Gemini' }
|
|
99
|
+
],
|
|
100
|
+
defaultValue: 'claude',
|
|
101
|
+
required: false
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
name: 'Temperature',
|
|
105
|
+
displayName: 'Temperature',
|
|
106
|
+
dataType: 'number',
|
|
107
|
+
valueSource: 'freeform',
|
|
108
|
+
defaultValue: 0.7,
|
|
109
|
+
required: false
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
name: 'RequiredVar',
|
|
113
|
+
displayName: 'Required Variable',
|
|
114
|
+
dataType: 'string',
|
|
115
|
+
valueSource: 'freeform',
|
|
116
|
+
required: true
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
};
|
|
120
|
+
it('should return empty result for null type schema', () => {
|
|
121
|
+
const result = resolver.resolveVariables(null, null, null, {});
|
|
122
|
+
expect(result).toEqual({ values: {}, sources: {} });
|
|
123
|
+
});
|
|
124
|
+
it('should return empty result for empty variables array', () => {
|
|
125
|
+
const schema = {
|
|
126
|
+
schemaVersion: '1.0',
|
|
127
|
+
variables: []
|
|
128
|
+
};
|
|
129
|
+
const result = resolver.resolveVariables(JSON.stringify(schema), null, null, {});
|
|
130
|
+
expect(result).toEqual({ values: {}, sources: {} });
|
|
131
|
+
});
|
|
132
|
+
it('should use type defaults when no overrides provided', () => {
|
|
133
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), null, null, { variables: { RequiredVar: 'required-value' } });
|
|
134
|
+
expect(result.values['AIConfiguration']).toBe('claude');
|
|
135
|
+
expect(result.sources['AIConfiguration']).toBe('type');
|
|
136
|
+
expect(result.values['Temperature']).toBe(0.7);
|
|
137
|
+
expect(result.sources['Temperature']).toBe('type');
|
|
138
|
+
});
|
|
139
|
+
it('should throw for missing required variable', () => {
|
|
140
|
+
expect(() => resolver.resolveVariables(JSON.stringify(baseTypeSchema), null, null, {})).toThrow(variable_resolver_1.VariableResolutionError);
|
|
141
|
+
});
|
|
142
|
+
it('should use run-level values over all others', () => {
|
|
143
|
+
const testConfig = {
|
|
144
|
+
variables: {
|
|
145
|
+
AIConfiguration: { exposed: true, defaultValue: 'test-default' }
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
const suiteConfig = {
|
|
149
|
+
variables: { AIConfiguration: 'gpt4' }
|
|
150
|
+
};
|
|
151
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), JSON.stringify(testConfig), JSON.stringify(suiteConfig), { variables: { AIConfiguration: 'gemini', RequiredVar: 'value' } });
|
|
152
|
+
expect(result.values['AIConfiguration']).toBe('gemini');
|
|
153
|
+
expect(result.sources['AIConfiguration']).toBe('run');
|
|
154
|
+
});
|
|
155
|
+
it('should use suite-level values over test and type defaults', () => {
|
|
156
|
+
const testConfig = {
|
|
157
|
+
variables: {
|
|
158
|
+
AIConfiguration: { exposed: true, defaultValue: 'test-default' }
|
|
159
|
+
}
|
|
160
|
+
};
|
|
161
|
+
const suiteConfig = {
|
|
162
|
+
variables: { AIConfiguration: 'gpt4' }
|
|
163
|
+
};
|
|
164
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), JSON.stringify(testConfig), JSON.stringify(suiteConfig), { variables: { RequiredVar: 'value' } });
|
|
165
|
+
expect(result.values['AIConfiguration']).toBe('gpt4');
|
|
166
|
+
expect(result.sources['AIConfiguration']).toBe('suite');
|
|
167
|
+
});
|
|
168
|
+
it('should use test-level defaults over type defaults', () => {
|
|
169
|
+
const testConfig = {
|
|
170
|
+
variables: {
|
|
171
|
+
Temperature: { exposed: true, defaultValue: 0.5 }
|
|
172
|
+
}
|
|
173
|
+
};
|
|
174
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), JSON.stringify(testConfig), null, { variables: { RequiredVar: 'value' } });
|
|
175
|
+
expect(result.values['Temperature']).toBe(0.5);
|
|
176
|
+
expect(result.sources['Temperature']).toBe('test');
|
|
177
|
+
});
|
|
178
|
+
it('should skip variables marked as not exposed', () => {
|
|
179
|
+
const testConfig = {
|
|
180
|
+
variables: {
|
|
181
|
+
AIConfiguration: { exposed: false },
|
|
182
|
+
Temperature: { exposed: true }
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), JSON.stringify(testConfig), null, { variables: { RequiredVar: 'value' } });
|
|
186
|
+
expect(result.values['AIConfiguration']).toBeUndefined();
|
|
187
|
+
expect(result.values['Temperature']).toBe(0.7);
|
|
188
|
+
});
|
|
189
|
+
it('should respect locked variables and ignore run/suite overrides', () => {
|
|
190
|
+
const testConfig = {
|
|
191
|
+
variables: {
|
|
192
|
+
AIConfiguration: { exposed: true, defaultValue: 'claude', locked: true }
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
const suiteConfig = {
|
|
196
|
+
variables: { AIConfiguration: 'gpt4' }
|
|
197
|
+
};
|
|
198
|
+
const result = resolver.resolveVariables(JSON.stringify(baseTypeSchema), JSON.stringify(testConfig), JSON.stringify(suiteConfig), { variables: { AIConfiguration: 'gemini', RequiredVar: 'value' } });
|
|
199
|
+
// Should use test default despite run/suite trying to override
|
|
200
|
+
expect(result.values['AIConfiguration']).toBe('claude');
|
|
201
|
+
expect(result.sources['AIConfiguration']).toBe('test');
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
// ============================================================================
|
|
205
|
+
// Value Validation Tests
|
|
206
|
+
// ============================================================================
|
|
207
|
+
describe('validateValue', () => {
|
|
208
|
+
it('should accept valid static values', () => {
|
|
209
|
+
const varDef = {
|
|
210
|
+
name: 'Model',
|
|
211
|
+
displayName: 'Model',
|
|
212
|
+
dataType: 'string',
|
|
213
|
+
valueSource: 'static',
|
|
214
|
+
possibleValues: [
|
|
215
|
+
{ value: 'a' },
|
|
216
|
+
{ value: 'b' },
|
|
217
|
+
{ value: 'c' }
|
|
218
|
+
],
|
|
219
|
+
required: false
|
|
220
|
+
};
|
|
221
|
+
// Should not throw
|
|
222
|
+
expect(() => resolver.validateValue(varDef, undefined, 'a', 'run')).not.toThrow();
|
|
223
|
+
expect(() => resolver.validateValue(varDef, undefined, 'b', 'run')).not.toThrow();
|
|
224
|
+
});
|
|
225
|
+
it('should reject invalid static values', () => {
|
|
226
|
+
const varDef = {
|
|
227
|
+
name: 'Model',
|
|
228
|
+
displayName: 'Model',
|
|
229
|
+
dataType: 'string',
|
|
230
|
+
valueSource: 'static',
|
|
231
|
+
possibleValues: [
|
|
232
|
+
{ value: 'a' },
|
|
233
|
+
{ value: 'b' }
|
|
234
|
+
],
|
|
235
|
+
required: false
|
|
236
|
+
};
|
|
237
|
+
expect(() => resolver.validateValue(varDef, undefined, 'invalid', 'run'))
|
|
238
|
+
.toThrow(variable_resolver_1.VariableResolutionError);
|
|
239
|
+
});
|
|
240
|
+
it('should respect test-level restricted values', () => {
|
|
241
|
+
const varDef = {
|
|
242
|
+
name: 'Model',
|
|
243
|
+
displayName: 'Model',
|
|
244
|
+
dataType: 'string',
|
|
245
|
+
valueSource: 'static',
|
|
246
|
+
possibleValues: [
|
|
247
|
+
{ value: 'a' },
|
|
248
|
+
{ value: 'b' },
|
|
249
|
+
{ value: 'c' }
|
|
250
|
+
],
|
|
251
|
+
required: false
|
|
252
|
+
};
|
|
253
|
+
const testOverride = {
|
|
254
|
+
exposed: true,
|
|
255
|
+
restrictedValues: ['a', 'b'] // Only a and b allowed
|
|
256
|
+
};
|
|
257
|
+
// 'a' should be allowed
|
|
258
|
+
expect(() => resolver.validateValue(varDef, testOverride, 'a', 'run')).not.toThrow();
|
|
259
|
+
// 'c' should be rejected even though it's in type possibleValues
|
|
260
|
+
expect(() => resolver.validateValue(varDef, testOverride, 'c', 'run'))
|
|
261
|
+
.toThrow(variable_resolver_1.VariableResolutionError);
|
|
262
|
+
});
|
|
263
|
+
});
|
|
264
|
+
describe('validateDataType', () => {
|
|
265
|
+
it('should validate string type', () => {
|
|
266
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'string', valueSource: 'freeform', required: false };
|
|
267
|
+
expect(() => resolver.validateDataType(varDef, 'hello')).not.toThrow();
|
|
268
|
+
expect(() => resolver.validateDataType(varDef, 123)).toThrow(variable_resolver_1.VariableResolutionError);
|
|
269
|
+
});
|
|
270
|
+
it('should validate number type', () => {
|
|
271
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'number', valueSource: 'freeform', required: false };
|
|
272
|
+
expect(() => resolver.validateDataType(varDef, 123)).not.toThrow();
|
|
273
|
+
expect(() => resolver.validateDataType(varDef, 0.5)).not.toThrow();
|
|
274
|
+
expect(() => resolver.validateDataType(varDef, 'hello')).toThrow(variable_resolver_1.VariableResolutionError);
|
|
275
|
+
expect(() => resolver.validateDataType(varDef, NaN)).toThrow(variable_resolver_1.VariableResolutionError);
|
|
276
|
+
});
|
|
277
|
+
it('should validate boolean type', () => {
|
|
278
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'boolean', valueSource: 'freeform', required: false };
|
|
279
|
+
expect(() => resolver.validateDataType(varDef, true)).not.toThrow();
|
|
280
|
+
expect(() => resolver.validateDataType(varDef, false)).not.toThrow();
|
|
281
|
+
expect(() => resolver.validateDataType(varDef, 'true')).toThrow(variable_resolver_1.VariableResolutionError);
|
|
282
|
+
});
|
|
283
|
+
it('should validate date type', () => {
|
|
284
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'date', valueSource: 'freeform', required: false };
|
|
285
|
+
expect(() => resolver.validateDataType(varDef, new Date())).not.toThrow();
|
|
286
|
+
expect(() => resolver.validateDataType(varDef, '2025-01-13')).not.toThrow();
|
|
287
|
+
expect(() => resolver.validateDataType(varDef, 'not-a-date')).toThrow(variable_resolver_1.VariableResolutionError);
|
|
288
|
+
});
|
|
289
|
+
});
|
|
290
|
+
// ============================================================================
|
|
291
|
+
// Available Variables Tests
|
|
292
|
+
// ============================================================================
|
|
293
|
+
describe('getAvailableVariables', () => {
|
|
294
|
+
const typeSchema = {
|
|
295
|
+
schemaVersion: '1.0',
|
|
296
|
+
variables: [
|
|
297
|
+
{ name: 'Var1', displayName: 'Variable 1', dataType: 'string', valueSource: 'freeform', required: false },
|
|
298
|
+
{ name: 'Var2', displayName: 'Variable 2', dataType: 'number', valueSource: 'freeform', defaultValue: 10, required: false },
|
|
299
|
+
{ name: 'Var3', displayName: 'Variable 3', dataType: 'string', valueSource: 'static', possibleValues: [{ value: 'a' }, { value: 'b' }, { value: 'c' }], required: false }
|
|
300
|
+
]
|
|
301
|
+
};
|
|
302
|
+
it('should return empty array for null type schema', () => {
|
|
303
|
+
const result = resolver.getAvailableVariables(null, null);
|
|
304
|
+
expect(result).toEqual([]);
|
|
305
|
+
});
|
|
306
|
+
it('should return all variables when no test config', () => {
|
|
307
|
+
const result = resolver.getAvailableVariables(JSON.stringify(typeSchema), null);
|
|
308
|
+
expect(result).toHaveLength(3);
|
|
309
|
+
});
|
|
310
|
+
it('should filter out non-exposed variables', () => {
|
|
311
|
+
const testConfig = {
|
|
312
|
+
variables: {
|
|
313
|
+
Var1: { exposed: false },
|
|
314
|
+
Var2: { exposed: true }
|
|
315
|
+
}
|
|
316
|
+
};
|
|
317
|
+
const result = resolver.getAvailableVariables(JSON.stringify(typeSchema), JSON.stringify(testConfig));
|
|
318
|
+
expect(result).toHaveLength(2);
|
|
319
|
+
expect(result.find(v => v.name === 'Var1')).toBeUndefined();
|
|
320
|
+
expect(result.find(v => v.name === 'Var2')).toBeDefined();
|
|
321
|
+
expect(result.find(v => v.name === 'Var3')).toBeDefined();
|
|
322
|
+
});
|
|
323
|
+
it('should apply test-level default overrides', () => {
|
|
324
|
+
const testConfig = {
|
|
325
|
+
variables: {
|
|
326
|
+
Var2: { exposed: true, defaultValue: 20 }
|
|
327
|
+
}
|
|
328
|
+
};
|
|
329
|
+
const result = resolver.getAvailableVariables(JSON.stringify(typeSchema), JSON.stringify(testConfig));
|
|
330
|
+
const var2 = result.find(v => v.name === 'Var2');
|
|
331
|
+
expect(var2?.defaultValue).toBe(20);
|
|
332
|
+
});
|
|
333
|
+
it('should apply restricted values from test config', () => {
|
|
334
|
+
const testConfig = {
|
|
335
|
+
variables: {
|
|
336
|
+
Var3: { exposed: true, restrictedValues: ['a', 'b'] }
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
const result = resolver.getAvailableVariables(JSON.stringify(typeSchema), JSON.stringify(testConfig));
|
|
340
|
+
const var3 = result.find(v => v.name === 'Var3');
|
|
341
|
+
expect(var3?.possibleValues).toHaveLength(2);
|
|
342
|
+
expect(var3?.possibleValues?.map(pv => pv.value)).toEqual(['a', 'b']);
|
|
343
|
+
});
|
|
344
|
+
});
|
|
345
|
+
// ============================================================================
|
|
346
|
+
// CLI Parsing Tests
|
|
347
|
+
// ============================================================================
|
|
348
|
+
describe('parseCliValue', () => {
|
|
349
|
+
it('should parse string values', () => {
|
|
350
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'string', valueSource: 'freeform', required: false };
|
|
351
|
+
expect(resolver.parseCliValue(varDef, 'hello')).toBe('hello');
|
|
352
|
+
});
|
|
353
|
+
it('should parse number values', () => {
|
|
354
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'number', valueSource: 'freeform', required: false };
|
|
355
|
+
expect(resolver.parseCliValue(varDef, '123')).toBe(123);
|
|
356
|
+
expect(resolver.parseCliValue(varDef, '0.5')).toBe(0.5);
|
|
357
|
+
});
|
|
358
|
+
it('should throw for invalid number', () => {
|
|
359
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'number', valueSource: 'freeform', required: false };
|
|
360
|
+
expect(() => resolver.parseCliValue(varDef, 'not-a-number')).toThrow(variable_resolver_1.VariableResolutionError);
|
|
361
|
+
});
|
|
362
|
+
it('should parse boolean values', () => {
|
|
363
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'boolean', valueSource: 'freeform', required: false };
|
|
364
|
+
expect(resolver.parseCliValue(varDef, 'true')).toBe(true);
|
|
365
|
+
expect(resolver.parseCliValue(varDef, 'TRUE')).toBe(true);
|
|
366
|
+
expect(resolver.parseCliValue(varDef, '1')).toBe(true);
|
|
367
|
+
expect(resolver.parseCliValue(varDef, 'yes')).toBe(true);
|
|
368
|
+
expect(resolver.parseCliValue(varDef, 'false')).toBe(false);
|
|
369
|
+
expect(resolver.parseCliValue(varDef, '0')).toBe(false);
|
|
370
|
+
expect(resolver.parseCliValue(varDef, 'no')).toBe(false);
|
|
371
|
+
});
|
|
372
|
+
it('should throw for invalid boolean', () => {
|
|
373
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'boolean', valueSource: 'freeform', required: false };
|
|
374
|
+
expect(() => resolver.parseCliValue(varDef, 'maybe')).toThrow(variable_resolver_1.VariableResolutionError);
|
|
375
|
+
});
|
|
376
|
+
it('should parse date values', () => {
|
|
377
|
+
const varDef = { name: 'V', displayName: 'V', dataType: 'date', valueSource: 'freeform', required: false };
|
|
378
|
+
const result = resolver.parseCliValue(varDef, '2025-01-13');
|
|
379
|
+
expect(result).toBeInstanceOf(Date);
|
|
380
|
+
expect(result.getFullYear()).toBe(2025);
|
|
381
|
+
});
|
|
382
|
+
});
|
|
383
|
+
describe('parseCliVariables', () => {
|
|
384
|
+
it('should parse name=value format', () => {
|
|
385
|
+
const result = resolver.parseCliVariables(['name=value', 'other=123']);
|
|
386
|
+
expect(result).toEqual({ name: 'value', other: '123' });
|
|
387
|
+
});
|
|
388
|
+
it('should throw for invalid format (no equals)', () => {
|
|
389
|
+
expect(() => resolver.parseCliVariables(['invalid']))
|
|
390
|
+
.toThrow(variable_resolver_1.VariableResolutionError);
|
|
391
|
+
});
|
|
392
|
+
it('should handle values with equals signs', () => {
|
|
393
|
+
const result = resolver.parseCliVariables(['equation=a=b+c']);
|
|
394
|
+
expect(result).toEqual({ equation: 'a=b+c' });
|
|
395
|
+
});
|
|
396
|
+
it('should convert types when schema provided', () => {
|
|
397
|
+
const schema = {
|
|
398
|
+
schemaVersion: '1.0',
|
|
399
|
+
variables: [
|
|
400
|
+
{ name: 'num', displayName: 'Number', dataType: 'number', valueSource: 'freeform', required: false },
|
|
401
|
+
{ name: 'bool', displayName: 'Boolean', dataType: 'boolean', valueSource: 'freeform', required: false }
|
|
402
|
+
]
|
|
403
|
+
};
|
|
404
|
+
const result = resolver.parseCliVariables(['num=42', 'bool=true'], JSON.stringify(schema));
|
|
405
|
+
expect(result['num']).toBe(42);
|
|
406
|
+
expect(typeof result['num']).toBe('number');
|
|
407
|
+
expect(result['bool']).toBe(true);
|
|
408
|
+
expect(typeof result['bool']).toBe('boolean');
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
});
|
|
412
|
+
//# sourceMappingURL=variable-resolver.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"variable-resolver.test.js","sourceRoot":"","sources":["../../src/__tests__/variable-resolver.test.ts"],"names":[],"mappings":";AAAA;;GAEG;;AAEH,kEAAuF;AAQvF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,IAAI,QAA0B,CAAC;IAE/B,UAAU,CAAC,GAAG,EAAE;QACd,QAAQ,GAAG,IAAI,oCAAgB,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,uBAAuB;IACvB,+EAA+E;IAE/E,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,gBAAgB,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAA4B;gBACtC,aAAa,EAAE,KAAK;gBACpB,SAAS,EAAE;oBACT;wBACE,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,eAAe;wBAC5B,QAAQ,EAAE,QAAQ;wBAClB,WAAW,EAAE,UAAU;wBACvB,QAAQ,EAAE,KAAK;qBAChB;iBACF;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,MAAM,GAAG;gBACb,aAAa,EAAE,KAAK;gBACpB,SAAS,EAAE,EAAE;aACd,CAAC;YAEF,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;iBAC3D,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAAwB;gBAClC,SAAS,EAAE;oBACT,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE;iBACjD;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAA6B;gBACvC,SAAS,EAAE;oBACT,OAAO,EAAE,aAAa;iBACvB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,4BAA4B;IAC5B,+EAA+E;IAE/E,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,MAAM,cAAc,GAA4B;YAC9C,aAAa,EAAE,KAAK;YACpB,SAAS,EAAE;gBACT;oBACE,IAAI,EAAE,iBAAiB;oBACvB,WAAW,EAAE,kBAAkB;oBAC/B,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,QAAQ;oBACrB,cAAc,EAAE;wBACd,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;wBACpC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;wBACjC,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;qBACrC;oBACD,YAAY,EAAE,QAAQ;oBACtB,QAAQ,EAAE,KAAK;iBAChB;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,aAAa;oBAC1B,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,UAAU;oBACvB,YAAY,EAAE,GAAG;oBACjB,QAAQ,EAAE,KAAK;iBAChB;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,WAAW,EAAE,mBAAmB;oBAChC,QAAQ,EAAE,QAAQ;oBAClB,WAAW,EAAE,UAAU;oBACvB,QAAQ,EAAE,IAAI;iBACf;aACF;SACF,CAAC;QAEF,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YAC/D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;YAC9D,MAAM,MAAM,GAA4B;gBACtC,aAAa,EAAE,KAAK;gBACpB,SAAS,EAAE,EAAE;aACd,CAAC;YACF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;YACjF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;YAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,EACJ,IAAI,EACJ,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,gBAAgB,EAAE,EAAE,CACjD,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CACpC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,EACJ,IAAI,EACJ,EAAE,CACH,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,eAAe,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE;iBACjE;aACF,CAAC;YAEF,MAAM,WAAW,GAA6B;gBAC5C,SAAS,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;aACvC,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAC3B,EAAE,SAAS,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CACnE,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACxD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,GAAG,EAAE;YACnE,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,eAAe,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,cAAc,EAAE;iBACjE;aACF,CAAC;YAEF,MAAM,WAAW,GAA6B;gBAC5C,SAAS,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;aACvC,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAC3B,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CACxC,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;YAC3D,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE;iBAClD;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,EACJ,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CACxC,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,eAAe,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;oBACnC,WAAW,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;iBAC/B;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,EACJ,EAAE,SAAS,EAAE,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CACxC,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;YACzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;YACxE,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,eAAe,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE;iBACzE;aACF,CAAC;YAEF,MAAM,WAAW,GAA6B;gBAC5C,SAAS,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE;aACvC,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CACtC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAC3B,EAAE,SAAS,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,CACnE,CAAC;YAEF,+DAA+D;YAC/D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,yBAAyB;IACzB,+EAA+E;IAE/E,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;YAC3C,MAAM,MAAM,GAAG;gBACb,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,OAAO;gBACpB,QAAQ,EAAE,QAAiB;gBAC3B,WAAW,EAAE,QAAiB;gBAC9B,cAAc,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;iBACf;gBACD,QAAQ,EAAE,KAAK;aAChB,CAAC;YAEF,mBAAmB;YACnB,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAClF,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACpF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,MAAM,GAAG;gBACb,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,OAAO;gBACpB,QAAQ,EAAE,QAAiB;gBAC3B,WAAW,EAAE,QAAiB;gBAC9B,cAAc,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;iBACf;gBACD,QAAQ,EAAE,KAAK;aAChB,CAAC;YAEF,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;iBACtE,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,MAAM,GAAG;gBACb,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,OAAO;gBACpB,QAAQ,EAAE,QAAiB;gBAC3B,WAAW,EAAE,QAAiB;gBAC9B,cAAc,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;oBACd,EAAE,KAAK,EAAE,GAAG,EAAE;iBACf;gBACD,QAAQ,EAAE,KAAK;aAChB,CAAC;YAEF,MAAM,YAAY,GAAG;gBACnB,OAAO,EAAE,IAAI;gBACb,gBAAgB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,uBAAuB;aACrD,CAAC;YAEF,wBAAwB;YACxB,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAErF,iEAAiE;YACjE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;iBACnE,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAE/H,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACvE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAE/H,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;YAC1F,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,GAAG,EAAE;YACtC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAkB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAEhI,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACpE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;YACnC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAe,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAE7H,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC1E,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5E,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACjG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,4BAA4B;IAC5B,+EAA+E;IAE/E,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACrC,MAAM,UAAU,GAA4B;YAC1C,aAAa,EAAE,KAAK;YACpB,SAAS,EAAE;gBACT,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACzG,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBAC3H,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE;aAC1K;SACF,CAAC;QAEF,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,CAAC;YAChF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;YACjD,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;oBACxB,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;iBACxB;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAC3C,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAC3B,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC1D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE,EAAE;iBAC1C;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAC3C,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAC3B,CAAC;YAEF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,UAAU,GAAwB;gBACtC,SAAS,EAAE;oBACT,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE;iBACtD;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,qBAAqB,CAC3C,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAC1B,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAC3B,CAAC;YAEF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC7C,MAAM,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,+EAA+E;IAC/E,oBAAoB;IACpB,+EAA+E;IAE/E,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAC/H,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;YACpC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAC/H,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxD,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;YACzC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAiB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAC/H,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QAChG,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;YACrC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAkB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChI,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1D,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1D,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvD,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzD,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC5D,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxD,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;YAC1C,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAkB,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAChI,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACzF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,GAAG,EAAE;YAClC,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAe,EAAE,WAAW,EAAE,UAAmB,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;YAC7H,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;YAC5D,MAAM,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YACpC,MAAM,CAAE,MAAe,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;YACxC,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;YACvE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;iBAClD,OAAO,CAAC,2CAAuB,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;YACnD,MAAM,MAAM,GAA4B;gBACtC,aAAa,EAAE,KAAK;gBACpB,SAAS,EAAE;oBACT,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE;oBACpG,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE;iBACxG;aACF,CAAC;YAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CACvC,CAAC,QAAQ,EAAE,WAAW,CAAC,EACvB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CACvB,CAAC;YAEF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/B,MAAM,CAAC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,MAAM,CAAC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -253,6 +253,16 @@ export declare class AgentEvalDriver extends BaseTestDriver {
|
|
|
253
253
|
* @private
|
|
254
254
|
*/
|
|
255
255
|
private extractAgentOutput;
|
|
256
|
+
/**
|
|
257
|
+
* Build execution override object from turn params and resolved variables.
|
|
258
|
+
*
|
|
259
|
+
* Priority (highest to lowest):
|
|
260
|
+
* 1. Turn-level execution params (modelOverride, temperatureOverride, etc.)
|
|
261
|
+
* 2. Resolved variables (AIConfiguration, Temperature, etc.)
|
|
262
|
+
*
|
|
263
|
+
* @private
|
|
264
|
+
*/
|
|
265
|
+
private buildExecutionOverride;
|
|
256
266
|
/**
|
|
257
267
|
* Run oracles for multi-turn evaluation.
|
|
258
268
|
* @private
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AgentEvalDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/AgentEvalDriver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAmC,UAAU,EAAiB,MAAM,+BAA+B,CAAC;AAG3G,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EAIrB,gBAAgB,EAGnB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,EAAE,CAAC;IAEJ;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,GAAG,WAAW,GAAG,qBAAqB,CAAC;CAChF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,eAAe,CAAC,EAAE;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;IAEF;;;OAGG;IACH,gBAAgB,CAAC,EAAE,yBAAyB,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;IAExB;;OAEG;IACH,mBAAmB,CAAC,EAAE;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,KAAK,CAAC;YAClB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAC3B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;KACN,CAAC;IAEF;;OAEG;IACH,eAAe,CAAC,EAAE;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,OAAO,CAAC;KAC3B,CAAC,CAAC;IAEH;;OAEG;IACH,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,qBACa,eAAgB,SAAQ,cAAc;IAC/C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAuB;IAExE;;OAEG;IACH,OAAO,CAAC,oBAAoB,CAAuB;IAEnD;;;OAGG;IACa,oBAAoB,IAAI,OAAO;IAI/C;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAa9B;;;;;;;;;;;;;OAaG;IACU,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA0HrF;;;;;;;;;;;;OAYG;IACmB,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqH3E;;;OAGG;YACW,SAAS;IAMvB;;;;OAIG;YACW,YAAY;
|
|
1
|
+
{"version":3,"file":"AgentEvalDriver.d.ts","sourceRoot":"","sources":["../../src/drivers/AgentEvalDriver.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAmC,UAAU,EAAiB,MAAM,+BAA+B,CAAC;AAG3G,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EAIrB,gBAAgB,EAGnB,MAAM,UAAU,CAAC;AAElB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC5B;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,MAAM,CAAC,EAAE,MAAM,CAAC;KACnB,EAAE,CAAC;IAEJ;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAExC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,kBAAkB,CAAC,EAAE,iBAAiB,GAAG,WAAW,GAAG,qBAAqB,CAAC;CAChF;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,eAAe,CAAC,EAAE;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;IAEF;;;OAGG;IACH,gBAAgB,CAAC,EAAE,yBAAyB,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC3B;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEvC;;OAEG;IACH,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC;IAExB;;OAEG;IACH,mBAAmB,CAAC,EAAE;QAClB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,aAAa,CAAC,EAAE,KAAK,CAAC;YAClB,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;YAC3B,OAAO,EAAE,MAAM,CAAC;SACnB,CAAC,CAAC;KACN,CAAC;IAEF;;OAEG;IACH,eAAe,CAAC,EAAE;QACd,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC9B,CAAC;CACL;AAED;;GAEG;AACH,MAAM,WAAW,yBAAyB;IACtC;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;IAE5B;;OAEG;IACH,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAEzC;;OAEG;IACH,cAAc,CAAC,EAAE,KAAK,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,cAAc,EAAE,OAAO,CAAC;KAC3B,CAAC,CAAC;IAEH;;OAEG;IACH,uBAAuB,CAAC,EAAE,MAAM,EAAE,CAAC;CACtC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,qBACa,eAAgB,SAAQ,cAAc;IAC/C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAuB;IAExE;;OAEG;IACH,OAAO,CAAC,oBAAoB,CAAuB;IAEnD;;;OAGG;IACa,oBAAoB,IAAI,OAAO;IAI/C;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAa9B;;;;;;;;;;;;;OAaG;IACU,OAAO,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA0HrF;;;;;;;;;;;;OAYG;IACmB,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqH3E;;;OAGG;YACW,SAAS;IAMvB;;;;OAIG;YACW,YAAY;IA4F1B;;;;OAIG;YACW,iBAAiB;IA2G/B;;;OAGG;IACH,OAAO,CAAC,cAAc;IAiBtB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAQ5B;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAS1B;;;;;;;;OAQG;IACH,OAAO,CAAC,sBAAsB;IA6C9B;;;OAGG;YACW,sBAAsB;IAiCpC;;;OAGG;YACW,sBAAsB;IAmBpC;;;OAGG;YACW,qBAAqB;IA4BnC;;;OAGG;YACW,qBAAqB;IAwDnC;;;OAGG;YACW,uBAAuB;IAsDrC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAS3B;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,qBAAqB;CAiChC"}
|
|
@@ -357,7 +357,7 @@ let AgentEvalDriver = AgentEvalDriver_1 = class AgentEvalDriver extends BaseTest
|
|
|
357
357
|
? turn.inputPayload // First turn: use provided payload
|
|
358
358
|
: previousOutputPayload; // Subsequent turns: use previous output
|
|
359
359
|
this.logToTestRun(context, 'info', `Executing turn ${turnNumber} of ${turns.length}`);
|
|
360
|
-
// Execute single turn with cancellation token
|
|
360
|
+
// Execute single turn with cancellation token and resolved variables
|
|
361
361
|
const turnResult = await this.executeSingleTurn({
|
|
362
362
|
agent,
|
|
363
363
|
turn,
|
|
@@ -369,7 +369,8 @@ let AgentEvalDriver = AgentEvalDriver_1 = class AgentEvalDriver extends BaseTest
|
|
|
369
369
|
contextUser,
|
|
370
370
|
test,
|
|
371
371
|
testRun,
|
|
372
|
-
cancellationToken: abortController.signal
|
|
372
|
+
cancellationToken: abortController.signal,
|
|
373
|
+
resolvedVariables: context.resolvedVariables
|
|
373
374
|
});
|
|
374
375
|
agentRuns.push(turnResult.agentRun);
|
|
375
376
|
turnResults.push(turnResult);
|
|
@@ -416,6 +417,8 @@ let AgentEvalDriver = AgentEvalDriver_1 = class AgentEvalDriver extends BaseTest
|
|
|
416
417
|
const conversationName = this.buildConversationName(params.test.Name, params.testRun.Sequence, params.testRun.Tags, params.turnNumber, params.totalTurns);
|
|
417
418
|
// Get Entity ID for AI Agent Runs for proper FK linkage
|
|
418
419
|
const aiAgentRunsEntityId = this.getAIAgentRunsEntityId();
|
|
420
|
+
// Build override from turn execution params and resolved variables
|
|
421
|
+
const override = this.buildExecutionOverride(params.turn.executionParams, params.resolvedVariables);
|
|
419
422
|
// Build execution parameters with cancellation token and onAgentRunCreated callback
|
|
420
423
|
const runParams = {
|
|
421
424
|
agent: params.agent,
|
|
@@ -423,9 +426,7 @@ let AgentEvalDriver = AgentEvalDriver_1 = class AgentEvalDriver extends BaseTest
|
|
|
423
426
|
conversationMessages,
|
|
424
427
|
contextUser: params.contextUser,
|
|
425
428
|
payload: params.inputPayload, // Pass payload from previous turn
|
|
426
|
-
override
|
|
427
|
-
modelId: params.turn.executionParams.modelOverride
|
|
428
|
-
} : undefined,
|
|
429
|
+
override,
|
|
429
430
|
cancellationToken: params.cancellationToken, // Pass cancellation token to agent
|
|
430
431
|
// Callback to immediately link TestRun <-> AgentRun when AgentRun is created
|
|
431
432
|
onAgentRunCreated: async (agentRunId) => {
|
|
@@ -509,6 +510,47 @@ let AgentEvalDriver = AgentEvalDriver_1 = class AgentEvalDriver extends BaseTest
|
|
|
509
510
|
conversationId: agentRun.ConversationID
|
|
510
511
|
};
|
|
511
512
|
}
|
|
513
|
+
/**
|
|
514
|
+
* Build execution override object from turn params and resolved variables.
|
|
515
|
+
*
|
|
516
|
+
* Priority (highest to lowest):
|
|
517
|
+
* 1. Turn-level execution params (modelOverride, temperatureOverride, etc.)
|
|
518
|
+
* 2. Resolved variables (AIConfiguration, Temperature, etc.)
|
|
519
|
+
*
|
|
520
|
+
* @private
|
|
521
|
+
*/
|
|
522
|
+
buildExecutionOverride(turnExecutionParams, resolvedVariables) {
|
|
523
|
+
const override = {};
|
|
524
|
+
// Apply resolved variables (lower priority)
|
|
525
|
+
if (resolvedVariables?.values) {
|
|
526
|
+
// AIConfiguration variable maps to aiConfigurationId
|
|
527
|
+
if (resolvedVariables.values['AIConfiguration']) {
|
|
528
|
+
override.aiConfigurationId = resolvedVariables.values['AIConfiguration'];
|
|
529
|
+
}
|
|
530
|
+
// Temperature variable maps to temperature override
|
|
531
|
+
if (resolvedVariables.values['Temperature'] !== undefined) {
|
|
532
|
+
override.temperature = resolvedVariables.values['Temperature'];
|
|
533
|
+
}
|
|
534
|
+
// MaxTokens variable maps to maxTokens override
|
|
535
|
+
if (resolvedVariables.values['MaxTokens'] !== undefined) {
|
|
536
|
+
override.maxTokens = resolvedVariables.values['MaxTokens'];
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
// Apply turn execution params (higher priority - overwrites variables)
|
|
540
|
+
if (turnExecutionParams) {
|
|
541
|
+
if (turnExecutionParams.modelOverride) {
|
|
542
|
+
override.modelId = turnExecutionParams.modelOverride;
|
|
543
|
+
}
|
|
544
|
+
if (turnExecutionParams.temperatureOverride !== undefined) {
|
|
545
|
+
override.temperature = turnExecutionParams.temperatureOverride;
|
|
546
|
+
}
|
|
547
|
+
if (turnExecutionParams.maxTokensOverride !== undefined) {
|
|
548
|
+
override.maxTokens = turnExecutionParams.maxTokensOverride;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
// Return undefined if no overrides
|
|
552
|
+
return Object.keys(override).length > 0 ? override : undefined;
|
|
553
|
+
}
|
|
512
554
|
/**
|
|
513
555
|
* Run oracles for multi-turn evaluation.
|
|
514
556
|
* @private
|