@mappies/miniwat-eval 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -13
- package/dist/index.cjs +435 -178
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +72 -51
- package/dist/index.d.ts +72 -51
- package/dist/index.js +434 -178
- package/dist/index.js.map +1 -1
- package/dist/main.js +410 -148
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +9 -4
- package/schemas/eval-config.schema.json +5 -0
- package/schemas/report.schema.json +12 -1
package/dist/index.d.cts
CHANGED
|
@@ -224,6 +224,29 @@ interface ITurnSpec {
|
|
|
224
224
|
expect: ITurnExpectation;
|
|
225
225
|
location: ISourceLocation;
|
|
226
226
|
}
|
|
227
|
+
interface IProtocolCapabilities {
|
|
228
|
+
toolUses: boolean;
|
|
229
|
+
toolResults: boolean;
|
|
230
|
+
tokenUsage: boolean;
|
|
231
|
+
}
|
|
232
|
+
export declare const RequestFormat: {
|
|
233
|
+
readonly Json: "json";
|
|
234
|
+
};
|
|
235
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
236
|
+
interface IRequestConfig {
|
|
237
|
+
format: RequestFormat;
|
|
238
|
+
value: object | undefined;
|
|
239
|
+
inputPath: string | undefined;
|
|
240
|
+
}
|
|
241
|
+
interface IProtocolConfig {
|
|
242
|
+
request: IRequestConfig;
|
|
243
|
+
capabilities: IProtocolCapabilities;
|
|
244
|
+
}
|
|
245
|
+
interface IWebSocketConfig {
|
|
246
|
+
endpoint: string;
|
|
247
|
+
connection: IConnectionConfig;
|
|
248
|
+
protocol: IProtocolConfig;
|
|
249
|
+
}
|
|
227
250
|
interface IConversationSpec {
|
|
228
251
|
version: number;
|
|
229
252
|
name: string;
|
|
@@ -231,6 +254,7 @@ interface IConversationSpec {
|
|
|
231
254
|
turns: ITurnSpec[];
|
|
232
255
|
conversation: IConversationExpectation | undefined;
|
|
233
256
|
location: ISourceLocation;
|
|
257
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
258
|
secrets: string[];
|
|
235
259
|
}
|
|
236
260
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
284
|
location: SourceLocation;
|
|
261
285
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
286
|
}
|
|
287
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
288
|
+
toolUses: boolean;
|
|
289
|
+
toolResults: boolean;
|
|
290
|
+
tokenUsage: boolean;
|
|
291
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
292
|
+
}
|
|
293
|
+
declare class RequestConfig implements IRequestConfig {
|
|
294
|
+
format: RequestFormat;
|
|
295
|
+
value: object | undefined;
|
|
296
|
+
inputPath: string | undefined;
|
|
297
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
298
|
+
}
|
|
299
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
300
|
+
request: RequestConfig;
|
|
301
|
+
capabilities: ProtocolCapabilities;
|
|
302
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
303
|
+
}
|
|
304
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
305
|
+
endpoint: string;
|
|
306
|
+
connection: ConnectionConfig;
|
|
307
|
+
protocol: ProtocolConfig;
|
|
308
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
309
|
+
}
|
|
263
310
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
311
|
version: number;
|
|
265
312
|
name: string;
|
|
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
|
|
|
267
314
|
turns: TurnSpec[];
|
|
268
315
|
conversation: ConversationExpectation | undefined;
|
|
269
316
|
location: SourceLocation;
|
|
317
|
+
websocket: WebSocketConfig | undefined;
|
|
270
318
|
secrets: string[];
|
|
271
319
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
320
|
}
|
|
@@ -336,32 +384,11 @@ interface IReportConfig {
|
|
|
336
384
|
directory: string;
|
|
337
385
|
formats: ReportFormat[];
|
|
338
386
|
}
|
|
339
|
-
interface IProtocolCapabilities {
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
362
387
|
interface IEvalConfig {
|
|
363
388
|
version: number;
|
|
364
389
|
configPath: string;
|
|
390
|
+
parametersPath: string;
|
|
391
|
+
variables: Record<string, string>;
|
|
365
392
|
websocket: IWebSocketConfig;
|
|
366
393
|
execution: IExecutionConfig;
|
|
367
394
|
evaluator: IEvaluatorConfig;
|
|
@@ -406,32 +433,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
406
433
|
formats: ReportFormat[];
|
|
407
434
|
constructor(data?: Partial<IReportConfig>);
|
|
408
435
|
}
|
|
409
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
410
|
-
toolUses: boolean;
|
|
411
|
-
toolResults: boolean;
|
|
412
|
-
tokenUsage: boolean;
|
|
413
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
414
|
-
}
|
|
415
|
-
declare class RequestConfig implements IRequestConfig {
|
|
416
|
-
format: RequestFormat;
|
|
417
|
-
value: object | undefined;
|
|
418
|
-
inputPath: string | undefined;
|
|
419
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
420
|
-
}
|
|
421
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
422
|
-
request: RequestConfig;
|
|
423
|
-
capabilities: ProtocolCapabilities;
|
|
424
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
425
|
-
}
|
|
426
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
427
|
-
endpoint: string;
|
|
428
|
-
connection: ConnectionConfig;
|
|
429
|
-
protocol: ProtocolConfig;
|
|
430
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
431
|
-
}
|
|
432
436
|
declare class EvalConfig implements IEvalConfig {
|
|
433
437
|
version: number;
|
|
434
438
|
configPath: string;
|
|
439
|
+
parametersPath: string;
|
|
440
|
+
variables: Record<string, string>;
|
|
435
441
|
websocket: WebSocketConfig;
|
|
436
442
|
execution: ExecutionConfig;
|
|
437
443
|
evaluator: EvaluatorConfig;
|
|
@@ -673,26 +679,41 @@ declare class EvalConfigLoader {
|
|
|
673
679
|
private readonly cwd;
|
|
674
680
|
private readonly documentLoader;
|
|
675
681
|
private readonly schemaValidator;
|
|
682
|
+
private readonly variablesReader;
|
|
683
|
+
private readonly webSocketResolver;
|
|
676
684
|
private readonly environmentResolver;
|
|
677
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
685
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
678
686
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
687
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
679
688
|
private static applyReport;
|
|
680
689
|
private validateSecretReferences;
|
|
681
|
-
private validateInputPath;
|
|
682
690
|
private build;
|
|
683
691
|
private applyRepetition;
|
|
684
692
|
private static secrets;
|
|
685
693
|
private static section;
|
|
686
|
-
|
|
694
|
+
}
|
|
695
|
+
declare class ParametersLoader {
|
|
696
|
+
private readonly cwd;
|
|
697
|
+
constructor(cwd?: string);
|
|
698
|
+
load(path: string): {
|
|
699
|
+
path: string;
|
|
700
|
+
parameters: Record<string, string>;
|
|
701
|
+
};
|
|
702
|
+
private static error;
|
|
687
703
|
}
|
|
688
704
|
declare class ScenarioLoader {
|
|
689
705
|
private readonly cwd;
|
|
690
706
|
private readonly documentLoader;
|
|
691
707
|
private readonly schemaValidator;
|
|
692
708
|
private readonly conversationValidator;
|
|
709
|
+
private readonly variablesReader;
|
|
710
|
+
private readonly webSocketResolver;
|
|
693
711
|
private readonly environmentResolver;
|
|
694
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
712
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
695
713
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
714
|
+
private static shared;
|
|
715
|
+
private static variablesLocation;
|
|
716
|
+
private static unusedVariables;
|
|
696
717
|
discover(target: string, config: EvalConfig): string[];
|
|
697
718
|
private walk;
|
|
698
719
|
private build;
|
|
@@ -938,4 +959,4 @@ declare class ReportFileWriter {
|
|
|
938
959
|
path(fileName: string): string;
|
|
939
960
|
write(fileName: string, content: string): string;
|
|
940
961
|
}
|
|
941
|
-
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
|
962
|
+
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
package/dist/index.d.ts
CHANGED
|
@@ -224,6 +224,29 @@ interface ITurnSpec {
|
|
|
224
224
|
expect: ITurnExpectation;
|
|
225
225
|
location: ISourceLocation;
|
|
226
226
|
}
|
|
227
|
+
interface IProtocolCapabilities {
|
|
228
|
+
toolUses: boolean;
|
|
229
|
+
toolResults: boolean;
|
|
230
|
+
tokenUsage: boolean;
|
|
231
|
+
}
|
|
232
|
+
export declare const RequestFormat: {
|
|
233
|
+
readonly Json: "json";
|
|
234
|
+
};
|
|
235
|
+
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
236
|
+
interface IRequestConfig {
|
|
237
|
+
format: RequestFormat;
|
|
238
|
+
value: object | undefined;
|
|
239
|
+
inputPath: string | undefined;
|
|
240
|
+
}
|
|
241
|
+
interface IProtocolConfig {
|
|
242
|
+
request: IRequestConfig;
|
|
243
|
+
capabilities: IProtocolCapabilities;
|
|
244
|
+
}
|
|
245
|
+
interface IWebSocketConfig {
|
|
246
|
+
endpoint: string;
|
|
247
|
+
connection: IConnectionConfig;
|
|
248
|
+
protocol: IProtocolConfig;
|
|
249
|
+
}
|
|
227
250
|
interface IConversationSpec {
|
|
228
251
|
version: number;
|
|
229
252
|
name: string;
|
|
@@ -231,6 +254,7 @@ interface IConversationSpec {
|
|
|
231
254
|
turns: ITurnSpec[];
|
|
232
255
|
conversation: IConversationExpectation | undefined;
|
|
233
256
|
location: ISourceLocation;
|
|
257
|
+
websocket: IWebSocketConfig | undefined;
|
|
234
258
|
secrets: string[];
|
|
235
259
|
}
|
|
236
260
|
declare class ResponseExpectation implements IResponseExpectation {
|
|
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
|
|
|
260
284
|
location: SourceLocation;
|
|
261
285
|
constructor(data?: Partial<ITurnSpec>);
|
|
262
286
|
}
|
|
287
|
+
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
288
|
+
toolUses: boolean;
|
|
289
|
+
toolResults: boolean;
|
|
290
|
+
tokenUsage: boolean;
|
|
291
|
+
constructor(data?: Partial<IProtocolCapabilities>);
|
|
292
|
+
}
|
|
293
|
+
declare class RequestConfig implements IRequestConfig {
|
|
294
|
+
format: RequestFormat;
|
|
295
|
+
value: object | undefined;
|
|
296
|
+
inputPath: string | undefined;
|
|
297
|
+
constructor(data?: Partial<IRequestConfig>);
|
|
298
|
+
}
|
|
299
|
+
declare class ProtocolConfig implements IProtocolConfig {
|
|
300
|
+
request: RequestConfig;
|
|
301
|
+
capabilities: ProtocolCapabilities;
|
|
302
|
+
constructor(data?: Partial<IProtocolConfig>);
|
|
303
|
+
}
|
|
304
|
+
declare class WebSocketConfig implements IWebSocketConfig {
|
|
305
|
+
endpoint: string;
|
|
306
|
+
connection: ConnectionConfig;
|
|
307
|
+
protocol: ProtocolConfig;
|
|
308
|
+
constructor(data?: Partial<IWebSocketConfig>);
|
|
309
|
+
}
|
|
263
310
|
declare class ConversationSpec implements IConversationSpec {
|
|
264
311
|
version: number;
|
|
265
312
|
name: string;
|
|
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
|
|
|
267
314
|
turns: TurnSpec[];
|
|
268
315
|
conversation: ConversationExpectation | undefined;
|
|
269
316
|
location: SourceLocation;
|
|
317
|
+
websocket: WebSocketConfig | undefined;
|
|
270
318
|
secrets: string[];
|
|
271
319
|
constructor(data?: Partial<IConversationSpec>);
|
|
272
320
|
}
|
|
@@ -336,32 +384,11 @@ interface IReportConfig {
|
|
|
336
384
|
directory: string;
|
|
337
385
|
formats: ReportFormat[];
|
|
338
386
|
}
|
|
339
|
-
interface IProtocolCapabilities {
|
|
340
|
-
toolUses: boolean;
|
|
341
|
-
toolResults: boolean;
|
|
342
|
-
tokenUsage: boolean;
|
|
343
|
-
}
|
|
344
|
-
export declare const RequestFormat: {
|
|
345
|
-
readonly Json: "json";
|
|
346
|
-
};
|
|
347
|
-
export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
|
|
348
|
-
interface IRequestConfig {
|
|
349
|
-
format: RequestFormat;
|
|
350
|
-
value: object | undefined;
|
|
351
|
-
inputPath: string | undefined;
|
|
352
|
-
}
|
|
353
|
-
interface IProtocolConfig {
|
|
354
|
-
request: IRequestConfig;
|
|
355
|
-
capabilities: IProtocolCapabilities;
|
|
356
|
-
}
|
|
357
|
-
interface IWebSocketConfig {
|
|
358
|
-
endpoint: string;
|
|
359
|
-
connection: IConnectionConfig;
|
|
360
|
-
protocol: IProtocolConfig;
|
|
361
|
-
}
|
|
362
387
|
interface IEvalConfig {
|
|
363
388
|
version: number;
|
|
364
389
|
configPath: string;
|
|
390
|
+
parametersPath: string;
|
|
391
|
+
variables: Record<string, string>;
|
|
365
392
|
websocket: IWebSocketConfig;
|
|
366
393
|
execution: IExecutionConfig;
|
|
367
394
|
evaluator: IEvaluatorConfig;
|
|
@@ -406,32 +433,11 @@ declare class ReportConfig implements IReportConfig {
|
|
|
406
433
|
formats: ReportFormat[];
|
|
407
434
|
constructor(data?: Partial<IReportConfig>);
|
|
408
435
|
}
|
|
409
|
-
declare class ProtocolCapabilities implements IProtocolCapabilities {
|
|
410
|
-
toolUses: boolean;
|
|
411
|
-
toolResults: boolean;
|
|
412
|
-
tokenUsage: boolean;
|
|
413
|
-
constructor(data?: Partial<IProtocolCapabilities>);
|
|
414
|
-
}
|
|
415
|
-
declare class RequestConfig implements IRequestConfig {
|
|
416
|
-
format: RequestFormat;
|
|
417
|
-
value: object | undefined;
|
|
418
|
-
inputPath: string | undefined;
|
|
419
|
-
constructor(data?: Partial<IRequestConfig>);
|
|
420
|
-
}
|
|
421
|
-
declare class ProtocolConfig implements IProtocolConfig {
|
|
422
|
-
request: RequestConfig;
|
|
423
|
-
capabilities: ProtocolCapabilities;
|
|
424
|
-
constructor(data?: Partial<IProtocolConfig>);
|
|
425
|
-
}
|
|
426
|
-
declare class WebSocketConfig implements IWebSocketConfig {
|
|
427
|
-
endpoint: string;
|
|
428
|
-
connection: ConnectionConfig;
|
|
429
|
-
protocol: ProtocolConfig;
|
|
430
|
-
constructor(data?: Partial<IWebSocketConfig>);
|
|
431
|
-
}
|
|
432
436
|
declare class EvalConfig implements IEvalConfig {
|
|
433
437
|
version: number;
|
|
434
438
|
configPath: string;
|
|
439
|
+
parametersPath: string;
|
|
440
|
+
variables: Record<string, string>;
|
|
435
441
|
websocket: WebSocketConfig;
|
|
436
442
|
execution: ExecutionConfig;
|
|
437
443
|
evaluator: EvaluatorConfig;
|
|
@@ -673,26 +679,41 @@ declare class EvalConfigLoader {
|
|
|
673
679
|
private readonly cwd;
|
|
674
680
|
private readonly documentLoader;
|
|
675
681
|
private readonly schemaValidator;
|
|
682
|
+
private readonly variablesReader;
|
|
683
|
+
private readonly webSocketResolver;
|
|
676
684
|
private readonly environmentResolver;
|
|
677
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
685
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
678
686
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
687
|
+
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
679
688
|
private static applyReport;
|
|
680
689
|
private validateSecretReferences;
|
|
681
|
-
private validateInputPath;
|
|
682
690
|
private build;
|
|
683
691
|
private applyRepetition;
|
|
684
692
|
private static secrets;
|
|
685
693
|
private static section;
|
|
686
|
-
|
|
694
|
+
}
|
|
695
|
+
declare class ParametersLoader {
|
|
696
|
+
private readonly cwd;
|
|
697
|
+
constructor(cwd?: string);
|
|
698
|
+
load(path: string): {
|
|
699
|
+
path: string;
|
|
700
|
+
parameters: Record<string, string>;
|
|
701
|
+
};
|
|
702
|
+
private static error;
|
|
687
703
|
}
|
|
688
704
|
declare class ScenarioLoader {
|
|
689
705
|
private readonly cwd;
|
|
690
706
|
private readonly documentLoader;
|
|
691
707
|
private readonly schemaValidator;
|
|
692
708
|
private readonly conversationValidator;
|
|
709
|
+
private readonly variablesReader;
|
|
710
|
+
private readonly webSocketResolver;
|
|
693
711
|
private readonly environmentResolver;
|
|
694
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
712
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
695
713
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
714
|
+
private static shared;
|
|
715
|
+
private static variablesLocation;
|
|
716
|
+
private static unusedVariables;
|
|
696
717
|
discover(target: string, config: EvalConfig): string[];
|
|
697
718
|
private walk;
|
|
698
719
|
private build;
|
|
@@ -938,4 +959,4 @@ declare class ReportFileWriter {
|
|
|
938
959
|
path(fileName: string): string;
|
|
939
960
|
write(fileName: string, content: string): string;
|
|
940
961
|
}
|
|
941
|
-
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
|
962
|
+
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|