@mappies/miniwat-eval 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -224,6 +224,29 @@ interface ITurnSpec {
224
224
  expect: ITurnExpectation;
225
225
  location: ISourceLocation;
226
226
  }
227
+ interface IProtocolCapabilities {
228
+ toolUses: boolean;
229
+ toolResults: boolean;
230
+ tokenUsage: boolean;
231
+ }
232
+ export declare const RequestFormat: {
233
+ readonly Json: "json";
234
+ };
235
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
236
+ interface IRequestConfig {
237
+ format: RequestFormat;
238
+ value: object | undefined;
239
+ inputPath: string | undefined;
240
+ }
241
+ interface IProtocolConfig {
242
+ request: IRequestConfig;
243
+ capabilities: IProtocolCapabilities;
244
+ }
245
+ interface IWebSocketConfig {
246
+ endpoint: string;
247
+ connection: IConnectionConfig;
248
+ protocol: IProtocolConfig;
249
+ }
227
250
  interface IConversationSpec {
228
251
  version: number;
229
252
  name: string;
@@ -231,6 +254,7 @@ interface IConversationSpec {
231
254
  turns: ITurnSpec[];
232
255
  conversation: IConversationExpectation | undefined;
233
256
  location: ISourceLocation;
257
+ websocket: IWebSocketConfig | undefined;
234
258
  secrets: string[];
235
259
  }
236
260
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
260
284
  location: SourceLocation;
261
285
  constructor(data?: Partial<ITurnSpec>);
262
286
  }
287
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
288
+ toolUses: boolean;
289
+ toolResults: boolean;
290
+ tokenUsage: boolean;
291
+ constructor(data?: Partial<IProtocolCapabilities>);
292
+ }
293
+ declare class RequestConfig implements IRequestConfig {
294
+ format: RequestFormat;
295
+ value: object | undefined;
296
+ inputPath: string | undefined;
297
+ constructor(data?: Partial<IRequestConfig>);
298
+ }
299
+ declare class ProtocolConfig implements IProtocolConfig {
300
+ request: RequestConfig;
301
+ capabilities: ProtocolCapabilities;
302
+ constructor(data?: Partial<IProtocolConfig>);
303
+ }
304
+ declare class WebSocketConfig implements IWebSocketConfig {
305
+ endpoint: string;
306
+ connection: ConnectionConfig;
307
+ protocol: ProtocolConfig;
308
+ constructor(data?: Partial<IWebSocketConfig>);
309
+ }
263
310
  declare class ConversationSpec implements IConversationSpec {
264
311
  version: number;
265
312
  name: string;
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
267
314
  turns: TurnSpec[];
268
315
  conversation: ConversationExpectation | undefined;
269
316
  location: SourceLocation;
317
+ websocket: WebSocketConfig | undefined;
270
318
  secrets: string[];
271
319
  constructor(data?: Partial<IConversationSpec>);
272
320
  }
@@ -336,32 +384,11 @@ interface IReportConfig {
336
384
  directory: string;
337
385
  formats: ReportFormat[];
338
386
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
362
387
  interface IEvalConfig {
363
388
  version: number;
364
389
  configPath: string;
390
+ parametersPath: string;
391
+ variables: Record<string, string>;
365
392
  websocket: IWebSocketConfig;
366
393
  execution: IExecutionConfig;
367
394
  evaluator: IEvaluatorConfig;
@@ -406,32 +433,11 @@ declare class ReportConfig implements IReportConfig {
406
433
  formats: ReportFormat[];
407
434
  constructor(data?: Partial<IReportConfig>);
408
435
  }
409
- declare class ProtocolCapabilities implements IProtocolCapabilities {
410
- toolUses: boolean;
411
- toolResults: boolean;
412
- tokenUsage: boolean;
413
- constructor(data?: Partial<IProtocolCapabilities>);
414
- }
415
- declare class RequestConfig implements IRequestConfig {
416
- format: RequestFormat;
417
- value: object | undefined;
418
- inputPath: string | undefined;
419
- constructor(data?: Partial<IRequestConfig>);
420
- }
421
- declare class ProtocolConfig implements IProtocolConfig {
422
- request: RequestConfig;
423
- capabilities: ProtocolCapabilities;
424
- constructor(data?: Partial<IProtocolConfig>);
425
- }
426
- declare class WebSocketConfig implements IWebSocketConfig {
427
- endpoint: string;
428
- connection: ConnectionConfig;
429
- protocol: ProtocolConfig;
430
- constructor(data?: Partial<IWebSocketConfig>);
431
- }
432
436
  declare class EvalConfig implements IEvalConfig {
433
437
  version: number;
434
438
  configPath: string;
439
+ parametersPath: string;
440
+ variables: Record<string, string>;
435
441
  websocket: WebSocketConfig;
436
442
  execution: ExecutionConfig;
437
443
  evaluator: EvaluatorConfig;
@@ -673,26 +679,41 @@ declare class EvalConfigLoader {
673
679
  private readonly cwd;
674
680
  private readonly documentLoader;
675
681
  private readonly schemaValidator;
682
+ private readonly variablesReader;
683
+ private readonly webSocketResolver;
676
684
  private readonly environmentResolver;
677
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
685
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
678
686
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
687
+ static buildWebSocket(value: unknown): WebSocketConfig;
679
688
  private static applyReport;
680
689
  private validateSecretReferences;
681
- private validateInputPath;
682
690
  private build;
683
691
  private applyRepetition;
684
692
  private static secrets;
685
693
  private static section;
686
- private static child;
694
+ }
695
+ declare class ParametersLoader {
696
+ private readonly cwd;
697
+ constructor(cwd?: string);
698
+ load(path: string): {
699
+ path: string;
700
+ parameters: Record<string, string>;
701
+ };
702
+ private static error;
687
703
  }
688
704
  declare class ScenarioLoader {
689
705
  private readonly cwd;
690
706
  private readonly documentLoader;
691
707
  private readonly schemaValidator;
692
708
  private readonly conversationValidator;
709
+ private readonly variablesReader;
710
+ private readonly webSocketResolver;
693
711
  private readonly environmentResolver;
694
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
712
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
695
713
  load(target: string, config: EvalConfig): ConversationSpec[];
714
+ private static shared;
715
+ private static variablesLocation;
716
+ private static unusedVariables;
696
717
  discover(target: string, config: EvalConfig): string[];
697
718
  private walk;
698
719
  private build;
@@ -938,4 +959,4 @@ declare class ReportFileWriter {
938
959
  path(fileName: string): string;
939
960
  write(fileName: string, content: string): string;
940
961
  }
941
- export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
962
+ export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
package/dist/index.d.ts CHANGED
@@ -224,6 +224,29 @@ interface ITurnSpec {
224
224
  expect: ITurnExpectation;
225
225
  location: ISourceLocation;
226
226
  }
227
+ interface IProtocolCapabilities {
228
+ toolUses: boolean;
229
+ toolResults: boolean;
230
+ tokenUsage: boolean;
231
+ }
232
+ export declare const RequestFormat: {
233
+ readonly Json: "json";
234
+ };
235
+ export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
236
+ interface IRequestConfig {
237
+ format: RequestFormat;
238
+ value: object | undefined;
239
+ inputPath: string | undefined;
240
+ }
241
+ interface IProtocolConfig {
242
+ request: IRequestConfig;
243
+ capabilities: IProtocolCapabilities;
244
+ }
245
+ interface IWebSocketConfig {
246
+ endpoint: string;
247
+ connection: IConnectionConfig;
248
+ protocol: IProtocolConfig;
249
+ }
227
250
  interface IConversationSpec {
228
251
  version: number;
229
252
  name: string;
@@ -231,6 +254,7 @@ interface IConversationSpec {
231
254
  turns: ITurnSpec[];
232
255
  conversation: IConversationExpectation | undefined;
233
256
  location: ISourceLocation;
257
+ websocket: IWebSocketConfig | undefined;
234
258
  secrets: string[];
235
259
  }
236
260
  declare class ResponseExpectation implements IResponseExpectation {
@@ -260,6 +284,29 @@ declare class TurnSpec implements ITurnSpec {
260
284
  location: SourceLocation;
261
285
  constructor(data?: Partial<ITurnSpec>);
262
286
  }
287
+ declare class ProtocolCapabilities implements IProtocolCapabilities {
288
+ toolUses: boolean;
289
+ toolResults: boolean;
290
+ tokenUsage: boolean;
291
+ constructor(data?: Partial<IProtocolCapabilities>);
292
+ }
293
+ declare class RequestConfig implements IRequestConfig {
294
+ format: RequestFormat;
295
+ value: object | undefined;
296
+ inputPath: string | undefined;
297
+ constructor(data?: Partial<IRequestConfig>);
298
+ }
299
+ declare class ProtocolConfig implements IProtocolConfig {
300
+ request: RequestConfig;
301
+ capabilities: ProtocolCapabilities;
302
+ constructor(data?: Partial<IProtocolConfig>);
303
+ }
304
+ declare class WebSocketConfig implements IWebSocketConfig {
305
+ endpoint: string;
306
+ connection: ConnectionConfig;
307
+ protocol: ProtocolConfig;
308
+ constructor(data?: Partial<IWebSocketConfig>);
309
+ }
263
310
  declare class ConversationSpec implements IConversationSpec {
264
311
  version: number;
265
312
  name: string;
@@ -267,6 +314,7 @@ declare class ConversationSpec implements IConversationSpec {
267
314
  turns: TurnSpec[];
268
315
  conversation: ConversationExpectation | undefined;
269
316
  location: SourceLocation;
317
+ websocket: WebSocketConfig | undefined;
270
318
  secrets: string[];
271
319
  constructor(data?: Partial<IConversationSpec>);
272
320
  }
@@ -336,32 +384,11 @@ interface IReportConfig {
336
384
  directory: string;
337
385
  formats: ReportFormat[];
338
386
  }
339
- interface IProtocolCapabilities {
340
- toolUses: boolean;
341
- toolResults: boolean;
342
- tokenUsage: boolean;
343
- }
344
- export declare const RequestFormat: {
345
- readonly Json: "json";
346
- };
347
- export type RequestFormat = typeof RequestFormat[keyof typeof RequestFormat];
348
- interface IRequestConfig {
349
- format: RequestFormat;
350
- value: object | undefined;
351
- inputPath: string | undefined;
352
- }
353
- interface IProtocolConfig {
354
- request: IRequestConfig;
355
- capabilities: IProtocolCapabilities;
356
- }
357
- interface IWebSocketConfig {
358
- endpoint: string;
359
- connection: IConnectionConfig;
360
- protocol: IProtocolConfig;
361
- }
362
387
  interface IEvalConfig {
363
388
  version: number;
364
389
  configPath: string;
390
+ parametersPath: string;
391
+ variables: Record<string, string>;
365
392
  websocket: IWebSocketConfig;
366
393
  execution: IExecutionConfig;
367
394
  evaluator: IEvaluatorConfig;
@@ -406,32 +433,11 @@ declare class ReportConfig implements IReportConfig {
406
433
  formats: ReportFormat[];
407
434
  constructor(data?: Partial<IReportConfig>);
408
435
  }
409
- declare class ProtocolCapabilities implements IProtocolCapabilities {
410
- toolUses: boolean;
411
- toolResults: boolean;
412
- tokenUsage: boolean;
413
- constructor(data?: Partial<IProtocolCapabilities>);
414
- }
415
- declare class RequestConfig implements IRequestConfig {
416
- format: RequestFormat;
417
- value: object | undefined;
418
- inputPath: string | undefined;
419
- constructor(data?: Partial<IRequestConfig>);
420
- }
421
- declare class ProtocolConfig implements IProtocolConfig {
422
- request: RequestConfig;
423
- capabilities: ProtocolCapabilities;
424
- constructor(data?: Partial<IProtocolConfig>);
425
- }
426
- declare class WebSocketConfig implements IWebSocketConfig {
427
- endpoint: string;
428
- connection: ConnectionConfig;
429
- protocol: ProtocolConfig;
430
- constructor(data?: Partial<IWebSocketConfig>);
431
- }
432
436
  declare class EvalConfig implements IEvalConfig {
433
437
  version: number;
434
438
  configPath: string;
439
+ parametersPath: string;
440
+ variables: Record<string, string>;
435
441
  websocket: WebSocketConfig;
436
442
  execution: ExecutionConfig;
437
443
  evaluator: EvaluatorConfig;
@@ -673,26 +679,41 @@ declare class EvalConfigLoader {
673
679
  private readonly cwd;
674
680
  private readonly documentLoader;
675
681
  private readonly schemaValidator;
682
+ private readonly variablesReader;
683
+ private readonly webSocketResolver;
676
684
  private readonly environmentResolver;
677
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
685
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
678
686
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
687
+ static buildWebSocket(value: unknown): WebSocketConfig;
679
688
  private static applyReport;
680
689
  private validateSecretReferences;
681
- private validateInputPath;
682
690
  private build;
683
691
  private applyRepetition;
684
692
  private static secrets;
685
693
  private static section;
686
- private static child;
694
+ }
695
+ declare class ParametersLoader {
696
+ private readonly cwd;
697
+ constructor(cwd?: string);
698
+ load(path: string): {
699
+ path: string;
700
+ parameters: Record<string, string>;
701
+ };
702
+ private static error;
687
703
  }
688
704
  declare class ScenarioLoader {
689
705
  private readonly cwd;
690
706
  private readonly documentLoader;
691
707
  private readonly schemaValidator;
692
708
  private readonly conversationValidator;
709
+ private readonly variablesReader;
710
+ private readonly webSocketResolver;
693
711
  private readonly environmentResolver;
694
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
712
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
695
713
  load(target: string, config: EvalConfig): ConversationSpec[];
714
+ private static shared;
715
+ private static variablesLocation;
716
+ private static unusedVariables;
696
717
  discover(target: string, config: EvalConfig): string[];
697
718
  private walk;
698
719
  private build;
@@ -938,4 +959,4 @@ declare class ReportFileWriter {
938
959
  path(fileName: string): string;
939
960
  write(fileName: string, content: string): string;
940
961
  }
941
- export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
962
+ export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };