@mappies/miniwat-eval 0.1.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -362,6 +362,7 @@ interface IWebSocketConfig {
362
362
  interface IEvalConfig {
363
363
  version: number;
364
364
  configPath: string;
365
+ parametersPath: string;
365
366
  websocket: IWebSocketConfig;
366
367
  execution: IExecutionConfig;
367
368
  evaluator: IEvaluatorConfig;
@@ -432,6 +433,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
432
433
  declare class EvalConfig implements IEvalConfig {
433
434
  version: number;
434
435
  configPath: string;
436
+ parametersPath: string;
435
437
  websocket: WebSocketConfig;
436
438
  execution: ExecutionConfig;
437
439
  evaluator: EvaluatorConfig;
@@ -674,7 +676,7 @@ declare class EvalConfigLoader {
674
676
  private readonly documentLoader;
675
677
  private readonly schemaValidator;
676
678
  private readonly environmentResolver;
677
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
679
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
678
680
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
679
681
  private static applyReport;
680
682
  private validateSecretReferences;
@@ -685,13 +687,22 @@ declare class EvalConfigLoader {
685
687
  private static section;
686
688
  private static child;
687
689
  }
690
+ declare class ParametersLoader {
691
+ private readonly cwd;
692
+ constructor(cwd?: string);
693
+ load(path: string): {
694
+ path: string;
695
+ parameters: Record<string, string>;
696
+ };
697
+ private static error;
698
+ }
688
699
  declare class ScenarioLoader {
689
700
  private readonly cwd;
690
701
  private readonly documentLoader;
691
702
  private readonly schemaValidator;
692
703
  private readonly conversationValidator;
693
704
  private readonly environmentResolver;
694
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
705
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
695
706
  load(target: string, config: EvalConfig): ConversationSpec[];
696
707
  discover(target: string, config: EvalConfig): string[];
697
708
  private walk;
@@ -938,4 +949,4 @@ declare class ReportFileWriter {
938
949
  path(fileName: string): string;
939
950
  write(fileName: string, content: string): string;
940
951
  }
941
- export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
952
+ export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
package/dist/index.d.ts CHANGED
@@ -362,6 +362,7 @@ interface IWebSocketConfig {
362
362
  interface IEvalConfig {
363
363
  version: number;
364
364
  configPath: string;
365
+ parametersPath: string;
365
366
  websocket: IWebSocketConfig;
366
367
  execution: IExecutionConfig;
367
368
  evaluator: IEvaluatorConfig;
@@ -432,6 +433,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
432
433
  declare class EvalConfig implements IEvalConfig {
433
434
  version: number;
434
435
  configPath: string;
436
+ parametersPath: string;
435
437
  websocket: WebSocketConfig;
436
438
  execution: ExecutionConfig;
437
439
  evaluator: EvaluatorConfig;
@@ -674,7 +676,7 @@ declare class EvalConfigLoader {
674
676
  private readonly documentLoader;
675
677
  private readonly schemaValidator;
676
678
  private readonly environmentResolver;
677
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
679
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
678
680
  load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
679
681
  private static applyReport;
680
682
  private validateSecretReferences;
@@ -685,13 +687,22 @@ declare class EvalConfigLoader {
685
687
  private static section;
686
688
  private static child;
687
689
  }
690
+ declare class ParametersLoader {
691
+ private readonly cwd;
692
+ constructor(cwd?: string);
693
+ load(path: string): {
694
+ path: string;
695
+ parameters: Record<string, string>;
696
+ };
697
+ private static error;
698
+ }
688
699
  declare class ScenarioLoader {
689
700
  private readonly cwd;
690
701
  private readonly documentLoader;
691
702
  private readonly schemaValidator;
692
703
  private readonly conversationValidator;
693
704
  private readonly environmentResolver;
694
- constructor(environment?: Record<string, string | undefined>, cwd?: string);
705
+ constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
695
706
  load(target: string, config: EvalConfig): ConversationSpec[];
696
707
  discover(target: string, config: EvalConfig): string[];
697
708
  private walk;
@@ -938,4 +949,4 @@ declare class ReportFileWriter {
938
949
  path(fileName: string): string;
939
950
  write(fileName: string, content: string): string;
940
951
  }
941
- export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
952
+ export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
package/dist/index.js CHANGED
@@ -438,6 +438,7 @@ var WebSocketConfig = class {
438
438
  var EvalConfig = class {
439
439
  version;
440
440
  configPath;
441
+ parametersPath;
441
442
  websocket;
442
443
  execution;
443
444
  evaluator;
@@ -446,6 +447,7 @@ var EvalConfig = class {
446
447
  constructor(data) {
447
448
  this.version = data?.version ?? 1;
448
449
  this.configPath = data?.configPath ?? "";
450
+ this.parametersPath = data?.parametersPath ?? "";
449
451
  this.websocket = new WebSocketConfig(data?.websocket);
450
452
  this.execution = new ExecutionConfig(data?.execution);
451
453
  this.evaluator = new EvaluatorConfig(data?.evaluator);
@@ -916,12 +918,14 @@ var eval_config_schema_default = {
916
918
  }
917
919
  };
918
920
  // src/yaml/environmentResolver.ts
919
- const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
921
+ const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
920
922
  const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
921
923
  var EnvironmentResolver = class EnvironmentResolver {
922
924
  environment;
923
- constructor(environment = process.env) {
925
+ parameters;
926
+ constructor(environment = process.env, parameters = {}) {
924
927
  this.environment = environment;
928
+ this.parameters = parameters;
925
929
  }
926
930
  resolve(value, document, issues, secrets = []) {
927
931
  return this.resolveValue(value, [], document, issues, secrets);
@@ -934,10 +938,11 @@ var EnvironmentResolver = class EnvironmentResolver {
934
938
  }
935
939
  resolveValue(value, path, document, issues, secrets) {
936
940
  if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
937
- const resolved = this.environment[name];
941
+ const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
938
942
  if (resolved === void 0) {
943
+ const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
939
944
  issues.push(new ValidationIssue({
940
- message: `environment variable '${name}' is not set`,
945
+ message: `${source} '${name}' is not set`,
941
946
  location: document.locate(path)
942
947
  }));
943
948
  return reference;
@@ -1111,9 +1116,9 @@ var EvalConfigLoader = class EvalConfigLoader {
1111
1116
  documentLoader = new YamlDocumentLoader();
1112
1117
  schemaValidator = new SchemaValidator(eval_config_schema_default);
1113
1118
  environmentResolver;
1114
- constructor(environment = process.env, cwd = process.cwd()) {
1119
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1115
1120
  this.cwd = cwd;
1116
- this.environmentResolver = new EnvironmentResolver(environment);
1121
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1117
1122
  }
1118
1123
  load(configPath, override, report = new ReportOverride()) {
1119
1124
  const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
@@ -1150,7 +1155,7 @@ var EvalConfigLoader = class EvalConfigLoader {
1150
1155
  const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
1151
1156
  if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
1152
1157
  return [new ValidationIssue({
1153
- message: "must be an environment reference such as ${OPENAI_API_KEY}",
1158
+ message: "must be a reference such as ${OPENAI_API_KEY}",
1154
1159
  location: document.locate(["evaluator", "apiKey"])
1155
1160
  })];
1156
1161
  }
@@ -1275,6 +1280,40 @@ var EvalConfigLoader = class EvalConfigLoader {
1275
1280
  return isRecord(value) ? value[key] : void 0;
1276
1281
  }
1277
1282
  };
1283
+ // src/config/parametersLoader.ts
1284
+ var ParametersLoader = class ParametersLoader {
1285
+ cwd;
1286
+ constructor(cwd = process.cwd()) {
1287
+ this.cwd = cwd;
1288
+ }
1289
+ load(path) {
1290
+ const absolute = resolve(this.cwd, path);
1291
+ const shown = relative(this.cwd, absolute) || path;
1292
+ let text;
1293
+ let value;
1294
+ try {
1295
+ text = readFileSync(absolute, "utf8");
1296
+ } catch {
1297
+ throw ParametersLoader.error(`parameters file not found: ${shown}`);
1298
+ }
1299
+ try {
1300
+ value = JSON.parse(text);
1301
+ } catch {
1302
+ throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
1303
+ }
1304
+ if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
1305
+ return {
1306
+ path: absolute,
1307
+ parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
1308
+ };
1309
+ }
1310
+ static error(message) {
1311
+ return new EvalError({
1312
+ category: ErrorCategory.Configuration,
1313
+ message
1314
+ });
1315
+ }
1316
+ };
1278
1317
  // schemas/conversation.schema.json
1279
1318
  var conversation_schema_default = {
1280
1319
  $schema: "http://json-schema.org/draft-07/schema#",
@@ -1455,9 +1494,9 @@ var ScenarioLoader = class {
1455
1494
  schemaValidator = new SchemaValidator(conversation_schema_default);
1456
1495
  conversationValidator = new ConversationValidator();
1457
1496
  environmentResolver;
1458
- constructor(environment = process.env, cwd = process.cwd()) {
1497
+ constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
1459
1498
  this.cwd = cwd;
1460
- this.environmentResolver = new EnvironmentResolver(environment);
1499
+ this.environmentResolver = new EnvironmentResolver(environment, parameters);
1461
1500
  }
1462
1501
  load(target, config) {
1463
1502
  const files = this.discover(resolve(this.cwd, target), config);
@@ -3156,6 +3195,7 @@ var JsonReporter = class JsonReporter {
3156
3195
  const { websocket, execution, evaluator, report } = config;
3157
3196
  return {
3158
3197
  path: this.path(config.configPath),
3198
+ parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
3159
3199
  websocket: {
3160
3200
  endpoint: websocket.endpoint,
3161
3201
  subprotocols: [...websocket.connection.subprotocols],
@@ -3338,6 +3378,6 @@ var ReportFileWriter = class {
3338
3378
  return path;
3339
3379
  }
3340
3380
  };
3341
- export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
3381
+ export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
3342
3382
 
3343
3383
  //# sourceMappingURL=index.js.map