@mappies/miniwat-eval 0.1.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -6
- package/dist/index.cjs +51 -10
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +14 -3
- package/dist/index.d.ts +14 -3
- package/dist/index.js +50 -10
- package/dist/index.js.map +1 -1
- package/dist/main.js +69 -23
- package/dist/main.js.map +1 -1
- package/package.json +2 -5
- package/schemas/report.schema.json +1 -0
package/dist/index.d.cts
CHANGED
|
@@ -362,6 +362,7 @@ interface IWebSocketConfig {
|
|
|
362
362
|
interface IEvalConfig {
|
|
363
363
|
version: number;
|
|
364
364
|
configPath: string;
|
|
365
|
+
parametersPath: string;
|
|
365
366
|
websocket: IWebSocketConfig;
|
|
366
367
|
execution: IExecutionConfig;
|
|
367
368
|
evaluator: IEvaluatorConfig;
|
|
@@ -432,6 +433,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
|
|
|
432
433
|
declare class EvalConfig implements IEvalConfig {
|
|
433
434
|
version: number;
|
|
434
435
|
configPath: string;
|
|
436
|
+
parametersPath: string;
|
|
435
437
|
websocket: WebSocketConfig;
|
|
436
438
|
execution: ExecutionConfig;
|
|
437
439
|
evaluator: EvaluatorConfig;
|
|
@@ -674,7 +676,7 @@ declare class EvalConfigLoader {
|
|
|
674
676
|
private readonly documentLoader;
|
|
675
677
|
private readonly schemaValidator;
|
|
676
678
|
private readonly environmentResolver;
|
|
677
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
679
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
678
680
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
679
681
|
private static applyReport;
|
|
680
682
|
private validateSecretReferences;
|
|
@@ -685,13 +687,22 @@ declare class EvalConfigLoader {
|
|
|
685
687
|
private static section;
|
|
686
688
|
private static child;
|
|
687
689
|
}
|
|
690
|
+
declare class ParametersLoader {
|
|
691
|
+
private readonly cwd;
|
|
692
|
+
constructor(cwd?: string);
|
|
693
|
+
load(path: string): {
|
|
694
|
+
path: string;
|
|
695
|
+
parameters: Record<string, string>;
|
|
696
|
+
};
|
|
697
|
+
private static error;
|
|
698
|
+
}
|
|
688
699
|
declare class ScenarioLoader {
|
|
689
700
|
private readonly cwd;
|
|
690
701
|
private readonly documentLoader;
|
|
691
702
|
private readonly schemaValidator;
|
|
692
703
|
private readonly conversationValidator;
|
|
693
704
|
private readonly environmentResolver;
|
|
694
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
705
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
695
706
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
696
707
|
discover(target: string, config: EvalConfig): string[];
|
|
697
708
|
private walk;
|
|
@@ -938,4 +949,4 @@ declare class ReportFileWriter {
|
|
|
938
949
|
path(fileName: string): string;
|
|
939
950
|
write(fileName: string, content: string): string;
|
|
940
951
|
}
|
|
941
|
-
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
|
952
|
+
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
package/dist/index.d.ts
CHANGED
|
@@ -362,6 +362,7 @@ interface IWebSocketConfig {
|
|
|
362
362
|
interface IEvalConfig {
|
|
363
363
|
version: number;
|
|
364
364
|
configPath: string;
|
|
365
|
+
parametersPath: string;
|
|
365
366
|
websocket: IWebSocketConfig;
|
|
366
367
|
execution: IExecutionConfig;
|
|
367
368
|
evaluator: IEvaluatorConfig;
|
|
@@ -432,6 +433,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
|
|
|
432
433
|
declare class EvalConfig implements IEvalConfig {
|
|
433
434
|
version: number;
|
|
434
435
|
configPath: string;
|
|
436
|
+
parametersPath: string;
|
|
435
437
|
websocket: WebSocketConfig;
|
|
436
438
|
execution: ExecutionConfig;
|
|
437
439
|
evaluator: EvaluatorConfig;
|
|
@@ -674,7 +676,7 @@ declare class EvalConfigLoader {
|
|
|
674
676
|
private readonly documentLoader;
|
|
675
677
|
private readonly schemaValidator;
|
|
676
678
|
private readonly environmentResolver;
|
|
677
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
679
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
678
680
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
679
681
|
private static applyReport;
|
|
680
682
|
private validateSecretReferences;
|
|
@@ -685,13 +687,22 @@ declare class EvalConfigLoader {
|
|
|
685
687
|
private static section;
|
|
686
688
|
private static child;
|
|
687
689
|
}
|
|
690
|
+
declare class ParametersLoader {
|
|
691
|
+
private readonly cwd;
|
|
692
|
+
constructor(cwd?: string);
|
|
693
|
+
load(path: string): {
|
|
694
|
+
path: string;
|
|
695
|
+
parameters: Record<string, string>;
|
|
696
|
+
};
|
|
697
|
+
private static error;
|
|
698
|
+
}
|
|
688
699
|
declare class ScenarioLoader {
|
|
689
700
|
private readonly cwd;
|
|
690
701
|
private readonly documentLoader;
|
|
691
702
|
private readonly schemaValidator;
|
|
692
703
|
private readonly conversationValidator;
|
|
693
704
|
private readonly environmentResolver;
|
|
694
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string);
|
|
705
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
695
706
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
696
707
|
discover(target: string, config: EvalConfig): string[];
|
|
697
708
|
private walk;
|
|
@@ -938,4 +949,4 @@ declare class ReportFileWriter {
|
|
|
938
949
|
path(fileName: string): string;
|
|
939
950
|
write(fileName: string, content: string): string;
|
|
940
951
|
}
|
|
941
|
-
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
|
952
|
+
export { ActiveTurn, type AgentEvent, ArgumentDifference, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, type IActiveTurn, type IAgentConnection, type IArgumentDifference, type IAttemptExecution, type IAttemptScenarioResult, type IAttemptTestResult, type IConnectionConfig, type IConversationExpectation, type IConversationSpec, type IErrorEvent, type IEvalConfig, type IEvalError, type IEvalRunResult, type IEvaluationOutcome, type IEvaluationRequest, type IEvaluationResponse, type IEvaluationTranscriptEntry, type IEvaluator, type IEvaluatorConfig, type IEvaluatorThresholds, type IExecutionConfig, type IExecutionOverride, type IExecutionTimeouts, type IProtocolCapabilities, type IProtocolConfig, type IReportConfig, type IReportOverride, type IRequestConfig, type IResponseChunkEvent, type IResponseExpectation, type IResponseMessageEvent, type IScenarioResult, type ISemanticCheck, type ISourceLocation, type IStatusEvent, type IStreamEndEvent, type ITestResult, type ITokenUsageSummary, type IToolExpectation, type IToolMessageEvent, type ITurnExecution, type ITurnExpectation, type ITurnOutcome, type ITurnSpec, type IValidationIssue, type IWebSocketClient, type IWebSocketConfig, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, type ProviderImporter, ReportConfig, ReportFileWriter, ReportOverride, RequestConfig, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, type WebSocketClientFactory, WebSocketConfig };
|
package/dist/index.js
CHANGED
|
@@ -438,6 +438,7 @@ var WebSocketConfig = class {
|
|
|
438
438
|
var EvalConfig = class {
|
|
439
439
|
version;
|
|
440
440
|
configPath;
|
|
441
|
+
parametersPath;
|
|
441
442
|
websocket;
|
|
442
443
|
execution;
|
|
443
444
|
evaluator;
|
|
@@ -446,6 +447,7 @@ var EvalConfig = class {
|
|
|
446
447
|
constructor(data) {
|
|
447
448
|
this.version = data?.version ?? 1;
|
|
448
449
|
this.configPath = data?.configPath ?? "";
|
|
450
|
+
this.parametersPath = data?.parametersPath ?? "";
|
|
449
451
|
this.websocket = new WebSocketConfig(data?.websocket);
|
|
450
452
|
this.execution = new ExecutionConfig(data?.execution);
|
|
451
453
|
this.evaluator = new EvaluatorConfig(data?.evaluator);
|
|
@@ -916,12 +918,14 @@ var eval_config_schema_default = {
|
|
|
916
918
|
}
|
|
917
919
|
};
|
|
918
920
|
// src/yaml/environmentResolver.ts
|
|
919
|
-
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g;
|
|
921
|
+
const REFERENCE = /\$\{([A-Za-z_][A-Za-z0-9_-]*)\}/g;
|
|
920
922
|
const SECRET_NAME = /TOKEN|SECRET|PASSWORD|KEY/i;
|
|
921
923
|
var EnvironmentResolver = class EnvironmentResolver {
|
|
922
924
|
environment;
|
|
923
|
-
|
|
925
|
+
parameters;
|
|
926
|
+
constructor(environment = process.env, parameters = {}) {
|
|
924
927
|
this.environment = environment;
|
|
928
|
+
this.parameters = parameters;
|
|
925
929
|
}
|
|
926
930
|
resolve(value, document, issues, secrets = []) {
|
|
927
931
|
return this.resolveValue(value, [], document, issues, secrets);
|
|
@@ -934,10 +938,11 @@ var EnvironmentResolver = class EnvironmentResolver {
|
|
|
934
938
|
}
|
|
935
939
|
resolveValue(value, path, document, issues, secrets) {
|
|
936
940
|
if (typeof value === "string") return value.replace(REFERENCE, (reference, name) => {
|
|
937
|
-
const resolved = this.environment[name];
|
|
941
|
+
const resolved = Object.hasOwn(this.parameters, name) ? this.parameters[name] : this.environment[name];
|
|
938
942
|
if (resolved === void 0) {
|
|
943
|
+
const source = Object.keys(this.parameters).length > 0 ? "parameter or environment variable" : "environment variable";
|
|
939
944
|
issues.push(new ValidationIssue({
|
|
940
|
-
message:
|
|
945
|
+
message: `${source} '${name}' is not set`,
|
|
941
946
|
location: document.locate(path)
|
|
942
947
|
}));
|
|
943
948
|
return reference;
|
|
@@ -1111,9 +1116,9 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1111
1116
|
documentLoader = new YamlDocumentLoader();
|
|
1112
1117
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1113
1118
|
environmentResolver;
|
|
1114
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1119
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1115
1120
|
this.cwd = cwd;
|
|
1116
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1121
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1117
1122
|
}
|
|
1118
1123
|
load(configPath, override, report = new ReportOverride()) {
|
|
1119
1124
|
const document = this.documentLoader.load(configPath, relative(this.cwd, configPath));
|
|
@@ -1150,7 +1155,7 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1150
1155
|
const apiKey = EvalConfigLoader.section(document.value, "evaluator").apiKey;
|
|
1151
1156
|
if (typeof apiKey !== "string" || EnvironmentResolver.hasReference(apiKey)) return [];
|
|
1152
1157
|
return [new ValidationIssue({
|
|
1153
|
-
message: "must be
|
|
1158
|
+
message: "must be a reference such as ${OPENAI_API_KEY}",
|
|
1154
1159
|
location: document.locate(["evaluator", "apiKey"])
|
|
1155
1160
|
})];
|
|
1156
1161
|
}
|
|
@@ -1275,6 +1280,40 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1275
1280
|
return isRecord(value) ? value[key] : void 0;
|
|
1276
1281
|
}
|
|
1277
1282
|
};
|
|
1283
|
+
// src/config/parametersLoader.ts
|
|
1284
|
+
var ParametersLoader = class ParametersLoader {
|
|
1285
|
+
cwd;
|
|
1286
|
+
constructor(cwd = process.cwd()) {
|
|
1287
|
+
this.cwd = cwd;
|
|
1288
|
+
}
|
|
1289
|
+
load(path) {
|
|
1290
|
+
const absolute = resolve(this.cwd, path);
|
|
1291
|
+
const shown = relative(this.cwd, absolute) || path;
|
|
1292
|
+
let text;
|
|
1293
|
+
let value;
|
|
1294
|
+
try {
|
|
1295
|
+
text = readFileSync(absolute, "utf8");
|
|
1296
|
+
} catch {
|
|
1297
|
+
throw ParametersLoader.error(`parameters file not found: ${shown}`);
|
|
1298
|
+
}
|
|
1299
|
+
try {
|
|
1300
|
+
value = JSON.parse(text);
|
|
1301
|
+
} catch {
|
|
1302
|
+
throw ParametersLoader.error(`parameters file ${shown} is not valid JSON`);
|
|
1303
|
+
}
|
|
1304
|
+
if (!isRecord(value)) throw ParametersLoader.error(`parameters file ${shown} must contain a JSON object`);
|
|
1305
|
+
return {
|
|
1306
|
+
path: absolute,
|
|
1307
|
+
parameters: Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === "string"))
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
static error(message) {
|
|
1311
|
+
return new EvalError({
|
|
1312
|
+
category: ErrorCategory.Configuration,
|
|
1313
|
+
message
|
|
1314
|
+
});
|
|
1315
|
+
}
|
|
1316
|
+
};
|
|
1278
1317
|
// schemas/conversation.schema.json
|
|
1279
1318
|
var conversation_schema_default = {
|
|
1280
1319
|
$schema: "http://json-schema.org/draft-07/schema#",
|
|
@@ -1455,9 +1494,9 @@ var ScenarioLoader = class {
|
|
|
1455
1494
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1456
1495
|
conversationValidator = new ConversationValidator();
|
|
1457
1496
|
environmentResolver;
|
|
1458
|
-
constructor(environment = process.env, cwd = process.cwd()) {
|
|
1497
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1459
1498
|
this.cwd = cwd;
|
|
1460
|
-
this.environmentResolver = new EnvironmentResolver(environment);
|
|
1499
|
+
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1461
1500
|
}
|
|
1462
1501
|
load(target, config) {
|
|
1463
1502
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
@@ -3156,6 +3195,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3156
3195
|
const { websocket, execution, evaluator, report } = config;
|
|
3157
3196
|
return {
|
|
3158
3197
|
path: this.path(config.configPath),
|
|
3198
|
+
parametersPath: config.parametersPath === "" ? void 0 : this.path(config.parametersPath),
|
|
3159
3199
|
websocket: {
|
|
3160
3200
|
endpoint: websocket.endpoint,
|
|
3161
3201
|
subprotocols: [...websocket.connection.subprotocols],
|
|
@@ -3338,6 +3378,6 @@ var ReportFileWriter = class {
|
|
|
3338
3378
|
return path;
|
|
3339
3379
|
}
|
|
3340
3380
|
};
|
|
3341
|
-
export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
|
|
3381
|
+
export { ActiveTurn, AgentEventType, ArgumentDifference, ArgumentDifferenceKind, AssertionRunner, AttemptExecution, AttemptScenarioResult, AttemptTestResult, CONFIG_FILE_NAME, ConcurrencyLimiter, ConfigLocator, ConnectionConfig, ConsoleReporter, ConversationEvaluator, ConversationExpectation, ConversationRunner, ConversationSpec, DEFAULT_CONFIG, ErrorCategory, ErrorEvent, EvalConfig, EvalConfigLoader, EvalError, EvalRunResult, EvaluationOutcome, EvaluationRequest, EvaluationResponse, EvaluationScope, EvaluationTranscriptEntry, EvaluatorAgentFactory, EvaluatorConfig, EvaluatorOutputParser, EvaluatorPromptBuilder, EvaluatorProvider, EvaluatorThresholds, ExecutionConfig, ExecutionOverride, ExecutionTimeouts, ExitCodeCalculator, FlakyPolicy, INTERRUPTED_EXIT_CODE, JSON_REPORT_FILE, JSON_REPORT_VERSION, JsonReporter, LlmEvaluator, ParametersLoader, ProtocolCapabilities, ProtocolConfig, REDACTED, ReportConfig, ReportFileWriter, ReportFormat, ReportOverride, RequestConfig, RequestFormat, ResponseChunkEvent, ResponseEvaluator, ResponseExpectation, ResponseMessageEvent, ResultAggregator, ResultStatus, RunStatus, ScenarioLoader, ScenarioResult, ScenarioRunner, ScenarioScheduler, SecretRedactor, SemanticCheck, SourceLocation, StatusEvent, StreamEndEvent, TestKind, TestResult, TokenUsageAggregator, TokenUsageSummary, ToolArgumentMatcher, ToolExpectation, ToolMessageEvent, ToolUsageEvaluator, TurnExecution, TurnExpectation, TurnOutcome, TurnRunner, TurnSpec, UnavailableEvaluator, ValidationIssue, WebSocketAgentConnection, WebSocketConfig, applyScore };
|
|
3342
3382
|
|
|
3343
3383
|
//# sourceMappingURL=index.js.map
|