@mappies/miniwat-eval 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/index.cjs +393 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +9 -2
- package/dist/index.d.ts +10 -2
- package/dist/index.js +393 -44
- package/dist/index.js.map +1 -1
- package/dist/main.js +393 -44
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
- package/schemas/conversation.schema.json +35 -1
- package/schemas/eval-config.schema.json +6 -1
- package/schemas/report.schema.json +1 -0
package/dist/index.d.cts
CHANGED
|
@@ -250,6 +250,7 @@ interface IWebSocketConfig {
|
|
|
250
250
|
interface IConversationSpec {
|
|
251
251
|
version: number;
|
|
252
252
|
name: string;
|
|
253
|
+
runName: string;
|
|
253
254
|
description: string;
|
|
254
255
|
turns: ITurnSpec[];
|
|
255
256
|
conversation: IConversationExpectation | undefined;
|
|
@@ -310,6 +311,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
|
|
|
310
311
|
declare class ConversationSpec implements IConversationSpec {
|
|
311
312
|
version: number;
|
|
312
313
|
name: string;
|
|
314
|
+
runName: string;
|
|
313
315
|
description: string;
|
|
314
316
|
turns: TurnSpec[];
|
|
315
317
|
conversation: ConversationExpectation | undefined;
|
|
@@ -384,11 +386,12 @@ interface IReportConfig {
|
|
|
384
386
|
directory: string;
|
|
385
387
|
formats: ReportFormat[];
|
|
386
388
|
}
|
|
389
|
+
type Variables = Record<string, string | string[]>;
|
|
387
390
|
interface IEvalConfig {
|
|
388
391
|
version: number;
|
|
389
392
|
configPath: string;
|
|
390
393
|
parametersPath: string;
|
|
391
|
-
variables:
|
|
394
|
+
variables: Variables;
|
|
392
395
|
websocket: IWebSocketConfig;
|
|
393
396
|
execution: IExecutionConfig;
|
|
394
397
|
evaluator: IEvaluatorConfig;
|
|
@@ -437,7 +440,7 @@ declare class EvalConfig implements IEvalConfig {
|
|
|
437
440
|
version: number;
|
|
438
441
|
configPath: string;
|
|
439
442
|
parametersPath: string;
|
|
440
|
-
variables:
|
|
443
|
+
variables: Variables;
|
|
441
444
|
websocket: WebSocketConfig;
|
|
442
445
|
execution: ExecutionConfig;
|
|
443
446
|
evaluator: EvaluatorConfig;
|
|
@@ -707,10 +710,13 @@ declare class ScenarioLoader {
|
|
|
707
710
|
private readonly schemaValidator;
|
|
708
711
|
private readonly conversationValidator;
|
|
709
712
|
private readonly variablesReader;
|
|
713
|
+
private readonly runWithReader;
|
|
710
714
|
private readonly webSocketResolver;
|
|
711
715
|
private readonly environmentResolver;
|
|
712
716
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
713
717
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
718
|
+
private resolveRun;
|
|
719
|
+
private static byRun;
|
|
714
720
|
private static shared;
|
|
715
721
|
private static variablesLocation;
|
|
716
722
|
private static unusedVariables;
|
|
@@ -909,6 +915,7 @@ declare class ConsoleReporter {
|
|
|
909
915
|
constructor(verbose: boolean, colour: boolean);
|
|
910
916
|
header(run: RunHeader): string;
|
|
911
917
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
918
|
+
static title(spec: ConversationSpec): string;
|
|
912
919
|
summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
|
|
913
920
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
914
921
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ITokenUsage, Message, ResponseMessage, TokenUsage, ToolResult, ToolUse } from "@mappies/miniwat-agent-core";
|
|
2
|
+
import "yaml";
|
|
2
3
|
import { ClientOptions } from "configurapi-handler-ws";
|
|
3
4
|
import { Agent } from "@mappies/miniwat-agent";
|
|
4
5
|
interface ITurnExecution {
|
|
@@ -250,6 +251,7 @@ interface IWebSocketConfig {
|
|
|
250
251
|
interface IConversationSpec {
|
|
251
252
|
version: number;
|
|
252
253
|
name: string;
|
|
254
|
+
runName: string;
|
|
253
255
|
description: string;
|
|
254
256
|
turns: ITurnSpec[];
|
|
255
257
|
conversation: IConversationExpectation | undefined;
|
|
@@ -310,6 +312,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
|
|
|
310
312
|
declare class ConversationSpec implements IConversationSpec {
|
|
311
313
|
version: number;
|
|
312
314
|
name: string;
|
|
315
|
+
runName: string;
|
|
313
316
|
description: string;
|
|
314
317
|
turns: TurnSpec[];
|
|
315
318
|
conversation: ConversationExpectation | undefined;
|
|
@@ -384,11 +387,12 @@ interface IReportConfig {
|
|
|
384
387
|
directory: string;
|
|
385
388
|
formats: ReportFormat[];
|
|
386
389
|
}
|
|
390
|
+
type Variables = Record<string, string | string[]>;
|
|
387
391
|
interface IEvalConfig {
|
|
388
392
|
version: number;
|
|
389
393
|
configPath: string;
|
|
390
394
|
parametersPath: string;
|
|
391
|
-
variables:
|
|
395
|
+
variables: Variables;
|
|
392
396
|
websocket: IWebSocketConfig;
|
|
393
397
|
execution: IExecutionConfig;
|
|
394
398
|
evaluator: IEvaluatorConfig;
|
|
@@ -437,7 +441,7 @@ declare class EvalConfig implements IEvalConfig {
|
|
|
437
441
|
version: number;
|
|
438
442
|
configPath: string;
|
|
439
443
|
parametersPath: string;
|
|
440
|
-
variables:
|
|
444
|
+
variables: Variables;
|
|
441
445
|
websocket: WebSocketConfig;
|
|
442
446
|
execution: ExecutionConfig;
|
|
443
447
|
evaluator: EvaluatorConfig;
|
|
@@ -707,10 +711,13 @@ declare class ScenarioLoader {
|
|
|
707
711
|
private readonly schemaValidator;
|
|
708
712
|
private readonly conversationValidator;
|
|
709
713
|
private readonly variablesReader;
|
|
714
|
+
private readonly runWithReader;
|
|
710
715
|
private readonly webSocketResolver;
|
|
711
716
|
private readonly environmentResolver;
|
|
712
717
|
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
|
|
713
718
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
719
|
+
private resolveRun;
|
|
720
|
+
private static byRun;
|
|
714
721
|
private static shared;
|
|
715
722
|
private static variablesLocation;
|
|
716
723
|
private static unusedVariables;
|
|
@@ -909,6 +916,7 @@ declare class ConsoleReporter {
|
|
|
909
916
|
constructor(verbose: boolean, colour: boolean);
|
|
910
917
|
header(run: RunHeader): string;
|
|
911
918
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
919
|
+
static title(spec: ConversationSpec): string;
|
|
912
920
|
summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
|
|
913
921
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
914
922
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|