@mappies/miniwat-eval 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.cts CHANGED
@@ -250,6 +250,7 @@ interface IWebSocketConfig {
250
250
  interface IConversationSpec {
251
251
  version: number;
252
252
  name: string;
253
+ runName: string;
253
254
  description: string;
254
255
  turns: ITurnSpec[];
255
256
  conversation: IConversationExpectation | undefined;
@@ -310,6 +311,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
310
311
  declare class ConversationSpec implements IConversationSpec {
311
312
  version: number;
312
313
  name: string;
314
+ runName: string;
313
315
  description: string;
314
316
  turns: TurnSpec[];
315
317
  conversation: ConversationExpectation | undefined;
@@ -384,11 +386,12 @@ interface IReportConfig {
384
386
  directory: string;
385
387
  formats: ReportFormat[];
386
388
  }
389
+ type Variables = Record<string, string | string[]>;
387
390
  interface IEvalConfig {
388
391
  version: number;
389
392
  configPath: string;
390
393
  parametersPath: string;
391
- variables: Record<string, string>;
394
+ variables: Variables;
392
395
  websocket: IWebSocketConfig;
393
396
  execution: IExecutionConfig;
394
397
  evaluator: IEvaluatorConfig;
@@ -437,7 +440,7 @@ declare class EvalConfig implements IEvalConfig {
437
440
  version: number;
438
441
  configPath: string;
439
442
  parametersPath: string;
440
- variables: Record<string, string>;
443
+ variables: Variables;
441
444
  websocket: WebSocketConfig;
442
445
  execution: ExecutionConfig;
443
446
  evaluator: EvaluatorConfig;
@@ -707,10 +710,13 @@ declare class ScenarioLoader {
707
710
  private readonly schemaValidator;
708
711
  private readonly conversationValidator;
709
712
  private readonly variablesReader;
713
+ private readonly runWithReader;
710
714
  private readonly webSocketResolver;
711
715
  private readonly environmentResolver;
712
716
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
713
717
  load(target: string, config: EvalConfig): ConversationSpec[];
718
+ private resolveRun;
719
+ private static byRun;
714
720
  private static shared;
715
721
  private static variablesLocation;
716
722
  private static unusedVariables;
@@ -903,21 +909,42 @@ type RunHeader = {
903
909
  scenarios: number;
904
910
  concurrency: number;
905
911
  };
912
+ type ConsoleOptions = {
913
+ nameWidth?: number;
914
+ columns?: number;
915
+ thresholds?: {
916
+ response: number;
917
+ conversation: number;
918
+ };
919
+ };
906
920
  declare class ConsoleReporter {
907
921
  private readonly verbose;
908
922
  private readonly colour;
909
- constructor(verbose: boolean, colour: boolean);
923
+ private readonly nameWidth;
924
+ private readonly columns;
925
+ private readonly thresholds;
926
+ constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
927
+ static nameWidth(specs: ConversationSpec[]): number;
910
928
  header(run: RunHeader): string;
911
929
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
930
+ static title(spec: ConversationSpec): string;
912
931
  summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
913
932
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
914
933
  static finalLine(run: EvalRunResult, exitCode: number): string;
915
934
  static duration(ms: number): string;
916
- private problems;
935
+ static wrap(text: string, width: number): string[];
936
+ private section;
937
+ private rule;
938
+ private blocks;
939
+ private attemptBlock;
940
+ private testBlock;
941
+ private entryBody;
942
+ private static expected;
943
+ private paragraph;
944
+ private bullet;
917
945
  private static testTitle;
918
946
  private static tally;
919
- private status;
920
- private static counts;
947
+ private mark;
921
948
  private static usage;
922
949
  private formatAttempt;
923
950
  private formatTest;
package/dist/index.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { ITokenUsage, Message, ResponseMessage, TokenUsage, ToolResult, ToolUse } from "@mappies/miniwat-agent-core";
2
+ import "yaml";
2
3
  import { ClientOptions } from "configurapi-handler-ws";
3
4
  import { Agent } from "@mappies/miniwat-agent";
4
5
  interface ITurnExecution {
@@ -250,6 +251,7 @@ interface IWebSocketConfig {
250
251
  interface IConversationSpec {
251
252
  version: number;
252
253
  name: string;
254
+ runName: string;
253
255
  description: string;
254
256
  turns: ITurnSpec[];
255
257
  conversation: IConversationExpectation | undefined;
@@ -310,6 +312,7 @@ declare class WebSocketConfig implements IWebSocketConfig {
310
312
  declare class ConversationSpec implements IConversationSpec {
311
313
  version: number;
312
314
  name: string;
315
+ runName: string;
313
316
  description: string;
314
317
  turns: TurnSpec[];
315
318
  conversation: ConversationExpectation | undefined;
@@ -384,11 +387,12 @@ interface IReportConfig {
384
387
  directory: string;
385
388
  formats: ReportFormat[];
386
389
  }
390
+ type Variables = Record<string, string | string[]>;
387
391
  interface IEvalConfig {
388
392
  version: number;
389
393
  configPath: string;
390
394
  parametersPath: string;
391
- variables: Record<string, string>;
395
+ variables: Variables;
392
396
  websocket: IWebSocketConfig;
393
397
  execution: IExecutionConfig;
394
398
  evaluator: IEvaluatorConfig;
@@ -437,7 +441,7 @@ declare class EvalConfig implements IEvalConfig {
437
441
  version: number;
438
442
  configPath: string;
439
443
  parametersPath: string;
440
- variables: Record<string, string>;
444
+ variables: Variables;
441
445
  websocket: WebSocketConfig;
442
446
  execution: ExecutionConfig;
443
447
  evaluator: EvaluatorConfig;
@@ -707,10 +711,13 @@ declare class ScenarioLoader {
707
711
  private readonly schemaValidator;
708
712
  private readonly conversationValidator;
709
713
  private readonly variablesReader;
714
+ private readonly runWithReader;
710
715
  private readonly webSocketResolver;
711
716
  private readonly environmentResolver;
712
717
  constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>);
713
718
  load(target: string, config: EvalConfig): ConversationSpec[];
719
+ private resolveRun;
720
+ private static byRun;
714
721
  private static shared;
715
722
  private static variablesLocation;
716
723
  private static unusedVariables;
@@ -903,21 +910,42 @@ type RunHeader = {
903
910
  scenarios: number;
904
911
  concurrency: number;
905
912
  };
913
+ type ConsoleOptions = {
914
+ nameWidth?: number;
915
+ columns?: number;
916
+ thresholds?: {
917
+ response: number;
918
+ conversation: number;
919
+ };
920
+ };
906
921
  declare class ConsoleReporter {
907
922
  private readonly verbose;
908
923
  private readonly colour;
909
- constructor(verbose: boolean, colour: boolean);
924
+ private readonly nameWidth;
925
+ private readonly columns;
926
+ private readonly thresholds;
927
+ constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
928
+ static nameWidth(specs: ConversationSpec[]): number;
910
929
  header(run: RunHeader): string;
911
930
  scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
931
+ static title(spec: ConversationSpec): string;
912
932
  summary(run: EvalRunResult, exitCode: number, reportPaths?: string[]): string;
913
933
  static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
914
934
  static finalLine(run: EvalRunResult, exitCode: number): string;
915
935
  static duration(ms: number): string;
916
- private problems;
936
+ static wrap(text: string, width: number): string[];
937
+ private section;
938
+ private rule;
939
+ private blocks;
940
+ private attemptBlock;
941
+ private testBlock;
942
+ private entryBody;
943
+ private static expected;
944
+ private paragraph;
945
+ private bullet;
917
946
  private static testTitle;
918
947
  private static tally;
919
- private status;
920
- private static counts;
948
+ private mark;
921
949
  private static usage;
922
950
  private formatAttempt;
923
951
  private formatTest;