@mappies/miniwat-eval 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -5
- package/dist/index.cjs +234 -100
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +40 -12
- package/dist/index.d.ts +40 -12
- package/dist/index.js +235 -101
- package/dist/index.js.map +1 -1
- package/dist/main.js +269 -115
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -670,7 +670,8 @@ interface IEvaluator {
|
|
|
670
670
|
export declare const CONFIG_FILE_NAME = "eval-config.yml";
|
|
671
671
|
declare class ConfigLocator {
|
|
672
672
|
private readonly cwd;
|
|
673
|
-
|
|
673
|
+
private readonly displayBase;
|
|
674
|
+
constructor(cwd?: string, displayBase?: string);
|
|
674
675
|
locate(target: string, explicitConfig: string): string;
|
|
675
676
|
private static searchUpward;
|
|
676
677
|
private static isProjectRoot;
|
|
@@ -680,12 +681,13 @@ declare class ConfigLocator {
|
|
|
680
681
|
export declare const DEFAULT_CONFIG: EvalConfig;
|
|
681
682
|
declare class EvalConfigLoader {
|
|
682
683
|
private readonly cwd;
|
|
684
|
+
private readonly displayBase;
|
|
683
685
|
private readonly documentLoader;
|
|
684
686
|
private readonly schemaValidator;
|
|
685
687
|
private readonly variablesReader;
|
|
686
688
|
private readonly webSocketResolver;
|
|
687
689
|
private readonly environmentResolver;
|
|
688
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
690
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
689
691
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
690
692
|
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
691
693
|
private static applyReport;
|
|
@@ -697,7 +699,8 @@ declare class EvalConfigLoader {
|
|
|
697
699
|
}
|
|
698
700
|
declare class ParametersLoader {
|
|
699
701
|
private readonly cwd;
|
|
700
|
-
|
|
702
|
+
private readonly displayBase;
|
|
703
|
+
constructor(cwd?: string, displayBase?: string);
|
|
701
704
|
load(path: string): {
|
|
702
705
|
path: string;
|
|
703
706
|
parameters: Record<string, string>;
|
|
@@ -706,6 +709,7 @@ declare class ParametersLoader {
|
|
|
706
709
|
}
|
|
707
710
|
declare class ScenarioLoader {
|
|
708
711
|
private readonly cwd;
|
|
712
|
+
private readonly displayBase;
|
|
709
713
|
private readonly documentLoader;
|
|
710
714
|
private readonly schemaValidator;
|
|
711
715
|
private readonly conversationValidator;
|
|
@@ -713,7 +717,7 @@ declare class ScenarioLoader {
|
|
|
713
717
|
private readonly runWithReader;
|
|
714
718
|
private readonly webSocketResolver;
|
|
715
719
|
private readonly environmentResolver;
|
|
716
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
720
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
717
721
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
718
722
|
private resolveRun;
|
|
719
723
|
private static byRun;
|
|
@@ -909,10 +913,22 @@ type RunHeader = {
|
|
|
909
913
|
scenarios: number;
|
|
910
914
|
concurrency: number;
|
|
911
915
|
};
|
|
916
|
+
type ConsoleOptions = {
|
|
917
|
+
nameWidth?: number;
|
|
918
|
+
columns?: number;
|
|
919
|
+
thresholds?: {
|
|
920
|
+
response: number;
|
|
921
|
+
conversation: number;
|
|
922
|
+
};
|
|
923
|
+
};
|
|
912
924
|
declare class ConsoleReporter {
|
|
913
925
|
private readonly verbose;
|
|
914
926
|
private readonly colour;
|
|
915
|
-
|
|
927
|
+
private readonly nameWidth;
|
|
928
|
+
private readonly columns;
|
|
929
|
+
private readonly thresholds;
|
|
930
|
+
constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
|
|
931
|
+
static nameWidth(specs: ConversationSpec[]): number;
|
|
916
932
|
header(run: RunHeader): string;
|
|
917
933
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
918
934
|
static title(spec: ConversationSpec): string;
|
|
@@ -920,11 +936,21 @@ declare class ConsoleReporter {
|
|
|
920
936
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
921
937
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
|
922
938
|
static duration(ms: number): string;
|
|
923
|
-
|
|
939
|
+
static wrap(text: string, width: number): string[];
|
|
940
|
+
private section;
|
|
941
|
+
private rule;
|
|
942
|
+
private blocks;
|
|
943
|
+
private attemptBlock;
|
|
944
|
+
private testBlock;
|
|
945
|
+
private entryBody;
|
|
946
|
+
private static actual;
|
|
947
|
+
private capped;
|
|
948
|
+
private static expected;
|
|
949
|
+
private paragraph;
|
|
950
|
+
private bullet;
|
|
924
951
|
private static testTitle;
|
|
925
952
|
private static tally;
|
|
926
|
-
private
|
|
927
|
-
private static counts;
|
|
953
|
+
private mark;
|
|
928
954
|
private static usage;
|
|
929
955
|
private formatAttempt;
|
|
930
956
|
private formatTest;
|
|
@@ -943,22 +969,24 @@ export declare const JSON_REPORT_VERSION = 1;
|
|
|
943
969
|
declare class JsonReporter {
|
|
944
970
|
private readonly redactor;
|
|
945
971
|
private readonly cwd;
|
|
972
|
+
private readonly displayBase;
|
|
946
973
|
private readonly tokens;
|
|
947
|
-
constructor(redactor: SecretRedactor, cwd: string);
|
|
974
|
+
constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
|
|
948
975
|
render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
|
|
949
976
|
private configuration;
|
|
950
977
|
private scenario;
|
|
951
978
|
private attempt;
|
|
952
979
|
private static turn;
|
|
953
|
-
private
|
|
954
|
-
private
|
|
955
|
-
private
|
|
980
|
+
private test;
|
|
981
|
+
private identity;
|
|
982
|
+
private location;
|
|
956
983
|
private static counts;
|
|
957
984
|
private static tally;
|
|
958
985
|
private static usage;
|
|
959
986
|
private static tokenUsage;
|
|
960
987
|
private static error;
|
|
961
988
|
private path;
|
|
989
|
+
private file;
|
|
962
990
|
}
|
|
963
991
|
declare class ReportFileWriter {
|
|
964
992
|
private readonly directory;
|
package/dist/index.d.ts
CHANGED
|
@@ -671,7 +671,8 @@ interface IEvaluator {
|
|
|
671
671
|
export declare const CONFIG_FILE_NAME = "eval-config.yml";
|
|
672
672
|
declare class ConfigLocator {
|
|
673
673
|
private readonly cwd;
|
|
674
|
-
|
|
674
|
+
private readonly displayBase;
|
|
675
|
+
constructor(cwd?: string, displayBase?: string);
|
|
675
676
|
locate(target: string, explicitConfig: string): string;
|
|
676
677
|
private static searchUpward;
|
|
677
678
|
private static isProjectRoot;
|
|
@@ -681,12 +682,13 @@ declare class ConfigLocator {
|
|
|
681
682
|
export declare const DEFAULT_CONFIG: EvalConfig;
|
|
682
683
|
declare class EvalConfigLoader {
|
|
683
684
|
private readonly cwd;
|
|
685
|
+
private readonly displayBase;
|
|
684
686
|
private readonly documentLoader;
|
|
685
687
|
private readonly schemaValidator;
|
|
686
688
|
private readonly variablesReader;
|
|
687
689
|
private readonly webSocketResolver;
|
|
688
690
|
private readonly environmentResolver;
|
|
689
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
691
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
690
692
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
691
693
|
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
692
694
|
private static applyReport;
|
|
@@ -698,7 +700,8 @@ declare class EvalConfigLoader {
|
|
|
698
700
|
}
|
|
699
701
|
declare class ParametersLoader {
|
|
700
702
|
private readonly cwd;
|
|
701
|
-
|
|
703
|
+
private readonly displayBase;
|
|
704
|
+
constructor(cwd?: string, displayBase?: string);
|
|
702
705
|
load(path: string): {
|
|
703
706
|
path: string;
|
|
704
707
|
parameters: Record<string, string>;
|
|
@@ -707,6 +710,7 @@ declare class ParametersLoader {
|
|
|
707
710
|
}
|
|
708
711
|
declare class ScenarioLoader {
|
|
709
712
|
private readonly cwd;
|
|
713
|
+
private readonly displayBase;
|
|
710
714
|
private readonly documentLoader;
|
|
711
715
|
private readonly schemaValidator;
|
|
712
716
|
private readonly conversationValidator;
|
|
@@ -714,7 +718,7 @@ declare class ScenarioLoader {
|
|
|
714
718
|
private readonly runWithReader;
|
|
715
719
|
private readonly webSocketResolver;
|
|
716
720
|
private readonly environmentResolver;
|
|
717
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
721
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
718
722
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
719
723
|
private resolveRun;
|
|
720
724
|
private static byRun;
|
|
@@ -910,10 +914,22 @@ type RunHeader = {
|
|
|
910
914
|
scenarios: number;
|
|
911
915
|
concurrency: number;
|
|
912
916
|
};
|
|
917
|
+
type ConsoleOptions = {
|
|
918
|
+
nameWidth?: number;
|
|
919
|
+
columns?: number;
|
|
920
|
+
thresholds?: {
|
|
921
|
+
response: number;
|
|
922
|
+
conversation: number;
|
|
923
|
+
};
|
|
924
|
+
};
|
|
913
925
|
declare class ConsoleReporter {
|
|
914
926
|
private readonly verbose;
|
|
915
927
|
private readonly colour;
|
|
916
|
-
|
|
928
|
+
private readonly nameWidth;
|
|
929
|
+
private readonly columns;
|
|
930
|
+
private readonly thresholds;
|
|
931
|
+
constructor(verbose: boolean, colour: boolean, options?: ConsoleOptions);
|
|
932
|
+
static nameWidth(specs: ConversationSpec[]): number;
|
|
917
933
|
header(run: RunHeader): string;
|
|
918
934
|
scenario(result: ScenarioResult, repeat: number, tokenUsageCapability: boolean): string;
|
|
919
935
|
static title(spec: ConversationSpec): string;
|
|
@@ -921,11 +937,21 @@ declare class ConsoleReporter {
|
|
|
921
937
|
static footer(run: EvalRunResult, exitCode: number, reportPaths: string[]): string;
|
|
922
938
|
static finalLine(run: EvalRunResult, exitCode: number): string;
|
|
923
939
|
static duration(ms: number): string;
|
|
924
|
-
|
|
940
|
+
static wrap(text: string, width: number): string[];
|
|
941
|
+
private section;
|
|
942
|
+
private rule;
|
|
943
|
+
private blocks;
|
|
944
|
+
private attemptBlock;
|
|
945
|
+
private testBlock;
|
|
946
|
+
private entryBody;
|
|
947
|
+
private static actual;
|
|
948
|
+
private capped;
|
|
949
|
+
private static expected;
|
|
950
|
+
private paragraph;
|
|
951
|
+
private bullet;
|
|
925
952
|
private static testTitle;
|
|
926
953
|
private static tally;
|
|
927
|
-
private
|
|
928
|
-
private static counts;
|
|
954
|
+
private mark;
|
|
929
955
|
private static usage;
|
|
930
956
|
private formatAttempt;
|
|
931
957
|
private formatTest;
|
|
@@ -944,22 +970,24 @@ export declare const JSON_REPORT_VERSION = 1;
|
|
|
944
970
|
declare class JsonReporter {
|
|
945
971
|
private readonly redactor;
|
|
946
972
|
private readonly cwd;
|
|
973
|
+
private readonly displayBase;
|
|
947
974
|
private readonly tokens;
|
|
948
|
-
constructor(redactor: SecretRedactor, cwd: string);
|
|
975
|
+
constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
|
|
949
976
|
render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
|
|
950
977
|
private configuration;
|
|
951
978
|
private scenario;
|
|
952
979
|
private attempt;
|
|
953
980
|
private static turn;
|
|
954
|
-
private
|
|
955
|
-
private
|
|
956
|
-
private
|
|
981
|
+
private test;
|
|
982
|
+
private identity;
|
|
983
|
+
private location;
|
|
957
984
|
private static counts;
|
|
958
985
|
private static tally;
|
|
959
986
|
private static usage;
|
|
960
987
|
private static tokenUsage;
|
|
961
988
|
private static error;
|
|
962
989
|
private path;
|
|
990
|
+
private file;
|
|
963
991
|
}
|
|
964
992
|
declare class ReportFileWriter {
|
|
965
993
|
private readonly directory;
|