@mappies/miniwat-eval 1.4.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/index.cjs +52 -26
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +16 -8
- package/dist/index.d.ts +16 -8
- package/dist/index.js +52 -26
- package/dist/index.js.map +1 -1
- package/dist/main.js +77 -37
- package/dist/main.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -670,7 +670,8 @@ interface IEvaluator {
|
|
|
670
670
|
export declare const CONFIG_FILE_NAME = "eval-config.yml";
|
|
671
671
|
declare class ConfigLocator {
|
|
672
672
|
private readonly cwd;
|
|
673
|
-
|
|
673
|
+
private readonly displayBase;
|
|
674
|
+
constructor(cwd?: string, displayBase?: string);
|
|
674
675
|
locate(target: string, explicitConfig: string): string;
|
|
675
676
|
private static searchUpward;
|
|
676
677
|
private static isProjectRoot;
|
|
@@ -680,12 +681,13 @@ declare class ConfigLocator {
|
|
|
680
681
|
export declare const DEFAULT_CONFIG: EvalConfig;
|
|
681
682
|
declare class EvalConfigLoader {
|
|
682
683
|
private readonly cwd;
|
|
684
|
+
private readonly displayBase;
|
|
683
685
|
private readonly documentLoader;
|
|
684
686
|
private readonly schemaValidator;
|
|
685
687
|
private readonly variablesReader;
|
|
686
688
|
private readonly webSocketResolver;
|
|
687
689
|
private readonly environmentResolver;
|
|
688
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
690
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
689
691
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
690
692
|
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
691
693
|
private static applyReport;
|
|
@@ -697,7 +699,8 @@ declare class EvalConfigLoader {
|
|
|
697
699
|
}
|
|
698
700
|
declare class ParametersLoader {
|
|
699
701
|
private readonly cwd;
|
|
700
|
-
|
|
702
|
+
private readonly displayBase;
|
|
703
|
+
constructor(cwd?: string, displayBase?: string);
|
|
701
704
|
load(path: string): {
|
|
702
705
|
path: string;
|
|
703
706
|
parameters: Record<string, string>;
|
|
@@ -706,6 +709,7 @@ declare class ParametersLoader {
|
|
|
706
709
|
}
|
|
707
710
|
declare class ScenarioLoader {
|
|
708
711
|
private readonly cwd;
|
|
712
|
+
private readonly displayBase;
|
|
709
713
|
private readonly documentLoader;
|
|
710
714
|
private readonly schemaValidator;
|
|
711
715
|
private readonly conversationValidator;
|
|
@@ -713,7 +717,7 @@ declare class ScenarioLoader {
|
|
|
713
717
|
private readonly runWithReader;
|
|
714
718
|
private readonly webSocketResolver;
|
|
715
719
|
private readonly environmentResolver;
|
|
716
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
720
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
717
721
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
718
722
|
private resolveRun;
|
|
719
723
|
private static byRun;
|
|
@@ -939,6 +943,8 @@ declare class ConsoleReporter {
|
|
|
939
943
|
private attemptBlock;
|
|
940
944
|
private testBlock;
|
|
941
945
|
private entryBody;
|
|
946
|
+
private static actual;
|
|
947
|
+
private capped;
|
|
942
948
|
private static expected;
|
|
943
949
|
private paragraph;
|
|
944
950
|
private bullet;
|
|
@@ -963,22 +969,24 @@ export declare const JSON_REPORT_VERSION = 1;
|
|
|
963
969
|
declare class JsonReporter {
|
|
964
970
|
private readonly redactor;
|
|
965
971
|
private readonly cwd;
|
|
972
|
+
private readonly displayBase;
|
|
966
973
|
private readonly tokens;
|
|
967
|
-
constructor(redactor: SecretRedactor, cwd: string);
|
|
974
|
+
constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
|
|
968
975
|
render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
|
|
969
976
|
private configuration;
|
|
970
977
|
private scenario;
|
|
971
978
|
private attempt;
|
|
972
979
|
private static turn;
|
|
973
|
-
private
|
|
974
|
-
private
|
|
975
|
-
private
|
|
980
|
+
private test;
|
|
981
|
+
private identity;
|
|
982
|
+
private location;
|
|
976
983
|
private static counts;
|
|
977
984
|
private static tally;
|
|
978
985
|
private static usage;
|
|
979
986
|
private static tokenUsage;
|
|
980
987
|
private static error;
|
|
981
988
|
private path;
|
|
989
|
+
private file;
|
|
982
990
|
}
|
|
983
991
|
declare class ReportFileWriter {
|
|
984
992
|
private readonly directory;
|
package/dist/index.d.ts
CHANGED
|
@@ -671,7 +671,8 @@ interface IEvaluator {
|
|
|
671
671
|
export declare const CONFIG_FILE_NAME = "eval-config.yml";
|
|
672
672
|
declare class ConfigLocator {
|
|
673
673
|
private readonly cwd;
|
|
674
|
-
|
|
674
|
+
private readonly displayBase;
|
|
675
|
+
constructor(cwd?: string, displayBase?: string);
|
|
675
676
|
locate(target: string, explicitConfig: string): string;
|
|
676
677
|
private static searchUpward;
|
|
677
678
|
private static isProjectRoot;
|
|
@@ -681,12 +682,13 @@ declare class ConfigLocator {
|
|
|
681
682
|
export declare const DEFAULT_CONFIG: EvalConfig;
|
|
682
683
|
declare class EvalConfigLoader {
|
|
683
684
|
private readonly cwd;
|
|
685
|
+
private readonly displayBase;
|
|
684
686
|
private readonly documentLoader;
|
|
685
687
|
private readonly schemaValidator;
|
|
686
688
|
private readonly variablesReader;
|
|
687
689
|
private readonly webSocketResolver;
|
|
688
690
|
private readonly environmentResolver;
|
|
689
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
691
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
690
692
|
load(configPath: string, override: ExecutionOverride, report?: ReportOverride): EvalConfig;
|
|
691
693
|
static buildWebSocket(value: unknown): WebSocketConfig;
|
|
692
694
|
private static applyReport;
|
|
@@ -698,7 +700,8 @@ declare class EvalConfigLoader {
|
|
|
698
700
|
}
|
|
699
701
|
declare class ParametersLoader {
|
|
700
702
|
private readonly cwd;
|
|
701
|
-
|
|
703
|
+
private readonly displayBase;
|
|
704
|
+
constructor(cwd?: string, displayBase?: string);
|
|
702
705
|
load(path: string): {
|
|
703
706
|
path: string;
|
|
704
707
|
parameters: Record<string, string>;
|
|
@@ -707,6 +710,7 @@ declare class ParametersLoader {
|
|
|
707
710
|
}
|
|
708
711
|
declare class ScenarioLoader {
|
|
709
712
|
private readonly cwd;
|
|
713
|
+
private readonly displayBase;
|
|
710
714
|
private readonly documentLoader;
|
|
711
715
|
private readonly schemaValidator;
|
|
712
716
|
private readonly conversationValidator;
|
|
@@ -714,7 +718,7 @@ declare class ScenarioLoader {
|
|
|
714
718
|
private readonly runWithReader;
|
|
715
719
|
private readonly webSocketResolver;
|
|
716
720
|
private readonly environmentResolver;
|
|
717
|
-
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string
|
|
721
|
+
constructor(environment?: Record<string, string | undefined>, cwd?: string, parameters?: Record<string, string>, displayBase?: string);
|
|
718
722
|
load(target: string, config: EvalConfig): ConversationSpec[];
|
|
719
723
|
private resolveRun;
|
|
720
724
|
private static byRun;
|
|
@@ -940,6 +944,8 @@ declare class ConsoleReporter {
|
|
|
940
944
|
private attemptBlock;
|
|
941
945
|
private testBlock;
|
|
942
946
|
private entryBody;
|
|
947
|
+
private static actual;
|
|
948
|
+
private capped;
|
|
943
949
|
private static expected;
|
|
944
950
|
private paragraph;
|
|
945
951
|
private bullet;
|
|
@@ -964,22 +970,24 @@ export declare const JSON_REPORT_VERSION = 1;
|
|
|
964
970
|
declare class JsonReporter {
|
|
965
971
|
private readonly redactor;
|
|
966
972
|
private readonly cwd;
|
|
973
|
+
private readonly displayBase;
|
|
967
974
|
private readonly tokens;
|
|
968
|
-
constructor(redactor: SecretRedactor, cwd: string);
|
|
975
|
+
constructor(redactor: SecretRedactor, cwd: string, displayBase?: string);
|
|
969
976
|
render(run: EvalRunResult, config: EvalConfig, unsuccessfulCount: number, exitCode: number): string;
|
|
970
977
|
private configuration;
|
|
971
978
|
private scenario;
|
|
972
979
|
private attempt;
|
|
973
980
|
private static turn;
|
|
974
|
-
private
|
|
975
|
-
private
|
|
976
|
-
private
|
|
981
|
+
private test;
|
|
982
|
+
private identity;
|
|
983
|
+
private location;
|
|
977
984
|
private static counts;
|
|
978
985
|
private static tally;
|
|
979
986
|
private static usage;
|
|
980
987
|
private static tokenUsage;
|
|
981
988
|
private static error;
|
|
982
989
|
private path;
|
|
990
|
+
private file;
|
|
983
991
|
}
|
|
984
992
|
declare class ReportFileWriter {
|
|
985
993
|
private readonly directory;
|
package/dist/index.js
CHANGED
|
@@ -680,15 +680,17 @@ const CONFIG_FILE_NAME = "eval-config.yml";
|
|
|
680
680
|
const PROJECT_ROOT_MARKERS = [".git", "package.json"];
|
|
681
681
|
var ConfigLocator = class ConfigLocator {
|
|
682
682
|
cwd;
|
|
683
|
-
|
|
683
|
+
displayBase;
|
|
684
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
684
685
|
this.cwd = cwd;
|
|
686
|
+
this.displayBase = displayBase;
|
|
685
687
|
}
|
|
686
688
|
locate(target, explicitConfig) {
|
|
687
689
|
if (explicitConfig !== "") {
|
|
688
690
|
const path = resolve(this.cwd, explicitConfig);
|
|
689
691
|
if (!ConfigLocator.isFile(path)) throw new EvalError({
|
|
690
692
|
category: ErrorCategory.Configuration,
|
|
691
|
-
message: `configuration file not found: ${explicitConfig}`
|
|
693
|
+
message: `configuration file not found: ${relative(this.displayBase, path) || explicitConfig}`
|
|
692
694
|
});
|
|
693
695
|
return path;
|
|
694
696
|
}
|
|
@@ -697,7 +699,7 @@ var ConfigLocator = class ConfigLocator {
|
|
|
697
699
|
const found = ConfigLocator.searchUpward(start) ?? ConfigLocator.candidate(this.cwd);
|
|
698
700
|
if (found === void 0) throw new EvalError({
|
|
699
701
|
category: ErrorCategory.Configuration,
|
|
700
|
-
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.
|
|
702
|
+
message: `no ${CONFIG_FILE_NAME} found above ${relative(this.displayBase, targetPath) || "."} or in the working directory; pass --config <path>`
|
|
701
703
|
});
|
|
702
704
|
return found;
|
|
703
705
|
}
|
|
@@ -1292,17 +1294,19 @@ var WebSocketResolver = class WebSocketResolver {
|
|
|
1292
1294
|
// src/config/evalConfigLoader.ts
|
|
1293
1295
|
var EvalConfigLoader = class EvalConfigLoader {
|
|
1294
1296
|
cwd;
|
|
1297
|
+
displayBase;
|
|
1295
1298
|
documentLoader = new YamlDocumentLoader();
|
|
1296
1299
|
schemaValidator = new SchemaValidator(eval_config_schema_default);
|
|
1297
1300
|
variablesReader = new VariablesReader();
|
|
1298
1301
|
webSocketResolver = new WebSocketResolver();
|
|
1299
1302
|
environmentResolver;
|
|
1300
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1303
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1301
1304
|
this.cwd = cwd;
|
|
1305
|
+
this.displayBase = displayBase;
|
|
1302
1306
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1303
1307
|
}
|
|
1304
1308
|
load(configPath, override, report = new ReportOverride()) {
|
|
1305
|
-
const document = this.documentLoader.load(configPath, relative(this.
|
|
1309
|
+
const document = this.documentLoader.load(configPath, relative(this.displayBase, configPath));
|
|
1306
1310
|
const issues = [...document.errors];
|
|
1307
1311
|
if (issues.length === 0) issues.push(...this.validateSecretReferences(document));
|
|
1308
1312
|
const named = [];
|
|
@@ -1458,12 +1462,14 @@ var EvalConfigLoader = class EvalConfigLoader {
|
|
|
1458
1462
|
// src/config/parametersLoader.ts
|
|
1459
1463
|
var ParametersLoader = class ParametersLoader {
|
|
1460
1464
|
cwd;
|
|
1461
|
-
|
|
1465
|
+
displayBase;
|
|
1466
|
+
constructor(cwd = process.cwd(), displayBase = cwd) {
|
|
1462
1467
|
this.cwd = cwd;
|
|
1468
|
+
this.displayBase = displayBase;
|
|
1463
1469
|
}
|
|
1464
1470
|
load(path) {
|
|
1465
1471
|
const absolute = resolve(this.cwd, path);
|
|
1466
|
-
const shown = relative(this.
|
|
1472
|
+
const shown = relative(this.displayBase, absolute) || path;
|
|
1467
1473
|
let text;
|
|
1468
1474
|
let value;
|
|
1469
1475
|
try {
|
|
@@ -1922,6 +1928,7 @@ var RunWithReader = class RunWithReader {
|
|
|
1922
1928
|
const SCENARIO_EXTENSIONS = [".yml", ".yaml"];
|
|
1923
1929
|
var ScenarioLoader = class ScenarioLoader {
|
|
1924
1930
|
cwd;
|
|
1931
|
+
displayBase;
|
|
1925
1932
|
documentLoader = new YamlDocumentLoader();
|
|
1926
1933
|
schemaValidator = new SchemaValidator(conversation_schema_default);
|
|
1927
1934
|
conversationValidator = new ConversationValidator();
|
|
@@ -1929,18 +1936,19 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
1929
1936
|
runWithReader = new RunWithReader();
|
|
1930
1937
|
webSocketResolver = new WebSocketResolver();
|
|
1931
1938
|
environmentResolver;
|
|
1932
|
-
constructor(environment = process.env, cwd = process.cwd(), parameters = {}) {
|
|
1939
|
+
constructor(environment = process.env, cwd = process.cwd(), parameters = {}, displayBase = cwd) {
|
|
1933
1940
|
this.cwd = cwd;
|
|
1941
|
+
this.displayBase = displayBase;
|
|
1934
1942
|
this.environmentResolver = new EnvironmentResolver(environment, parameters);
|
|
1935
1943
|
}
|
|
1936
1944
|
load(target, config) {
|
|
1937
1945
|
const files = this.discover(resolve(this.cwd, target), config);
|
|
1938
|
-
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.
|
|
1946
|
+
const configDocument = existsSync(config.configPath) ? this.documentLoader.load(config.configPath, relative(this.displayBase, config.configPath)) : void 0;
|
|
1939
1947
|
const issues = [];
|
|
1940
1948
|
const connectionIssues = [];
|
|
1941
1949
|
const specs = [];
|
|
1942
1950
|
for (const file of files) {
|
|
1943
|
-
const document = this.documentLoader.load(file, relative(this.
|
|
1951
|
+
const document = this.documentLoader.load(file, relative(this.displayBase, file));
|
|
1944
1952
|
const fileIssues = [...document.errors];
|
|
1945
1953
|
const fileSecrets = [];
|
|
1946
1954
|
const used = /* @__PURE__ */ new Set();
|
|
@@ -2070,13 +2078,13 @@ var ScenarioLoader = class ScenarioLoader {
|
|
|
2070
2078
|
discover(target, config) {
|
|
2071
2079
|
if (!existsSync(target)) throw new EvalError({
|
|
2072
2080
|
category: ErrorCategory.Specification,
|
|
2073
|
-
message: `target not found: ${relative(this.
|
|
2081
|
+
message: `target not found: ${relative(this.displayBase, target) || "."}`
|
|
2074
2082
|
});
|
|
2075
2083
|
const excluded = (path) => path === config.configPath || config.report.directory !== "" && path === config.report.directory;
|
|
2076
2084
|
const files = statSync(target).isDirectory() ? this.walk(target, excluded) : [target].filter((file) => !excluded(file));
|
|
2077
2085
|
if (files.length === 0) throw new EvalError({
|
|
2078
2086
|
category: ErrorCategory.Specification,
|
|
2079
|
-
message: `no scenario files found in ${relative(this.
|
|
2087
|
+
message: `no scenario files found in ${relative(this.displayBase, target) || "."}`
|
|
2080
2088
|
});
|
|
2081
2089
|
return files;
|
|
2082
2090
|
}
|
|
@@ -3698,17 +3706,18 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3698
3706
|
const lines = [` test at ${location.file}:${location.line}:${location.column}`, ` ${this.mark(test.status)} ${title} › ${ConsoleReporter.testTitle(test)}`];
|
|
3699
3707
|
const failing = test.history.map((entry, index) => ({
|
|
3700
3708
|
entry,
|
|
3709
|
+
run: result.attempts[index],
|
|
3701
3710
|
attempt: result.attempts[index]?.attempt ?? index + 1
|
|
3702
3711
|
})).filter(({ entry }) => entry.status !== ResultStatus.Pass);
|
|
3703
3712
|
const several = test.history.length > 1;
|
|
3704
|
-
for (const { entry, attempt } of failing) {
|
|
3713
|
+
for (const { entry, run, attempt } of failing) {
|
|
3705
3714
|
if (several) lines.push("", ` Attempt ${attempt} ${this.mark(entry.status)} ${entry.status}`);
|
|
3706
|
-
lines.push(...this.entryBody(spec, entry, several ? 5 : 3));
|
|
3715
|
+
lines.push(...this.entryBody(spec, entry, run, several ? 5 : 3));
|
|
3707
3716
|
}
|
|
3708
3717
|
lines.push("", ` Attempts ${test.history.length}: ${test.passed} passed, ${test.failed} failed, ${test.errors} errors (${test.required} required)`);
|
|
3709
3718
|
return lines;
|
|
3710
3719
|
}
|
|
3711
|
-
entryBody(spec, entry, indent) {
|
|
3720
|
+
entryBody(spec, entry, attempt, indent) {
|
|
3712
3721
|
const pad = " ".repeat(indent);
|
|
3713
3722
|
const evaluation = entry.evaluation;
|
|
3714
3723
|
if (evaluation === void 0) {
|
|
@@ -3722,11 +3731,23 @@ var ConsoleReporter = class ConsoleReporter {
|
|
|
3722
3731
|
const threshold = this.thresholds === void 0 ? "" : `, threshold ${(entry.kind === TestKind.Conversation ? this.thresholds.conversation : this.thresholds.response).toFixed(2)}`;
|
|
3723
3732
|
const expected = ConsoleReporter.expected(spec, entry);
|
|
3724
3733
|
const lines = [`${pad}Score ${evaluation.score.toFixed(2)}${threshold}`];
|
|
3734
|
+
const actual = ConsoleReporter.actual(entry, attempt);
|
|
3725
3735
|
if (expected !== void 0) lines.push("", `${pad}Expected`, ...this.paragraph(expected, indent + 2));
|
|
3736
|
+
if (actual !== void 0) lines.push("", `${pad}Actual`, ...this.capped(actual, indent + 2));
|
|
3726
3737
|
if (evaluation.explanation !== "") lines.push("", `${pad}Explanation`, ...this.paragraph(evaluation.explanation, indent + 2));
|
|
3727
|
-
if (evaluation.evidence.length > 0) lines.push("", `${pad}Evidence`, ...evaluation.evidence.flatMap((item) => this.bullet(item, indent + 2)));
|
|
3728
3738
|
return lines;
|
|
3729
3739
|
}
|
|
3740
|
+
static actual(entry, attempt) {
|
|
3741
|
+
const turns = attempt?.execution.turns ?? [];
|
|
3742
|
+
if (entry.kind === TestKind.Conversation) return turns.length === 0 ? void 0 : turns.map((turn, index) => `Turn ${index + 1}: ${turn.response}`).join("\n");
|
|
3743
|
+
return turns[entry.turn - 1]?.response;
|
|
3744
|
+
}
|
|
3745
|
+
capped(text, indent) {
|
|
3746
|
+
const lines = text.trim() === "" ? [`${" ".repeat(indent)}(empty)`] : this.paragraph(text, indent);
|
|
3747
|
+
if (lines.length <= 12) return lines;
|
|
3748
|
+
const more = lines.length - 12;
|
|
3749
|
+
return [...lines.slice(0, 12), `${" ".repeat(indent)}… ${more} more ${more === 1 ? "line" : "lines"} in the JSON report`];
|
|
3750
|
+
}
|
|
3730
3751
|
static expected(spec, entry) {
|
|
3731
3752
|
if (entry.kind === TestKind.Conversation) return spec.conversation?.criteria;
|
|
3732
3753
|
return spec.turns[entry.turn - 1]?.expect.response?.criteria;
|
|
@@ -3826,10 +3847,12 @@ const JSON_REPORT_VERSION = 1;
|
|
|
3826
3847
|
var JsonReporter = class JsonReporter {
|
|
3827
3848
|
redactor;
|
|
3828
3849
|
cwd;
|
|
3850
|
+
displayBase;
|
|
3829
3851
|
tokens = new TokenUsageAggregator();
|
|
3830
|
-
constructor(redactor, cwd) {
|
|
3852
|
+
constructor(redactor, cwd, displayBase = cwd) {
|
|
3831
3853
|
this.redactor = redactor;
|
|
3832
3854
|
this.cwd = cwd;
|
|
3855
|
+
this.displayBase = displayBase;
|
|
3833
3856
|
}
|
|
3834
3857
|
render(run, config, unsuccessfulCount, exitCode) {
|
|
3835
3858
|
const report = {
|
|
@@ -3900,7 +3923,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3900
3923
|
return {
|
|
3901
3924
|
name: result.spec.name,
|
|
3902
3925
|
runWith: result.spec.runName === "" ? void 0 : result.spec.runName,
|
|
3903
|
-
file: result.spec.location.file,
|
|
3926
|
+
file: this.file(result.spec.location.file),
|
|
3904
3927
|
status: result.status,
|
|
3905
3928
|
connection: {
|
|
3906
3929
|
endpoint: websocket.endpoint,
|
|
@@ -3910,7 +3933,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3910
3933
|
agentUsage: JsonReporter.usage(result.agentUsage),
|
|
3911
3934
|
evaluatorUsage: JsonReporter.usage(result.evaluatorUsage),
|
|
3912
3935
|
tests: result.tests.map((test) => ({
|
|
3913
|
-
...
|
|
3936
|
+
...this.identity(test),
|
|
3914
3937
|
status: test.status,
|
|
3915
3938
|
...JsonReporter.counts(test),
|
|
3916
3939
|
history: test.history.map((entry, index) => ({
|
|
@@ -3935,7 +3958,7 @@ var JsonReporter = class JsonReporter {
|
|
|
3935
3958
|
agentUsage: JsonReporter.usage(this.tokens.agent([attempt], capability)),
|
|
3936
3959
|
evaluatorUsage: JsonReporter.usage(this.tokens.evaluator([attempt])),
|
|
3937
3960
|
turns: execution.turns.map((turn, index) => JsonReporter.turn(index + 1, spec.turns[index].input, turn)),
|
|
3938
|
-
tests: attempt.tests.map((test) =>
|
|
3961
|
+
tests: attempt.tests.map((test) => this.test(test, config))
|
|
3939
3962
|
};
|
|
3940
3963
|
}
|
|
3941
3964
|
static turn(number, input, turn) {
|
|
@@ -3960,11 +3983,11 @@ var JsonReporter = class JsonReporter {
|
|
|
3960
3983
|
tokenUsage: turn.tokenUsage === void 0 ? void 0 : JsonReporter.tokenUsage(turn.tokenUsage)
|
|
3961
3984
|
};
|
|
3962
3985
|
}
|
|
3963
|
-
|
|
3986
|
+
test(test, config) {
|
|
3964
3987
|
const thresholds = config.evaluator.thresholds;
|
|
3965
3988
|
const evaluation = test.evaluation;
|
|
3966
3989
|
return {
|
|
3967
|
-
...
|
|
3990
|
+
...this.identity(test),
|
|
3968
3991
|
status: test.status,
|
|
3969
3992
|
details: [...test.details],
|
|
3970
3993
|
evaluation: evaluation === void 0 ? void 0 : {
|
|
@@ -3977,17 +4000,17 @@ var JsonReporter = class JsonReporter {
|
|
|
3977
4000
|
evaluatorUsage: JsonReporter.usage(test.evaluatorUsage)
|
|
3978
4001
|
};
|
|
3979
4002
|
}
|
|
3980
|
-
|
|
4003
|
+
identity(test) {
|
|
3981
4004
|
return {
|
|
3982
4005
|
kind: test.kind,
|
|
3983
4006
|
turn: test.turn,
|
|
3984
4007
|
title: test.title,
|
|
3985
|
-
location:
|
|
4008
|
+
location: this.location(test.location)
|
|
3986
4009
|
};
|
|
3987
4010
|
}
|
|
3988
|
-
|
|
4011
|
+
location(location) {
|
|
3989
4012
|
return {
|
|
3990
|
-
file: location.file,
|
|
4013
|
+
file: this.file(location.file),
|
|
3991
4014
|
line: location.line,
|
|
3992
4015
|
column: location.column,
|
|
3993
4016
|
path: location.path
|
|
@@ -4034,6 +4057,9 @@ var JsonReporter = class JsonReporter {
|
|
|
4034
4057
|
path(path) {
|
|
4035
4058
|
return relative(this.cwd, path) || ".";
|
|
4036
4059
|
}
|
|
4060
|
+
file(shown) {
|
|
4061
|
+
return shown === "" ? shown : this.path(resolve(this.displayBase, shown));
|
|
4062
|
+
}
|
|
4037
4063
|
};
|
|
4038
4064
|
// src/reporting/reportFileWriter.ts
|
|
4039
4065
|
var ReportFileWriter = class {
|