@m4trix/evals 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-simple.cjs +8 -0
- package/dist/cli-simple.cjs.map +1 -1
- package/dist/cli-simple.js +8 -0
- package/dist/cli-simple.js.map +1 -1
- package/dist/cli.cjs +8 -0
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +8 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +19 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli-simple.cjs
CHANGED
|
@@ -587,6 +587,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
587
587
|
const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
|
|
588
588
|
const rerunPassed = [];
|
|
589
589
|
for (let r = 0; r < reruns; r++) {
|
|
590
|
+
const evaluatorRunId = `run-${crypto.randomUUID()}`;
|
|
590
591
|
const started = Date.now();
|
|
591
592
|
const evaluatorScores = [];
|
|
592
593
|
let testCaseError;
|
|
@@ -613,6 +614,11 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
613
614
|
input: testCaseItem.testCase.getInput(),
|
|
614
615
|
ctx,
|
|
615
616
|
output,
|
|
617
|
+
meta: {
|
|
618
|
+
triggerId: task.triggerId,
|
|
619
|
+
runId: evaluatorRunId,
|
|
620
|
+
datasetId: task.datasetId
|
|
621
|
+
},
|
|
616
622
|
logDiff,
|
|
617
623
|
log
|
|
618
624
|
})
|
|
@@ -1072,6 +1078,7 @@ var EffectRunner = class {
|
|
|
1072
1078
|
(sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
|
|
1073
1079
|
0
|
|
1074
1080
|
);
|
|
1081
|
+
const triggerId = request.triggerId ?? `trg-${crypto.randomUUID()}`;
|
|
1075
1082
|
const runId = `run-${crypto.randomUUID()}`;
|
|
1076
1083
|
const artifactPath = createArtifactPath(
|
|
1077
1084
|
this.config.artifactDirectory,
|
|
@@ -1113,6 +1120,7 @@ var EffectRunner = class {
|
|
|
1113
1120
|
await effect.Effect.runPromise(
|
|
1114
1121
|
effect.Queue.offer(this.runQueue, {
|
|
1115
1122
|
runId,
|
|
1123
|
+
triggerId,
|
|
1116
1124
|
datasetId: request.datasetId,
|
|
1117
1125
|
dataset: dataset.dataset,
|
|
1118
1126
|
evaluators: selectedEvaluators,
|