@m4trix/evals 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-simple.cjs +8 -0
- package/dist/cli-simple.cjs.map +1 -1
- package/dist/cli-simple.js +8 -0
- package/dist/cli-simple.js.map +1 -1
- package/dist/cli.cjs +8 -0
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +8 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +19 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1057,6 +1057,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
1057
1057
|
const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
|
|
1058
1058
|
const rerunPassed = [];
|
|
1059
1059
|
for (let r = 0; r < reruns; r++) {
|
|
1060
|
+
const evaluatorRunId = `run-${crypto.randomUUID()}`;
|
|
1060
1061
|
const started = Date.now();
|
|
1061
1062
|
const evaluatorScores = [];
|
|
1062
1063
|
let testCaseError;
|
|
@@ -1083,6 +1084,11 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
1083
1084
|
input: testCaseItem.testCase.getInput(),
|
|
1084
1085
|
ctx,
|
|
1085
1086
|
output,
|
|
1087
|
+
meta: {
|
|
1088
|
+
triggerId: task.triggerId,
|
|
1089
|
+
runId: evaluatorRunId,
|
|
1090
|
+
datasetId: task.datasetId
|
|
1091
|
+
},
|
|
1086
1092
|
logDiff,
|
|
1087
1093
|
log
|
|
1088
1094
|
})
|
|
@@ -1542,6 +1548,7 @@ var EffectRunner = class {
|
|
|
1542
1548
|
(sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
|
|
1543
1549
|
0
|
|
1544
1550
|
);
|
|
1551
|
+
const triggerId = request.triggerId ?? `trg-${crypto.randomUUID()}`;
|
|
1545
1552
|
const runId = `run-${crypto.randomUUID()}`;
|
|
1546
1553
|
const artifactPath = createArtifactPath(
|
|
1547
1554
|
this.config.artifactDirectory,
|
|
@@ -1583,6 +1590,7 @@ var EffectRunner = class {
|
|
|
1583
1590
|
await effect.Effect.runPromise(
|
|
1584
1591
|
effect.Queue.offer(this.runQueue, {
|
|
1585
1592
|
runId,
|
|
1593
|
+
triggerId,
|
|
1586
1594
|
datasetId: request.datasetId,
|
|
1587
1595
|
dataset: dataset.dataset,
|
|
1588
1596
|
evaluators: selectedEvaluators,
|