@m4trix/evals 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-simple.cjs +8 -0
- package/dist/cli-simple.cjs.map +1 -1
- package/dist/cli-simple.js +8 -0
- package/dist/cli-simple.js.map +1 -1
- package/dist/cli.cjs +8 -0
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +8 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +19 -1
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/cli.cjs
CHANGED
|
@@ -1287,6 +1287,7 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
1287
1287
|
const reruns = typeof testCaseItem.testCase.getReruns === "function" ? testCaseItem.testCase.getReruns() : 1;
|
|
1288
1288
|
const rerunPassed = [];
|
|
1289
1289
|
for (let r = 0; r < reruns; r++) {
|
|
1290
|
+
const evaluatorRunId = `run-${crypto.randomUUID()}`;
|
|
1290
1291
|
const started = Date.now();
|
|
1291
1292
|
const evaluatorScores = [];
|
|
1292
1293
|
let testCaseError;
|
|
@@ -1313,6 +1314,11 @@ function processOneTestCase(task, testCaseItem, totalEvaluations, publishEvent,
|
|
|
1313
1314
|
input: testCaseItem.testCase.getInput(),
|
|
1314
1315
|
ctx,
|
|
1315
1316
|
output,
|
|
1317
|
+
meta: {
|
|
1318
|
+
triggerId: task.triggerId,
|
|
1319
|
+
runId: evaluatorRunId,
|
|
1320
|
+
datasetId: task.datasetId
|
|
1321
|
+
},
|
|
1316
1322
|
logDiff,
|
|
1317
1323
|
log
|
|
1318
1324
|
})
|
|
@@ -1802,6 +1808,7 @@ var EffectRunner = class {
|
|
|
1802
1808
|
(sum, tc) => sum + (typeof tc.testCase.getReruns === "function" ? tc.testCase.getReruns() : 1),
|
|
1803
1809
|
0
|
|
1804
1810
|
);
|
|
1811
|
+
const triggerId = request.triggerId ?? `trg-${crypto.randomUUID()}`;
|
|
1805
1812
|
const runId = `run-${crypto.randomUUID()}`;
|
|
1806
1813
|
const artifactPath = createArtifactPath(
|
|
1807
1814
|
this.config.artifactDirectory,
|
|
@@ -1843,6 +1850,7 @@ var EffectRunner = class {
|
|
|
1843
1850
|
await effect.Effect.runPromise(
|
|
1844
1851
|
effect.Queue.offer(this.runQueue, {
|
|
1845
1852
|
runId,
|
|
1853
|
+
triggerId,
|
|
1846
1854
|
datasetId: request.datasetId,
|
|
1847
1855
|
dataset: dataset.dataset,
|
|
1848
1856
|
evaluators: selectedEvaluators,
|