@leostera/evalkit 0.7.0 → 0.9.0

package/dist/{src-b2Wnmzbb.mjs → src-BrfIXfO2.mjs}

@@ -1,6 +1,7 @@
+import { n as assertNoRemovedOptions } from "./upgrade-60SIRius.mjs";
 import * as Schema from "effect/Schema";
 //#region ../core/src/identity.ts
-const ResourceKindSchema = Schema.Literal("run", "trial", "artifact");
+const ResourceKindSchema = Schema.Literal("run", "artifact");
 /** Human-authored project-local identifier; never a generated resource URI. */
 function authorId(value) {
 	if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(value)) throw new Error(`ID must be lowercase kebab-case: ${value}`);
@@ -8,7 +9,7 @@ function authorId(value) {
 }
 /** RFC 4122 textual UUID. EvalKit accepts v4/v7 and other valid UUID versions. */
 const UuidSchema = Schema.String.pipe(Schema.pattern(/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
-const ResourceUriSchema = Schema.String.pipe(Schema.pattern(/^evalkit:(run|trial|artifact):[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
+const ResourceUriSchema = Schema.String.pipe(Schema.pattern(/^evalkit:(run|artifact):[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i));
 function resourceUriSchema(kind) {
 	return Schema.String.pipe(Schema.pattern(new RegExp(`^evalkit:${kind}:[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, "i")));
 }
@@ -16,7 +17,7 @@ function resourceUri(kind, uuid) {
 	return `evalkit:${kind}:${uuid}`;
 }
 function parseResourceUri(value, expectedKind) {
-	const match = /^evalkit:(run|trial|artifact):(.+)$/.exec(value);
+	const match = /^evalkit:(run|artifact):(.+)$/.exec(value);
 	if (!match) throw new Error(`Invalid EvalKit resource URI: ${value}`);
 	const kind = match[1];
 	if (expectedKind && kind !== expectedKind) throw new Error(`Expected an EvalKit ${expectedKind} URI; received ${value}`);
@@ -101,10 +102,10 @@ function relativeDatasetPath(value, label) {
 	return parts.length ? parts.join("/") : ".";
 }
 /**
-* Declares named data that scenarios use: files placed in each trial's
+* Declares named data that scenarios use: files placed in each run's
 * workspace, a value for the agent and scorers, or both. Static datasets give
 * `files`, `value` or `valueFile`; computed datasets give `onLoad` (cached per
-* process) and optionally `onRun` (per trial).
+* process) and optionally `onRun` (per run).
 */
 function defineDataset(definition) {
 	return markDefinitionSite(checkedDataset(definition));
@@ -219,9 +220,8 @@ function catalogEntry(evaluation, suite, options = {}) {
 		agent: catalogAgent(evaluation.agent, at),
 		...judgeAgent ? { judgeAgent } : {},
 		datasets: (evaluation.datasets ?? []).map((dataset) => catalogDataset(dataset, at)),
-		trialCount: policy.trials ?? 1,
+		repeat: evaluation.repeat ?? 1,
 		policy: {
-			trials: policy.trials ?? 1,
 			...policy.timeoutMs !== void 0 ? { timeoutMs: policy.timeoutMs } : {},
 			...policy.failfast !== void 0 ? { failfast: policy.failfast } : {}
 		},
@@ -317,6 +317,10 @@ function shapeOf(value) {
 }
 //#endregion
 //#region ../core/src/config.ts
+const DEFAULT_ATTACHMENT_LIMITS = {
+	maxFileBytes: 5e6,
+	maxRunBytes: 2e7
+};
 /** Left out of candidate snapshots unless `artifacts.exclude` says otherwise. */
 const DEFAULT_ARTIFACT_EXCLUDE = [
 	".git",
@@ -332,6 +336,18 @@ function defineConfig(configuration) {
 	return configuration;
 }
 //#endregion
+//#region ../core/src/usage.ts
+/**
+* The tokens a turn counts toward the `tokens` metric: `inputTokens +
+* outputTokens` (cached input included), or `totalTokens` when neither is
+* reported. Undefined when the turn reported no usage.
+*/
+function turnTokens(usage) {
+	if (!usage) return void 0;
+	if (usage.inputTokens !== void 0 || usage.outputTokens !== void 0) return (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
+	return usage.totalTokens;
+}
+//#endregion
 //#region ../core/src/canonical-json.ts
 /** Canonical JSON: keys sorted recursively, array order preserved. Rejects non-JSON inputs. */
 function canonicalParameters(value) {
@@ -556,7 +572,9 @@ function turnResultEvents(result, turn, now) {
 //#endregion
 //#region ../core/src/schema.ts
 /** Runtime schemas for serialized EvalKit report and API boundaries. */
-const RunStatusSchema = Schema.Literal("running", "completed", "failed", "cancelled");
+const RunStatusSchema = Schema.Literal("running", "passed", "failed", "errored", "cancelled");
+/** The report schema version this EvalKit writes and reads. */
+const REPORT_SCHEMA_VERSION = 6;
 const RecordedErrorSchema = Schema.Struct({
 	name: Schema.String,
 	message: Schema.String,
@@ -594,8 +612,19 @@ const RunSourceSchema = Schema.Struct({
 		url: Schema.optional(Schema.String)
 	}))
 });
+const BatchSchema = Schema.Struct({
+	id: Schema.String,
+	index: Schema.Number,
+	size: Schema.Number
+});
+const RunDatasetSchema = Schema.Struct({
+	id: Schema.String,
+	visibility: Schema.Literal("candidate", "evaluator"),
+	source: Schema.Literal("static", "computed"),
+	digest: Schema.String
+});
 const RunMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4, 5),
+	schemaVersion: Schema.Literal(6),
 	runUri: resourceUriSchema("run"),
 	evalId: Schema.String,
 	suiteId: Schema.optional(Schema.String),
@@ -607,37 +636,21 @@ const RunMetadataSchema = Schema.Struct({
 	process: Schema.optional(Schema.Struct({
 		pid: Schema.Number,
 		hostname: Schema.String
-	}))
-});
-const TrialMetadataSchema = Schema.Struct({
-	schemaVersion: Schema.Literal(2, 3, 4, 5),
-	runUri: resourceUriSchema("run"),
-	trialUri: resourceUriSchema("trial"),
-	trialIndex: Schema.Number,
-	evalId: Schema.String,
-	parameters: Schema.optional(ParametersSchema),
-	matrix: Schema.optional(MatrixCellSchema),
-	aut: Schema.optional(AutIdentitySchema),
-	startedAt: Schema.String,
-	datasets: Schema.optional(Schema.mutable(Schema.Array(Schema.Struct({
-		id: Schema.String,
-		visibility: Schema.Literal("candidate", "evaluator"),
-		source: Schema.Literal("static", "computed"),
-		digest: Schema.String
-	}))))
+	})),
+	batch: Schema.optional(BatchSchema),
+	/** The effective snapshot ignore patterns: the project's, then the scenario's. */
+	workspaceIgnore: Schema.optional(Schema.mutable(Schema.Array(Schema.String))),
+	datasets: Schema.optional(Schema.mutable(Schema.Array(RunDatasetSchema)))
 });
-/** On-disk manifests include a lifecycle status; older finalized manifests may still say running. */
+/** On-disk manifests include a lifecycle status. */
 const RunManifestSchema = Schema.Struct({
 	...RunMetadataSchema.fields,
 	status: Schema.optional(RunStatusSchema)
 });
-const TrialManifestSchema = Schema.Struct({
-	...TrialMetadataSchema.fields,
-	status: Schema.optional(RunStatusSchema)
-});
 const UsageSchema = Schema.Struct({
 	inputTokens: Schema.optional(Schema.Number),
 	outputTokens: Schema.optional(Schema.Number),
+	cachedInputTokens: Schema.optional(Schema.Number),
 	totalTokens: Schema.optional(Schema.Number)
 });
 const JudgeRunInfoSchema = Schema.Struct({
@@ -687,7 +700,7 @@ const MetricResultSchema = Schema.Struct({
 	builtIn: Schema.optional(Schema.Literal(true)),
 	error: Schema.optional(RecordedErrorSchema)
 });
-const TrialScoringSchema = Schema.Struct({
+const RunScoringSchema = Schema.Struct({
 	results: Schema.mutable(Schema.Array(ScoreResultSchema)),
 	metrics: Schema.optional(Schema.mutable(Schema.Array(MetricResultSchema))),
 	checkpoints: Schema.optional(Schema.mutable(Schema.Array(CheckpointResultSchema))),
@@ -700,22 +713,18 @@ const ArtifactEntrySchema = Schema.Struct({
 	kind: Schema.Literal("file", "directory"),
 	size: Schema.optional(Schema.Number)
 });
-const TrialSummarySchema = Schema.Struct({
-	status: RunStatusSchema,
-	endedAt: Schema.String,
-	durationMs: Schema.optional(Schema.Number),
-	scoring: Schema.optional(TrialScoringSchema),
-	artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),
-	error: Schema.optional(RecordedErrorSchema)
+const AttachmentEntrySchema = Schema.Struct({
+	name: Schema.String,
+	size: Schema.Number,
+	truncatedFrom: Schema.optional(Schema.Number)
 });
 const RunSummarySchema = Schema.Struct({
 	status: RunStatusSchema,
 	endedAt: Schema.String,
 	durationMs: Schema.optional(Schema.Number),
-	trialCount: Schema.Number,
-	passed: Schema.Number,
-	failed: Schema.Number,
-	cancelled: Schema.optional(Schema.Number),
+	scoring: Schema.optional(RunScoringSchema),
+	artifacts: Schema.optional(Schema.mutable(Schema.Array(ArtifactEntrySchema))),
+	attachments: Schema.optional(Schema.mutable(Schema.Array(AttachmentEntrySchema))),
 	error: Schema.optional(RecordedErrorSchema)
 });
 const autEvent = {
@@ -775,9 +784,9 @@ const autEvent = {
 	})
 };
 const runnerEvent = {
-	trialStarted: Schema.Struct({
+	runStarted: Schema.Struct({
 		source: Schema.Literal("runner"),
-		kind: Schema.Literal("trial-started"),
+		kind: Schema.Literal("run-started"),
 		timestamp: Schema.String
 	}),
 	stepStarted: Schema.Struct({
@@ -841,9 +850,9 @@ const runnerEvent = {
 		error: RecordedErrorSchema,
 		timestamp: Schema.String
 	}),
-	trialCompleted: Schema.Struct({
+	runCompleted: Schema.Struct({
 		source: Schema.Literal("runner"),
-		kind: Schema.Literal("trial-completed"),
+		kind: Schema.Literal("run-completed"),
 		timestamp: Schema.String
 	}),
 	error: Schema.Struct({
@@ -900,7 +909,7 @@ function expectToolCall(expected) {
 	};
 }
 /**
-* A user message. Pass a function to compute it for each trial, e.g. from a
+* A user message. Pass a function to compute it for each run, e.g. from a
 * matrix axis: `user(({ parameters }) => `Set up OCaml with ${parameters.toolchain}.`)`.
 */
 function user(message) {
@@ -916,8 +925,8 @@ function agent(expectation) {
 	};
 }
 /**
-* Marks a scorer as not applicable to this trial. A skipped result neither
-* passes nor fails the trial and is left out of score averages.
+* Marks a scorer as not applicable to this run. A skipped result neither
+* passes nor fails the run and is left out of score averages.
 */
 function skip(reason) {
 	if (!reason.trim()) throw new Error("skip() needs a reason");
@@ -926,6 +935,7 @@ function skip(reason) {
 		reason
 	};
 }
+/** A scorer in code. Its `evidence` type carries through to `checkpoint(rule)`. */
 function predicate(name, run, options = {}) {
 	if (!name.trim()) throw new Error("Predicate name must not be empty");
 	if (options.passAt !== void 0 && !(Number.isFinite(options.passAt) && options.passAt >= 0 && options.passAt <= 1)) throw new Error(`Predicate ${name} passAt must be a number from 0 to 1; received ${options.passAt}`);
@@ -936,6 +946,7 @@ function predicate(name, run, options = {}) {
 		...options
 	});
 }
+/** A built-in metric's name is a type error here, as well as at definition time. */
 function metric(name, run, options = {}) {
 	if (!name.trim()) throw new Error("Metric name must not be empty");
 	return markDefinitionSite({
@@ -955,8 +966,9 @@ function judge(name, options) {
 }
 function defineEval(definition) {
 	authoringId(definition);
-	if ("fixtures" in definition) throw new Error(`Eval ${definition.id} uses fixtures, which were replaced by datasets: [defineDataset(...)]`);
+	assertNoRemovedOptions(definition);
 	if (definition.metadata !== void 0) toJsonObject(definition.metadata, `${definition.id}.metadata`);
+	for (const [index, pattern] of (definition.workspace?.ignore ?? []).entries()) if (typeof pattern !== "string" || pattern.replace(/^!/, "").trim() === "") throw new Error(`Eval ${definition.id} workspace.ignore[${index}] must be a non-empty glob, got ${JSON.stringify(pattern)}`);
 	const datasetIds = /* @__PURE__ */ new Set();
 	for (const [index, dataset] of (definition.datasets ?? []).entries()) {
 		if (!isDataset(dataset)) throw new Error(`Eval ${definition.id} datasets[${index}] is not a dataset; create it with defineDataset(...)`);
@@ -1045,7 +1057,16 @@ function registerEvals(registrations) {
 		catalog: (options) => evals.map((evaluation) => catalogEntry(evaluation, suiteIdByEvalId.get(authoringId(evaluation)), options))
 	};
 }
-/** Metrics EvalKit records for every trial; scenario metrics may not reuse these names. */
+/**
+* Metrics EvalKit records for every run; scenario metrics may not reuse
+* these names.
+*
+* - `agent time` (ms): from the run's first event to its last one before
+*   final scoring.
+* - `tokens`: the sum of `turnTokens(usage)` over the turns that reported
+*   usage, cached input included. Absent when no turn reported any.
+* - `tool calls`: the agent's `tool-call` events.
+*/
 const BUILT_IN_METRICS = [
 	"agent time",
 	"tokens",
@@ -1063,4 +1084,4 @@ function recordError(error) {
 	};
 }
 //#endregion
-export { ResourceUriSchema as $, RunSummarySchema as A, defineEvalMatrix as B, JudgeRunInfoSchema as C, RunMetadataSchema as D, RunManifestSchema as E, TrialScoringSchema as F, NotJsonError as G, DEFAULT_ARTIFACT_EXCLUDE as H, TrialSummarySchema as I, defineDataset as J, toJson as K, UsageSchema as L, TrajectoryEventSchema as M, TrialManifestSchema as N, RunSourceSchema as O, TrialMetadataSchema as P, ResourceKindSchema as Q, normalizeAutEvent as R, JsonValueSchema as S, RecordedErrorSchema as T, defineConfig as U, canonicalParameters as V, catalogEntry as W, relativeDatasetPath as X, isDataset as Y, definitionSite as Z, parseTrajectoryJsonl as _, authoringId as a, resourceUuid as at, AutIdentitySchema as b, defineSuite as c, metric as d, UuidSchema as et, predicate as f, user as g, skip as h, agentRuntimes as i, resourceUriSchema as it, ScoreResultSchema as j, RunStatusSchema as k, expectToolCall as l, registerEvals as m, DEFAULT_AGENT_RUNTIMES as n, parseResourceUri as nt, defineAgent as o, recordError as p, toJsonObject as q, agent as r, resourceUri as rt, defineEval as s, BUILT_IN_METRICS as t, authorId as tt, judge as u, ApiErrorSchema as v, MetricResultSchema as w, CheckpointResultSchema as x, ArtifactEntrySchema as y, turnResultEvents as z };
+export { relativeDatasetPath as $, RunManifestSchema as A, turnResultEvents as B, CheckpointResultSchema as C, REPORT_SCHEMA_VERSION as D, MetricResultSchema as E, RunSummarySchema as F, DEFAULT_ATTACHMENT_LIMITS as G, canonicalParameters as H, ScoreResultSchema as I, NotJsonError as J, defineConfig as K, TrajectoryEventSchema as L, RunScoringSchema as M, RunSourceSchema as N, RecordedErrorSchema as O, RunStatusSchema as P, isDataset as Q, UsageSchema as R, BatchSchema as S, JudgeRunInfoSchema as T, turnTokens as U, defineEvalMatrix as V, DEFAULT_ARTIFACT_EXCLUDE as W, toJsonObject as X, toJson as Y, defineDataset as Z, parseTrajectoryJsonl as _, authoringId as a, parseResourceUri as at, AttachmentEntrySchema as b, defineSuite as c, resourceUuid as ct, metric as d, definitionSite as et, predicate as f, user as g, skip as h, agentRuntimes as i, authorId as it, RunMetadataSchema as j, RunDatasetSchema as k, expectToolCall as l, registerEvals as m, DEFAULT_AGENT_RUNTIMES as n, ResourceUriSchema as nt, defineAgent as o, resourceUri as ot, recordError as p, catalogEntry as q, agent as r, UuidSchema as rt, defineEval as s, resourceUriSchema as st, BUILT_IN_METRICS as t, ResourceKindSchema as tt, judge as u, ApiErrorSchema as v, JsonValueSchema as w, AutIdentitySchema as x, ArtifactEntrySchema as y, normalizeAutEvent as z };