@leostera/evalkit 0.0.1

package/dist/index-BwNYw4-_.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index-BwNYw4-_.d.mts","names":[],"sources":["../../core/src/identity.ts","../../core/src/matrix.ts","../../core/src/config.ts","../../core/src/schema.ts","../../core/src/trajectory.ts","../../core/src/index.ts"],"mappings":";;cAEa,oBAAkB,OAAA;KACnB,sBAAsB,mBAAmB;;iBAGrC,SAAS;;cAOZ,YAAU,OAAA,cAAA,OAAA;KAKX,cAAc,WAAW;KAEzB,YAAY,cAAc,eAAe,2BACxC;cAEA,mBAAiB,OAAA,cAAA,OAAA;iBAMd,kBAAkB,cAAc,cAAc,MAAM,QAAK,OAAA,cAAA,OAAA;iBAWzD,YAAY,cAAc,cACxC,MAAM,OACN,MAAM,OACL,YAAY;iBAIC,iBAAiB,cAAc,cAC7C,eACA,eAAe;EACZ,MAAM;EAAO,MAAM;EAAM,KAAK,YAAY;;iBAY/B,aAAa,eAAe,eAAe,eAAe;;;;iBCtD1D,oBAAoB,OAAO;KAqB/B;;EAEV;;EAEA,aAAa,SAAS,wBAAwB;;EAE9C,YAAY;;KAEF;EACV;;EAEA;EACA,MAAM;EACN,YAAY;;KAEF;EACV;EACA;EACA,gBAAgB;EAChB,YAAY,SAAS,wBAAwB;EAC7C,WAAW;;EAEX,iBAAiB;;EAEjB,mBAAmB;;KAET,aAAa;EACvB;EACA,MAAM,YAAY;EAClB,MAAM,YAAY,kBAAkB,iBAAiB;;iBAWvC,uBAAuB,UAAU,sBAC/C,YAAY,IACX,aAAa;;;KCpEJ;;EAEV;EACA;EACA;;EAEA,iBAAiB;EACjB,WAAW;;EAEX,SAAS,KAAK;IAA0C;;EACxD;IAAc;IAAsB;IAAiB;;EACrD;EACA;;;iBAIc,mBAAmB,UAAU,eAC3C,eAAe,IACd;;;;cChBU,iBAAe,OAAA;cAOf,qBAAmB,OAAA;;;;;cAMnB,mBAAiB,OAAA;;;;;;cAQjB,iBAAiB,OAAO,OAAO;cAoB/B,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;cAWjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;cAanB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;cAIjB,qBAAmB,OAAA;;;;;;;;;;;;;;;;;;;;cAKnB,aAAW,OAAA;;;;;cAMX,oBAAkB,OAAA;;;;;;;;;;;;;;cAalB,mBAAiB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAWjB,wBAAsB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAgBtB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cASlB,qBAAmB,OAAA;;;;;cAKnB,oBAAkB,OAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cASlB,kBAAgB,OAAA;;;;;;;;;;;;;cAgJhB,uBAAuB,OAAO,OAAO;cAGrC,gBAAc,OAAA;;;;;;;iBClSX,qBAAqB,mBAAmB;;;KCF5C;KACA,YACV,gBAAgB;GAAiB,cAAc;;KACrC;GAAgB,cAAc;;KAE9B;;EAEV;EACA;EACA;EACA;;KAGU;;;;;KAMA;EACV;EACA,gBAAgB;;KAGN,gBAAgB,QAAQ,OAAO,kBAAkB;KAEjD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACN;EAAiB;;EAEjB;EACA;EACA,SAAS;EACT;;EAGA;EACA;EACA;EACA,WAAW;EACX;;EAGA;EACA;EACA;EACA,QAAQ;EACR;;EAEA;EAAsB;EAAc;;EAEpC;EACA;EACA;EACA;EACA,QAAQ;;EAER;EAAmB,SAAS;EAAW;;EACvC;EAAe,OAAO;EAAe;;KAE/B;EACN;EAAuB;;EACvB;EAAiC;EAAc;;EAC/C;EAAmC;EAAc;;EACjD;EAAiC;EAAc;;EAE/C;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAwB;EAAgB;;EAExC;EACA;EACA;EACA;;EAGA;EACA;EACA,OAAO;EACP;;EAEA;EAAyB;;EACzB;EAAe,OAAO;EAAe;;KAE/B,mBACT;EAAa;MAAoB;EAAgB;;KAExC;;EAEV;;KAGU,qBAAqB;KACrB,qBAAqB;KAErB;EACV;EACA;EACA;EACA;EACA,UAAU;;EAEV,aAAa;EACb,UAAU;;EAEV,WAAW;;EAEX,oBAAoB;;KAGV,iBAAiB,KAC3B;KAIU,WAAW;WACZ,WAAW;;WAEX,WAAW;EACpB,MAAM;IACJ,SAAS;IACT,UAAU;IACV,UAAU,OAAO,oBAAoB;MACnC,QAAQ,WAAW;;KAGb,WAAW;;EAErB,KAAK,SAAS,WAAW;EACzB,SAAS;;;;;;iBAOK,kBAAkB,UAAU,YAAY,OAAO,IAAI;KAKvD;KAEA;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA;EACA;EACA,YAAY;;KAGF;EACV;EACA,OACE,SAAS,iBACR,UAAU,YAAY,QAAQ,UAAU;;KAGjC,UACV,mBAAmB,cAAc,gBAAgB;;;;;iBAenC,UACd,aACA,UAAS,QAAQ,KAAK,qCACrB;iBASa,KACd,aACA,SAAS,KAAK,+BACb;iBAIa,WACd,cACA,cACA,YAAY,oBACX;iBAIa,QAAQ,QAAQ,2BAA2B;KAI/C;EACV;EACA;;KAGU;EACV;EACA,aAAa;;KAGH;EACV;EACA,kBAAkB;IAAa;;EAC/B,6BAA6B,QAAQ;IAAY;;IAC/C;IACA;;EAEF;IACE;IACA;IACA;IACA,WAAW;IACX;IACA,SAAS;;EAEX;;KAGU;EACV;EACA;EACA;IAAY;IAAc,YAAY;;;KAE5B,iBACV,WAAW,YAAY,cAAc;;iBAGvB,eACd,UAAU,iCACT;iBAMa,KAAK,kBAAkB;iBAIvB,MAAM,aAAa,aAAa;KAIpC;EACV,QAAQ;EACR,QAAQ;;EAER,SAAS;;KAGC;EAGN;EACA;EACA;EACA,WAAW;;EAEX,QAAQ;;KAGF;EACV;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;;KAGE;EACV,WAAW;EACX,WAAW;;KAGD;EACV,SAAS;EACT;IAAc,iBAAiB;;EAC/B,WAAW;;EAEX,OAAO;;KAGG;EACV;EACA;EACA;EACA,IACE,SAAS,2BACE,aAAa,kBAAkB;;KAGlC;EACV;EACA;EACA;;EAEA;;KAGU,cAAc,kBAAkB;iBAE5B,UACd,cACA,KAAK,wBACL,UAAS,KAAK,sCACb;iBAIa,MACd,cACA,SAAS,KAAK,2CACb;KAMS;EACV;;EAEA;;EAEA;;KAGU,eAAe,eAAe,aAAa;EACrD;EACA;EACA,OAAO;;EAEP,QAAQ;EACR,WAAW;EACX,YAAY;EACZ,SAAS;EACT,SAAS;EACT,WAAW;;iBAGG,iBAAiB,UAAU,gBAAgB,YAAY,IAAI;KAuB/D,UACV,wBAAwB,4BAA4B;EAEpD;EACA;EACA,OAAO;;iBAGO,kBAAkB,eAAe,WAC/C,OAAO,SACN;KAKS,mBAAmB,iBAAiB,YAAY;iBAE5C,YAAY;EAAS;;KAkBzB;;EAEV,gBAAgB;EAChB,iBAAiB;EACjB,mBAAmB;EACnB,IAAI,cAAc;EAClB,SAAS,cAAc;EACvB,YAAY;EACZ,iBAAiB;EACjB,WAAW;;KAGD;EACV;EACA;EACA;;KAGU;EACV;EACA;EACA;;KAGU;EACV,MAAM;EACN;EACA;EACA,aAAa;;KAGH;EACV;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA,UAAU;MAAQ,MAAM;MAAkB;;;EAE5C,UAAU;EACV;EACA,SAAS;IAAQ;IAAc,MAAM;;;;;;;iBAOvB,oBACR,gCAAgC,oBACtC,eAAe,iBAAiB;KAyGtB;KAEA;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;KAGU;EACV;EACA;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,aAAa;EACb;IAAW;IAAY;;EACvB,MAAM;EACN;;KAGU;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;EACX,QAAQ;EACR;EACA,QAAQ;EACR;IAAoB;IAAoB;;;KAG9B;EACV,SAAS;EACT,cAAc;EACd;EACA;EACA;;KAGU;;EAEV;EACA;EACA;;KAGU;EACV,QAAQ;EACR;EACA;EACA,UAAU;EACV,YAAY;EACZ,QAAQ;;KAGE;EACV,QAAQ;EACR;EACA;EACA;EACA;EACA;EACA,QAAQ;;KAGE;EACV,YAAY,OAAO,kBAAkB;EACrC,YAAY,SAAS,eAAe;;EAEpC,cAAc,cAAc,MAAM,aAAa;EAC/C,SAAS,SAAS,eAAe;;KAGvB;WACD;EACT,WAAW,UAAU,gBAAgB,QAAQ;EAC7C,SAAS,SAAS,aAAa;;KAGrB;EACV,SAAS,UAAU,cAAc,QAAQ;;KAG/B;EACV;EACA,QAAQ;EACR;EACA,UAAU;EACV;EACA;EACA,QAAQ;EACR;EACA;EACA,UAAU;EACV,QAAQ;;KAGE;EACV;EACA;EACA;EACA;;KAGU,YAAY;EACtB;EACA;EACA;EACA;EACA,mBAAmB;;EAEnB,SAAS;;iBAOK,YAAY,iBAAiB"}

package/dist/index.d.mts

@@ -0,0 +1,12 @@
+import { $ as TrialMetadata, $t as Uuid, A as FixtureContext, At as RecordedErrorSchema, B as ReportStore, Bt as TrialSummarySchema, C as EvalRegistryMetadata, Ct as parseTrajectoryJsonl, D as ExpectToolCallStep, Dt as CheckpointResultSchema, E as EvaluatorWorkspace, Et as AutIdentitySchema, F as JsonValue, Ft as ScoreResultSchema, G as RunWriter, Gt as EvalMatrixCell, H as RunResult, Ht as EvalkitConfig, I as JudgeRule, It as TrajectoryEventSchema, J as ScoreValue, Jt as defineEvalMatrix, K as RunnerEvent, Kt as MatrixSelection, L as JudgeRunInfo, Lt as TrialManifestSchema, M as InlineFixture, Mt as RunMetadataSchema, N as JsonObject, Nt as RunStatusSchema, O as FileFixture, Ot as JsonValueSchema, P as JsonPrimitive, Pt as RunSummarySchema, Q as TranscriptStep, Qt as ResourceUriSchema, R as PredicateScorer, Rt as TrialMetadataSchema, S as EvalRegistry, St as user, T as EvalSuiteMetadata, Tt as ArtifactEntrySchema, U as RunStatus, Ut as defineConfig, V as RunMetadata, Vt as UsageSchema, W as RunSummary, Wt as EvalMatrix, X as ScoringRule, Xt as ResourceKindSchema, Y as ScoringContext, Yt as ResourceKind, Z as TrajectoryEvent, Zt as ResourceUri, _ as EvalCatalogEntry, _t as inlineFile, a as AggregateScoring, an as resourceUuid, at as Usage, b as EvalPolicy, bt as recordError, c as AutAdapter, ct as agent, d as AutIdentity, dt as defineEval, en as UuidSchema, et as TrialResult, f as AutSession, ft as defineSuite, g as DynamicFixture, gt as file, h as DirectoryFixture, ht as expectToolCall, i as AgentStep, in as resourceUriSchema, it as TurnView, j as FixtureVisibility, jt as RunManifestSchema, k as Fixture, kt as JudgeRunInfoSchema, l as AutContext, lt as authoringId, m as CheckpointResult, mt as dynamic, n as AgentRuntimeName, nn as parseResourceUri, nt as TrialSummary, o as ArtifactEntry, ot as UserStep, p as CandidateWorkspace, pt as directory, q as ScoreResult, qt as canonicalParameters, r as AgentRuntimes, rn as resourceUri, rt as TrialWriter, s as ArtifactView, st as WorkspaceView, t as AgentRuntime, tn as authorId, tt as TrialScoring, u as AutEvent, ut as defineAgent, v as EvalCatalogFixture, vt as judge, w as EvalSuite, wt as ApiErrorSchema, x as EvalRegistration, xt as registerEvals, y as EvalDefinition, yt as predicate, z as RecordedError, zt as TrialScoringSchema } from "./index-BwNYw4-_.mjs";
+//#region ../agents/src/index.d.ts
+type PiAutOptions = {
+  command?: string;
+  /** Additional non-secret Pi arguments, such as an explicit model selection. */
+  args?: string[];
+};
+/** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
+export declare function piAgent(options?: PiAutOptions): AutAdapter;
+//#endregion
+export { AgentRuntime, AgentRuntimeName, AgentRuntimes, AgentStep, AggregateScoring, ApiErrorSchema, ArtifactEntry, ArtifactEntrySchema, ArtifactView, AutAdapter, AutContext, AutEvent, AutIdentity, AutIdentitySchema, AutSession, CandidateWorkspace, CheckpointResult, CheckpointResultSchema, DirectoryFixture, DynamicFixture, EvalCatalogEntry, EvalCatalogFixture, EvalDefinition, type EvalMatrix, type EvalMatrixCell, EvalPolicy, EvalRegistration, EvalRegistry, EvalRegistryMetadata, EvalSuite, EvalSuiteMetadata, type EvalkitConfig, EvaluatorWorkspace, ExpectToolCallStep, FileFixture, Fixture, FixtureContext, FixtureVisibility, InlineFixture, JsonObject, JsonPrimitive, JsonValue, JsonValueSchema, JudgeRule, JudgeRunInfo, JudgeRunInfoSchema, type MatrixSelection, PredicateScorer, RecordedError, RecordedErrorSchema, ReportStore, ResourceKind, ResourceKindSchema, ResourceUri, ResourceUriSchema, RunManifestSchema, RunMetadata, RunMetadataSchema, RunResult, RunStatus, RunStatusSchema, RunSummary, RunSummarySchema, RunWriter, RunnerEvent, ScoreResult, ScoreResultSchema, ScoreValue, ScoringContext, ScoringRule, TrajectoryEvent, TrajectoryEventSchema, TranscriptStep, TrialManifestSchema, TrialMetadata, TrialMetadataSchema, TrialResult, TrialScoring, TrialScoringSchema, TrialSummary, TrialSummarySchema, TrialWriter, TurnView, Usage, UsageSchema, UserStep, Uuid, UuidSchema, WorkspaceView, agent, authorId, authoringId, canonicalParameters, defineAgent, defineConfig, defineEval, defineEvalMatrix, defineSuite, directory, dynamic, expectToolCall, file, inlineFile, judge, parseResourceUri, parseTrajectoryJsonl, predicate, recordError, registerEvals, resourceUri, resourceUriSchema, resourceUuid, user };
+//# sourceMappingURL=index.d.mts.map

package/dist/index.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.mts","names":[],"sources":["../../agents/src/index.ts"],"mappings":";;KAcY;EACV;;EAEA;;;wBAIc,QAAQ,UAAS,eAAoB"}

package/dist/index.mjs

@@ -0,0 +1,79 @@
+import { A as TrialManifestSchema, B as authorId, C as RecordedErrorSchema, D as RunSummarySchema, E as RunStatusSchema, F as canonicalParameters, H as resourceUri, I as defineEvalMatrix, L as ResourceKindSchema, M as TrialScoringSchema, N as TrialSummarySchema, O as ScoreResultSchema, P as UsageSchema, R as ResourceUriSchema, S as JudgeRunInfoSchema, T as RunMetadataSchema, U as resourceUriSchema, V as parseResourceUri, W as resourceUuid, _ as ApiErrorSchema, a as defineSuite, b as CheckpointResultSchema, c as expectToolCall, d as judge, f as predicate, g as parseTrajectoryJsonl, h as user, i as defineEval, j as TrialMetadataSchema, k as TrajectoryEventSchema, l as file, m as registerEvals, n as authoringId, o as directory, p as recordError, r as defineAgent, s as dynamic, t as agent, u as inlineFile, v as ArtifactEntrySchema, w as RunManifestSchema, x as JsonValueSchema, y as AutIdentitySchema, z as UuidSchema } from "./src-CopGdrwX.mjs";
+//#region ../core/src/config.ts
+/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */
+function defineConfig(configuration) {
+	return configuration;
+}
+//#endregion
+//#region ../agents/src/index.ts
+/** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */
+function piAgent(options = {}) {
+	const command = options.command ?? "pi";
+	return defineAgent({
+		identity: {
+			name: "Pi Agent",
+			kind: "process",
+			id: "pi-agent"
+		},
+		runtimes: { local: {
+			kind: "process",
+			configuration: { command }
+		} },
+		async start({ context, onEvent }) {
+			await onEvent({
+				kind: "started",
+				timestamp: (/* @__PURE__ */ new Date()).toISOString()
+			});
+			let turn = 0;
+			return {
+				async send(message) {
+					turn += 1;
+					await onEvent({
+						kind: "turn-started",
+						turn,
+						timestamp: (/* @__PURE__ */ new Date()).toISOString()
+					});
+					const process = Bun.spawn([
+						command,
+						"--print",
+						"--no-session",
+						"--no-tools",
+						...options.args ?? [],
+						message
+					], {
+						cwd: context.workspace.root,
+						stdout: "pipe",
+						stderr: "pipe"
+					});
+					const [stdout, stderr, exitCode] = await Promise.all([
+						new Response(process.stdout).text(),
+						new Response(process.stderr).text(),
+						process.exited
+					]);
+					if (exitCode !== 0) throw new Error(`Pi exited with ${exitCode}: ${stderr.trim()}`);
+					await onEvent({
+						kind: "message",
+						role: "assistant",
+						content: stdout.trim(),
+						timestamp: (/* @__PURE__ */ new Date()).toISOString()
+					});
+					await onEvent({
+						kind: "turn-completed",
+						turn,
+						timestamp: (/* @__PURE__ */ new Date()).toISOString()
+					});
+				},
+				async close() {
+					await onEvent({
+						kind: "completed",
+						timestamp: (/* @__PURE__ */ new Date()).toISOString()
+					});
+				}
+			};
+		}
+	});
+}
+//#endregion
+export { ApiErrorSchema, ArtifactEntrySchema, AutIdentitySchema, CheckpointResultSchema, JsonValueSchema, JudgeRunInfoSchema, RecordedErrorSchema, ResourceKindSchema, ResourceUriSchema, RunManifestSchema, RunMetadataSchema, RunStatusSchema, RunSummarySchema, ScoreResultSchema, TrajectoryEventSchema, TrialManifestSchema, TrialMetadataSchema, TrialScoringSchema, TrialSummarySchema, UsageSchema, UuidSchema, agent, authorId, authoringId, canonicalParameters, defineAgent, defineConfig, defineEval, defineEvalMatrix, defineSuite, directory, dynamic, expectToolCall, file, inlineFile, judge, parseResourceUri, parseTrajectoryJsonl, piAgent, predicate, recordError, registerEvals, resourceUri, resourceUriSchema, resourceUuid, user };
+
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","names":[],"sources":["../../core/src/config.ts","../../agents/src/index.ts"],"sourcesContent":["import type { EvalDefinition, EvalRegistry } from './index.js';\nimport type { EvalMatrixDefinition } from './matrix.js';\n\nexport type EvalkitConfig = {\n  /** Paths are relative to evalkit.config.js/ts, not the invoking shell directory. */\n  testDir?: string;\n  include?: readonly string[];\n  exclude?: readonly string[];\n  /** Explicit definitions are an alternative to discovery (e.g. hosted registries). */\n  evals?: readonly EvalDefinition[];\n  registry?: EvalRegistry;\n  /** Applied to every discovered eval. Values are forwarded, never provider-interpreted. */\n  matrix?: Omit<EvalMatrixDefinition, 'evals' | 'id'> & { id?: string };\n  execution?: { concurrency?: number; trials?: number; maxCells?: number };\n  reportDir?: string;\n  sandboxDir?: string;\n};\n\n/** Pure, typed project configuration. Loading/discovery belongs to the CLI. */\nexport function defineConfig<const T extends EvalkitConfig>(\n  configuration: T,\n): T {\n  return configuration;\n}\n","import { defineAgent, type AutAdapter } from '@evalkit/core';\n\nexport type AgentsSdkAutOptions = {\n  endpoint: string;\n  agent: string;\n  getToken?: () => Promise<string | undefined> | string | undefined;\n  instanceName?: (context: { runId: string; trialId: string }) => string;\n};\n\n/** Deferred remote Agents SDK transport. */\nexport function agentsSdk(_options: AgentsSdkAutOptions): AutAdapter {\n  throw new Error('Agents SDK transport is not implemented yet');\n}\n\nexport type PiAutOptions = {\n  command?: string;\n  /** Additional non-secret Pi arguments, such as an explicit model selection. */\n  args?: string[];\n};\n\n/** A local, one-prompt-per-turn AUT backed by Pi's non-interactive print mode. */\nexport function piAgent(options: PiAutOptions = {}): AutAdapter {\n  const command = options.command ?? 'pi';\n  return defineAgent({\n    identity: {\n      name: 'Pi Agent',\n      kind: 'process',\n      id: 'pi-agent',\n    },\n    runtimes: { local: { kind: 'process', configuration: { command } } },\n    async start({ context, onEvent }) {\n      await onEvent({ kind: 'started', timestamp: new Date().toISOString() });\n      let turn = 0;\n      return {\n        async send(message: string) {\n          turn += 1;\n          await onEvent({\n            kind: 'turn-started',\n            turn,\n            timestamp: new Date().toISOString(),\n          });\n          const process = Bun.spawn(\n            [\n              command,\n              '--print',\n              '--no-session',\n              '--no-tools',\n              ...(options.args ?? []),\n              message,\n            ],\n            { cwd: context.workspace.root, stdout: 'pipe', stderr: 'pipe' },\n          );\n          const [stdout, stderr, exitCode] = await Promise.all([\n            new Response(process.stdout).text(),\n            new Response(process.stderr).text(),\n            process.exited,\n          ]);\n          if (exitCode !== 0)\n            throw new Error(`Pi exited with ${exitCode}: ${stderr.trim()}`);\n          await onEvent({\n            kind: 'message',\n            role: 'assistant',\n            content: stdout.trim(),\n            timestamp: new Date().toISOString(),\n          });\n          await onEvent({\n            kind: 'turn-completed',\n            turn,\n            timestamp: new Date().toISOString(),\n          });\n        },\n        async close() {\n          await onEvent({\n            kind: 'completed',\n            timestamp: new Date().toISOString(),\n          });\n        },\n      };\n    },\n  });\n}\n"],"mappings":";;;AAmBA,SAAgB,aACd,eACG;CACH,OAAO;AACT;;;;ACFA,SAAgB,QAAQ,UAAwB,CAAC,GAAe;CAC9D,MAAM,UAAU,QAAQ,WAAW;CACnC,OAAO,YAAY;EACjB,UAAU;GACR,MAAM;GACN,MAAM;GACN,IAAI;EACN;EACA,UAAU,EAAE,OAAO;GAAE,MAAM;GAAW,eAAe,EAAE,QAAQ;EAAE,EAAE;EACnE,MAAM,MAAM,EAAE,SAAS,WAAW;GAChC,MAAM,QAAQ;IAAE,MAAM;IAAW,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;GAAE,CAAC;GACtE,IAAI,OAAO;GACX,OAAO;IACL,MAAM,KAAK,SAAiB;KAC1B,QAAQ;KACR,MAAM,QAAQ;MACZ,MAAM;MACN;MACA,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;KACpC,CAAC;KACD,MAAM,UAAU,IAAI,MAClB;MACE;MACA;MACA;MACA;MACA,GAAI,QAAQ,QAAQ,CAAC;MACrB;KACF,GACA;MAAE,KAAK,QAAQ,UAAU;MAAM,QAAQ;MAAQ,QAAQ;KAAO,CAChE;KACA,MAAM,CAAC,QAAQ,QAAQ,YAAY,MAAM,QAAQ,IAAI;MACnD,IAAI,SAAS,QAAQ,MAAM,CAAC,CAAC,KAAK;MAClC,IAAI,SAAS,QAAQ,MAAM,CAAC,CAAC,KAAK;MAClC,QAAQ;KACV,CAAC;KACD,IAAI,aAAa,GACf,MAAM,IAAI,MAAM,kBAAkB,SAAS,IAAI,OAAO,KAAK,GAAG;KAChE,MAAM,QAAQ;MACZ,MAAM;MACN,MAAM;MACN,SAAS,OAAO,KAAK;MACrB,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;KACpC,CAAC;KACD,MAAM,QAAQ;MACZ,MAAM;MACN;MACA,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;KACpC,CAAC;IACH;IACA,MAAM,QAAQ;KACZ,MAAM,QAAQ;MACZ,MAAM;MACN,4BAAW,IAAI,KAAK,EAAA,CAAE,YAAY;KACpC,CAAC;IACH;GACF;EACF;CACF,CAAC;AACH"}

package/dist/runner.d.mts

@@ -0,0 +1,290 @@
+import { B as ReportStore, F as JsonValue, Gt as EvalMatrixCell, H as RunResult, Kt as MatrixSelection, N as JsonObject, V as RunMetadata, Wt as EvalMatrix, Z as TrajectoryEvent, n as AgentRuntimeName, y as EvalDefinition } from "./index-BwNYw4-_.mjs";
+import { Effect } from "effect";
+//#region ../runner/src/errors.d.ts
+declare const AutExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "AutExecutionError";
+} & Readonly<A>;
+export declare class AutExecutionError extends AutExecutionError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const FixtureError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "FixtureError";
+} & Readonly<A>;
+export declare class FixtureError extends FixtureError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const ReportError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "ReportError";
+} & Readonly<A>;
+export declare class ReportError extends ReportError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const ScoringError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "ScoringError";
+} & Readonly<A>;
+export declare class ScoringError extends ScoringError_base<{
+  cause: unknown;
+  message?: string;
+}> {}
+declare const CheckpointExecutionError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "CheckpointExecutionError";
+} & Readonly<A>;
+export declare class CheckpointExecutionError extends CheckpointExecutionError_base<{
+  step: number;
+  name: string;
+  message: string;
+  cause: unknown;
+}> {}
+//#endregion
+//#region ../runner/src/local-report-store.d.ts
+/** Persists each run as a local tree rooted below `rootDirectory`. */
+export declare function localReportStore(rootDirectory: string): ReportStore;
+//#endregion
+//#region ../runner/src/local-report-reader.d.ts
+export declare const readRunManifest: (root: string, runId: string) => Promise<{
+  readonly schemaVersion: 2 | 3;
+  readonly runUri: string;
+  readonly evalId: string;
+  readonly suiteId?: string | undefined;
+  readonly parameters?: {
+    readonly [x: string]: JsonValue;
+  } | undefined;
+  readonly matrix?: {
+    readonly id: string;
+    readonly cellKey: string;
+  } | undefined;
+  readonly aut?: {
+    readonly name?: string | undefined;
+    readonly kind: string;
+    readonly id: string;
+    readonly version?: string | undefined;
+  } | undefined;
+  readonly startedAt: string;
+  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
+}>;
+export declare const readRunSummary: (root: string, runId: string) => Promise<{
+  readonly status: "cancelled" | "completed" | "failed" | "running";
+  readonly endedAt: string;
+  readonly durationMs?: number | undefined;
+  readonly trialCount: number;
+  readonly passed: number;
+  readonly failed: number;
+  readonly error?: {
+    readonly name: string;
+    readonly message: string;
+    readonly stack?: string | undefined;
+  } | undefined;
+}>;
+export declare const readTrialManifest: (root: string, runId: string, trialId: string) => Promise<{
+  readonly schemaVersion: 2 | 3;
+  readonly runUri: string;
+  readonly trialUri: string;
+  readonly trialIndex: number;
+  readonly evalId: string;
+  readonly parameters?: {
+    readonly [x: string]: JsonValue;
+  } | undefined;
+  readonly matrix?: {
+    readonly id: string;
+    readonly cellKey: string;
+  } | undefined;
+  readonly aut?: {
+    readonly name?: string | undefined;
+    readonly kind: string;
+    readonly id: string;
+    readonly version?: string | undefined;
+  } | undefined;
+  readonly startedAt: string;
+  readonly status?: "cancelled" | "completed" | "failed" | "running" | undefined;
+}>;
+export declare const readTrialSummary: (root: string, runId: string, trialId: string) => Promise<{
+  readonly status: "cancelled" | "completed" | "failed" | "running";
+  readonly endedAt: string;
+  readonly durationMs?: number | undefined;
+  readonly scoring?: {
+    readonly results: {
+      readonly name: string;
+      readonly kind: "judge" | "predicate";
+      readonly value?: number | undefined;
+      readonly passed?: boolean | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+      readonly judge?: {
+        readonly agent?: {
+          readonly id: string;
+          readonly kind: string;
+          readonly name?: string | undefined;
+          readonly version?: string | undefined;
+        } | undefined;
+        readonly usage?: {
+          readonly inputTokens?: number | undefined;
+          readonly outputTokens?: number | undefined;
+          readonly totalTokens?: number | undefined;
+        } | undefined;
+        readonly events?: JsonValue[] | undefined;
+      } | undefined;
+      readonly durationMs: number;
+      readonly error?: {
+        readonly name: string;
+        readonly message: string;
+        readonly stack?: string | undefined;
+      } | undefined;
+    }[];
+    readonly checkpoints?: {
+      readonly step: number;
+      readonly kind: "expect-tool-call" | "judge" | "predicate";
+      readonly name: string;
+      readonly status: "error" | "failed" | "passed" | "skipped";
+      readonly value?: number | undefined;
+      readonly passed?: boolean | undefined;
+      readonly explanation?: string | undefined;
+      readonly evidence?: JsonValue | undefined;
+      readonly judge?: {
+        readonly agent?: {
+          readonly id: string;
+          readonly kind: string;
+          readonly name?: string | undefined;
+          readonly version?: string | undefined;
+        } | undefined;
+        readonly usage?: {
+          readonly inputTokens?: number | undefined;
+          readonly outputTokens?: number | undefined;
+          readonly totalTokens?: number | undefined;
+        } | undefined;
+        readonly events?: JsonValue[] | undefined;
+      } | undefined;
+      readonly durationMs?: number | undefined;
+      readonly error?: {
+        readonly name: string;
+        readonly message: string;
+        readonly stack?: string | undefined;
+      } | undefined;
+      readonly matchedToolCall?: {
+        readonly eventIndex: number;
+        readonly id: string;
+      } | undefined;
+    }[] | undefined;
+    readonly skippedScorers?: string[] | undefined;
+    readonly overall?: number | undefined;
+    readonly passed: boolean;
+  } | undefined;
+  readonly artifacts?: {
+    readonly path: string;
+    readonly kind: "directory" | "file";
+    readonly size?: number | undefined;
+  }[] | undefined;
+  readonly error?: {
+    readonly name: string;
+    readonly message: string;
+    readonly stack?: string | undefined;
+  } | undefined;
+}>;
+export declare const readTrialScoring: (root: string, runId: string, trialId: string) => Promise<{
+  readonly results: {
+    readonly name: string;
+    readonly kind: "judge" | "predicate";
+    readonly value?: number | undefined;
+    readonly passed?: boolean | undefined;
+    readonly explanation?: string | undefined;
+    readonly evidence?: JsonValue | undefined;
+    readonly judge?: {
+      readonly agent?: {
+        readonly id: string;
+        readonly kind: string;
+        readonly name?: string | undefined;
+        readonly version?: string | undefined;
+      } | undefined;
+      readonly usage?: {
+        readonly inputTokens?: number | undefined;
+        readonly outputTokens?: number | undefined;
+        readonly totalTokens?: number | undefined;
+      } | undefined;
+      readonly events?: JsonValue[] | undefined;
+    } | undefined;
+    readonly durationMs: number;
+    readonly error?: {
+      readonly name: string;
+      readonly message: string;
+      readonly stack?: string | undefined;
+    } | undefined;
+  }[];
+  readonly checkpoints?: {
+    readonly step: number;
+    readonly kind: "expect-tool-call" | "judge" | "predicate";
+    readonly name: string;
+    readonly status: "error" | "failed" | "passed" | "skipped";
+    readonly value?: number | undefined;
+    readonly passed?: boolean | undefined;
+    readonly explanation?: string | undefined;
+    readonly evidence?: JsonValue | undefined;
+    readonly judge?: {
+      readonly agent?: {
+        readonly id: string;
+        readonly kind: string;
+        readonly name?: string | undefined;
+        readonly version?: string | undefined;
+      } | undefined;
+      readonly usage?: {
+        readonly inputTokens?: number | undefined;
+        readonly outputTokens?: number | undefined;
+        readonly totalTokens?: number | undefined;
+      } | undefined;
+      readonly events?: JsonValue[] | undefined;
+    } | undefined;
+    readonly durationMs?: number | undefined;
+    readonly error?: {
+      readonly name: string;
+      readonly message: string;
+      readonly stack?: string | undefined;
+    } | undefined;
+    readonly matchedToolCall?: {
+      readonly eventIndex: number;
+      readonly id: string;
+    } | undefined;
+  }[] | undefined;
+  readonly skippedScorers?: string[] | undefined;
+  readonly overall?: number | undefined;
+  readonly passed: boolean;
+}>;
+export declare function readTrialEvents(root: string, runId: string, trialId: string): Promise<TrajectoryEvent[]>;
+//#endregion
+//#region ../runner/src/matrix.d.ts
+/** Bounded workers pull cells from the iterator; no Cartesian job array is allocated. */
+export declare function runMatrix(matrix: EvalMatrix, options: Omit<RunEvalOptions, 'runId' | 'trialId' | 'parameters' | 'matrix'> & {
+  selection?: MatrixSelection;
+  onResult?: (cell: EvalMatrixCell, result: RunResult) => void | Promise<void>;
+}): Effect.Effect<{
+  cells: number;
+  passed: number;
+  failed: number;
+}, unknown>;
+//#endregion
+//#region ../runner/src/index.d.ts
+export type RunEvalOptions = {
+  report: ReportStore;
+  runId?: string;
+  trialId?: string;
+  suiteId?: string;
+  /** Overrides the eval policy's requested number of independent trials. */
+  trials?: number;
+  /** Parameters made available to the AUT for this invocation. */
+  parameters?: JsonObject;
+  /** Matrix provenance retained with every aggregate and trial manifest. */
+  matrix?: RunMetadata['matrix'];
+  /** Selects a declared AUT runtime such as local, sandbox, or remote. */
+  runtime?: AgentRuntimeName;
+  /** Maximum number of trials from this aggregate run executing concurrently. */
+  concurrency?: number;
+  /** Optional shared limiter for coordinating trials across aggregate runs. */
+  semaphore?: Effect.Semaphore;
+  /** Base directory for persistent local trial workspaces. Omit for an ephemeral OS temp workspace. */
+  workspaceRoot?: string;
+  now?: () => Date;
+};
+/** Effect-native aggregate eval execution entry point. */
+export declare function runEval(definition: EvalDefinition, options: RunEvalOptions): Effect.Effect<RunResult, unknown>;
+//#endregion
+//# sourceMappingURL=runner.d.mts.map

package/dist/runner.d.mts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.mts","names":["_$_evalkit_core0.JsonValue","_$_evalkit_core0.TrajectoryEvent"],"sources":["../../runner/src/errors.ts","../../runner/src/local-report-store.ts","../../runner/src/local-report-reader.ts","../../runner/src/matrix.ts","../../runner/src/index.ts"],"mappings":";;;;;;qBAEa,0BAA0B;EACrC;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBAGW,oBAAoB;EAC/B;EACA;;;;;qBAGW,qBAAqB;EAChC;EACA;;;;;qBASW,iCAAiC;EAG5C;EACA;EACA;EACA;;;;;wBCmGc,iBAAiB,wBAAwB;;;qBC/G5C,kBAAe,cAAgB,kBAAe;;;;;;0BAA9CA;;;;;;;;;;;;;;;qBAEA,iBAAc,cAAgB,kBAAe;;;;;;;;;;;;;qBAE7C,oBAAiB,cAChB,eACC,oBACE;;;;;;;0BAHJA;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;;;;;0BAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBASA,mBAAgB,cACf,eACC,oBACE;;;;;;;wBAHJA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;wBASS,gBACpB,cACA,eACA,kBAAe,QAHKC;;;;wBC3CN,UACd,QAAQ,YACR,SAAS,KACP;EAGA,YAAY;EACZ,YACE,MAAM,gBACN,QAAQ,qBACE;IAEb,OAAO;EAAS;EAAe;EAAgB;;;;YCoBtC;EACV,QAAQ;EACR;EACA;EACA;;EAEA;;EAEA,aAAa;;EAEb,SAAS;;EAET,UAAU;;EAEV;;EAEA,YAAY,OAAO;;EAEnB;EACA,YAAY;;;wBASE,QACd,YAAY,gBACZ,SAAS,iBACR,OAAO,OAAO"}

package/dist/runner.mjs

@@ -0,0 +1,2 @@
+import { a as readTrialEvents, c as readTrialSummary, d as CheckpointExecutionError, f as FixtureError, i as readRunSummary, l as localReportStore, m as ScoringError, n as runMatrix, o as readTrialManifest, p as ReportError, r as readRunManifest, s as readTrialScoring, t as runEval, u as AutExecutionError } from "./src-DJDem6nZ.mjs";
+export { AutExecutionError, CheckpointExecutionError, FixtureError, ReportError, ScoringError, localReportStore, readRunManifest, readRunSummary, readTrialEvents, readTrialManifest, readTrialScoring, readTrialSummary, runEval, runMatrix };