anpord 0.1.28 → 0.1.29

package/dist/api-context.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_api_mocks = require("./api-mocks-BsUGpsim.cjs");
+const require_api_mocks = require("./api-mocks-p7nRxMS0.cjs");
 let effect = require("effect");
 let node_fs_promises = require("node:fs/promises");
 //#region src/mock-api/context.ts

package/dist/api-context.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.cjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.cjs";
 //#region src/mock-api/context.d.ts
 declare const apiContext: {
   url: (name: string) => Promise<string>;

package/dist/api-context.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.mjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.mjs";
 //#region src/mock-api/context.d.ts
 declare const apiContext: {
   url: (name: string) => Promise<string>;

package/dist/api-context.mjs

@@ -1,4 +1,4 @@
-import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-DH1C7any.mjs";
+import { n as API_MANIFEST, o as ApiCall, s as ApiManifest, t as API_JOURNAL } from "./api-mocks-DDs4SgSW.mjs";
 import { Schema } from "effect";
 import { readFile } from "node:fs/promises";
 //#region src/mock-api/context.ts

package/dist/{api-mocks-CBa3MWVP.d.cts → api-mocks-Bp5izg-M.d.cts}

@@ -11,7 +11,7 @@ declare const ValidationCall: Schema.Struct<{
   index: Schema.refine<number, typeof Schema.NonNegative>;
   method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
   startedAt: typeof Schema.Number;
-  durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+  durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
   input: Schema.Struct<{
     text: Schema.filter<typeof Schema.String>;
     format: Schema.Literal<["text", "json"]>;
@@ -39,7 +39,7 @@ declare const EvalValidation: Schema.Struct<{
   kind: Schema.Literal<["code", "judge", "command"]>;
   status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
   startedAt: Schema.NullOr<typeof Schema.Number>;
-  durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+  durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
   message: Schema.filter<typeof Schema.String>;
   exitCode: Schema.NullOr<typeof Schema.Int>;
   truncated: typeof Schema.Boolean;
@@ -91,7 +91,7 @@ declare const EvalValidation: Schema.Struct<{
     index: Schema.refine<number, typeof Schema.NonNegative>;
     method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
     startedAt: typeof Schema.Number;
-    durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+    durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
     input: Schema.Struct<{
       text: Schema.filter<typeof Schema.String>;
       format: Schema.Literal<["text", "json"]>;

package/dist/{api-mocks-CBa3MWVP.d.mts → api-mocks-Bp5izg-M.d.mts}

@@ -11,7 +11,7 @@ declare const ValidationCall: Schema.Struct<{
   index: Schema.refine<number, typeof Schema.NonNegative>;
   method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
   startedAt: typeof Schema.Number;
-  durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+  durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
   input: Schema.Struct<{
     text: Schema.filter<typeof Schema.String>;
     format: Schema.Literal<["text", "json"]>;
@@ -39,7 +39,7 @@ declare const EvalValidation: Schema.Struct<{
   kind: Schema.Literal<["code", "judge", "command"]>;
   status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
   startedAt: Schema.NullOr<typeof Schema.Number>;
-  durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+  durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
   message: Schema.filter<typeof Schema.String>;
   exitCode: Schema.NullOr<typeof Schema.Int>;
   truncated: typeof Schema.Boolean;
@@ -91,7 +91,7 @@ declare const EvalValidation: Schema.Struct<{
     index: Schema.refine<number, typeof Schema.NonNegative>;
     method: Schema.Literal<["answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"]>;
     startedAt: typeof Schema.Number;
-    durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
+    durationMs: Schema.NullOr<Schema.transform<typeof Schema.Number, Schema.refine<number, typeof Schema.NonNegative>>>;
     input: Schema.Struct<{
       text: Schema.filter<typeof Schema.String>;
       format: Schema.Literal<["text", "json"]>;

package/dist/{api-mocks-DH1C7any.mjs → api-mocks-DDs4SgSW.mjs}

@@ -35,6 +35,12 @@ const REPORTED_LIMITS = {
 	budget: 192e3,
 	text: 64e3
 };
+const DurationMs = Schema.transform(Schema.Number, Schema.NonNegativeInt, {
+	decode: (value) => Math.max(0, Math.round(value)),
+	encode: (value) => value,
+	strict: true
+});
+const validationDuration = (startedAt, finished) => startedAt === null ? null : Math.max(0, Math.round(finished - startedAt));
 const valueWithin = (limit) => Schema.Struct({
 	text: Schema.String.pipe(Schema.maxLength(limit)),
 	format: Schema.Literal("text", "json"),
@@ -45,7 +51,7 @@ const callOf = (value) => Schema.Struct({
 	index: Schema.NonNegativeInt,
 	method: Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: Schema.Number,
-	durationMs: Schema.NullOr(Schema.NonNegativeInt),
+	durationMs: Schema.NullOr(DurationMs),
 	input: value,
 	output: value,
 	error: Schema.NullOr(value)
@@ -57,7 +63,7 @@ const validationOf = (value, call) => Schema.Struct({
 	kind: Schema.Literal("code", "judge", "command"),
 	status: Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
 	startedAt: Schema.NullOr(Schema.Number),
-	durationMs: Schema.NullOr(Schema.NonNegativeInt),
+	durationMs: Schema.NullOr(DurationMs),
 	message: Schema.String.pipe(Schema.maxLength(2e3)),
 	exitCode: Schema.NullOr(Schema.Int),
 	truncated: Schema.Boolean,
@@ -100,6 +106,30 @@ const unavailableValue = {
 	state: "unavailable",
 	truncated: false
 };
+const withoutText = (value) => value.text === "" ? value : {
+	...unavailableValue,
+	format: value.format,
+	truncated: true
+};
+const orNull = (value) => value === null ? null : withoutText(value);
+const validationWithoutEvidence = (value) => ({
+	...value,
+	calls: value.calls.map((call) => ({
+		...call,
+		error: orNull(call.error),
+		input: withoutText(call.input),
+		output: withoutText(call.output)
+	})),
+	error: orNull(value.error),
+	input: withoutText(value.input),
+	logs: value.logs.map((log) => ({
+		...log,
+		value: withoutText(log.value)
+	})),
+	output: withoutText(value.output),
+	truncated: true,
+	...value.metadata === void 0 ? {} : { metadata: withoutText(value.metadata) }
+});
 const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
 	let remaining = limits.budget;
 	return (value, format = "json") => {
@@ -182,4 +212,4 @@ var ApiCall = class extends Schema.Class("ApiCall")({
 	logs: Schema.Array(ReportedValue).pipe(Schema.maxItems(64))
 }) {};
 //#endregion
-export { unavailableValue as _, API_REPORTED_LIMITS as a, validationSnapshot as b, ApiProgram as c, EVIDENCE_LIMITS as d, EvalValidations as f, VALIDATION_TEXT_LIMIT as g, VALIDATION_FRAME as h, API_READY as i, CLI_JOURNAL as l, ReportedValidation as m, API_MANIFEST as n, ApiCall as o, REPORTED_LIMITS as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, MCP_JOURNAL as u, validationCapture as v, redactSecrets as x, validationExecution as y };
+export { redactSecrets as C, validationWithoutEvidence as S, unavailableValue as _, API_REPORTED_LIMITS as a, validationExecution as b, ApiProgram as c, EVIDENCE_LIMITS as d, EvalValidations as f, VALIDATION_TEXT_LIMIT as g, VALIDATION_FRAME as h, API_READY as i, CLI_JOURNAL as l, ReportedValidation as m, API_MANIFEST as n, ApiCall as o, REPORTED_LIMITS as p, API_PROGRAM as r, ApiManifest as s, API_JOURNAL as t, MCP_JOURNAL as u, validationCapture as v, validationSnapshot as x, validationDuration as y };

package/dist/{api-mocks-BsUGpsim.cjs → api-mocks-p7nRxMS0.cjs}

@@ -35,6 +35,12 @@ const REPORTED_LIMITS = {
 	budget: 192e3,
 	text: 64e3
 };
+const DurationMs = effect.Schema.transform(effect.Schema.Number, effect.Schema.NonNegativeInt, {
+	decode: (value) => Math.max(0, Math.round(value)),
+	encode: (value) => value,
+	strict: true
+});
+const validationDuration = (startedAt, finished) => startedAt === null ? null : Math.max(0, Math.round(finished - startedAt));
 const valueWithin = (limit) => effect.Schema.Struct({
 	text: effect.Schema.String.pipe(effect.Schema.maxLength(limit)),
 	format: effect.Schema.Literal("text", "json"),
@@ -45,7 +51,7 @@ const callOf = (value) => effect.Schema.Struct({
 	index: effect.Schema.NonNegativeInt,
 	method: effect.Schema.Literal("answer", "transcript", "turns", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: effect.Schema.Number,
-	durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
+	durationMs: effect.Schema.NullOr(DurationMs),
 	input: value,
 	output: value,
 	error: effect.Schema.NullOr(value)
@@ -57,7 +63,7 @@ const validationOf = (value, call) => effect.Schema.Struct({
 	kind: effect.Schema.Literal("code", "judge", "command"),
 	status: effect.Schema.Literal("queued", "running", "passed", "failed", "error", "skipped"),
 	startedAt: effect.Schema.NullOr(effect.Schema.Number),
-	durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
+	durationMs: effect.Schema.NullOr(DurationMs),
 	message: effect.Schema.String.pipe(effect.Schema.maxLength(2e3)),
 	exitCode: effect.Schema.NullOr(effect.Schema.Int),
 	truncated: effect.Schema.Boolean,
@@ -100,6 +106,30 @@ const unavailableValue = {
 	state: "unavailable",
 	truncated: false
 };
+const withoutText = (value) => value.text === "" ? value : {
+	...unavailableValue,
+	format: value.format,
+	truncated: true
+};
+const orNull = (value) => value === null ? null : withoutText(value);
+const validationWithoutEvidence = (value) => ({
+	...value,
+	calls: value.calls.map((call) => ({
+		...call,
+		error: orNull(call.error),
+		input: withoutText(call.input),
+		output: withoutText(call.output)
+	})),
+	error: orNull(value.error),
+	input: withoutText(value.input),
+	logs: value.logs.map((log) => ({
+		...log,
+		value: withoutText(log.value)
+	})),
+	output: withoutText(value.output),
+	truncated: true,
+	...value.metadata === void 0 ? {} : { metadata: withoutText(value.metadata) }
+});
 const validationCapture = (enabled = true, secrets = [], limits = EVIDENCE_LIMITS) => {
 	let remaining = limits.budget;
 	return (value, format = "json") => {
@@ -296,6 +326,12 @@ Object.defineProperty(exports, "validationCapture", {
 		return validationCapture;
 	}
 });
+Object.defineProperty(exports, "validationDuration", {
+	enumerable: true,
+	get: function() {
+		return validationDuration;
+	}
+});
 Object.defineProperty(exports, "validationExecution", {
 	enumerable: true,
 	get: function() {
@@ -308,3 +344,9 @@ Object.defineProperty(exports, "validationSnapshot", {
 		return validationSnapshot;
 	}
 });
+Object.defineProperty(exports, "validationWithoutEvidence", {
+	enumerable: true,
+	get: function() {
+		return validationWithoutEvidence;
+	}
+});

package/dist/api-runtime.cjs

@@ -1,4 +1,4 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_runtime = require("./runtime-BcX6BpI5.cjs");
+const require_runtime = require("./runtime-V4yw0sb-.cjs");
 exports.runApiServers = require_runtime.runApiServers;
 exports.withApi = require_runtime.withApi;

package/dist/api-runtime.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.cjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.cjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.cjs";
 //#region src/mock-api/runtime.d.ts
 declare const withApi: <A>({ api, run }: {

package/dist/api-runtime.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.mjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.mjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.mjs";
 //#region src/mock-api/runtime.d.ts
 declare const withApi: <A>({ api, run }: {

package/dist/api-runtime.mjs

@@ -1,2 +1,2 @@
-import { n as withApi, t as runApiServers } from "./runtime-CjDUik7_.mjs";
+import { n as withApi, t as runApiServers } from "./runtime-DxvthKib.mjs";
 export { runApiServers, withApi };

package/dist/api.cjs

@@ -1,5 +1,5 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_runtime = require("./runtime-BcX6BpI5.cjs");
+const require_runtime = require("./runtime-V4yw0sb-.cjs");
 exports.api = require_runtime.api;
 exports.endpoint = require_runtime.endpoint;
 exports.withApi = require_runtime.withApi;

package/dist/api.d.cts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.cjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.cjs";
 import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.cjs";
 import { withApi } from "./api-runtime.cjs";
 export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };

package/dist/api.d.mts

@@ -1,4 +1,4 @@
-import { t as ApiCall } from "./api-mocks-CBa3MWVP.mjs";
+import { t as ApiCall } from "./api-mocks-Bp5izg-M.mjs";
 import { a as endpoint, i as api, n as ApiHandlerContext, r as EndpointDefinition, t as ApiDefinition } from "./define-NZvmQuIv.mjs";
 import { withApi } from "./api-runtime.mjs";
 export { type ApiCall, type ApiDefinition, type ApiHandlerContext, type EndpointDefinition, api, endpoint, withApi };

package/dist/api.mjs

@@ -1,2 +1,2 @@
-import { i as endpoint, n as withApi, r as api } from "./runtime-CjDUik7_.mjs";
+import { i as endpoint, n as withApi, r as api } from "./runtime-DxvthKib.mjs";
 export { api, endpoint, withApi };

package/dist/bin.cjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 const require_eval_judges = require("./eval-judges-CO0ds-HQ.cjs");
-const require_api_mocks = require("./api-mocks-BsUGpsim.cjs");
-const require_errors = require("./errors-D1L76YFH.cjs");
-const require_compiler = require("./compiler-DKWUVkxB.cjs");
+const require_api_mocks = require("./api-mocks-p7nRxMS0.cjs");
+const require_errors = require("./errors-C1vD70D6.cjs");
+const require_compiler = require("./compiler-8DrTWGac.cjs");
 const require_eval_turns = require("./eval-turns-B9dd8Vfb.cjs");
 const require_sandbox_env = require("./sandbox-env-B8cLIcku.cjs");
 let effect = require("effect");
@@ -17,7 +17,7 @@ let node_os = require("node:os");
 let node_child_process = require("node:child_process");
 let node_crypto = require("node:crypto");
 //#region package.json
-var version$1 = "0.1.28";
+var version$1 = "0.1.29";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -1795,6 +1795,21 @@ const profileOfRequest = (profile) => profile === void 0 ? null : {
 //#endregion
 //#region ../eval/src/domain/journal-entries.ts
 const JOURNAL_OUTPUT_LIMIT = 4e3;
+const cut = (text) => text.length > JOURNAL_OUTPUT_LIMIT ? `${text.slice(0, JOURNAL_OUTPUT_LIMIT)}\n[the rest was cut before this was sent, ${text.length} characters in all]` : text;
+const cutIfPresent = (text) => text === void 0 ? text : cut(text);
+const cutToJournal = (event) => {
+	if (event._tag === "Command") return {
+		...event,
+		output: cut(event.output)
+	};
+	if (event._tag === "ToolCall") return {
+		...event,
+		error: cutIfPresent(event.error),
+		input: cut(event.input),
+		output: cutIfPresent(event.output)
+	};
+	return event;
+};
 const millisOrNull = (at) => at ?? null;
 const asEntries = (event) => {
 	if (event._tag === "Command") return [{
@@ -3968,6 +3983,10 @@ const TRANSCRIPT_PATH = (home) => `${home}/.anpord-transcript.txt`;
 const TURNS_PATH = (home) => `${home}/.anpord-turns.json`;
 //#endregion
 //#region ../eval/src/domain/journal.ts
+const relativeToWorkspace = (workspace, events) => events.map((event) => event._tag === "FileChange" ? {
+	...event,
+	paths: event.paths.map((path) => node_path.posix.isAbsolute(path) ? node_path.posix.relative(workspace, path) : path)
+} : event);
 const commandsIn = (events) => events.filter((event) => event._tag === "Command").length;
 const filesIn = (events) => [...new Set(events.flatMap((event) => event._tag === "FileChange" ? event.paths : []))];
 const failedCommandsIn = (events) => events.filter((event) => event._tag === "Command" && event.exitCode !== null && event.exitCode !== 0).length;
@@ -4118,7 +4137,7 @@ const completeValidations = (records, message, finished, error, exitCode) => {
 			exitCode,
 			message: message ?? "Validator did not return a complete result",
 			error: record.error ?? error,
-			durationMs: record.startedAt === null ? null : Math.max(0, finished - record.startedAt)
+			durationMs: require_api_mocks.validationDuration(record.startedAt, finished)
 		};
 		if (record.status === "queued" || record.status === "running") return {
 			...record,
@@ -4135,7 +4154,7 @@ const legacyValidation = (record, execution, finished, capture) => {
 		status: raw === null ? "error" : resultStatus(raw.passed),
 		message: (raw?.message ?? (raw === null ? "Validator returned no valid result" : "")).slice(0, 2e3),
 		output: raw === null ? record.output : capture(raw),
-		durationMs: Math.max(0, finished - (record.startedAt ?? finished)),
+		durationMs: require_api_mocks.validationDuration(record.startedAt ?? finished, finished),
 		exitCode: execution.exitCode,
 		logs: [{
 			index: 0,
@@ -5496,7 +5515,7 @@ const AgentTrialLive = effect.Layer.effect(AgentTrial, effect.Effect.gen(functio
 		const apiEvents = yield* api.collect();
 		yield* effect.Stream.fromIterable(apiEvents).pipe(sink.through, effect.Stream.runDrain);
 		yield* api.check;
-		const events = [...conversation.events, ...apiEvents];
+		const events = relativeToWorkspace(request.workspace, [...conversation.events, ...apiEvents]);
 		const modelFinished = yield* effect.Clock.currentTimeMillis;
 		const artifacts = yield* captureArtifacts(sandbox, request.workspace, filesIn(events), secrets);
 		const scored = yield* scorer.score({
@@ -5876,7 +5895,7 @@ const brokenBy = (cause, durationMs, events) => ({
 	timedOut: effect.Option.exists(effect.Cause.failureOption(cause), (error) => error instanceof TrialTimedOut)
 });
 const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	ordinal,
 	outcome: result.outcome,
 	runId,
@@ -5884,7 +5903,7 @@ const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { p
 	usage: result.usage,
 	userSpend: result.userSpend
 } } : { payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	failure: result.reason,
 	ordinal,
 	runId,
@@ -5903,7 +5922,7 @@ const UNSCORED = {
 	voidFields: []
 };
 const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	ordinal,
 	outcome: UNSCORED,
 	runId,
@@ -5911,6 +5930,17 @@ const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
 	usage: null,
 	userSpend: null
 } });
+const withoutJournal = (result) => ({
+	...result,
+	events: []
+});
+const withoutEvidence = (result) => result.kind === "scored" ? {
+	...result,
+	outcome: {
+		...result.outcome,
+		validations: result.outcome.validations.map(require_api_mocks.validationWithoutEvidence)
+	}
+} : result;
 const verdictOf = (result) => result.kind === "scored" ? result.outcome : {
 	failure: result.reason,
 	status: result.timedOut ? "timed out" : "void",
@@ -5936,6 +5966,10 @@ const labelOfRequest = (variant) => formatVariant({
 	model: variant.model,
 	profile: variant.profile?.name ?? null
 });
+const casesAtOnce = effect.Config.string("EVAL_LOCAL_CASES_AT_ONCE").pipe(effect.Config.withDefault("1"), effect.Config.map((value) => value === "unbounded" ? "unbounded" : Number.parseInt(value, 10)), effect.Config.validate({
+	message: "EVAL_LOCAL_CASES_AT_ONCE must be a positive number or unbounded",
+	validation: (value) => value === "unbounded" || value > 0
+}));
 const localConfig = effect.ConfigProvider.fromEnv().pipe(effect.ConfigProvider.orElse(() => effect.ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))));
 const printed = (lines) => lines.length === 0 ? effect.Effect.void : note(lines.join("\n"));
 const runVariant = (request, variant, transcript, options) => effect.Effect.gen(function* () {
@@ -5998,7 +6032,7 @@ const runVariant = (request, variant, transcript, options) => effect.Effect.gen(
 			variant: label
 		});
 	});
-	return yield* effect.Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency: 1 });
+	return yield* effect.Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency: yield* casesAtOnce });
 }).pipe(effect.Effect.provide(options.credentials === void 0 || options.credentials.size === 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), effect.Effect.scoped);
 const runLocally = (request, options = {}) => effect.Effect.gen(function* () {
 	const transcript = yield* makeTranscriber(stderrStyle());
@@ -6267,9 +6301,19 @@ const untilClosed = (batchId) => require_errors.AnpordApi.pipe(effect.Effect.fla
 	const refused = require_errors.asAnpordError(error);
 	return refused.status === CLOSED_BATCH ? effect.Effect.fail(new BatchClosed({ message: refused.message })) : effect.Effect.void;
 }), effect.Effect.repeat(effect.Schedule.spaced(LOCAL_BEAT_EVERY)), effect.Effect.zipRight(effect.Effect.never));
-const reportTrial = (api, result, ordinal, runId) => {
+const TOO_LARGE = 413;
+const tooLarge = (error) => _effect_platform.HttpClientError.isHttpClientError(error) && error._tag === "ResponseError" && error.response.status === TOO_LARGE;
+const SHRUNK = [{
+	gaveUp: "its journal",
+	shrink: withoutJournal
+}, {
+	gaveUp: "its journal or the evidence its checks captured",
+	shrink: (result) => withoutEvidence(withoutJournal(result))
+}];
+const reportTrial = (api, result, ordinal, runId, trial) => {
 	const send = (request) => retryTransient(api.runner.report(request));
-	return send(reportRequest(result, ordinal, runId)).pipe(effect.Effect.catchTag("HttpApiDecodeError", (refused) => result.kind === "broken" ? send(unscoredForOlderServer(result, ordinal, runId)) : effect.Effect.fail(refused)));
+	const sendTrial = (sent) => send(reportRequest(sent, ordinal, runId)).pipe(effect.Effect.catchTag("HttpApiDecodeError", (refused) => sent.kind === "broken" ? send(unscoredForOlderServer(sent, ordinal, runId)) : effect.Effect.fail(refused)));
+	return SHRUNK.reduce((attempt, rung) => attempt.pipe(effect.Effect.catchIf(tooLarge, () => sendTrial(rung.shrink(result)).pipe(effect.Effect.zipRight(note(`A trial of ${trial} was too large for Anpord to take, so its verdict was recorded without ${rung.gaveUp}.`))))), sendTrial(result));
 };
 const recordInto = (label, request, ui, started) => effect.Effect.gen(function* () {
 	const api = yield* require_errors.AnpordApi;
@@ -6282,7 +6326,10 @@ const recordInto = (label, request, ui, started) => effect.Effect.gen(function*
 	return {
 		cases: yield* runLocally(request, {
 			credentials: leases,
-			onTrial: (slot, result, ordinal) => reportTrial(api, result, ordinal, runIdOf(slot)).pipe(effect.Effect.catchAll((error) => note(`A trial of ${slot.caseId} on ${labelOfRequest(slot.variant)} was not recorded, so it shows as void. ${require_errors.asAnpordError(error).message}`)))
+			onTrial: (slot, result, ordinal) => {
+				const trial = `${slot.caseId} on ${labelOfRequest(slot.variant)}`;
+				return reportTrial(api, result, ordinal, runIdOf(slot), trial).pipe(effect.Effect.catchAll((error) => note(`A trial of ${trial} was not recorded, so it shows as void. ${require_errors.asAnpordError(error).message}`)));
+			}
 		}),
 		costs: yield* finishBatch(started.id),
 		link: effect.Option.some(link)

package/dist/bin.mjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { i as HarnessUsage, r as HarnessEvent } from "./eval-judges-DVxvZQl5.mjs";
-import { b as validationSnapshot, c as ApiProgram, d as EVIDENCE_LIMITS, g as VALIDATION_TEXT_LIMIT, h as VALIDATION_FRAME, i as API_READY, m as ReportedValidation, o as ApiCall, r as API_PROGRAM, s as ApiManifest, t as API_JOURNAL, v as validationCapture, x as redactSecrets, y as validationExecution } from "./api-mocks-DH1C7any.mjs";
-import { C as VersionNumber, S as PromptId, _ as EvalHarness, a as apiKeyConfig, b as Permission, c as AnpordApi, d as IdempotencyKey, g as EvalBatch, i as ClientLayer, o as webUrlConfig, p as WEB_ORIGIN, r as asAnpordError, s as tokenMatcher, u as neverAnswered, v as isSettledTrial, x as ChannelName, y as COST_COMPONENT_LABELS } from "./errors-DaYv6_WJ.mjs";
-import { f as DEFAULT_TIMEOUT_MS$1, o as EvalSandbox, p as EvalCaseId, r as compileEvalEffect, u as EvalTrigger } from "./compiler-DDc7Towh.mjs";
+import { C as redactSecrets, S as validationWithoutEvidence, b as validationExecution, c as ApiProgram, d as EVIDENCE_LIMITS, g as VALIDATION_TEXT_LIMIT, h as VALIDATION_FRAME, i as API_READY, m as ReportedValidation, o as ApiCall, r as API_PROGRAM, s as ApiManifest, t as API_JOURNAL, v as validationCapture, x as validationSnapshot, y as validationDuration } from "./api-mocks-DDs4SgSW.mjs";
+import { C as VersionNumber, S as PromptId, _ as EvalHarness, a as apiKeyConfig, b as Permission, c as AnpordApi, d as IdempotencyKey, g as EvalBatch, i as ClientLayer, o as webUrlConfig, p as WEB_ORIGIN, r as asAnpordError, s as tokenMatcher, u as neverAnswered, v as isSettledTrial, x as ChannelName, y as COST_COMPONENT_LABELS } from "./errors-H6woEW0d.mjs";
+import { f as DEFAULT_TIMEOUT_MS$1, o as EvalSandbox, p as EvalCaseId, r as compileEvalEffect, u as EvalTrigger } from "./compiler-dmc6AOx6.mjs";
 import { t as DEFAULT_USER_MODEL } from "./eval-turns-JXtnOI0D.mjs";
 import { a as TRANSCRIPT_ENV, i as PREPARE_VALUE_ENV, n as CACHE_RESTORED_ENV, o as TURNS_ENV, r as PREPARE_RESULT_MARKER, t as ANSWER_ENV } from "./sandbox-env-s-smXj2C.mjs";
 import { Array as Array$1, Cause, Chunk, Clock, Config, ConfigProvider, Console, Context, Data, Deferred, Duration, Effect, Either, Exit, Fiber, JSONSchema, Layer, Option, ParseResult, Predicate, Random, Redacted, Ref, Schedule, ScheduleDecision, Schema, Scope, Stream, pipe } from "effect";
@@ -17,7 +17,7 @@ import { homedir, tmpdir } from "node:os";
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 //#region package.json
-var version$1 = "0.1.28";
+var version$1 = "0.1.29";
 //#endregion
 //#region ../template/src/extract.ts
 /** Reads escapes too, so text the renderer leaves literal is never reported as
@@ -1795,6 +1795,21 @@ const profileOfRequest = (profile) => profile === void 0 ? null : {
 //#endregion
 //#region ../eval/src/domain/journal-entries.ts
 const JOURNAL_OUTPUT_LIMIT = 4e3;
+const cut = (text) => text.length > JOURNAL_OUTPUT_LIMIT ? `${text.slice(0, JOURNAL_OUTPUT_LIMIT)}\n[the rest was cut before this was sent, ${text.length} characters in all]` : text;
+const cutIfPresent = (text) => text === void 0 ? text : cut(text);
+const cutToJournal = (event) => {
+	if (event._tag === "Command") return {
+		...event,
+		output: cut(event.output)
+	};
+	if (event._tag === "ToolCall") return {
+		...event,
+		error: cutIfPresent(event.error),
+		input: cut(event.input),
+		output: cutIfPresent(event.output)
+	};
+	return event;
+};
 const millisOrNull = (at) => at ?? null;
 const asEntries = (event) => {
 	if (event._tag === "Command") return [{
@@ -3968,6 +3983,10 @@ const TRANSCRIPT_PATH = (home) => `${home}/.anpord-transcript.txt`;
 const TURNS_PATH = (home) => `${home}/.anpord-turns.json`;
 //#endregion
 //#region ../eval/src/domain/journal.ts
+const relativeToWorkspace = (workspace, events) => events.map((event) => event._tag === "FileChange" ? {
+	...event,
+	paths: event.paths.map((path) => posix.isAbsolute(path) ? posix.relative(workspace, path) : path)
+} : event);
 const commandsIn = (events) => events.filter((event) => event._tag === "Command").length;
 const filesIn = (events) => [...new Set(events.flatMap((event) => event._tag === "FileChange" ? event.paths : []))];
 const failedCommandsIn = (events) => events.filter((event) => event._tag === "Command" && event.exitCode !== null && event.exitCode !== 0).length;
@@ -4118,7 +4137,7 @@ const completeValidations = (records, message, finished, error, exitCode) => {
 			exitCode,
 			message: message ?? "Validator did not return a complete result",
 			error: record.error ?? error,
-			durationMs: record.startedAt === null ? null : Math.max(0, finished - record.startedAt)
+			durationMs: validationDuration(record.startedAt, finished)
 		};
 		if (record.status === "queued" || record.status === "running") return {
 			...record,
@@ -4135,7 +4154,7 @@ const legacyValidation = (record, execution, finished, capture) => {
 		status: raw === null ? "error" : resultStatus(raw.passed),
 		message: (raw?.message ?? (raw === null ? "Validator returned no valid result" : "")).slice(0, 2e3),
 		output: raw === null ? record.output : capture(raw),
-		durationMs: Math.max(0, finished - (record.startedAt ?? finished)),
+		durationMs: validationDuration(record.startedAt ?? finished, finished),
 		exitCode: execution.exitCode,
 		logs: [{
 			index: 0,
@@ -5496,7 +5515,7 @@ const AgentTrialLive = Layer.effect(AgentTrial, Effect.gen(function* () {
 		const apiEvents = yield* api.collect();
 		yield* Stream.fromIterable(apiEvents).pipe(sink.through, Stream.runDrain);
 		yield* api.check;
-		const events = [...conversation.events, ...apiEvents];
+		const events = relativeToWorkspace(request.workspace, [...conversation.events, ...apiEvents]);
 		const modelFinished = yield* Clock.currentTimeMillis;
 		const artifacts = yield* captureArtifacts(sandbox, request.workspace, filesIn(events), secrets);
 		const scored = yield* scorer.score({
@@ -5876,7 +5895,7 @@ const brokenBy = (cause, durationMs, events) => ({
 	timedOut: Option.exists(Cause.failureOption(cause), (error) => error instanceof TrialTimedOut)
 });
 const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	ordinal,
 	outcome: result.outcome,
 	runId,
@@ -5884,7 +5903,7 @@ const reportRequest = (result, ordinal, runId) => result.kind === "scored" ? { p
 	usage: result.usage,
 	userSpend: result.userSpend
 } } : { payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	failure: result.reason,
 	ordinal,
 	runId,
@@ -5903,7 +5922,7 @@ const UNSCORED = {
 	voidFields: []
 };
 const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
-	events: result.events,
+	events: result.events.map(cutToJournal),
 	ordinal,
 	outcome: UNSCORED,
 	runId,
@@ -5911,6 +5930,17 @@ const unscoredForOlderServer = (result, ordinal, runId) => ({ payload: {
 	usage: null,
 	userSpend: null
 } });
+const withoutJournal = (result) => ({
+	...result,
+	events: []
+});
+const withoutEvidence = (result) => result.kind === "scored" ? {
+	...result,
+	outcome: {
+		...result.outcome,
+		validations: result.outcome.validations.map(validationWithoutEvidence)
+	}
+} : result;
 const verdictOf = (result) => result.kind === "scored" ? result.outcome : {
 	failure: result.reason,
 	status: result.timedOut ? "timed out" : "void",
@@ -5936,6 +5966,10 @@ const labelOfRequest = (variant) => formatVariant({
 	model: variant.model,
 	profile: variant.profile?.name ?? null
 });
+const casesAtOnce = Config.string("EVAL_LOCAL_CASES_AT_ONCE").pipe(Config.withDefault("1"), Config.map((value) => value === "unbounded" ? "unbounded" : Number.parseInt(value, 10)), Config.validate({
+	message: "EVAL_LOCAL_CASES_AT_ONCE must be a positive number or unbounded",
+	validation: (value) => value === "unbounded" || value > 0
+}));
 const localConfig = ConfigProvider.fromEnv().pipe(ConfigProvider.orElse(() => ConfigProvider.fromMap(/* @__PURE__ */ new Map([["ANPORD_LOCAL_SANDBOX", "true"]]))));
 const printed = (lines) => lines.length === 0 ? Effect.void : note(lines.join("\n"));
 const runVariant = (request, variant, transcript, options) => Effect.gen(function* () {
@@ -5998,7 +6032,7 @@ const runVariant = (request, variant, transcript, options) => Effect.gen(functio
 			variant: label
 		});
 	});
-	return yield* Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency: 1 });
+	return yield* Effect.forEach(request.cases.flatMap((subject) => ordinals.map((ordinal) => [subject, ordinal])), ([subject, ordinal]) => attempt(subject, ordinal), { concurrency: yield* casesAtOnce });
 }).pipe(Effect.provide(options.credentials === void 0 || options.credentials.size === 0 ? EvalLocalLive : evalLocalWith(credentialResolverFrom(options.credentials))), Effect.scoped);
 const runLocally = (request, options = {}) => Effect.gen(function* () {
 	const transcript = yield* makeTranscriber(stderrStyle());
@@ -6267,9 +6301,19 @@ const untilClosed = (batchId) => AnpordApi.pipe(Effect.flatMap((api) => api.runn
 	const refused = asAnpordError(error);
 	return refused.status === CLOSED_BATCH ? Effect.fail(new BatchClosed({ message: refused.message })) : Effect.void;
 }), Effect.repeat(Schedule.spaced(LOCAL_BEAT_EVERY)), Effect.zipRight(Effect.never));
-const reportTrial = (api, result, ordinal, runId) => {
+const TOO_LARGE = 413;
+const tooLarge = (error) => HttpClientError.isHttpClientError(error) && error._tag === "ResponseError" && error.response.status === TOO_LARGE;
+const SHRUNK = [{
+	gaveUp: "its journal",
+	shrink: withoutJournal
+}, {
+	gaveUp: "its journal or the evidence its checks captured",
+	shrink: (result) => withoutEvidence(withoutJournal(result))
+}];
+const reportTrial = (api, result, ordinal, runId, trial) => {
 	const send = (request) => retryTransient(api.runner.report(request));
-	return send(reportRequest(result, ordinal, runId)).pipe(Effect.catchTag("HttpApiDecodeError", (refused) => result.kind === "broken" ? send(unscoredForOlderServer(result, ordinal, runId)) : Effect.fail(refused)));
+	const sendTrial = (sent) => send(reportRequest(sent, ordinal, runId)).pipe(Effect.catchTag("HttpApiDecodeError", (refused) => sent.kind === "broken" ? send(unscoredForOlderServer(sent, ordinal, runId)) : Effect.fail(refused)));
+	return SHRUNK.reduce((attempt, rung) => attempt.pipe(Effect.catchIf(tooLarge, () => sendTrial(rung.shrink(result)).pipe(Effect.zipRight(note(`A trial of ${trial} was too large for Anpord to take, so its verdict was recorded without ${rung.gaveUp}.`))))), sendTrial(result));
 };
 const recordInto = (label, request, ui, started) => Effect.gen(function* () {
 	const api = yield* AnpordApi;
@@ -6282,7 +6326,10 @@ const recordInto = (label, request, ui, started) => Effect.gen(function* () {
 	return {
 		cases: yield* runLocally(request, {
 			credentials: leases,
-			onTrial: (slot, result, ordinal) => reportTrial(api, result, ordinal, runIdOf(slot)).pipe(Effect.catchAll((error) => note(`A trial of ${slot.caseId} on ${labelOfRequest(slot.variant)} was not recorded, so it shows as void. ${asAnpordError(error).message}`)))
+			onTrial: (slot, result, ordinal) => {
+				const trial = `${slot.caseId} on ${labelOfRequest(slot.variant)}`;
+				return reportTrial(api, result, ordinal, runIdOf(slot), trial).pipe(Effect.catchAll((error) => note(`A trial of ${trial} was not recorded, so it shows as void. ${asAnpordError(error).message}`)));
+			}
 		}),
 		costs: yield* finishBatch(started.id),
 		link: Option.some(link)