anpord 0.1.21 → 0.1.22

package/dist/{evals-api-CK3F1KoI.mjs → evals-api-NLUUV9tM.mjs}

@@ -117,6 +117,8 @@ var ApiKeyAuthentication = class extends HttpApiMiddleware.Tag()("@anpord/schema
 const PROMPT_LIMIT = 16384;
 const VERIFY_LIMIT = 8192;
 const NAME_LIMIT = 200;
+const TAG_LIMIT = 40;
+const TAGS_PER_CASE = 12;
 const VARIABLE_VALUE_LIMIT = 2048;
 const EvalPrompt = Schema.String.pipe(Schema.maxLength(PROMPT_LIMIT), Schema.annotations({
 	description: "What the agent is asked to do.",
@@ -133,6 +135,11 @@ const EvalCaseName = Schema.String.pipe(Schema.maxLength(NAME_LIMIT), Schema.ann
 	identifier: "EvalCaseName",
 	message: () => `A case name must be at most ${NAME_LIMIT} characters.`
 }));
+Schema.Array(Schema.String.pipe(Schema.minLength(1), Schema.maxLength(TAG_LIMIT))).pipe(Schema.maxItems(TAGS_PER_CASE), Schema.annotations({
+	description: "What a case is grouped under.",
+	identifier: "EvalCaseTags",
+	message: () => `A case may carry at most ${TAGS_PER_CASE} tags of ${TAG_LIMIT} characters.`
+}));
 const EvalVariableValue = Schema.String.pipe(Schema.maxLength(VARIABLE_VALUE_LIMIT), Schema.annotations({
 	identifier: "EvalVariableValue",
 	message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
@@ -536,6 +543,28 @@ const EvalRunPage = Schema.Struct({
 	runs: Schema.Array(EvalRunSummary),
 	total: Schema.Int
 });
+const EvalCaseSummary = Schema.Struct({
+	cellKey: Schema.String,
+	distribution: EvalDistribution,
+	harness: Schema.String,
+	lastRunAtMillis: Schema.Int,
+	lastRunId: Schema.String,
+	model: Schema.String,
+	name: Schema.String,
+	runCount: Schema.Int,
+	suite: Schema.NullOr(Schema.String),
+	tags: Schema.Array(Schema.String)
+}).annotations({
+	description: "A case as the list shows it, with its newest run.",
+	identifier: "EvalCaseSummary"
+});
+const EvalCasePage = Schema.Struct({
+	cases: Schema.Array(EvalCaseSummary),
+	tags: Schema.Array(Schema.String)
+}).annotations({
+	description: "Cases and every tag they carry between them.",
+	identifier: "EvalCasePage"
+});
 const EvalCellHistoryEntry = Schema.Struct({
 	trigger: Schema.optionalWith(Schema.NullOr(EvalTrigger), { default: () => null }),
 	distribution: EvalDistribution,
@@ -727,6 +756,13 @@ const PublicEvalSandbox = Schema.Literal(...HOSTED_SANDBOXES).annotations({
 	description: "The hosted sandbox a task runs in.",
 	identifier: "PublicEvalSandbox"
 });
+const ListCasesRequest = Schema.Struct({
+	limit: Schema.optional(Schema.Int),
+	tag: Schema.optional(Schema.NullOr(Schema.String))
+}).annotations({
+	description: "Which cases to read.",
+	identifier: "ListCasesRequest"
+});
 const ListEvalsRequest = Schema.Struct({
 	cursor: Schema.optional(Schema.NullOr(EvalPageCursor)),
 	limit: Schema.optional(Schema.Int)
@@ -768,6 +804,23 @@ const PublicStartEvalRequest = Schema.Struct({
 	description: `Start a grid with at most 100 total case, task, and trial combinations.`,
 	identifier: "StartEvalRequest"
 });
+const CredentialLeaseRequest = Schema.Struct({
+	harness: EvalHarness,
+	id: Schema.String
+}).annotations({
+	description: "Ask for the credentials a run the caller is executing needs, for the harness it names.",
+	identifier: "CredentialLeaseRequest"
+});
+const CredentialLease = Schema.Struct({
+	expiresAt: Schema.DateTimeUtc,
+	values: Schema.Record({
+		key: Schema.String,
+		value: Schema.String
+	})
+}).annotations({
+	description: "Credentials for one run, held in memory and never written down. Short-lived: start another run rather than keeping these.",
+	identifier: "CredentialLease"
+});
 const ReportedTrial = Schema.Struct({
 	caseName: EvalCaseName,
 	events: Schema.Array(HarnessEvent),
@@ -787,6 +840,6 @@ const ReportTrialRequest = Schema.Struct({
 	description: "A trial the caller ran, named by the run it belongs to.",
 	identifier: "ReportTrialRequest"
 });
-var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(Schema.Void).annotate(OpenApi.Summary, "Report a trial run outside the platform").annotate(OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(OpenApi.Summary, "Watch an eval run as it moves").annotate(OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
+var PublicEvalsGroup = class extends HttpApiGroup.make("evals").add(HttpApiEndpoint.post("list", "/evals.list").setPayload(ListEvalsRequest).addSuccess(EvalRunPage).annotate(OpenApi.Summary, "List eval runs").annotate(OpenApi.Description, "Newest first. Pass the `next` cursor from a response to read the page after it; a null `next` means there are no more.")).add(HttpApiEndpoint.post("cases", "/evals.cases").setPayload(ListCasesRequest).addSuccess(EvalCasePage).annotate(OpenApi.Summary, "List cases").annotate(OpenApi.Description, "One row per case, carrying its newest run. Pass `tag` to read only the cases carrying it.")).add(HttpApiEndpoint.post("start", "/evals.start").setPayload(PublicStartEvalRequest).addSuccess(StartedEval).annotate(OpenApi.Summary, "Start an eval run").annotate(OpenApi.Description, `Starts the grid and returns its id while trials continue in the background. ${PROFILE_HARNESS_RULE}`)).add(HttpApiEndpoint.post("credentials", "/evals.credentials").setPayload(CredentialLeaseRequest).addSuccess(CredentialLease).annotate(OpenApi.Summary, "Lease the credentials a local run needs").annotate(OpenApi.Description, "For a run started with executeLocally, so the machine running it holds no credentials of its own. Returns the organization credential for one harness, expiring in minutes. Sandbox credentials are never leased: a local run opens no cloud sandbox.")).add(HttpApiEndpoint.post("reportTrial", "/evals.reportTrial").setPayload(ReportTrialRequest).addSuccess(Schema.Void).annotate(OpenApi.Summary, "Report a trial run outside the platform").annotate(OpenApi.Description, "For a run started with executeLocally. The result is recorded as reported: it was produced somewhere the platform cannot inspect, so it is marked and kept out of baselines.")).add(HttpApiEndpoint.post("finishRun", "/evals.finishRun").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Settle a run whose trials the caller ran").annotate(OpenApi.Description, "Settles the run from the trials reported so far. A run left unsettled is swept like any other abandoned work.")).add(HttpApiEndpoint.post("artifact", "/evals.artifact").setPayload(EvalArtifactRequest).addSuccess(EvalArtifact)).add(HttpApiEndpoint.post("get", "/evals.get").setPayload(EvalRunRequest).addSuccess(EvalRun).annotate(OpenApi.Summary, "Get an eval run")).add(HttpApiEndpoint.post("subscription", "/evals.subscription").setPayload(EvalRunRequest).addSuccess(RunSubscription).annotate(OpenApi.Summary, "Watch an eval run as it moves").annotate(OpenApi.Description, "Returns a read-only token scoped to this run, for following it in real time. Short-lived: request another when it expires.")).add(HttpApiEndpoint.post("cellHistory", "/evals.cellHistory").setPayload(EvalCellRequest).addSuccess(Schema.Array(EvalCellHistoryEntry)).annotate(OpenApi.Summary, "List a cell's history").annotate(OpenApi.Description, "Returns the 20 most recent results.")).add(HttpApiEndpoint.post("rerunCell", "/evals.rerunCell").setPayload(Schema.extend(EvalRunRequest, Schema.extend(EvalCellRequest, RerunCellRequest))).addSuccess(StartedEval).annotate(OpenApi.Summary, "Rerun one cell")).add(HttpApiEndpoint.post("models", "/evals.models").setPayload(EvalModelsRequest).addSuccess(ModelCatalogue).annotate(OpenApi.Summary, "List models available to the harness").annotate(OpenApi.Description, "The command harness has no catalogue of its own, so its list is empty: the model is whatever the profile's run command reads from ANPORD_MODEL.")).addError(BadRequest).addError(Conflict).addError(Forbidden).addError(NotFound).middleware(ApiKeyAuthentication).annotate(OpenApi.Title, "Evals").annotate(OpenApi.Description, "Run cases across harness, model, and sandbox combinations and compare the results with their baselines.") {};
 //#endregion
 export { CredentialValues as S, Forbidden as _, EvalSandbox as a, CredentialIntegration as b, HarnessProfile as c, profileFitsHarness as d, EvalSourceFiles as f, Conflict as g, BadRequest as h, EvalRun as i, PROFILE_LIMITS as l, ApiKeyAuthentication as m, PublicStartEvalRequest as n, EvalValidator as o, EvalTrigger as p, HarnessEvent as r, EnvName as s, PublicEvalsGroup as t, ProfilePath as u, NotFound as v, CredentialScope as x, CredentialConnection as y };

package/dist/index.cjs

@@ -1,7 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_client = require("./client-BZDftC1x.cjs");
+const require_client = require("./client-CWVAt_wk.cjs");
 const require_errors = require("./errors-BX1wry8K.cjs");
-const require_compiler = require("./compiler-Cy85vNbp.cjs");
+const require_compiler = require("./compiler-B1UrHfCL.cjs");
 const require_source = require("./source-aW5nSo2h.cjs");
 let effect = require("effect");
 let _effect_platform = require("@effect/platform");
@@ -413,6 +413,7 @@ exports.EvalTimeout = EvalTimeout;
 exports.McpCallSchema = McpCallSchema;
 exports.MissingApiKey = require_errors.MissingApiKey;
 exports.empty = require_source.empty;
+exports.evalCase = require_compiler.evalCase;
 exports.files = require_source.files;
 exports.layer = require_client.layer;
 exports.make = require_client.make;

package/dist/index.d.cts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-CsGxLUZq.cjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-Dvd_z5ge.cjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.cjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-CE9VurDH.cjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as SingleCaseDefinition, h as ValidatorResult, i as EvalDefinition, l as PrepareContext, m as ValidatorContext, n as CommandResult, o as ExecOptions, p as Validator, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-C1Np-yYM.cjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-M2G4zvnS.cjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.cjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.cjs";
 import { empty, files, repo } from "./source.cjs";
@@ -348,10 +348,11 @@ declare class AnpordError extends Error {
 //#endregion
 //#region src/evals/define.d.ts
 declare function suite<const Definition extends EvalDefinition>(definition: Definition): Definition;
+declare function evalCase(definition: SingleCaseDefinition): EvalDefinition;
 //#endregion
 //#region src/index.d.ts
 type StartEvalRequest = Parameters<EvalsSurface["start"]>[0];
 type EvalCase = StartEvalRequest["cases"][number];
 type EvalTaskRequest = StartEvalRequest["tasks"][number];
 //#endregion
-export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, CliCallSchema, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, type EvalArtifact, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSandbox, type EvalSource, type EvalTask, type EvalTaskDefinition, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalTrigger, type EvalUsage, type EvalValidation, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, type HarnessRef, type McpCall, McpCallSchema, MissingApiKey, type ModelCatalogue, type Prepare, type PrepareContext, type PrepareValue, type ProfileRef, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type ValidationCall, type ValidationValue, type Validator, type ValidatorContext, type ValidatorResult, type WaitOptions, empty, files, layer, make, repo, suite };
+export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, CliCallSchema, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, type EvalArtifact, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSandbox, type EvalSource, type EvalTask, type EvalTaskDefinition, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalTrigger, type EvalUsage, type EvalValidation, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, type HarnessRef, type McpCall, McpCallSchema, MissingApiKey, type ModelCatalogue, type Prepare, type PrepareContext, type PrepareValue, type ProfileRef, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type ValidationCall, type ValidationValue, type Validator, type ValidatorContext, type ValidatorResult, type WaitOptions, empty, evalCase, files, layer, make, repo, suite };

package/dist/index.d.mts

@@ -1,6 +1,6 @@
-import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-C7iEhiDm.mjs";
-import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-D-h67MVJ.mjs";
-import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-Dw2Ek-mx.mjs";
+import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-EdYxRHX0.mjs";
+import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as SingleCaseDefinition, h as ValidatorResult, i as EvalDefinition, l as PrepareContext, m as ValidatorContext, n as CommandResult, o as ExecOptions, p as Validator, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-B1rRGWeL.mjs";
+import { _ as EvalVerdict, a as EvalDistribution, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as EvalArtifact, u as EvalRunSummary } from "./evals-M2G4zvnS.mjs";
 import { n as McpCallSchema, t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as CliCallSchema, t as CliCall } from "./calls-6Z1EoEa-.mjs";
 import { empty, files, repo } from "./source.mjs";
@@ -348,10 +348,11 @@ declare class AnpordError extends Error {
 //#endregion
 //#region src/evals/define.d.ts
 declare function suite<const Definition extends EvalDefinition>(definition: Definition): Definition;
+declare function evalCase(definition: SingleCaseDefinition): EvalDefinition;
 //#endregion
 //#region src/index.d.ts
 type StartEvalRequest = Parameters<EvalsSurface["start"]>[0];
 type EvalCase = StartEvalRequest["cases"][number];
 type EvalTaskRequest = StartEvalRequest["tasks"][number];
 //#endregion
-export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, CliCallSchema, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, type EvalArtifact, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSandbox, type EvalSource, type EvalTask, type EvalTaskDefinition, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalTrigger, type EvalUsage, type EvalValidation, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, type HarnessRef, type McpCall, McpCallSchema, MissingApiKey, type ModelCatalogue, type Prepare, type PrepareContext, type PrepareValue, type ProfileRef, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type ValidationCall, type ValidationValue, type Validator, type ValidatorContext, type ValidatorResult, type WaitOptions, empty, files, layer, make, repo, suite };
+export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, CliCallSchema, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, type EvalArtifact, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSandbox, type EvalSource, type EvalTask, type EvalTaskDefinition, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalTrigger, type EvalUsage, type EvalValidation, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, type HarnessRef, type McpCall, McpCallSchema, MissingApiKey, type ModelCatalogue, type Prepare, type PrepareContext, type PrepareValue, type ProfileRef, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type ValidationCall, type ValidationValue, type Validator, type ValidatorContext, type ValidatorResult, type WaitOptions, empty, evalCase, files, layer, make, repo, suite };

package/dist/index.mjs

@@ -1,6 +1,6 @@
-import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-D0l3kwWV.mjs";
+import { i as make, n as DEFAULT_BASE_URL, r as layer, t as AnpordApi } from "./client-CfPf0hIc.mjs";
 import { i as tokenMatcher, n as MissingApiKey, r as asAnpordError, t as AnpordError } from "./errors-B0YknR5V.mjs";
-import { a as suite, i as sourceUrlOf, t as compileDefinition } from "./compiler-DSQRHvXS.mjs";
+import { a as sourceUrlOf, i as evalCase, o as suite, t as compileDefinition } from "./compiler-D9jidpZ0.mjs";
 import { n as files, r as repo, t as empty } from "./source-BP11pzVV.mjs";
 import { Cache, Cause, Clock, Context, Data, DateTime, Duration, Effect, Exit, Layer, ManagedRuntime, Option, Redacted, Schema } from "effect";
 import { FetchHttpClient } from "@effect/platform";
@@ -402,4 +402,4 @@ const CliCallSchema = Schema.Struct({
 	output: Schema.optional(Schema.Unknown)
 });
 //#endregion
-export { Anpord, AnpordApi, AnpordError, CliCallSchema, DEFAULT_BASE_URL, EvalAborted, EvalTimeout, McpCallSchema, MissingApiKey, empty, files, layer, make, repo, suite };
+export { Anpord, AnpordApi, AnpordError, CliCallSchema, DEFAULT_BASE_URL, EvalAborted, EvalTimeout, McpCallSchema, MissingApiKey, empty, evalCase, files, layer, make, repo, suite };

package/dist/source.d.cts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-Dw2Ek-mx.cjs";
+import { f as EvalSource } from "./evals-M2G4zvnS.cjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/source.d.mts

@@ -1,4 +1,4 @@
-import { f as EvalSource } from "./evals-Dw2Ek-mx.mjs";
+import { f as EvalSource } from "./evals-M2G4zvnS.mjs";
 //#region src/evals/source.d.ts
 declare const repo: (spec: string) => EvalSource;
 declare const files: (contents: Readonly<Record<string, string>>) => EvalSource;

package/dist/{types-D-h67MVJ.d.mts → types-B1rRGWeL.d.mts}

@@ -1,7 +1,7 @@
 import "./errors-aGrd21jy.mjs";
-import { t as ApiCall } from "./api-mocks-BenVx0e8.mjs";
-import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-CTCoSuB6.mjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-Dw2Ek-mx.mjs";
+import { t as ApiCall } from "./api-mocks-B8a93t2t.mjs";
+import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-Uwe0G7PS.mjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-M2G4zvnS.mjs";
 import { t as McpCall } from "./calls-DJGZJPrF.mjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.mjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.mjs";
@@ -21,8 +21,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       readonly kind: "empty";
     } | {
       readonly kind: "repo";
-      readonly url: string;
       readonly ref: string | null;
+      readonly url: string;
     } | {
       readonly kind: "files";
       readonly files: {
@@ -39,8 +39,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     } | null | undefined;
     readonly user?: {
       readonly kind: "simulated";
-      readonly goal: string;
       readonly prompt: string;
+      readonly goal: string;
     } | {
       readonly kind: "scripted";
       readonly replies: readonly string[];
@@ -59,8 +59,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly content: string;
       }[] | undefined;
     }) | {
-      readonly name: string;
       readonly kind: "judged";
+      readonly name: string;
       readonly capture?: boolean | undefined;
       readonly sourceFiles?: readonly {
         readonly path: string;
@@ -76,12 +76,12 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         }[] | undefined;
       }[];
       readonly judges: readonly ({
-        readonly name: string;
         readonly kind: "judge";
+        readonly provider: "openai";
         readonly harness?: undefined;
+        readonly name: string;
         readonly model: string;
         readonly prompt: string;
-        readonly provider: "openai";
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -89,12 +89,12 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly threshold: number;
         readonly timeoutMs: number;
       } | {
-        readonly name: string;
         readonly kind: "judge";
-        readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen";
+        readonly provider?: undefined;
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly name: string;
         readonly model: string;
         readonly prompt: string;
-        readonly provider?: undefined;
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -178,7 +178,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         }>;
       }>, Schema.Struct<{
         provider: Schema.optional<typeof Schema.Never>;
-        harness: Schema.SchemaClass<"cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", never>;
+        harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
         kind: Schema.Literal<["judge"]>;
         name: Schema.filter<Schema.filter<typeof Schema.String>>;
         model: Schema.filter<Schema.filter<typeof Schema.String>>;
@@ -204,19 +204,19 @@ declare const PublicStartEvalRequest: Schema.Struct<{
   name: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
   prompt: Schema.refine<string, typeof Schema.String>;
   tasks: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
-    readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "command";
+    readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor" | "command";
     readonly model: string;
     readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
     readonly profile?: {
+      readonly run?: string | undefined;
+      readonly files: {
+        readonly [x: string]: string;
+      };
       readonly name: string;
       readonly env?: {
         readonly [x: string]: string;
       } | undefined;
-      readonly files: {
-        readonly [x: string]: string;
-      };
       readonly install?: string | undefined;
-      readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
   }, Schema.Struct<{
@@ -316,6 +316,9 @@ interface EvalCaseBase {
   readonly name: string;
   readonly prepare?: Prepare | null;
   readonly source?: DeclaredSource;
+  /** How this case is grouped. A case may carry several, and retagging one
+   * keeps its baseline, because tags are not part of its identity. */
+  readonly tags?: readonly string[];
   readonly user?: EvalUser;
   readonly variables?: Readonly<Record<string, string>>;
 }
@@ -326,6 +329,12 @@ type EvalCaseDefinition = EvalCaseBase & ({
   readonly validate?: never;
   readonly verify: string;
 });
+/** One case and what to run it on, for an eval that measures a single thing. */
+type SingleCaseDefinition = EvalCaseDefinition & {
+  readonly prompt: string;
+  readonly tasks: readonly EvalTaskDefinition[];
+  readonly trials: number;
+};
 interface EvalDefinition {
   readonly api?: readonly ApiDefinition[];
   readonly captureSource?: boolean;
@@ -340,4 +349,4 @@ interface EvalDefinition {
   readonly trials: number;
 }
 //#endregion
-export { EvalTaskDefinition as a, Prepare as c, ProfileRef as d, Validator as f, PublicStartEvalRequest as h, EvalDefinition as i, PrepareContext as l, ValidatorResult as m, CommandResult as n, ExecOptions as o, ValidatorContext as p, EvalCaseDefinition as r, HarnessRef as s, CaseCache as t, PrepareValue as u };
+export { EvalTaskDefinition as a, Prepare as c, ProfileRef as d, SingleCaseDefinition as f, PublicStartEvalRequest as g, ValidatorResult as h, EvalDefinition as i, PrepareContext as l, ValidatorContext as m, CommandResult as n, ExecOptions as o, Validator as p, EvalCaseDefinition as r, HarnessRef as s, CaseCache as t, PrepareValue as u };

package/dist/{types-Dvd_z5ge.d.cts → types-C1Np-yYM.d.cts}

@@ -1,7 +1,7 @@
 import "./errors-aGrd21jy.cjs";
-import { t as ApiCall } from "./api-mocks-BenVx0e8.cjs";
-import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-CTCoSuB6.cjs";
-import { f as EvalSource, o as EvalHarness } from "./evals-Dw2Ek-mx.cjs";
+import { t as ApiCall } from "./api-mocks-B8a93t2t.cjs";
+import { a as EvalJudge, i as EvalUser, r as EvalTurn } from "./eval-turns-Uwe0G7PS.cjs";
+import { f as EvalSource, o as EvalHarness } from "./evals-M2G4zvnS.cjs";
 import { t as McpCall } from "./calls-DJGZJPrF.cjs";
 import { n as McpServerDefinition } from "./define-HK0eeSNn.cjs";
 import { t as ApiDefinition } from "./define-NZvmQuIv.cjs";
@@ -21,8 +21,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
       readonly kind: "empty";
     } | {
       readonly kind: "repo";
-      readonly url: string;
       readonly ref: string | null;
+      readonly url: string;
     } | {
       readonly kind: "files";
       readonly files: {
@@ -39,8 +39,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
     } | null | undefined;
     readonly user?: {
       readonly kind: "simulated";
-      readonly goal: string;
       readonly prompt: string;
+      readonly goal: string;
     } | {
       readonly kind: "scripted";
       readonly replies: readonly string[];
@@ -59,8 +59,8 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly content: string;
       }[] | undefined;
     }) | {
-      readonly name: string;
       readonly kind: "judged";
+      readonly name: string;
       readonly capture?: boolean | undefined;
       readonly sourceFiles?: readonly {
         readonly path: string;
@@ -76,12 +76,12 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         }[] | undefined;
       }[];
       readonly judges: readonly ({
-        readonly name: string;
         readonly kind: "judge";
+        readonly provider: "openai";
         readonly harness?: undefined;
+        readonly name: string;
         readonly model: string;
         readonly prompt: string;
-        readonly provider: "openai";
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -89,12 +89,12 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         readonly threshold: number;
         readonly timeoutMs: number;
       } | {
-        readonly name: string;
         readonly kind: "judge";
-        readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen";
+        readonly provider?: undefined;
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly name: string;
         readonly model: string;
         readonly prompt: string;
-        readonly provider?: undefined;
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -178,7 +178,7 @@ declare const PublicStartEvalRequest: Schema.Struct<{
         }>;
       }>, Schema.Struct<{
         provider: Schema.optional<typeof Schema.Never>;
-        harness: Schema.SchemaClass<"cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen", never>;
+        harness: Schema.SchemaClass<"codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor", never>;
         kind: Schema.Literal<["judge"]>;
         name: Schema.filter<Schema.filter<typeof Schema.String>>;
         model: Schema.filter<Schema.filter<typeof Schema.String>>;
@@ -204,19 +204,19 @@ declare const PublicStartEvalRequest: Schema.Struct<{
   name: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
   prompt: Schema.refine<string, typeof Schema.String>;
   tasks: Schema.filter<Schema.filter<Schema.Array$<Schema.refine<{
-    readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "command";
+    readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor" | "command";
     readonly model: string;
     readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
     readonly profile?: {
+      readonly run?: string | undefined;
+      readonly files: {
+        readonly [x: string]: string;
+      };
       readonly name: string;
       readonly env?: {
         readonly [x: string]: string;
       } | undefined;
-      readonly files: {
-        readonly [x: string]: string;
-      };
       readonly install?: string | undefined;
-      readonly run?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
   }, Schema.Struct<{
@@ -316,6 +316,9 @@ interface EvalCaseBase {
   readonly name: string;
   readonly prepare?: Prepare | null;
   readonly source?: DeclaredSource;
+  /** How this case is grouped. A case may carry several, and retagging one
+   * keeps its baseline, because tags are not part of its identity. */
+  readonly tags?: readonly string[];
   readonly user?: EvalUser;
   readonly variables?: Readonly<Record<string, string>>;
 }
@@ -326,6 +329,12 @@ type EvalCaseDefinition = EvalCaseBase & ({
   readonly validate?: never;
   readonly verify: string;
 });
+/** One case and what to run it on, for an eval that measures a single thing. */
+type SingleCaseDefinition = EvalCaseDefinition & {
+  readonly prompt: string;
+  readonly tasks: readonly EvalTaskDefinition[];
+  readonly trials: number;
+};
 interface EvalDefinition {
   readonly api?: readonly ApiDefinition[];
   readonly captureSource?: boolean;
@@ -340,4 +349,4 @@ interface EvalDefinition {
   readonly trials: number;
 }
 //#endregion
-export { EvalTaskDefinition as a, Prepare as c, ProfileRef as d, Validator as f, PublicStartEvalRequest as h, EvalDefinition as i, PrepareContext as l, ValidatorResult as m, CommandResult as n, ExecOptions as o, ValidatorContext as p, EvalCaseDefinition as r, HarnessRef as s, CaseCache as t, PrepareValue as u };
+export { EvalTaskDefinition as a, Prepare as c, ProfileRef as d, SingleCaseDefinition as f, PublicStartEvalRequest as g, ValidatorResult as h, EvalDefinition as i, PrepareContext as l, ValidatorContext as m, CommandResult as n, ExecOptions as o, Validator as p, EvalCaseDefinition as r, HarnessRef as s, CaseCache as t, PrepareValue as u };

package/dist/validator-runtime.d.cts

@@ -1,4 +1,4 @@
-import { f as Validator } from "./types-Dvd_z5ge.cjs";
+import { p as Validator } from "./types-C1Np-yYM.cjs";
 //#region src/evals/validator-runtime.d.ts
 declare const runValidators: (checks: readonly {
   index: number;

package/dist/validator-runtime.d.mts

@@ -1,4 +1,4 @@
-import { f as Validator } from "./types-D-h67MVJ.mjs";
+import { p as Validator } from "./types-B1rRGWeL.mjs";
 //#region src/evals/validator-runtime.d.ts
 declare const runValidators: (checks: readonly {
   index: number;

package/dist/validators.d.cts

@@ -1,4 +1,4 @@
-import { a as EvalJudge, i as EvalUser, n as EvalSimulatedUser, o as EvalJudgment, r as EvalTurn, t as EvalScriptedUser } from "./eval-turns-CTCoSuB6.cjs";
+import { a as EvalJudge, i as EvalUser, n as EvalSimulatedUser, o as EvalJudgment, r as EvalTurn, t as EvalScriptedUser } from "./eval-turns-Uwe0G7PS.cjs";
 import { Schema } from "effect";
 //#region ../schema/src/domain/eval-classifiers.d.ts
 declare const EvalClassifier: Schema.filter<Schema.Struct<{

package/dist/validators.d.mts

@@ -1,4 +1,4 @@
-import { a as EvalJudge, i as EvalUser, n as EvalSimulatedUser, o as EvalJudgment, r as EvalTurn, t as EvalScriptedUser } from "./eval-turns-CTCoSuB6.mjs";
+import { a as EvalJudge, i as EvalUser, n as EvalSimulatedUser, o as EvalJudgment, r as EvalTurn, t as EvalScriptedUser } from "./eval-turns-Uwe0G7PS.mjs";
 import { Schema } from "effect";
 //#region ../schema/src/domain/eval-classifiers.d.ts
 declare const EvalClassifier: Schema.filter<Schema.Struct<{

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anpord",
-  "version": "0.1.21",
+  "version": "0.1.22",
   "description": "TypeScript SDK and CLI for coding agent evals and prompt management.",
   "license": "MIT",
   "type": "module",