anpord 0.1.13 → 0.1.17

package/dist/{eval-validations-_MlnQ_uv.cjs → eval-validations-BdlPhudg.cjs}

@@ -1,4 +1,4 @@
-const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
+const require_eval_judges = require("./eval-judges-1zF4PVqV.cjs");
 let effect = require("effect");
 //#region ../schema/src/domain/eval-validations.ts
 const VALIDATION_TEXT_LIMIT = 16e3;
@@ -12,7 +12,7 @@ const ValidationValue = effect.Schema.Struct({
 }).annotations({ identifier: "ValidationValue" });
 const ValidationCall = effect.Schema.Struct({
 	index: effect.Schema.NonNegativeInt,
-	method: effect.Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"),
+	method: effect.Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: effect.Schema.Number,
 	durationMs: effect.Schema.NullOr(effect.Schema.NonNegativeInt),
 	input: ValidationValue,
@@ -122,6 +122,12 @@ Object.defineProperty(exports, "VALIDATION_FRAME", {
 		return VALIDATION_FRAME;
 	}
 });
+Object.defineProperty(exports, "ValidationValue", {
+	enumerable: true,
+	get: function() {
+		return ValidationValue;
+	}
+});
 Object.defineProperty(exports, "unavailableValue", {
 	enumerable: true,
 	get: function() {

package/dist/{eval-validations-BRdoZrTQ.mjs → eval-validations-BvX9dkYh.mjs}

@@ -1,4 +1,4 @@
-import { n as EvalJudgment } from "./eval-judges-CwTvTq7N.mjs";
+import { n as EvalJudgment } from "./eval-judges-Bee_ABJt.mjs";
 import { Schema } from "effect";
 //#region ../schema/src/domain/eval-validations.ts
 const VALIDATION_TEXT_LIMIT = 16e3;
@@ -12,7 +12,7 @@ const ValidationValue = Schema.Struct({
 }).annotations({ identifier: "ValidationValue" });
 const ValidationCall = Schema.Struct({
 	index: Schema.NonNegativeInt,
-	method: Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"),
+	method: Schema.Literal("answer", "transcript", "readText", "exists", "exec", "cli.calls", "api.calls", "api.url", "mcp.calls"),
 	startedAt: Schema.Number,
 	durationMs: Schema.NullOr(Schema.NonNegativeInt),
 	input: ValidationValue,
@@ -110,4 +110,4 @@ const validationExecution = (identity, startedAt) => ({
 	logs: []
 });
 //#endregion
-export { validationExecution as a, validationCapture as i, VALIDATION_FRAME as n, validationSnapshot as o, unavailableValue as r, EvalValidations as t };
+export { validationCapture as a, unavailableValue as i, VALIDATION_FRAME as n, validationExecution as o, ValidationValue as r, validationSnapshot as s, EvalValidations as t };

package/dist/eval.cjs

@@ -1,4 +1,7 @@
 Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
-const require_compiler = require("./compiler-uwb1pkdK.cjs");
+const require_compiler = require("./compiler-CYwJNlT4.cjs");
+exports.compileDefinition = require_compiler.compileDefinition;
+exports.compileDefinitionEffect = require_compiler.compileDefinitionEffect;
 exports.compileEval = require_compiler.compileEval;
 exports.compileEvalEffect = require_compiler.compileEvalEffect;
+exports.compileRefEffect = require_compiler.compileRefEffect;

package/dist/eval.d.cts

@@ -1,4 +1,4 @@
-import { h as PublicStartEvalRequest, t as CaseCache } from "./types-ISsteq95.cjs";
+import { h as PublicStartEvalRequest, i as EvalDefinition } from "./types-C2nRDJ95.cjs";
 import { Effect } from "effect";
 //#region src/evals/profile-errors.d.ts
 declare const ProfileDirectoryUnreadable_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
@@ -79,12 +79,45 @@ declare class ProfileStepNotSupported extends ProfileStepNotSupported_base<{
   get message(): string;
 }
 //#endregion
+//#region src/evals/runner-source.d.ts
+interface DefinitionRef {
+  readonly entry: string;
+  readonly exportName: string | null;
+}
+//#endregion
 //#region src/evals/compiler.d.ts
-declare const compileEvalEffect: (path: string) => Effect.Effect<{
-  cases: {
-    validator: ({
+declare const compileRefEffect: (ref: DefinitionRef) => Effect.Effect<{
+  readonly trigger?: {
+    readonly source: "api" | "dashboard" | "cli" | "ci" | "mcp";
+    readonly url?: string | undefined;
+  } | undefined;
+  readonly name?: string | undefined;
+  readonly prompt: string;
+  readonly cases: readonly {
+    readonly source?: {
+      readonly kind: "empty";
+    } | {
+      readonly url: string;
+      readonly kind: "repo";
+      readonly ref: string | null;
+    } | {
+      readonly kind: "files";
+      readonly files: {
+        readonly [x: string]: string;
+      };
+    } | undefined;
+    readonly cache?: {
+      readonly key: string;
+      readonly path: string;
+    } | undefined;
+    readonly name: string;
+    readonly prepare?: {
+      readonly source: string;
       readonly name: string;
+    } | null | undefined;
+    readonly validator?: ({
       readonly source: string;
+      readonly name: string;
       readonly capture?: boolean | undefined;
       readonly manifest?: readonly {
         readonly name: string;
@@ -104,8 +137,8 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
         readonly content: string;
       }[] | undefined;
       readonly checks: readonly {
-        readonly name: string;
         readonly source: string;
+        readonly name: string;
         readonly capture?: boolean | undefined;
         readonly manifest?: readonly {
           readonly name: string;
@@ -115,10 +148,10 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
       readonly judges: readonly ({
         readonly name: string;
         readonly kind: "judge";
+        readonly provider: "openai";
         readonly harness?: undefined;
         readonly model: string;
-        readonly provider: "openai";
-        readonly rubric: string;
+        readonly prompt: string;
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -128,10 +161,121 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
       } | {
         readonly name: string;
         readonly kind: "judge";
-        readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen";
+        readonly provider?: undefined;
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
         readonly model: string;
+        readonly prompt: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      })[];
+    } | null | undefined;
+    readonly variables?: {
+      readonly [x: string]: string;
+    } | undefined;
+    readonly verify: string | null;
+  }[];
+  readonly tasks: readonly {
+    readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor" | "command";
+    readonly model: string;
+    readonly profile?: {
+      readonly run?: string | undefined;
+      readonly name: string;
+      readonly files: {
+        readonly [x: string]: string;
+      };
+      readonly env?: {
+        readonly [x: string]: string;
+      } | undefined;
+      readonly install?: string | undefined;
+      readonly systemPrompt?: string | undefined;
+    } | undefined;
+    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
+  }[];
+  readonly trials: number;
+}, Error | import("effect/ParseResult").ParseError | import("effect/ConfigError").ConfigError | ProfileTooManyFiles | ProfileDirectoryUnreadable | ProfileTooLarge | ProfileFileTooLarge | ProfilePathInvalid | ProfileManifestInvalid | ProfileManifestOutside | CommandProfileNeedsRun | ProfileStepNotSupported, never>;
+declare const compileDefinitionEffect: (definition: EvalDefinition) => Effect.Effect<{
+  readonly trigger?: {
+    readonly source: "api" | "dashboard" | "cli" | "ci" | "mcp";
+    readonly url?: string | undefined;
+  } | undefined;
+  readonly name?: string | undefined;
+  readonly prompt: string;
+  readonly cases: readonly {
+    readonly source?: {
+      readonly kind: "empty";
+    } | {
+      readonly url: string;
+      readonly kind: "repo";
+      readonly ref: string | null;
+    } | {
+      readonly kind: "files";
+      readonly files: {
+        readonly [x: string]: string;
+      };
+    } | undefined;
+    readonly cache?: {
+      readonly key: string;
+      readonly path: string;
+    } | undefined;
+    readonly name: string;
+    readonly prepare?: {
+      readonly source: string;
+      readonly name: string;
+    } | null | undefined;
+    readonly validator?: ({
+      readonly source: string;
+      readonly name: string;
+      readonly capture?: boolean | undefined;
+      readonly manifest?: readonly {
+        readonly name: string;
+        readonly index: number;
+      }[] | undefined;
+    } & {
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+    }) | {
+      readonly name: string;
+      readonly kind: "judged";
+      readonly capture?: boolean | undefined;
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+      readonly checks: readonly {
+        readonly source: string;
+        readonly name: string;
+        readonly capture?: boolean | undefined;
+        readonly manifest?: readonly {
+          readonly name: string;
+          readonly index: number;
+        }[] | undefined;
+      }[];
+      readonly judges: readonly ({
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider: "openai";
+        readonly harness?: undefined;
+        readonly model: string;
+        readonly prompt: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      } | {
+        readonly name: string;
+        readonly kind: "judge";
         readonly provider?: undefined;
-        readonly rubric: string;
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly model: string;
+        readonly prompt: string;
         readonly expected?: string | undefined;
         readonly choices: {
           readonly [x: string]: number;
@@ -139,22 +283,17 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
         readonly threshold: number;
         readonly timeoutMs: number;
       })[];
-    } | null;
-    variables: Readonly<Record<string, string>>;
-    verify: string | null;
-    name: string;
-    prepare: {
-      name: string;
-      source: string;
-    } | null;
-    cache?: CaseCache | undefined;
+    } | null | undefined;
+    readonly variables?: {
+      readonly [x: string]: string;
+    } | undefined;
+    readonly verify: string | null;
   }[];
-  name: string;
-  prompt: string;
-  tasks: {
-    readonly harness: "cursor" | "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "command";
+  readonly tasks: readonly {
+    readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor" | "command";
     readonly model: string;
     readonly profile?: {
+      readonly run?: string | undefined;
       readonly name: string;
       readonly files: {
         readonly [x: string]: string;
@@ -163,13 +302,124 @@ declare const compileEvalEffect: (path: string) => Effect.Effect<{
         readonly [x: string]: string;
       } | undefined;
       readonly install?: string | undefined;
+      readonly systemPrompt?: string | undefined;
+    } | undefined;
+    readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
+  }[];
+  readonly trials: number;
+}, Error | import("effect/ParseResult").ParseError | import("effect/ConfigError").ConfigError | ProfileTooManyFiles | ProfileDirectoryUnreadable | ProfileTooLarge | ProfileFileTooLarge | ProfilePathInvalid | ProfileManifestInvalid | ProfileManifestOutside | CommandProfileNeedsRun | ProfileStepNotSupported, never>;
+declare const compileDefinition: (definition: EvalDefinition) => Promise<PublicStartEvalRequest>;
+declare const compileEvalEffect: (path: string) => Effect.Effect<{
+  readonly trigger?: {
+    readonly source: "api" | "dashboard" | "cli" | "ci" | "mcp";
+    readonly url?: string | undefined;
+  } | undefined;
+  readonly name?: string | undefined;
+  readonly prompt: string;
+  readonly cases: readonly {
+    readonly source?: {
+      readonly kind: "empty";
+    } | {
+      readonly url: string;
+      readonly kind: "repo";
+      readonly ref: string | null;
+    } | {
+      readonly kind: "files";
+      readonly files: {
+        readonly [x: string]: string;
+      };
+    } | undefined;
+    readonly cache?: {
+      readonly key: string;
+      readonly path: string;
+    } | undefined;
+    readonly name: string;
+    readonly prepare?: {
+      readonly source: string;
+      readonly name: string;
+    } | null | undefined;
+    readonly validator?: ({
+      readonly source: string;
+      readonly name: string;
+      readonly capture?: boolean | undefined;
+      readonly manifest?: readonly {
+        readonly name: string;
+        readonly index: number;
+      }[] | undefined;
+    } & {
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+    }) | {
+      readonly name: string;
+      readonly kind: "judged";
+      readonly capture?: boolean | undefined;
+      readonly sourceFiles?: readonly {
+        readonly path: string;
+        readonly content: string;
+      }[] | undefined;
+      readonly checks: readonly {
+        readonly source: string;
+        readonly name: string;
+        readonly capture?: boolean | undefined;
+        readonly manifest?: readonly {
+          readonly name: string;
+          readonly index: number;
+        }[] | undefined;
+      }[];
+      readonly judges: readonly ({
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider: "openai";
+        readonly harness?: undefined;
+        readonly model: string;
+        readonly prompt: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      } | {
+        readonly name: string;
+        readonly kind: "judge";
+        readonly provider?: undefined;
+        readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor";
+        readonly model: string;
+        readonly prompt: string;
+        readonly expected?: string | undefined;
+        readonly choices: {
+          readonly [x: string]: number;
+        };
+        readonly threshold: number;
+        readonly timeoutMs: number;
+      })[];
+    } | null | undefined;
+    readonly variables?: {
+      readonly [x: string]: string;
+    } | undefined;
+    readonly verify: string | null;
+  }[];
+  readonly tasks: readonly {
+    readonly harness: "codex" | "opencode" | "pi" | "fx" | "claude" | "gemini" | "qwen" | "cursor" | "command";
+    readonly model: string;
+    readonly profile?: {
       readonly run?: string | undefined;
+      readonly name: string;
+      readonly files: {
+        readonly [x: string]: string;
+      };
+      readonly env?: {
+        readonly [x: string]: string;
+      } | undefined;
+      readonly install?: string | undefined;
       readonly systemPrompt?: string | undefined;
     } | undefined;
     readonly sandbox?: "daytona" | "e2b" | "upstash" | "modal" | "cloudflare" | "vercel" | undefined;
   }[];
-  trials: number;
-}, Error | import("effect/ConfigError").ConfigError | import("effect/ParseResult").ParseError | ProfileTooManyFiles | ProfileDirectoryUnreadable | ProfileTooLarge | ProfileFileTooLarge | ProfilePathInvalid | ProfileManifestInvalid | ProfileManifestOutside | CommandProfileNeedsRun | ProfileStepNotSupported, never>;
+  readonly trials: number;
+}, Error | import("effect/ParseResult").ParseError | import("effect/ConfigError").ConfigError | ProfileTooManyFiles | ProfileDirectoryUnreadable | ProfileTooLarge | ProfileFileTooLarge | ProfilePathInvalid | ProfileManifestInvalid | ProfileManifestOutside | CommandProfileNeedsRun | ProfileStepNotSupported, never>;
 declare const compileEval: (path: string) => Promise<PublicStartEvalRequest>;
 //#endregion
-export { compileEval, compileEvalEffect };
+export { compileDefinition, compileDefinitionEffect, compileEval, compileEvalEffect, compileRefEffect };