@open-insight/eval 0.0.0

package/dist/index-CjLj-Fvc.d.mts

@@ -0,0 +1,863 @@
+import { Brand, Cause, Context, Data, Effect, FileSystem, Layer, Path, PlatformError, Queue, Schema, Scope, Stream } from "effect";
+import { Agent, Sandbox } from "@open-insight/core/internal";
+import { ChildProcessHandle, ChildProcessSpawner, ExitCode } from "effect/unstable/process/ChildProcessSpawner";
+import { NodeServices } from "@effect/platform-node";
+import * as _Core from "@open-insight/core";
+import { Prompt, Response } from "effect/unstable/ai";
+import { Command } from "effect/unstable/process/ChildProcess";
+
+//#region src/utils/variant.d.ts
+type Bivariant<Fn extends (...args: never[]) => any> = {
+  bivarianceHack(...args: Parameters<Fn>): ReturnType<Fn>;
+}["bivarianceHack"];
+type Contravariant<T> = (arg: T) => void;
+type UnionToIntersection<T> = (T extends unknown ? Contravariant<T> : never) extends Contravariant<infer U> ? U : never;
+//#endregion
+//#region src/task/error.d.ts
+declare const TaskLoadError_base$1: Schema.Class<TaskLoadError$1, Schema.TaggedStruct<"TaskLoadError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class TaskLoadError$1 extends TaskLoadError_base$1 {}
+declare const GradeExecError_base: Schema.Class<GradeExecError, Schema.TaggedStruct<"GradeError", {
+  readonly name: Schema.String;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class GradeExecError extends GradeExecError_base {}
+declare const InvalidGradeResultError_base: Schema.Class<InvalidGradeResultError, Schema.TaggedStruct<"InvalidGradeResultError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class InvalidGradeResultError extends InvalidGradeResultError_base {}
+declare const TaskErrorReason: Schema.Union<readonly [typeof TaskLoadError$1, typeof GradeExecError, typeof InvalidGradeResultError]>;
+declare const TaskError_base: Schema.Class<TaskError, Schema.TaggedStruct<"TaskError", {
+  readonly reason: Schema.Union<readonly [typeof TaskLoadError$1, typeof GradeExecError, typeof InvalidGradeResultError]>;
+}>, import("effect/Cause").YieldableError>;
+declare class TaskError extends TaskError_base {
+  static load: (cause: unknown) => TaskError;
+  static gradeExec: (name: string) => (cause: unknown) => TaskError;
+  static gradeResult: (cause: Schema.SchemaError) => TaskError;
+}
+//#endregion
+//#region src/task/grade/builtin/command.d.ts
+declare const bash: (bash: string) => Exec$3<string>;
+declare namespace index_d_exports$9 {
+  export { Context$1 as Context, Exec$3 as Exec, Grader, Map, Result$3 as Result, ResultSchema, bash, run$1 as run };
+}
+type Context$1 = Sandbox.SandboxPromise & Readonly<{
+  trajectory: Agent.Trajectory;
+}>;
+type Exec$3<R = unknown> = Bivariant<(ctx: Context$1) => PromiseLike<R>>;
+type Grader<N extends string = string, R = unknown> = Readonly<{
+  name: N;
+  exec: Exec$3<R>;
+}> & Brand.Brand<N>;
+type Map<G> = UnionToIntersection<G extends Grader<infer N, infer R> ? Record<N, Exec$3<R>> : never>;
+declare const ResultSchema: Schema.$Record<Schema.String, Schema.Codec<Schema.Json, Schema.Json, never, never>>;
+type Result$3<G> = UnionToIntersection<G extends Grader<infer N, infer R> ? Record<N, R> : never>;
+/**
+ * Run a collection of graders with the given context.
+ */
+declare const run$1: (map: Map<Grader>) => (ctx: Context$1) => Effect.Effect<{
+  readonly [x: string]: Schema.Json;
+}, TaskError, never>;
+//#endregion
+//#region src/task/build.d.ts
+type ID = string;
+declare const MetadataSchema$1: Schema.Struct<{
+  readonly name: Schema.String;
+  readonly description: Schema.optional<Schema.String>;
+}>;
+type Metadata$1 = Schema.Schema.Type<typeof MetadataSchema$1>;
+type Task<G extends Grader = Grader> = Readonly<{
+  metadata: Metadata$1;
+  graders: Map<G>;
+  prompt: ReadonlyArray<Prompt.UserMessage>;
+  snapshot: Sandbox.Snapshot.Snapshot;
+  context: Sandbox.Context.Mode;
+  resources: Sandbox.ResourceLimits | null;
+  readonly _grader?: () => G;
+}>;
+type Tasks$1<T extends Task = Task> = ReadonlyArray<Effect.Effect<T, TaskError>>;
+type GraderOf<T> = T extends Task<infer G> ? G : never;
+type Builder$4<G extends Grader = Grader, H = never, R = never> = Effect.Effect<Partial<Task<G>>, TaskError, R> & {
+  _typestate?: Contravariant<H>;
+};
+declare const init$4: <T extends Task>(metadata: Metadata$1) => Builder$4<GraderOf<T>>;
+type HasPrompt = Brand.Brand<"prompt">;
+declare const withPrompt: (prompt: Readonly<[Prompt.UserMessage, ...Prompt.UserMessage[]]>) => <G extends Grader, H, R>(build: Builder$4<G, H, R>) => Builder$4<G, H | HasPrompt, R>;
+type HasContext = Brand.Brand<"context">;
+declare const withContext: (context: Sandbox.Context.Mode) => <G extends Grader, H, R>(build: Builder$4<G, H, R>) => Builder$4<G, H | HasContext, R>;
+type HasSnapshot = Brand.Brand<"snapshot">;
+declare const withSnapshot: (snapshot: Sandbox.Snapshot.Snapshot) => <G extends Grader, H, R>(build: Builder$4<G, H, R>) => Builder$4<G, H | HasSnapshot, R>;
+declare const withGrader: <N extends string, T>(name: N, exec: Exec$3<T>) => <G extends Grader, H, R>(build: Grader<N, T> extends G ? Builder$4<G, H, R> : never) => Builder$4<G, H | Grader<N, T>, R>;
+declare const withResources: (resources: Sandbox.ResourceLimits) => <G extends Grader, H, R>(build: Builder$4<G, H, R>) => Builder$4<G, H, R>;
+declare const build$6: <G extends Grader, R>(build: Builder$4<G, HasPrompt | HasContext | HasSnapshot | G, R>) => Effect.Effect<Task<G>, TaskError, R>;
+declare namespace file_d_exports {
+  export { fromDir };
+}
+declare const fromDir: <T extends Task>({
+  dir,
+  glob
+}: {
+  dir: string;
+  glob?: string;
+}) => Loader<T, FileSystem.FileSystem | Path.Path>;
+//#endregion
+//#region ../utils/dist/index.d.mts
+//#region src/spawn.d.ts
+declare namespace spawn_d_exports {
+  export { SpawnError, SpawnErrorReason, SpawnExitCodeError, SpawnService };
+}
+declare const SpawnExitCodeError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "SpawnExitCodeError";
+} & Readonly<A>;
+declare class SpawnExitCodeError extends SpawnExitCodeError_base<{
+  readonly exitCode: ExitCode;
+  readonly stdout: string;
+  readonly stderr: string;
+}> {
+  get message(): string;
+}
+type SpawnErrorReason = PlatformError.PlatformError | SpawnExitCodeError;
+declare const SpawnError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P] }>) => import("effect/Cause").YieldableError & {
+  readonly _tag: "SpawnError";
+} & Readonly<A>;
+declare class SpawnError extends SpawnError_base<{
+  readonly reason: SpawnErrorReason;
+}> {
+  get message(): string;
+  static platform: (err: PlatformError.PlatformError) => SpawnError;
+  static exit: ({
+    exitCode,
+    stdout,
+    stderr
+  }: {
+    exitCode: ExitCode;
+    stdout: string;
+    stderr: string;
+  }) => SpawnError;
+}
+declare const SpawnService_base: Context.ServiceClass<SpawnService, "packages/utils/SpawnService", {
+  spawn(command: Command): Effect.Effect<ChildProcessHandle, SpawnError, Scope.Scope>;
+  exitCode(command: Command): Effect.Effect<ExitCode, SpawnError>;
+  streamString(command: Command, options?: {
+    readonly includeStderr?: boolean | undefined;
+  }): Stream.Stream<string, SpawnError>;
+  streamLines(command: Command, options?: {
+    readonly includeStderr?: boolean | undefined;
+  }): Stream.Stream<string, SpawnError>;
+  string(command: Command, options?: {
+    readonly includeStderr?: boolean | undefined;
+  }): Effect.Effect<string, SpawnError>;
+  lines(command: Command, options?: {
+    readonly includeStderr?: boolean | undefined;
+  }): Effect.Effect<ReadonlyArray<string>, SpawnError>;
+}>;
+declare class SpawnService extends SpawnService_base {
+  static readonly layer: Layer.Layer<SpawnService, never, ChildProcessSpawner>;
+}
+declare namespace git_d_exports {
+  export { withGitRepo };
+}
+declare const withGitRepo: (repoURL: string) => <T extends Task>(exec: (repoPath: string) => Loader<T>) => Effect.Effect<Tasks<T>, import("effect/PlatformError").PlatformError | spawn_d_exports.SpawnError | TaskError, FileSystem.FileSystem | import("effect/Scope").Scope | spawn_d_exports.SpawnService>;
+//#endregion
+//#region src/task/load/iter.d.ts
+type TaskEffect<T extends Task> = Effect.Effect<T, TaskError>;
+declare const fromArray: <T extends Task>(tasks: ReadonlyArray<TaskEffect<T>>) => Loader<T>;
+declare const fromIterable: <T extends Task>(iterable: Iterable<TaskEffect<T>>) => Loader<T>;
+declare const fromAsyncIterable: <T extends Task>(iterable: AsyncIterable<TaskEffect<T>>) => Loader<T>;
+declare const fromStream: <T extends Task, E, R>(stream: Stream.Stream<TaskEffect<T>, E, R>) => Loader<T, R, E>;
+declare namespace index_d_exports$8 {
+  export { file_d_exports as File, git_d_exports as Git, Loader, Tasks, fromArray, fromAsyncIterable, fromIterable, fromStream };
+}
+type Tasks<T extends Task = Task> = ReadonlyArray<Effect.Effect<T, TaskError>>;
+type Loader<T extends Task = Task, R = never, E = TaskError> = Effect.Effect<Tasks<T>, E, R | Scope.Scope>;
+declare namespace index_d_exports$7 {
+  export { index_d_exports$9 as Grade, GradeExecError, Grader, GraderOf, ID, InvalidGradeResultError, index_d_exports$8 as Load, Metadata$1 as Metadata, MetadataSchema$1 as MetadataSchema, Task, TaskError, TaskErrorReason, TaskLoadError$1 as TaskLoadError, Tasks$1 as Tasks, build$6 as build, init$4 as init, withContext, withGrader, withPrompt, withResources, withSnapshot };
+}
+//#endregion
+//#region src/benchmark/error.d.ts
+declare const InitError_base$2: Schema.Class<InitError$2, Schema.TaggedStruct<"InitError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class InitError$2 extends InitError_base$2 {}
+declare const BenchmarkErrorReason: Schema.Union<readonly [typeof InitError$2]>;
+declare const BenchmarkError_base: Schema.Class<BenchmarkError, Schema.TaggedStruct<"BenchmarkError", {
+  readonly reason: Schema.Union<readonly [typeof InitError$2]>;
+}>, import("effect/Cause").YieldableError>;
+declare class BenchmarkError extends BenchmarkError_base {
+  static init: (cause: unknown) => BenchmarkError;
+}
+//#endregion
+//#region src/benchmark/build.d.ts
+type Metadata = Readonly<{
+  name: string;
+  description: string;
+}>;
+declare const MetadataSchema: Schema.Schema<Metadata>;
+type Benchmark<T extends Task = Task> = Readonly<{
+  tasks: Tasks$1<T>;
+  metadata: Metadata;
+}>;
+type TaskOf<B> = B extends Benchmark<infer T> ? T : never;
+type Builder$3<T extends Task = Task, H = never, R = never> = Effect.Effect<Partial<Benchmark<T>>, BenchmarkError, R> & {
+  _typestate?: Contravariant<H>;
+};
+declare const init$3: <T extends Task = Task>(metadata: Metadata) => Builder$3<T>;
+type HasTasks = Brand.Brand<"tasks">;
+declare const withTasks: <T extends Task, LR, LE>(loader: Loader<T, LR, LE>) => <H, R>(build: Builder$3<T, H, R>) => Builder$3<T, H | HasTasks, R | LR | Scope.Scope>;
+declare const build$5: <T extends Task, R>(build: Builder$3<T, HasTasks, R>) => Effect.Effect<Benchmark<T>, BenchmarkError, R>;
+declare namespace index_d_exports$6 {
+  export { Benchmark, BenchmarkError, BenchmarkErrorReason, InitError$2 as InitError, Metadata, MetadataSchema, TaskOf, build$5 as build, init$3 as init, withTasks };
+}
+//#endregion
+//#region src/harness/error.d.ts
+declare const InitError_base$1: Schema.Class<InitError$1, Schema.TaggedStruct<"InitError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class InitError$1 extends InitError_base$1 {}
+declare const HarnessErrorReason: Schema.Union<readonly [typeof InitError$1]>;
+declare const HarnessError_base: Schema.Class<HarnessError, Schema.TaggedStruct<"HarnessError", {
+  readonly reason: Schema.Union<readonly [typeof InitError$1]>;
+}>, import("effect/Cause").YieldableError>;
+declare class HarnessError extends HarnessError_base {
+  static init: (cause: unknown) => HarnessError;
+}
+//#endregion
+//#region src/harness/build.d.ts
+type Harness<T extends Task = Task> = Readonly<{
+  sandbox: Layer.Layer<Sandbox.ProviderService, HarnessError>;
+  agent: Layer.Layer<Agent.ProviderService, HarnessError>;
+}> & {
+  _T?: T;
+};
+type Builder$2<T extends Task = Task, H = never, R = never> = Effect.Effect<Partial<Harness<T>>, HarnessError, R> & {
+  _typestate?: Contravariant<H>;
+  _task?: T;
+};
+declare const init$2: <T extends Task = Task>() => Builder$2<T>;
+type HasSandboxProvider = Brand.Brand<"HasSandboxProvider">;
+declare const withSandboxProvider: <R>(provider: Effect.Effect<Sandbox.Provider, Sandbox.SandboxError, R>) => <T extends Task, H, BR>(builder: Builder$2<T, H, BR>) => Builder$2<T, H | HasSandboxProvider, R | BR>;
+type HasAgentProvider = Brand.Brand<"HasAgentProvider">;
+declare const withAgentProvider: <E, R>(provider: Effect.Effect<Agent.Provider, E, R>) => <T extends Task, H, BR>(builder: Builder$2<T, H, BR>) => Builder$2<T, H | HasAgentProvider, R | BR>;
+declare const build$4: <T extends Task, R>(build: Builder$2<T, HasSandboxProvider | HasAgentProvider, R>) => Effect.Effect<Harness<T>, HarnessError, R>;
+//#endregion
+//#region src/harness/config.d.ts
+type Config$1 = Readonly<{
+  readonly snapshotConcurrency?: number;
+  readonly trailConcurrency?: number;
+  readonly taskConcurrency?: number;
+}>;
+declare namespace index_d_exports$5 {
+  export { Config$1 as Config, Harness, HarnessError, HarnessErrorReason, InitError$1 as InitError, build$4 as build, init$2 as init, withAgentProvider, withSandboxProvider };
+}
+//#endregion
+//#region src/metric/schema.d.ts
+declare const GradeResultSchema: Schema.$Record<Schema.String, Schema.Codec<Schema.Json, Schema.Json, never, never>>;
+type GradeResult = Schema.Schema.Type<typeof GradeResultSchema>;
+type InputDelta = Data.TaggedEnum<{
+  Messages: {
+    messages: Array<Agent.Message>;
+  };
+  Grade: {
+    result: GradeResult;
+  };
+}>;
+declare const Grade: Data.TaggedEnum.ConstructorFrom<{
+    readonly _tag: "Grade";
+    readonly result: GradeResult;
+  }, "_tag">, Messages: Data.TaggedEnum.ConstructorFrom<{
+    readonly _tag: "Messages";
+    readonly messages: Array<Agent.Message>;
+  }, "_tag">;
+type Input$1 = Readonly<{
+  task: Task;
+  trajectory: Agent.Trajectory;
+  delta: InputDelta;
+}>;
+declare const TrajOutput_base: Schema.Class<TrajOutput, Schema.TaggedStruct<"TrajOutput", {
+  readonly name: Schema.String;
+  readonly task: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly description: Schema.optional<Schema.String>;
+  }>;
+  readonly result: Schema.Codec<Schema.Json, Schema.Json, never, never>;
+}>, {}>;
+declare class TrajOutput extends TrajOutput_base {}
+declare const TaskOutput_base: Schema.Class<TaskOutput, Schema.TaggedStruct<"TaskOutput", {
+  readonly name: Schema.String;
+  readonly task: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly description: Schema.optional<Schema.String>;
+  }>;
+  readonly result: Schema.Codec<Schema.Json, Schema.Json, never, never>;
+}>, {}>;
+declare class TaskOutput extends TaskOutput_base {}
+declare const BenchOutput_base: Schema.Class<BenchOutput, Schema.TaggedStruct<"BenchmarkOutput", {
+  readonly name: Schema.String;
+  readonly result: Schema.Codec<Schema.Json, Schema.Json, never, never>;
+}>, {}>;
+declare class BenchOutput extends BenchOutput_base {}
+declare const OutputSchema: Schema.Union<readonly [typeof TrajOutput, typeof TaskOutput, typeof BenchOutput]>;
+type Output = Schema.Schema.Type<typeof OutputSchema>;
+//#endregion
+//#region src/metric/error.d.ts
+declare const ExecError_base$1: Schema.Class<ExecError$1, Schema.TaggedStruct<"ExecError", {
+  readonly name: Schema.String;
+  readonly type: Schema.Union<readonly [Schema.Literal<"Trajectory">, Schema.Literal<"Task">, Schema.Literal<"Benchmark">]>;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class ExecError$1 extends ExecError_base$1 {}
+declare const MetricErrorReason: Schema.Union<readonly [typeof ExecError$1]>;
+declare const MetricError_base: Schema.Class<MetricError, Schema.TaggedStruct<"MetricError", {
+  readonly reason: Schema.Union<readonly [typeof ExecError$1]>;
+}>, import("effect/Cause").YieldableError>;
+declare class MetricError extends MetricError_base {
+  static exec: ({
+    name,
+    type
+  }: {
+    name: string;
+    type: "Trajectory" | "Task" | "Benchmark";
+  }) => (cause: unknown) => MetricError;
+  static taskExec: (name: string) => (cause: unknown) => MetricError;
+  static trajExec: (name: string) => (cause: unknown) => MetricError;
+  static benchExec: (name: string) => (cause: unknown) => MetricError;
+}
+declare namespace index_d_exports$4 {
+  export { AllFn$2 as AllFn, EachFn$2 as EachFn, Exec$2 as Exec, Metric$2 as Metric, ReduceFn$2 as ReduceFn, Result$2 as Result, all$2 as all, build$3 as build, buildAll$2 as buildAll, buildEach$2 as buildEach, buildReduce$2 as buildReduce, each$2 as each, reduce$2 as reduce };
+}
+type ReduceFn$2<R> = (prev: R, input: {
+  trajectory: Agent.Trajectory;
+  messages: ReadonlyArray<Agent.Message>;
+}) => PromiseLike<R> | R;
+type EachFn$2<R> = (input: {
+  trajectory: Agent.Trajectory;
+  messages: ReadonlyArray<Agent.Message>;
+}) => PromiseLike<R> | R;
+type AllFn$2<R> = (input: {
+  trajectory: Agent.Trajectory;
+}) => PromiseLike<R> | R;
+type ReduceExec$2<R = unknown> = {
+  init: R;
+  exec: Bivariant<ReduceFn$2<R>>;
+};
+type EachExec$2<R = unknown> = {
+  exec: Bivariant<EachFn$2<R>>;
+};
+type AllExec$2<R = unknown> = {
+  exec: Bivariant<AllFn$2<R>>;
+};
+type Exec$2<R = unknown> = Data.TaggedEnum<{
+  Reduce: ReduceExec$2<R>;
+  Each: EachExec$2<R>;
+  All: AllExec$2<R>;
+}>;
+type Metric$2<N extends string = string, R = unknown> = Readonly<{
+  name: N;
+  exec: Exec$2<R>;
+}>;
+declare const reduce$2: <N extends string, R>(name: N, init: R, exec: ReduceFn$2<R>) => Metric$2<N, R>;
+declare const each$2: <N extends string, R>(name: N, exec: EachFn$2<R>) => Metric$2<N, R>;
+declare const all$2: <N extends string, R>(name: N, exec: AllFn$2<R>) => Metric$2<N, R>;
+type Result$2<M> = UnionToIntersection<M extends Metric$2<infer N, infer R> ? Record<N, R> : never>;
+declare const buildReduce$2: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: ReduceExec$2;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TrajOutput | null, MetricError, never>;
+declare const buildEach$2: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: EachExec$2;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TrajOutput | null, MetricError, never>;
+declare const buildAll$2: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: AllExec$2;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TrajOutput | null, MetricError, never>;
+declare const build$3: (metric: Metric$2) => (args_0: Readonly<{
+  task: Task;
+  trajectory: Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TrajOutput | null, MetricError, never>;
+declare namespace index_d_exports$3 {
+  export { AllFn$1 as AllFn, EachFn$1 as EachFn, Exec$1 as Exec, Metric$1 as Metric, ReduceFn$1 as ReduceFn, Result$1 as Result, all$1 as all, build$2 as build, buildAll$1 as buildAll, buildEach$1 as buildEach, buildReduce$1 as buildReduce, each$1 as each, reduce$1 as reduce };
+}
+type ReduceFn$1<G extends Grader, R> = (prev: R, input: Result$3<G>) => PromiseLike<R> | R;
+type EachFn$1<G extends Grader, R> = (input: Result$3<G>) => PromiseLike<R> | R;
+type AllFn$1<G extends Grader, R> = (input: ReadonlyArray<Result$3<G>>) => PromiseLike<R> | R;
+type ReduceExec$1<G extends Grader = Grader, R = unknown> = {
+  init: R;
+  exec: Bivariant<(prev: R, input: Result$3<G>) => PromiseLike<R> | R>;
+};
+type EachExec$1<G extends Grader = Grader, R = unknown> = {
+  exec: Bivariant<EachFn$1<G, R>>;
+};
+type AllExec$1<G extends Grader = Grader, R = unknown> = {
+  exec: Bivariant<AllFn$1<G, R>>;
+};
+type Exec$1<G extends Grader, R> = Data.TaggedEnum<{
+  Reduce: ReduceExec$1<G, R>;
+  Each: EachExec$1<G, R>;
+  All: AllExec$1<G, R>;
+}>;
+type Metric$1<G extends Grader = Grader, N extends string = string, R = unknown> = Readonly<{
+  name: N;
+  exec: Exec$1<G, R>;
+}>;
+declare const reduce$1: <G extends Grader, N extends string, R>(name: N, init: R, exec: ReduceFn$1<G, R>) => Metric$1<G, N, R>;
+declare const each$1: <G extends Grader, N extends string, R>(name: N, exec: EachFn$1<G, R>) => Metric$1<G, N, R>;
+declare const all$1: <G extends Grader, N extends string, R>(name: N, exec: AllFn$1<G, R>) => Metric$1<G, N, R>;
+type Result$1<M> = UnionToIntersection<M extends Metric$1<infer _, infer N, infer R> ? Record<N, R> : never>;
+declare const buildReduce$1: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: ReduceExec$1;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TaskOutput | null, MetricError, never>;
+declare const buildEach$1: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: EachExec$1;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TaskOutput | null, MetricError, never>;
+declare const buildAll$1: ({
+  name,
+  exec,
+  trailCount
+}: {
+  name: string;
+  exec: AllExec$1;
+  trailCount: number;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TaskOutput | null, MetricError, never>;
+declare const build$2: ({
+  metric,
+  trailCount
+}: {
+  metric: Metric$1;
+  trailCount: number;
+}) => (args_0: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TaskOutput | null, MetricError, never>;
+declare namespace index_d_exports$2 {
+  export { AllFn, EachFn, Exec, Input, Inputs, Metric, ReduceFn, Result, all, build$1 as build, buildAll, buildEach, buildReduce, each, reduce };
+}
+type Input<TAM extends Metric$1 = Metric$1> = {
+  task: string;
+  input: Result$1<TAM>;
+};
+type Inputs<TAM extends Metric$1 = Metric$1> = Record<string, Result$1<TAM>>;
+type ReduceFn<TAM extends Metric$1, R> = (prev: R, input: Input<TAM>) => PromiseLike<R> | R;
+type EachFn<TAM extends Metric$1, R> = (input: Input<TAM>) => PromiseLike<R> | R;
+type AllFn<TAM extends Metric$1, R> = (input: Inputs<TAM>) => PromiseLike<R> | R;
+type ReduceExec<TAM extends Metric$1 = Metric$1, R = unknown> = {
+  init: R;
+  exec: Bivariant<ReduceFn<TAM, R>>;
+};
+type EachExec<TAM extends Metric$1 = Metric$1, R = unknown> = {
+  exec: Bivariant<EachFn<TAM, R>>;
+};
+type AllExec<TAM extends Metric$1 = Metric$1, R = unknown> = {
+  exec: Bivariant<AllFn<TAM, R>>;
+};
+type Exec<TAM extends Metric$1 = Metric$1, R = unknown> = Data.TaggedEnum<{
+  Reduce: ReduceExec<TAM, R>;
+  Each: EachExec<TAM, R>;
+  All: AllExec<TAM, R>;
+}>;
+type Metric<TAM extends Metric$1 = Metric$1, N extends string = string, R = unknown> = Readonly<{
+  name: N;
+  exec: Exec<TAM, R>;
+}>;
+declare const reduce: <TAM extends Metric$1, N extends string, R>(name: N, init: R, exec: ReduceFn<TAM, R>) => Metric<TAM, N, R>;
+declare const each: <TAM extends Metric$1, N extends string, R>(name: N, exec: EachFn<TAM, R>) => Metric<TAM, N, R>;
+declare const all: <TAM extends Metric$1, N extends string, R>(name: N, exec: AllFn<TAM, R>) => Metric<TAM, N, R>;
+type Result<M> = UnionToIntersection<M extends Metric<infer _, infer N, infer R> ? Record<N, R> : never>;
+declare const buildReduce: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: ReduceExec;
+}) => (input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput, MetricError, never>;
+declare const buildEach: ({
+  name,
+  exec
+}: {
+  name: string;
+  exec: EachExec;
+}) => (input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput, MetricError, never>;
+declare const buildAll: ({
+  name,
+  exec,
+  taskCount
+}: {
+  name: string;
+  exec: AllExec;
+  taskCount: number;
+}) => (input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput | null, MetricError, never>;
+declare const build$1: ({
+  metric,
+  taskCount
+}: {
+  metric: Metric;
+  taskCount: number;
+}) => ((input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput, MetricError, never>) | ((input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput, MetricError, never>) | ((input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput | null, MetricError, never>);
+//#endregion
+//#region src/metric/build.d.ts
+type Metrics<G extends Grader = Grader, TAM = Metric$1> = Readonly<{
+  trajectory: Array<Metric$2>;
+  task: Array<Metric$1>;
+  benchmark: Array<Metric>;
+}> & {
+  _G?: G;
+  _TAM?: TAM;
+};
+type Builder$1<G extends Grader, TAM> = Effect.Effect<Metrics<G, TAM>>;
+declare const init$1: <T extends Task>() => Builder$1<GraderOf<T>, never>;
+declare const withTrajReduce: <N extends string, R>(name: N, init: R, exec: ReduceFn$2<R>) => <G extends Grader, TAM>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+declare const withTrajEach: <N extends string, R>(name: N, exec: EachFn$2<R>) => <G extends Grader, TAM>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+declare const withTraj: <N extends string, R>(name: N, exec: AllFn$2<R>) => <G extends Grader, TAM>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+declare const withTaskReduce: <G extends Grader, N extends string, R>(name: N, init: R, exec: ReduceFn$1<G, R>) => <TAM>(builder: Builder$1<G, TAM>) => Builder$1<G, TAM | Metric$1<G, N, R>>;
+declare const withTaskEach: <G extends Grader, N extends string, R>(name: N, exec: EachFn$1<G, R>) => <TAM>(builder: Builder$1<G, TAM>) => Builder$1<G, TAM | Metric$1<G, N, R>>;
+declare const withTask: <G extends Grader, N extends string, R>(name: N, exec: AllFn$1<G, R>) => <TAM>(builder: Builder$1<G, TAM>) => Builder$1<G, TAM | Metric$1<G, N, R>>;
+declare const withBenchReduce: <TAM extends Metric$1, N extends string, R>(name: N, init: R, exec: ReduceFn<TAM, R>) => <G extends Grader>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+declare const withBenchEach: <TAM extends Metric$1, N extends string, R>(name: N, exec: EachFn<TAM, R>) => <G extends Grader>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+declare const withBenchmark$1: <TAM extends Metric$1, N extends string, R>(name: N, exec: AllFn<TAM, R>) => <G extends Grader>(build: Builder$1<G, TAM>) => Builder$1<G, TAM>;
+//#endregion
+//#region src/metric/stream.d.ts
+type InputStream<E, R> = Stream.Stream<Input$1, E, R>;
+type OutputQueue = Queue.Queue<Output, MetricError | Cause.Done>;
+declare const buildTrajMetricConsumer: ({
+  metrics: metricVariants,
+  queue
+}: {
+  metrics: ReadonlyArray<Metric$2>;
+  queue: OutputQueue;
+}) => (input: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TrajOutput[] | null, MetricError, never>;
+declare const buildTaskMetricConsumer: ({
+  metrics: metricVariants,
+  trailCount,
+  queue
+}: {
+  metrics: ReadonlyArray<Metric$1>;
+  trailCount: number;
+  queue: OutputQueue;
+}) => (input: Readonly<{
+  task: Task;
+  trajectory: _Core.Agent.Trajectory;
+  delta: InputDelta;
+}>) => Effect.Effect<TaskOutput[] | null, MetricError, never>;
+declare const buildBenchMetricConsumer: ({
+  metrics: metricVariants,
+  taskCount,
+  queue
+}: {
+  metrics: ReadonlyArray<Metric>;
+  taskCount: number;
+  queue: OutputQueue;
+}) => (input: Input<Readonly<{
+  name: string;
+  exec: {
+    readonly _tag: "All";
+    readonly exec: Bivariant<AllFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Each";
+    readonly exec: Bivariant<EachFn$1<Grader, unknown>>;
+  } | {
+    readonly _tag: "Reduce";
+    readonly init: unknown;
+    readonly exec: Bivariant<(prev: unknown, input: Record<string, unknown>) => unknown>;
+  };
+}>>) => Effect.Effect<BenchOutput[] | null, MetricError, never>;
+declare const transform: ({
+  metrics,
+  trailCount,
+  taskCount
+}: {
+  metrics: Metrics;
+  trailCount: number;
+  taskCount: number;
+}) => <E, R>(inputStream: InputStream<E, R>) => Stream.Stream<BenchOutput | TaskOutput | TrajOutput, E | Cause.Done<void> | MetricError, Exclude<Exclude<R, Scope.Scope>, Scope.Scope>>;
+declare namespace index_d_exports$1 {
+  export { BenchOutput, index_d_exports$2 as Benchmark, Builder$1 as Builder, ExecError$1 as ExecError, Grade, GradeResult, GradeResultSchema, Input$1 as Input, InputDelta, Messages, MetricError, MetricErrorReason, Metrics, Output, OutputSchema, index_d_exports$3 as Task, TaskOutput, index_d_exports$4 as Traj, TrajOutput, buildBenchMetricConsumer, buildTaskMetricConsumer, buildTrajMetricConsumer, init$1 as init, transform, withBenchEach, withBenchReduce, withBenchmark$1 as withBenchmark, withTask, withTaskEach, withTaskReduce, withTraj, withTrajEach, withTrajReduce };
+}
+//#endregion
+//#region src/exec/error.d.ts
+declare const InitError_base: Schema.Class<InitError, Schema.TaggedStruct<"InitError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class InitError extends InitError_base {}
+declare const TaskLoadError_base: Schema.Class<TaskLoadError, Schema.TaggedStruct<"TaskLoadError", {
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class TaskLoadError extends TaskLoadError_base {}
+declare const TaskInitError_base: Schema.Class<TaskInitError, Schema.TaggedStruct<"TaskInitError", {
+  readonly task: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly description: Schema.optional<Schema.String>;
+  }>;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class TaskInitError extends TaskInitError_base {}
+declare const TaskExecError_base: Schema.Class<TaskExecError, Schema.TaggedStruct<"TaskExecError", {
+  readonly task: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly description: Schema.optional<Schema.String>;
+  }>;
+  readonly trailIndex: Schema.Int;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class TaskExecError extends TaskExecError_base {}
+declare const EventTransportInitError_base: Schema.Class<EventTransportInitError, Schema.TaggedStruct<"EventTransportInitError", {
+  readonly transport: Schema.String;
+  readonly url: Schema.String;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class EventTransportInitError extends EventTransportInitError_base {}
+declare const EventTransportError_base: Schema.Class<EventTransportError, Schema.TaggedStruct<"EventTransportError", {
+  readonly transport: Schema.String;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class EventTransportError extends EventTransportError_base {}
+declare const SnapshotError_base: Schema.Class<SnapshotError, Schema.TaggedStruct<"SnapshotError", {
+  readonly task: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly description: Schema.optional<Schema.String>;
+  }>;
+  readonly snapshot: typeof Sandbox.Snapshot.Snapshot;
+  readonly cause: Schema.Defect;
+}>, import("effect/Cause").YieldableError>;
+declare class SnapshotError extends SnapshotError_base {}
+declare const ExecError_base: Schema.Class<ExecError, Schema.TaggedStruct<"ExecError", {
+  readonly reason: Schema.Union<readonly [typeof InitError, typeof TaskLoadError, typeof EventTransportInitError, typeof EventTransportError, typeof SnapshotError, typeof TaskInitError, typeof TaskExecError, typeof MetricError]>;
+}>, import("effect/Cause").YieldableError>;
+declare class ExecError extends ExecError_base {
+  static init: (cause: unknown) => ExecError;
+  static taskLoad: (cause: unknown) => ExecError;
+  static eventTransportInit: ({
+    transport,
+    url
+  }: {
+    transport: string;
+    url: string;
+  }) => (cause: unknown) => ExecError;
+  static eventTransport: ({
+    transport
+  }: {
+    transport: string;
+  }) => (cause: unknown) => ExecError;
+  static snapshot: ({
+    task: {
+      metadata,
+      snapshot
+    }
+  }: {
+    task: Task;
+  }) => (cause: unknown) => ExecError;
+  static taskInit: ({
+    task
+  }: {
+    task: Metadata$1;
+  }) => (cause: unknown) => ExecError;
+  static taskExec: ({
+    task,
+    trailIndex
+  }: {
+    task: Metadata$1;
+    trailIndex: number;
+  }) => (cause: unknown) => ExecError;
+  static metric: (cause: MetricError) => ExecError;
+}
+//#endregion
+//#region src/exec/event/schema.d.ts
+declare const EventSchema: Schema.TaggedUnion<{
+  readonly BenchmarkMetric: Schema.TaggedStruct<"BenchmarkMetric", {
+    readonly benchmark: Schema.String;
+    readonly output: typeof BenchOutput;
+  }>;
+  readonly Message: Schema.TaggedStruct<"Message", {
+    readonly benchmark: Schema.String;
+    readonly task: Schema.String;
+    readonly trail: Schema.Number;
+    readonly part: Schema.Codec<Response.StreamPart<{}>, Response.StreamPartEncoded, never, never>;
+  }>;
+  readonly TaskMetric: Schema.TaggedStruct<"TaskMetric", {
+    readonly benchmark: Schema.String;
+    readonly task: Schema.String;
+    readonly output: typeof TaskOutput;
+  }>;
+  readonly TrajMetric: Schema.TaggedStruct<"TrajMetric", {
+    readonly benchmark: Schema.String;
+    readonly task: Schema.String;
+    readonly trail: Schema.Number;
+    readonly output: typeof TrajOutput;
+  }>;
+}>;
+type Event = Schema.Schema.Type<typeof EventSchema>;
+type EventStream = Stream.Stream<Event, ExecError>;
+//#endregion
+//#region src/exec/event/service.d.ts
+type EventTransport = Readonly<{
+  send(options: Readonly<{
+    stream: EventStream;
+  }>): Effect.Effect<void, ExecError, Scope.Scope>;
+}>;
+//#endregion
+//#region src/exec/build.d.ts
+type Executor<T extends Task = Task> = Readonly<{
+  benchmark: Benchmark;
+  harness: Harness;
+  trailCount: number;
+  metrics: Metrics<GraderOf<T>> | null;
+  transport: EventTransport | null;
+}> & {
+  _T?: T;
+};
+type Builder<T extends Task = Task, H = never, R = never> = Effect.Effect<Partial<Executor<T>>, ExecError, R> & {
+  _typestate?: Contravariant<H>;
+  _task?: T;
+};
+declare const init: <T extends Task>() => Builder<T>;
+type HasBenchmark = Brand.Brand<"benchmark">;
+declare const withBenchmark: <T extends Task, E, BR>(benchmark: Effect.Effect<Benchmark<T>, E, BR>) => <H, R>(builder: Builder<T, H, R>) => Builder<T, H | HasBenchmark, R | BR>;
+type HasHarness = Brand.Brand<"harness">;
+declare const withHarness: <T extends Task, E, HR>(harness: Effect.Effect<Harness<T>, E, HR>) => <H, R>(builder: Builder<T, H, R>) => Builder<T, H | HasHarness, R | HR>;
+declare const withTrailCount: (trailCount: number) => <T extends Task, H, R>(builder: Builder<T, H, R>) => Builder<T, H, R>;
+declare const withMetrics: <T extends Task, TAM extends Metric$1 = never>(metrics: Effect.Effect<Metrics<GraderOf<T>, TAM>>) => <H, R>(builder: Builder<T, H, R>) => Builder<T, H, R>;
+declare const withTransport: <T extends Task, E, TR>(transport: Effect.Effect<EventTransport, E, TR>) => <H, R>(builder: Builder<T, H, R>) => Builder<T, H, R | TR>;
+declare const build: <T extends Task, R>(builder: Builder<T, HasBenchmark | HasHarness, R>) => Effect.Effect<Executor<T>, ExecError, R>;
+//#endregion
+//#region src/exec/config.d.ts
+type Config = Readonly<{
+  readonly harnessConfig?: Config$1;
+  readonly sandboxConfig?: Sandbox.Config;
+}>;
+//#endregion
+//#region src/exec/run.d.ts
+declare const run: <E, R>(executor: Effect.Effect<Executor, E, R>, config?: Readonly<{
+  readonly harnessConfig?: Config$1;
+  readonly sandboxConfig?: _Core.Sandbox.Config;
+}> | undefined) => Effect.Effect<void, E | ExecError, Exclude<Exclude<Exclude<R, NodeServices.NodeServices>, import("effect/unstable/http/HttpClient").HttpClient>, Scope.Scope>>;
+declare const runPromise: <E>(executor: Effect.Effect<Executor, E, Scope.Scope>, config?: Config) => Promise<void>;
+declare namespace index_d_exports {
+  export { Executor, build, init, run, runPromise, withBenchmark, withHarness, withMetrics, withTrailCount, withTransport };
+}
+//#endregion
+export { fromStream as $, all$2 as A, InitError$1 as B, all as C, each$1 as D, all$1 as E, build$4 as F, BenchmarkError as G, build$5 as H, init$2 as I, Loader as J, InitError$2 as K, withAgentProvider as L, reduce$2 as M, index_d_exports$5 as N, reduce$1 as O, Harness as P, fromIterable as Q, withSandboxProvider as R, withTrajReduce as S, reduce as T, init$3 as U, index_d_exports$6 as V, withTasks as W, fromArray as X, Tasks as Y, fromAsyncIterable as Z, withTask as _, build as a, withContext as at, withTraj as b, withHarness as c, withResources as ct, index_d_exports$1 as d, Grader as dt, withGitRepo as et, Metrics as f, Map as ft, withBenchmark$1 as g, bash as gt, withBenchReduce as h, index_d_exports$9 as ht, Executor as i, init$4 as it, each$2 as j, Metric$2 as k, withMetrics as l, withSnapshot as lt, withBenchEach as m, ResultSchema as mt, run as n, Task as nt, init as o, withGrader as ot, init$1 as p, Result$3 as pt, index_d_exports$7 as q, runPromise as r, build$6 as rt, withBenchmark as s, withPrompt as st, index_d_exports as t, fromDir as tt, withTrailCount as u, Exec$3 as ut, withTaskEach as v, each as w, withTrajEach as x, withTaskReduce as y, HarnessError as z };