@kortyx/agent 0.34.0 → 0.35.0

package/dist/index.js

@@ -6938,6 +6938,7 @@ var emitTelemetryEvent = (args) => {
 
 // src/telemetry/topology.ts
 var import_node_crypto2 = require("crypto");
+var import_eval_attribution = require("@kortyx/core/eval-attribution");
 var isRecord2 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
 var stableValue = (value) => {
   if (Array.isArray(value)) return value.map(stableValue);
@@ -7090,6 +7091,8 @@ var projectWorkflowTopology = (args) => {
   };
 };
 var prepareWorkflowTelemetry = (args) => {
+  if (args.sessionId)
+    (0, import_eval_attribution.associateEvalExecution)({ sessionId: args.sessionId, runId: args.runId });
   const telemetry = isRecord2(args.config.telemetry) ? args.config.telemetry : void 0;
   if (!telemetry?.reporter || !telemetry.environment || !telemetry.service) {
     return args.config;
@@ -10059,9 +10062,12 @@ var EvalStepResultSchema = import_zod7.z.object({
     EvalVerdictSchema.extend({ id: import_zod7.z.string(), text: import_zod7.z.string() })
   )
 }).strict();
+var runtimeExecution = import_zod7.z.object({ sessionId: import_zod7.z.string().min(1), runId: import_zod7.z.string().min(1) }).strict();
 var EvalCaseResultSchema = import_zod7.z.object({
   caseId: id2,
   repetition: import_zod7.z.number().int().positive(),
+  attemptId: import_zod7.z.uuid().optional(),
+  runtimeExecutions: import_zod7.z.array(runtimeExecution).optional(),
   sessionId: import_zod7.z.string(),
   status,
   durationMs: import_zod7.z.number().nonnegative(),
@@ -10110,6 +10116,7 @@ var EvalRunResultSchema = import_zod7.z.object({
 var EvalManifestSchema = import_zod7.z.object({
   schemaVersion: import_zod7.z.literal(1),
   studioJudging: import_zod7.z.literal(true).optional(),
+  attemptScheduling: import_zod7.z.literal(true).optional(),
   suites: import_zod7.z.array(EvalSuiteSchema),
   responders: import_zod7.z.array(
     import_zod7.z.object({
@@ -10123,6 +10130,13 @@ var EvalManifestSchema = import_zod7.z.object({
   judge: judge.optional()
 }).strict();
 var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
+  import_zod7.z.object({
+    type: import_zod7.z.literal("case-runtime-associated"),
+    caseId: id2,
+    repetition: import_zod7.z.number().int().positive(),
+    attemptId: import_zod7.z.uuid(),
+    execution: runtimeExecution
+  }).strict(),
   import_zod7.z.object({
     type: import_zod7.z.literal("run-started"),
     caseIds: import_zod7.z.array(id2).min(1),
@@ -10138,6 +10152,7 @@ var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
   }).strict(),
   import_zod7.z.object({
     type: import_zod7.z.literal("case-started"),
+    attemptId: import_zod7.z.uuid().optional(),
     caseId: id2,
     repetition: import_zod7.z.number().int().positive(),
     sessionId: import_zod7.z.string()
@@ -10157,11 +10172,17 @@ var EvalRemoteRunRequestSchema = import_zod7.z.object({
   judge: EvalJudgeIdentitySchema.optional(),
   caseIds: import_zod7.z.array(id2).min(1).max(100).optional(),
   repetitions: import_zod7.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod7.z.number().int().min(1).max(20).default(1)
+  concurrency: import_zod7.z.number().int().min(1).max(20).default(1),
+  attempt: import_zod7.z.object({
+    caseId: id2,
+    repetition: import_zod7.z.number().int().min(1).max(20),
+    evaluationId: import_zod7.z.uuid()
+  }).strict().optional()
 }).strict();
 var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
   grading: true,
-  judge: true
+  judge: true,
+  attempt: true
 }).extend({
   targetId: import_zod7.z.string().min(1).max(128),
   judge: import_zod7.z.enum(["studio", "app"]).default("studio")
@@ -10198,6 +10219,7 @@ function parseEvalSuite(value) {
 
 // src/evals/create-evals.ts
 var import_node_crypto5 = require("crypto");
+var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
 var import_zod8 = require("zod");
 
 // src/evals/evidence.ts
@@ -10683,6 +10705,7 @@ function createEvals(options) {
       return {
         schemaVersion: 1,
         studioJudging: true,
+        attemptScheduling: true,
         suites: clone(suites),
         responders: Object.entries(responders).map(([name, value]) => ({
           name,
@@ -10727,6 +10750,8 @@ function createEvals(options) {
       );
       if (!Number.isSafeInteger(selected.length * repetitions))
         throw new EvalConfigurationError("Too many case repetitions.");
+      if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
+        throw new EvalConfigurationError("Invalid scheduled attempt.");
       const studioGrading = args.grading === "studio";
       if (!studioGrading && !options.judge && selected.some(
         (item) => item.steps.some((step) => step.expect.criteria?.length)
@@ -10763,7 +10788,10 @@ function createEvals(options) {
       const emit = async (event) => {
         if (!args.onProgress || !reporting) return;
         try {
-          await args.onProgress(clone(event));
+          await (0, import_eval_attribution2.withEvalAttribution)(
+            void 0,
+            () => args.onProgress(clone(event))
+          );
         } catch {
           reporting = false;
           result.errors.push(failure("reporting"));
@@ -10778,138 +10806,220 @@ function createEvals(options) {
           controller.abort();
         }, defaults.caseTimeoutMs);
         const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
+        const attemptId = (0, import_node_crypto5.randomUUID)();
+        const runtimeExecutions = [];
         const attemptResult = {
           caseId: item.id,
           repetition,
+          attemptId,
+          runtimeExecutions,
           sessionId: `eval-${(0, import_node_crypto5.randomUUID)()}`,
           status: "passed",
           durationMs: 0,
           steps: [],
           errors: []
         };
-        let phase = "params";
-        let stepIndex;
-        const reportPhase = async (next) => {
-          phase = next;
-          if (args.includeActivity)
+        let associationProgress = Promise.resolve();
+        let acceptingExecutions = true;
+        const executeAttempt = async () => {
+          let phase = "params";
+          let stepIndex;
+          const reportPhase = async (next) => {
+            phase = next;
+            if (args.includeActivity)
+              await emit({
+                type: "case-progress",
+                caseId: item.id,
+                repetition,
+                phase,
+                ...stepIndex !== void 0 ? { stepIndex } : {}
+              });
+          };
+          let context;
+          let continuation;
+          let activeStep;
+          let activeStepReported = false;
+          const history = [];
+          const call = async (readyContext, command, activeSignal) => {
+            const run = (overrides) => executeEvalChat({
+              agent: options.agent,
+              command,
+              continuation,
+              history: clone(history),
+              sessionId: attemptResult.sessionId,
+              clientTurnId: (0, import_node_crypto5.randomUUID)(),
+              signal: activeSignal,
+              ...item.workflowId ? { workflowId: item.workflowId } : {},
+              ...overrides
+            });
+            const execution = options.execute ? await options.execute({
+              ...readyContext,
+              signal: activeSignal,
+              command,
+              continuation,
+              history: clone(history),
+              run
+            }) : await run();
+            continuation = execution.continuation;
+            const observation = EvalObservationSchema.parse(
+              execution.observation
+            );
+            if (observation.type === "interrupt" && execution.continuation === void 0)
+              throw new Error(
+                "A waiting interrupt requires a private continuation."
+              );
+            return {
+              observation,
+              ...continuation !== void 0 ? { continuation } : {}
+            };
+          };
+          try {
             await emit({
-              type: "case-progress",
+              type: "case-started",
+              attemptId,
               caseId: item.id,
               repetition,
-              phase,
-              ...stepIndex !== void 0 ? { stepIndex } : {}
+              sessionId: attemptResult.sessionId
             });
-        };
-        let context;
-        let continuation;
-        let activeStep;
-        let activeStepReported = false;
-        const history = [];
-        const call = async (readyContext, command, activeSignal) => {
-          const run = (overrides) => executeEvalChat({
-            agent: options.agent,
-            command,
-            continuation,
-            history: clone(history),
-            sessionId: attemptResult.sessionId,
-            clientTurnId: (0, import_node_crypto5.randomUUID)(),
-            signal: activeSignal,
-            ...item.workflowId ? { workflowId: item.workflowId } : {},
-            ...overrides
-          });
-          const execution = options.execute ? await options.execute({
-            ...readyContext,
-            signal: activeSignal,
-            command,
-            continuation,
-            history: clone(history),
-            run
-          }) : await run();
-          continuation = execution.continuation;
-          const observation = EvalObservationSchema.parse(
-            execution.observation
-          );
-          if (observation.type === "interrupt" && execution.continuation === void 0)
-            throw new Error(
-              "A waiting interrupt requires a private continuation."
-            );
-          return {
-            observation,
-            ...continuation !== void 0 ? { continuation } : {}
-          };
-        };
-        try {
-          await emit({
-            type: "case-started",
-            caseId: item.id,
-            repetition,
-            sessionId: attemptResult.sessionId
-          });
-          signal.throwIfAborted();
-          await reportPhase("params");
-          const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
-          const base = {
-            suiteId: suite.id,
-            case: item,
-            repetition,
-            sessionId: attemptResult.sessionId,
-            params,
-            signal
-          };
-          await reportPhase("setup");
-          const prepared = options.setup ? await options.setup(base) : void 0;
-          context = { ...base, prepared };
-          signal.throwIfAborted();
-          for (const [index, step] of item.steps.entries()) {
-            stepIndex = index;
-            activeStep = void 0;
-            activeStepReported = false;
             signal.throwIfAborted();
-            let command;
-            if (step.message !== void 0)
-              command = { type: "message", message: step.message };
-            else {
-              await reportPhase("responder");
-              const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
-              let response = step.resume;
-              if (handlerRef(response)) {
-                const responder = responders[response.using];
-                if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
-                  throw new Error("Incompatible responder.");
-                response = await (typeof responder === "function" ? responder : responder.respond)({
-                  ...context,
-                  interrupt: clone(interrupt),
-                  ...response.params !== void 0 ? { args: response.params } : {}
-                });
-              }
-              command = {
-                type: "resume",
-                // z.json() is inferred as optional under the package's relaxed
-                // declaration build. Runtime validation still requires value.
-                response: EvalResumeResponseSchema.parse(
-                  response
-                )
-              };
-            }
-            signal.throwIfAborted();
-            await reportPhase("execute");
-            const { observation } = await call(context, command, signal);
-            const input = command.type === "message" ? { message: command.message } : { resume: command.response };
-            const evaluated = {
-              index,
-              input,
-              expectation: clone(step.expect),
-              observation,
-              status: "passed",
-              criteria: []
+            await reportPhase("params");
+            const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
+            const base = {
+              suiteId: suite.id,
+              case: item,
+              repetition,
+              sessionId: attemptResult.sessionId,
+              params,
+              signal
             };
-            attemptResult.steps.push(evaluated);
-            activeStep = evaluated;
+            await reportPhase("setup");
+            const prepared = options.setup ? await options.setup(base) : void 0;
+            context = { ...base, prepared };
             signal.throwIfAborted();
-            if (observation.type === "error") throw new Error("Agent failed.");
-            if (observation.type === "cancelled") {
-              evaluated.status = "cancelled";
-              attemptResult.status = "cancelled";
+            for (const [index, step] of item.steps.entries()) {
+              stepIndex = index;
+              activeStep = void 0;
+              activeStepReported = false;
+              signal.throwIfAborted();
+              let command;
+              if (step.message !== void 0)
+                command = { type: "message", message: step.message };
+              else {
+                await reportPhase("responder");
+                const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
+                let response = step.resume;
+                if (handlerRef(response)) {
+                  const responder = responders[response.using];
+                  if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
+                    throw new Error("Incompatible responder.");
+                  response = await (typeof responder === "function" ? responder : responder.respond)({
+                    ...context,
+                    interrupt: clone(interrupt),
+                    ...response.params !== void 0 ? { args: response.params } : {}
+                  });
+                }
+                command = {
+                  type: "resume",
+                  // z.json() is inferred as optional under the package's relaxed
+                  // declaration build. Runtime validation still requires value.
+                  response: EvalResumeResponseSchema.parse(
+                    response
+                  )
+                };
+              }
+              signal.throwIfAborted();
+              await reportPhase("execute");
+              const { observation } = await call(context, command, signal);
+              const input = command.type === "message" ? { message: command.message } : { resume: command.response };
+              const evaluated = {
+                index,
+                input,
+                expectation: clone(step.expect),
+                observation,
+                status: "passed",
+                criteria: []
+              };
+              attemptResult.steps.push(evaluated);
+              activeStep = evaluated;
+              signal.throwIfAborted();
+              if (observation.type === "error")
+                throw new Error("Agent failed.");
+              if (observation.type === "cancelled") {
+                evaluated.status = "cancelled";
+                attemptResult.status = "cancelled";
+                await emit({
+                  type: "step-completed",
+                  caseId: item.id,
+                  repetition,
+                  step: evaluated
+                });
+                activeStepReported = true;
+                break;
+              }
+              if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
+                evaluated.status = "failed";
+                evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
+              } else {
+                const outputReason = missingOutputReason(
+                  step.expect.outputs,
+                  observation
+                );
+                if (outputReason) {
+                  evaluated.status = "failed";
+                  evaluated.reason = outputReason;
+                } else {
+                  const reference = step.expect.reference;
+                  if (reference !== void 0) {
+                    await reportPhase("reference");
+                    evaluated.reference = handlerRef(reference) ? import_zod8.z.json().parse(
+                      await references[reference.using]({
+                        ...context,
+                        observation: clone(observation),
+                        ...reference.params !== void 0 ? { args: reference.params } : {}
+                      })
+                    ) : clone(reference);
+                  }
+                  await reportPhase("grading");
+                  evaluated.evidence = createEvalJudgeEvidence(
+                    observation,
+                    suite.evidence,
+                    evidenceFilters
+                  );
+                  if (studioGrading && step.expect.criteria?.length) {
+                    evaluated.status = "ungraded";
+                    attemptResult.status = "ungraded";
+                  }
+                  for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
+                    signal.throwIfAborted();
+                    const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
+                    evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
+                    const verdict = EvalVerdictSchema.parse(
+                      await (0, import_eval_attribution2.withEvalAttribution)(
+                        void 0,
+                        () => options.judge.grade({
+                          criterion,
+                          onUsage: (usage) => {
+                            if (!signal.aborted) {
+                              evaluated.judgeUsage ??= [];
+                              evaluated.judgeUsage.push(clone(usage));
+                            }
+                          },
+                          input: clone(input),
+                          ...getEvalGradeEvidence(
+                            evaluated,
+                            attemptResult.steps.slice(0, -1)
+                          ),
+                          signal,
+                          ...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
+                        })
+                      )
+                    );
+                    signal.throwIfAborted();
+                    evaluated.criteria.push({ ...criterion, ...verdict });
+                    if (!verdict.passed) evaluated.status = "failed";
+                  }
+                }
+              }
               await emit({
                 type: "step-completed",
                 caseId: item.id,
@@ -10917,161 +11027,116 @@ function createEvals(options) {
                 step: evaluated
               });
               activeStepReported = true;
-              break;
-            }
-            if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
-              evaluated.status = "failed";
-              evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
-            } else {
-              const outputReason = missingOutputReason(
-                step.expect.outputs,
-                observation
-              );
-              if (outputReason) {
-                evaluated.status = "failed";
-                evaluated.reason = outputReason;
-              } else {
-                const reference = step.expect.reference;
-                if (reference !== void 0) {
-                  await reportPhase("reference");
-                  evaluated.reference = handlerRef(reference) ? import_zod8.z.json().parse(
-                    await references[reference.using]({
-                      ...context,
-                      observation: clone(observation),
-                      ...reference.params !== void 0 ? { args: reference.params } : {}
-                    })
-                  ) : clone(reference);
-                }
-                await reportPhase("grading");
-                evaluated.evidence = createEvalJudgeEvidence(
-                  observation,
-                  suite.evidence,
-                  evidenceFilters
-                );
-                if (studioGrading && step.expect.criteria?.length) {
-                  evaluated.status = "ungraded";
-                  attemptResult.status = "ungraded";
-                }
-                for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
-                  signal.throwIfAborted();
-                  const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
-                  evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
-                  const verdict = EvalVerdictSchema.parse(
-                    await options.judge.grade({
-                      criterion,
-                      onUsage: (usage) => {
-                        if (!signal.aborted) {
-                          evaluated.judgeUsage ??= [];
-                          evaluated.judgeUsage.push(clone(usage));
-                        }
-                      },
-                      input: clone(input),
-                      ...getEvalGradeEvidence(
-                        evaluated,
-                        attemptResult.steps.slice(0, -1)
-                      ),
-                      signal,
-                      ...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
-                    })
-                  );
-                  signal.throwIfAborted();
-                  evaluated.criteria.push({ ...criterion, ...verdict });
-                  if (!verdict.passed) evaluated.status = "failed";
-                }
+              if (evaluated.status === "failed") {
+                attemptResult.status = "failed";
+                break;
               }
+              history.push(
+                {
+                  role: "user",
+                  content: command.type === "message" ? command.message : JSON.stringify(command.response)
+                },
+                { role: "assistant", content: observation.text }
+              );
             }
-            await emit({
-              type: "step-completed",
-              caseId: item.id,
-              repetition,
-              step: evaluated
-            });
-            activeStepReported = true;
-            if (evaluated.status === "failed") {
-              attemptResult.status = "failed";
-              break;
+          } catch {
+            if (signal.aborted) {
+              attemptResult.status = timedOut ? "error" : "cancelled";
+              if (timedOut)
+                attemptResult.errors.push({
+                  phase,
+                  code: "EVAL_TIMEOUT",
+                  message: "Case exceeded its configured time limit."
+                });
+            } else {
+              attemptResult.status = "error";
+              attemptResult.errors.push(failure(phase));
             }
-            history.push(
-              {
-                role: "user",
-                content: command.type === "message" ? command.message : JSON.stringify(command.response)
-              },
-              { role: "assistant", content: observation.text }
-            );
-          }
-        } catch {
-          if (signal.aborted) {
-            attemptResult.status = timedOut ? "error" : "cancelled";
-            if (timedOut)
-              attemptResult.errors.push({
-                phase,
-                code: "EVAL_TIMEOUT",
-                message: "Case exceeded its configured time limit."
+            if (activeStep && !activeStepReported) {
+              activeStep.status = attemptResult.status;
+              activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
+              await emit({
+                type: "step-completed",
+                caseId: item.id,
+                repetition,
+                step: activeStep
               });
-          } else {
-            attemptResult.status = "error";
-            attemptResult.errors.push(failure(phase));
-          }
-          if (activeStep && !activeStepReported) {
-            activeStep.status = attemptResult.status;
-            activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
-            await emit({
-              type: "step-completed",
-              caseId: item.id,
-              repetition,
-              step: activeStep
-            });
-          }
-        } finally {
-          clearTimeout(timer);
-          if (context) {
-            await reportPhase("cleanup");
-            const cleanup = new AbortController();
-            const cleanupTimer = setTimeout(
-              () => cleanup.abort(),
-              defaults.cleanupTimeoutMs
-            );
-            try {
-              if (continuation !== void 0) {
-                try {
-                  const cancelled = await call(
-                    context,
-                    { type: "cancel" },
-                    cleanup.signal
-                  );
-                  if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
+            }
+          } finally {
+            clearTimeout(timer);
+            if (context) {
+              await reportPhase("cleanup");
+              const cleanup = new AbortController();
+              const cleanupTimer = setTimeout(
+                () => cleanup.abort(),
+                defaults.cleanupTimeoutMs
+              );
+              try {
+                if (continuation !== void 0) {
+                  try {
+                    const cancelled = await call(
+                      context,
+                      { type: "cancel" },
+                      cleanup.signal
+                    );
+                    if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
+                      attemptResult.errors.push(
+                        failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
+                      );
+                    cleanup.signal.throwIfAborted();
+                  } catch {
                     attemptResult.errors.push(
                       failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
                     );
+                  }
+                }
+                try {
+                  await options.teardown?.({
+                    ...context,
+                    signal: cleanup.signal,
+                    result: clone(attemptResult)
+                  });
                   cleanup.signal.throwIfAborted();
                 } catch {
-                  attemptResult.errors.push(
-                    failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
-                  );
+                  attemptResult.errors.push(failure("cleanup"));
                 }
+              } finally {
+                clearTimeout(cleanupTimer);
               }
-              try {
-                await options.teardown?.({
-                  ...context,
-                  signal: cleanup.signal,
-                  result: clone(attemptResult)
-                });
-                cleanup.signal.throwIfAborted();
-              } catch {
-                attemptResult.errors.push(failure("cleanup"));
-              }
-            } finally {
-              clearTimeout(cleanupTimer);
+              if (attemptResult.errors.some((error) => error.phase === "cleanup"))
+                attemptResult.status = "error";
             }
-            if (attemptResult.errors.some((error) => error.phase === "cleanup"))
-              attemptResult.status = "error";
+            attemptResult.durationMs = Date.now() - started;
           }
-          attemptResult.durationMs = Date.now() - started;
-        }
-        await emit({ type: "case-completed", result: attemptResult });
-        return attemptResult;
+          acceptingExecutions = false;
+          await associationProgress;
+          await emit({ type: "case-completed", result: attemptResult });
+          return attemptResult;
+        };
+        return (0, import_eval_attribution2.withEvalAttribution)(
+          {
+            attemptId,
+            onExecution: (execution) => {
+              if (!acceptingExecutions || runtimeExecutions.some(
+                (entry) => entry.runId === execution.runId && entry.sessionId === execution.sessionId
+              ))
+                return;
+              runtimeExecutions.push(execution);
+              associationProgress = associationProgress.then(
+                () => emit({
+                  type: "case-runtime-associated",
+                  caseId: item.id,
+                  repetition,
+                  attemptId,
+                  execution
+                })
+              );
+            }
+          },
+          executeAttempt
+        );
       };
-      const total = selected.length * repetitions;
+      const total = args.attempt ? 1 : selected.length * repetitions;
       if (args.includeActivity)
         await emit({
           type: "run-started",
@@ -11085,8 +11150,11 @@ function createEvals(options) {
         Array.from({ length: Math.min(concurrency, total) }, async () => {
           while (cursor < total) {
             const index = cursor++;
-            const item = selected[Math.floor(index / repetitions)];
-            completed[index] = await attempt(item, index % repetitions + 1);
+            const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
+            completed[index] = await attempt(
+              item,
+              args.attempt?.repetition ?? index % repetitions + 1
+            );
           }
         })
       );
@@ -11396,6 +11464,7 @@ function createEvalRouteHandler({
   const digest = (value) => (0, import_node_crypto6.createHash)("sha256").update(value).digest();
   const expected = digest(`Bearer ${serviceKey}`);
   let active = 0;
+  const evaluations = /* @__PURE__ */ new Map();
   return async (request) => {
     if (!(0, import_node_crypto6.timingSafeEqual)(
       expected,
@@ -11452,14 +11521,32 @@ function createEvalRouteHandler({
         { error: "App judge changed. Refresh before running." },
         { status: 409 }
       );
-    if (active >= maxActiveRuns)
+    if (body.attempt && (!manifest.attemptScheduling || !suite.cases.some((item) => item.id === body.attempt?.caseId) || body.caseIds && !body.caseIds.includes(body.attempt.caseId) || body.attempt.repetition > body.repetitions))
+      return Response.json(
+        { error: "Unsupported or invalid scheduled attempt." },
+        { status: 409 }
+      );
+    const group = body.attempt && evaluations.get(body.attempt.evaluationId);
+    if (group && (group.concurrency !== body.concurrency || group.active >= group.concurrency))
+      return Response.json(
+        { error: "Evaluation concurrency exhausted or changed." },
+        { status: 429 }
+      );
+    if (!group && active >= maxActiveRuns)
       return Response.json(
         { error: "Eval executor is busy." },
         { status: 429 }
       );
     if (request.signal.aborted)
       return Response.json({ error: "Request cancelled." }, { status: 409 });
-    active++;
+    if (body.attempt) {
+      const state = group ?? { active: 0, concurrency: body.concurrency };
+      if (!group) {
+        active++;
+        evaluations.set(body.attempt.evaluationId, state);
+      }
+      state.active++;
+    } else active++;
     const controller = new AbortController();
     const requestSignal = request.signal;
     const abortRequest = () => controller.abort();
@@ -11475,6 +11562,7 @@ function createEvalRouteHandler({
         };
         void evals.run({
           suiteId: body.suiteId,
+          ...body.attempt ? { attempt: body.attempt } : {},
           ...body.grading ? { grading: body.grading } : {},
           ...body.judge ? { judgeIdentity: body.judge } : {},
           repetitions: body.repetitions,
@@ -11487,7 +11575,14 @@ function createEvalRouteHandler({
           () => send({ type: "error", message: "Eval runner failed." })
         ).finally(() => {
           requestSignal.removeEventListener("abort", abortRequest);
-          active--;
+          if (body.attempt) {
+            const state = evaluations.get(body.attempt.evaluationId);
+            state.active--;
+            if (!state.active) {
+              evaluations.delete(body.attempt.evaluationId);
+              active--;
+            }
+          } else active--;
           if (!controller.signal.aborted) output.close();
         });
       },