@apo-ai/sdk 0.2.2 → 0.3.0

package/dist/agent-task/integrations/ai-sdk.d.ts

@@ -1,5 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-DtbhYQV6.js';
-import '../../types-C-nrDEVf.js';
+import { A as AgentTaskTraceContext } from '../../tracing-BE53Q8Ru.js';
 
 /**
  * Vercel AI SDK tracing integration for apo.

package/dist/agent-task/integrations/anthropic.d.ts

@@ -1,5 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-DtbhYQV6.js';
-import '../../types-C-nrDEVf.js';
+import { A as AgentTaskTraceContext } from '../../tracing-BE53Q8Ru.js';
 
 /**
  * Anthropic JS SDK tracing integration for apo.

package/dist/agent-task/integrations/openai.d.ts

@@ -1,5 +1,4 @@
-import { A as AgentTaskTraceContext } from '../../tracing-DtbhYQV6.js';
-import '../../types-C-nrDEVf.js';
+import { A as AgentTaskTraceContext } from '../../tracing-BE53Q8Ru.js';
 
 /**
  * OpenAI JS SDK tracing integration for apo.

package/dist/agent-task/public.d.ts

@@ -1,8 +1,7 @@
-import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-DtbhYQV6.js';
+import { A as AgentTaskTraceContext, a as AgentTaskTraceOptions } from '../tracing-BE53Q8Ru.js';
 export { CreateApoTracerOptions, createApoTracer } from './integrations/ai-sdk.js';
 export { CreateApoOpenAIOptions, createApoOpenAI } from './integrations/openai.js';
 export { CreateApoAnthropicOptions, createApoAnthropic } from './integrations/anthropic.js';
-import '../types-C-nrDEVf.js';
 
 type TaskDefinition<TAdapterName extends string = string, TDeliverable extends string = string> = {
     id: string;
@@ -1053,6 +1052,85 @@ type RunTaskOptions = {
 };
 declare function runTask(taskDir: string, options?: RunTaskOptions): Promise<TaskRunResult>;
 
+/**
+ * Issue #159: replay Phase 2 of a completed Run against its stored
+ * Deliverables — without re-running the agent.
+ *
+ * The backend serves what replay reads (run detail, Deliverable bodies, the
+ * pinned Task Definition source, the canonical trace projection) and stores
+ * what replay produces (a judgment record — submitted by the CLI, not here).
+ * The eval source itself is executed HERE, never in the backend: the
+ * definition revision's contract is that the backend stores source as
+ * private data and never executes it.
+ *
+ * Rules this module enforces on itself:
+ * - whole-run replay: the full registered check set runs every sample;
+ * - the Run must be terminal (passed/failed) and every Deliverable ready —
+ *   a missing Deliverable must surface as a refusal, not a low score;
+ * - scoring defaults to the Run's pinned definition revision; an explicit
+ *   different revision is allowed but reported as unpinned so it can never
+ *   be misread as "the same eval, better judge".
+ */
+
+/** Backend coordinates for every read replay performs. */
+interface RejudgeEndpoints {
+    /** Backend base URL, e.g. ``"http://localhost:8000"``. */
+    backendUrl: string;
+    /** Project API key (Authorization: Bearer …). Never recorded on judgments. */
+    authToken: string;
+}
+interface RejudgeOptions {
+    /** Judge config for ``t.judge`` calls; the API key stays client-side. */
+    judge?: JudgeConfig;
+    /** Judge the same Deliverables N times for a per-criterion stability measure. */
+    samples?: number;
+    /** Score against this revision instead of the Run's pinned one. */
+    definitionRevisionId?: string;
+    /**
+     * Local checkout of the task directory. The stored revision pins only the
+     * eval file, so relative imports and `files/` fixtures resolve from here.
+     */
+    taskDir?: string;
+    /** Max wait for the canonical trace projection (default 30s). */
+    projectionDeadlineMs?: number;
+    /** Progress sink for human-facing operators (CLI prints to stderr). */
+    onProgress?: (message: string) => void;
+}
+/** Per-check pass counts across samples — "fails 2 of 5", not "failed". */
+interface RejudgeCheckStability {
+    check_id: string;
+    passes: number;
+    samples: number;
+}
+interface RejudgeOutcome {
+    runId: string;
+    taskId: string;
+    /** Revision the checks were actually scored against. */
+    definitionRevisionId: string;
+    /** False when an explicit --definition-revision moved the target. */
+    definitionRevisionIsPinned: boolean;
+    /** Resolved judge config actually used (model + base URL, never the key). */
+    judge: {
+        model: string;
+        baseURL?: string;
+    } | null;
+    samples: number;
+    /** Full check results from the primary (first) sample. */
+    checks: EvaluationItemResult[];
+    pass: boolean;
+    /** Only meaningful with samples > 1; ordered like the primary sample. */
+    stability: RejudgeCheckStability[];
+    /** False when the trace projection was unreadable — trajectory assertions were unsupported. */
+    traceSnapshotAvailable: boolean;
+    /** The local task dir replay ran against, or null for an isolated scaffold. */
+    taskDirUsed: string | null;
+}
+/** A deliberate, operator-facing replay failure (refusals, bad state). */
+declare class RejudgeError extends Error {
+    constructor(message: string);
+}
+declare function rejudgeTaskRun(runId: string, endpoints: RejudgeEndpoints, options?: RejudgeOptions): Promise<RejudgeOutcome>;
+
 type AgentTaskCliOptions = {
     task?: string;
     dir?: string;
@@ -1325,4 +1403,4 @@ declare function canonicalManifestJson(manifest: TaskRevisionManifestV1): string
 /** SHA-256 (lowercase hex) of the canonical manifest JSON's UTF-8 bytes. */
 declare function contentSha256(manifest: TaskRevisionManifestV1): string;
 
-export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, type JudgeConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, type RegisterApoTracingOptions, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };
+export { type AISDKResult, type AdapterDefinition, type AdapterRuntimeState, type AdapterSession, type AgentTaskCliOptions, type AgentTaskRunConfiguration, AgentTaskRunError, type AgentTaskRunSummary, type AgentTaskRuntime, AgentTaskTraceContext, AgentTaskTraceOptions, type AgentTurnResult, type AnthropicMessage, type ApoRunContext, ApoSpanProcessor, type ArtifactUploadConfig, type CheckContext, type CheckGroup, type CleanupContext, type CollectDeliverablesContext, type CollectedDeliverables, type DeliverableDefinition, type DeliverableSummary, type DeliverableValidationResult, type DescribeRegistration, type FileArtifact, type FileArtifactOptions, type FileEntry, type Flow, type FlowEvent, FlowView, type InitializeContext, type JudgeConfig, type LoadedTask, type ManifestFile, type ManifestFileInput, type Matcher, type ModeClass, type NameMatcher, type OpenAIMessage, type PreparedTaskResult, type RegisterApoTracingOptions, type RejudgeCheckStability, type RejudgeEndpoints, RejudgeError, type RejudgeOptions, type RejudgeOutcome, type RunTaskOptions, type StartSessionContext, TEST_METHOD_NAMES, type TaskConfig, type TaskDefinition, type TaskEvaluationResult, TaskFiles, type TaskRevisionManifestV1, type TaskRunResult, type TaskScope, type TaskTranscript, type TaskTranscriptTurn, type TestContext, type TestRegistration, type ToolCallOptions, type ToolCallStatus, type TurnContext, type TurnFn, type TurnRecord, type TypedAdapterDefinition, type ValueMatcher, buildManifest, canonicalManifestJson, contentSha256, defineAdapter, defineTask, describe, discoverAgentTaskDirs, equals, fileArtifact, filePaths, fromAISDK, fromAnthropicMessages, fromOpenAIMessages, getActiveApoRun, getTaskTurn, includes, isFileArtifact, loadTask, loadTaskRuntime, matchValue, matches, normalizeManifestPath, parseAgentTaskCliArgs, persistFileArtifacts, registerApoTracing, rejudgeTaskRun, resetApoTracing, resetFlowChecks, resetTaskRegistry, resetTaskTurn, resolveTurn, runAgentTaskCli, runTask, runTaskDir, satisfies, sha256Hex, similarity, task, defineCheck as test, turn, withApoRun, withApoRunSync };

package/dist/agent-task/public.js

@@ -919,7 +919,7 @@ function attachAdapter(taskDefinition, adapter) {
 }
 
 // src/agent-task/task/loadTask.ts
-import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2 } from "fs";
+import { copyFileSync as copyFileSync2, readdirSync, statSync, existsSync as existsSync2, unlinkSync as unlinkSync2, writeFileSync } from "fs";
 import { join, resolve, basename as basename2 } from "path";
 import { pathToFileURL as pathToFileURL2 } from "url";
 
@@ -966,6 +966,48 @@ async function loadTask(taskDir) {
     evalFileName: basename2(taskFilePath)
   };
 }
+async function loadEvalSource(fileName, content, opts = {}) {
+  const dir = opts.siblingDir ?? opts.isolatedDir;
+  if (!dir) {
+    throw new Error("loadEvalSource requires a siblingDir or isolatedDir to materialize next to");
+  }
+  const tempPath = join(
+    dir,
+    `${fileName}.rejudge-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`
+  );
+  writeFileSync(tempPath, content);
+  try {
+    const { task: task2, adapter, inlineChecks, moduleUrl } = await loadTaskDefinition(tempPath);
+    validateTaskDefinition(task2, dir);
+    if (!inlineChecks && !existsSync2(join(dir, "checks.ts"))) {
+      throw new Error(
+        `The pinned eval did not register checks: legacy two-file tasks need their checks module, which the stored definition revision does not include`
+      );
+    }
+    return {
+      task: task2,
+      adapter,
+      taskDir: dir,
+      files: loadFilesDirectory(dir),
+      checksPath: resolveChecksPath(task2, dir),
+      inlineChecks,
+      moduleUrl,
+      evalFileName: fileName
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    if (!opts.siblingDir && /Failed to load task definition/.test(message)) {
+      throw new Error(
+        `Replay needs the task directory: the pinned eval imports sibling files that are not part of the stored definition revision. Point the command at a checkout of the task (--task-dir) and retry. Underlying error: ${message}`
+      );
+    }
+    throw error;
+  } finally {
+    if (existsSync2(tempPath)) {
+      unlinkSync2(tempPath);
+    }
+  }
+}
 async function loadTaskDefinition(taskFilePath) {
   const tempModulePath = `${taskFilePath}.${Date.now()}-${Math.random().toString(36).slice(2)}.ts`;
   try {
@@ -2315,12 +2357,247 @@ function buildTraceRunOptions(loaded, tracing) {
   };
 }
 
+// src/agent-task/run/rejudge.ts
+import { mkdtempSync, rmSync, writeFileSync as writeFileSync2 } from "fs";
+import { tmpdir } from "os";
+import { basename as basename4, dirname, join as join2 } from "path";
+import { fileURLToPath } from "url";
+var RejudgeError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "RejudgeError";
+  }
+};
+var TERMINAL_VERDICT_STATUSES = /* @__PURE__ */ new Set(["passed", "failed"]);
+var MAX_SAMPLES = 50;
+async function rejudgeTaskRun(runId, endpoints, options = {}) {
+  const api = createApiClient(endpoints);
+  const progress = options.onProgress ?? (() => {
+  });
+  const detail = await api.get(`/v1/agent-task-runs/${runId}`);
+  if (!TERMINAL_VERDICT_STATUSES.has(detail.status)) {
+    throw new RejudgeError(
+      `Run ${runId} is '${detail.status}', not a completed run with a verdict \u2014 there is nothing to replay against yet`
+    );
+  }
+  const manifest = await api.get(
+    `/v1/agent-task-runs/${runId}/deliverables`
+  );
+  const items = manifest.items ?? [];
+  const notReady = items.filter((item) => item.status !== "ready");
+  if (notReady.length > 0) {
+    const names = notReady.map((item) => `${item.name} (${item.status})`).join(", ");
+    throw new RejudgeError(
+      `Refusing to replay run ${runId}: deliverables are not ready: ${names}. A missing deliverable must not render as a low score \u2014 re-run the agent instead.`
+    );
+  }
+  const requestedRevision = options.definitionRevisionId ?? detail.task_definition?.id ?? null;
+  if (!requestedRevision) {
+    throw new RejudgeError(
+      `Run ${runId} has no pinned Task Definition Revision \u2014 its checks cannot be replayed`
+    );
+  }
+  const sourceQuery = options.definitionRevisionId && options.definitionRevisionId !== detail.task_definition?.id ? `?revision=${encodeURIComponent(options.definitionRevisionId)}` : "";
+  progress(`fetching definition source (${requestedRevision})`);
+  const source = await api.get(
+    `/v1/agent-task-runs/${runId}/definition-source${sourceQuery}`
+  );
+  const revisionId = source.task_definition_revision_id;
+  const revisionIsPinned = revisionId === (detail.task_definition?.id ?? null);
+  const evalFile = source.files[0];
+  if (!evalFile) {
+    throw new RejudgeError(`Definition revision ${revisionId} carries no source files`);
+  }
+  progress("fetching stored deliverables");
+  const artifactDir = mkdtempSync(join2(tmpdir(), "apo-rejudge-artifacts-"));
+  let scaffoldDir = null;
+  try {
+    const taskDir = options.taskDir ?? null;
+    scaffoldDir = taskDir ? null : isolatedScaffoldDir();
+    const deliverables = await fetchDeliverables(api, runId, items, artifactDir);
+    progress("reading canonical trace projection");
+    const snapshotResult = await readCanonicalSnapshot2(
+      endpoints,
+      runId,
+      options.projectionDeadlineMs
+    );
+    if (!snapshotResult.available) {
+      progress(
+        "trace projection unavailable \u2014 trajectory assertions (t.calledTool, \u2026) will be recorded as unsupported rather than replayed"
+      );
+    }
+    const snapshot = snapshotResult.snapshot;
+    const loaded = await loadEvalSource(evalFile.path, evalFile.content, {
+      siblingDir: taskDir,
+      isolatedDir: scaffoldDir
+    });
+    progress(`loaded eval ${evalFile.path} from revision ${revisionId}`);
+    const missingDeliverables = loaded.task.deliverables.filter(
+      (name) => !Object.prototype.hasOwnProperty.call(deliverables, name)
+    );
+    if (missingDeliverables.length > 0) {
+      throw new RejudgeError(
+        `Refusing to replay run ${runId}: stored deliverables are incomplete; missing ${missingDeliverables.join(", ")}. Re-run the agent instead.`
+      );
+    }
+    const samples = Math.max(1, Math.min(options.samples ?? 1, MAX_SAMPLES));
+    const validation = validateDeliverables(
+      loaded.task,
+      deliverables,
+      loaded.adapter.deliverables
+    );
+    const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
+    const passCounts = /* @__PURE__ */ new Map();
+    let primary = [];
+    for (let sample = 0; sample < samples; sample++) {
+      if (samples > 1) progress(`judging sample ${sample + 1}/${samples}`);
+      const results = await runChecksOnce(loaded, proxied, snapshot, options.judge);
+      if (sample === 0) primary = results;
+      for (const result of results) {
+        passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
+      }
+    }
+    const stability = primary.map((result) => ({
+      check_id: result.id,
+      passes: passCounts.get(result.id) ?? 0,
+      samples
+    }));
+    const aggregate = aggregateResult(primary);
+    return {
+      runId,
+      taskId: detail.task_id,
+      definitionRevisionId: revisionId,
+      definitionRevisionIsPinned: revisionIsPinned,
+      judge: options.judge ? {
+        model: options.judge.model,
+        baseURL: options.judge.baseURL ?? process.env.OPENROUTER_BASE_URL
+      } : null,
+      samples,
+      checks: aggregate.checks,
+      pass: aggregate.pass,
+      stability,
+      traceSnapshotAvailable: snapshotResult.available,
+      taskDirUsed: taskDir
+    };
+  } finally {
+    rmSync(artifactDir, { recursive: true, force: true });
+    if (scaffoldDir) rmSync(scaffoldDir, { recursive: true, force: true });
+  }
+}
+async function runChecksOnce(loaded, deliverables, snapshot, judgeConfig) {
+  if (loaded.inlineChecks) {
+    return runTraceChecks({
+      snapshot,
+      deliverables,
+      files: loaded.files,
+      task: loaded.task,
+      judgeConfig,
+      moduleUrl: loaded.moduleUrl,
+      displayFile: loaded.evalFileName
+    });
+  }
+  return loadAndRunFlowChecks(
+    loaded.checksPath,
+    {
+      snapshot,
+      deliverables,
+      files: loaded.files,
+      task: loaded.task,
+      judgeConfig
+    },
+    {}
+  );
+}
+async function fetchDeliverables(api, runId, items, artifactDir) {
+  const deliverables = {};
+  for (const [index, item] of items.entries()) {
+    if (item.kind === "artifact") {
+      const bytes = await api.getBytes(`/v1/agent-task-runs/${runId}/deliverables/${item.id}`);
+      const displayFilename = item.display_filename || item.name;
+      const safeFilename = basename4(displayFilename) || "artifact";
+      const path = join2(artifactDir, `artifact-${index}-${safeFilename}`);
+      writeFileSync2(path, bytes);
+      deliverables[item.name] = Object.freeze({
+        kind: "apo.file-artifact",
+        path,
+        mediaType: item.media_type ?? "application/octet-stream",
+        displayFilename
+      });
+    } else {
+      deliverables[item.name] = await api.getJson(
+        `/v1/agent-task-runs/${runId}/deliverables/${item.id}`
+      );
+    }
+  }
+  return deliverables;
+}
+async function readCanonicalSnapshot2(endpoints, runId, deadlineMs) {
+  try {
+    const snapshot = await readTaskRunProjection({
+      endpoint: endpoints.backendUrl,
+      authToken: endpoints.authToken,
+      taskRunId: runId,
+      deadlineMs: deadlineMs ?? 15e3
+    });
+    return { available: true, snapshot };
+  } catch {
+    return { available: false, snapshot: degenerateSnapshot(runId) };
+  }
+}
+function degenerateSnapshot(runId) {
+  return {
+    schemaVersion: 1,
+    projectionVersion: 1,
+    source: "local",
+    trace: { traceId: `rejudge-${runId}`, complete: true },
+    capabilities: {
+      messages: "unavailable",
+      tools: "unavailable",
+      errors: "unavailable",
+      timing: "unavailable",
+      skills: "unavailable",
+      subagents: "unavailable"
+    },
+    observations: []
+  };
+}
+function isolatedScaffoldDir() {
+  const here = dirname(fileURLToPath(import.meta.url));
+  return mkdtempSync(join2(here, ".rejudge-scaffold-"));
+}
+function createApiClient(endpoints) {
+  const base = endpoints.backendUrl.replace(/\/$/, "");
+  const headers = { Authorization: `Bearer ${endpoints.authToken}` };
+  async function request(path) {
+    const response = await fetch(`${base}${path}`, { headers });
+    if (!response.ok) {
+      const body = await response.text().catch(() => "");
+      throw new RejudgeError(
+        `${path} failed (${response.status}): ${body.slice(0, 200) || response.statusText}`
+      );
+    }
+    return response;
+  }
+  return {
+    async get(path) {
+      return await request(path).then((r) => r.json());
+    },
+    async getJson(path) {
+      return request(path).then((r) => r.json());
+    },
+    async getBytes(path) {
+      const buffer = await request(path).then((r) => r.arrayBuffer());
+      return Buffer.from(new Uint8Array(buffer));
+    }
+  };
+}
+
 // src/agent-task/cli.ts
 import { relative, resolve as resolve3 } from "path";
 
 // src/agent-task/discovery.ts
 import { existsSync as existsSync3, readdirSync as readdirSync2, statSync as statSync3 } from "fs";
-import { join as join2, resolve as resolve2 } from "path";
+import { join as join3, resolve as resolve2 } from "path";
 function discoverAgentTaskDirs(rootDir) {
   const absoluteRoot = resolve2(rootDir);
   if (!existsSync3(absoluteRoot) || !statSync3(absoluteRoot).isDirectory()) {
@@ -2336,7 +2613,7 @@ function walkForTaskDirs(currentDir, discovered) {
     return;
   }
   for (const entry of readdirSync2(currentDir)) {
-    const absolutePath = join2(currentDir, entry);
+    const absolutePath = join3(currentDir, entry);
     if (!statSync3(absolutePath).isDirectory()) {
       continue;
     }
@@ -2998,6 +3275,7 @@ export {
   AgentTaskRunError,
   ApoSpanProcessor,
   FlowView,
+  RejudgeError,
   TEST_METHOD_NAMES,
   TaskFiles,
   buildManifest,
@@ -3028,6 +3306,7 @@ export {
   parseAgentTaskCliArgs,
   persistFileArtifacts,
   registerApoTracing,
+  rejudgeTaskRun,
   resetApoTracing,
   resetFlowChecks,
   resetTaskRegistry,