@apo-ai/sdk 0.3.0 → 0.5.0

package/dist/agent-task/public.js

@@ -235,7 +235,8 @@ function levenshtein(a, b) {
 }
 
 // src/agent-task/checks/judge.ts
-var JUDGE_SYSTEM_PROMPT = 'You are an evaluation judge. Evaluate the given value(s) against the instruction. Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
+var JUDGE_RESPONSE_CONTRACT = 'Respond with ONLY a JSON object: {"pass": true/false, "reasoning": "your reasoning"}';
+var JUDGE_SYSTEM_PROMPT = `You are an evaluation judge. Evaluate the given value(s) against the instruction. ${JUDGE_RESPONSE_CONTRACT}`;
 function formatValue(value, depth = 0) {
   const indent = "  ".repeat(depth);
   if (value === null || value === void 0) return String(value);
@@ -314,12 +315,11 @@ async function callJudge(args) {
   const apiKey = args.apiKey ?? process.env.OPENROUTER_API_KEY ?? process.env.OPENAI_API_KEY;
   const deliverableText = `Values to evaluate:
 ${formatJudgeValues(args.values)}`;
-  const instructionText = `Instruction:
-${args.instruction}`;
-  const systemPromptText = `${JUDGE_SYSTEM_PROMPT}
+  const { briefingText, instructionText } = assembleBriefing(args);
+  const systemPromptText = `${briefingText}
 
 ${deliverableText}`;
-  const cacheKey = `${args.model}\0${deliverableText}`;
+  const cacheKey = `${args.model}\0${briefingText}\0${deliverableText}`;
   return runWithSharedPrefix(cacheKey, async () => {
     const startedAt = Date.now();
     const response = await fetch(`${baseURL}/chat/completions`, {
@@ -334,7 +334,7 @@ ${deliverableText}`;
           {
             role: "system",
             content: [
-              { type: "text", text: JUDGE_SYSTEM_PROMPT },
+              { type: "text", text: briefingText },
               {
                 type: "text",
                 text: deliverableText,
@@ -384,6 +384,33 @@ ${deliverableText}`;
     };
   });
 }
+function assembleBriefing(args) {
+  if (!args.prompt) {
+    return {
+      briefingText: JUDGE_SYSTEM_PROMPT,
+      instructionText: `Instruction:
+${args.instruction}`
+    };
+  }
+  const ctx = {
+    taskId: args.context?.taskId ?? "",
+    checkName: args.context?.checkName ?? "",
+    ...args.context?.taskDescription !== void 0 ? { taskDescription: args.context.taskDescription } : {},
+    ...args.context?.deliverableNames !== void 0 && args.context.deliverableNames.length > 0 ? { deliverableNames: args.context.deliverableNames } : {},
+    instruction: args.instruction
+  };
+  const built = args.prompt(ctx);
+  const system = built.system?.trim();
+  const user = built.user?.trim();
+  return {
+    // The SDK appends its own response contract to any custom briefing.
+    briefingText: system ? `${system}
+
+${JUDGE_RESPONSE_CONTRACT}` : JUDGE_SYSTEM_PROMPT,
+    instructionText: user ?? `Instruction:
+${args.instruction}`
+  };
+}
 
 // src/agent-task/checks/t.ts
 function resolveJudgeConfig(judgeConfig, override) {
@@ -392,7 +419,8 @@ function resolveJudgeConfig(judgeConfig, override) {
   return {
     model,
     baseURL: override?.baseURL ?? judgeConfig?.baseURL,
-    apiKey: override?.apiKey ?? judgeConfig?.apiKey
+    apiKey: override?.apiKey ?? judgeConfig?.apiKey,
+    prompt: override?.prompt ?? judgeConfig?.prompt
   };
 }
 var TEST_METHOD_NAMES = [
@@ -421,6 +449,67 @@ function matchName(name, m) {
   if (m instanceof RegExp) return m.test(name);
   return m(name);
 }
+function createJudgeMethod(rec, judgeConfig, judgeScope) {
+  return async (values, instruction, opts) => {
+    const label = opts?.label ?? "judge";
+    const location = rec.captureLocation();
+    const valueArray = Array.isArray(values) ? values : [values];
+    const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
+    if (!effective) {
+      rec.record(
+        label,
+        false,
+        "No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
+        { evaluator_type: "llm", location }
+      );
+      return;
+    }
+    const context2 = judgeScopeToContext(judgeScope);
+    try {
+      const { pass, reasoning, judge } = await callJudge({
+        values: valueArray,
+        instruction,
+        model: effective.model,
+        baseURL: effective.baseURL,
+        apiKey: effective.apiKey,
+        prompt: effective.prompt,
+        ...context2 ? { context: context2 } : {}
+      });
+      rec.record(label, pass, reasoning, {
+        evaluator_type: "llm",
+        judge,
+        expected: instruction,
+        // Store the raw evaluated value (not a truncated string) so the
+        // dashboard can render it with the right viewer — Markdown for
+        // prose, a JSON tree for structured objects.
+        received: valueArray.length === 1 ? valueArray[0] : valueArray,
+        location
+      });
+    } catch (error) {
+      rec.record(
+        label,
+        false,
+        `judge failed: ${error instanceof Error ? error.message : String(error)}`,
+        {
+          evaluator_type: "llm",
+          expected: instruction,
+          received: valueArray.length === 1 ? valueArray[0] : valueArray,
+          location
+        }
+      );
+    }
+  };
+}
+function judgeScopeToContext(scope) {
+  if (!scope) return void 0;
+  const deliverableNames = scope.readDeliverableNames?.();
+  return {
+    taskId: scope.taskId,
+    checkName: scope.checkName,
+    ...scope.taskDescription !== void 0 ? { taskDescription: scope.taskDescription } : {},
+    ...deliverableNames && deliverableNames.length > 0 ? { deliverableNames } : {}
+  };
+}
 var CAPABILITY_LABELS = {
   messages: "message",
   tools: "tool",
@@ -429,7 +518,7 @@ var CAPABILITY_LABELS = {
   skills: "skill",
   subagents: "subagent"
 };
-function createTraceTestContext(view, rec, judgeConfig) {
+function createTraceTestContext(view, rec, judgeConfig, judgeScope) {
   const unsupported = (id, capability) => {
     const label = CAPABILITY_LABELS[capability];
     rec.record(id, false, `${label} evidence is unavailable in this trace projection`, {
@@ -613,49 +702,9 @@ function createTraceTestContext(view, rec, judgeConfig) {
         { expected: matcher.label, received: describeValue(value) }
       );
     },
-    async judge(values, instruction, opts) {
-      const label = opts?.label ?? "judge";
-      const location = rec.captureLocation();
-      const valueArray = Array.isArray(values) ? values : [values];
-      const effective = resolveJudgeConfig(judgeConfig, opts?.judge);
-      if (!effective) {
-        rec.record(
-          label,
-          false,
-          "No judge model configured. Set one of:\n\u2022 OPENROUTER_MODEL + OPENROUTER_API_KEY (OpenRouter \u2014 works with 200+ models, one account)\n\u2022 OPENAI_MODEL + OPENAI_API_KEY (OpenAI direct)\nOr pass { judge } to runTask() programmatically.",
-          { evaluator_type: "llm", location }
-        );
-        return;
-      }
-      try {
-        const { pass, reasoning, judge } = await callJudge({
-          values: valueArray,
-          instruction,
-          model: effective.model,
-          baseURL: effective.baseURL,
-          apiKey: effective.apiKey
-        });
-        rec.record(label, pass, reasoning, {
-          evaluator_type: "llm",
-          judge,
-          expected: instruction,
-          received: valueArray.length === 1 ? valueArray[0] : valueArray,
-          location
-        });
-      } catch (error) {
-        rec.record(
-          label,
-          false,
-          `judge failed: ${error instanceof Error ? error.message : String(error)}`,
-          {
-            evaluator_type: "llm",
-            expected: instruction,
-            received: valueArray.length === 1 ? valueArray[0] : valueArray,
-            location
-          }
-        );
-      }
-    }
+    // judge does not consult trace capabilities; the scope carries the
+    // task/check frame for prompt builders (#161).
+    judge: createJudgeMethod(rec, judgeConfig, judgeScope)
   };
 }
 function matchesTraceToolCall(call, name, opts) {
@@ -787,14 +836,21 @@ function describe(id, nameOrFn, fn) {
 async function runTraceChecks(args) {
   const view = new TraceView(args.snapshot);
   const locate = args.moduleUrl && args.displayFile ? (stack) => parseCheckLocation(stack, args.moduleUrl, args.displayFile) : void 0;
+  const taskMeta = readTaskMeta(args.task);
   const results = await Promise.all(
     registry.map(async (check) => {
       const rec = new Recorder(locate);
-      const t = createTraceTestContext(view, rec, args.judgeConfig);
+      const reads = trackDeliverableReads(args.deliverables);
+      const t = createTraceTestContext(view, rec, args.judgeConfig, {
+        taskId: taskMeta.id ?? "",
+        ...taskMeta.description !== void 0 ? { taskDescription: taskMeta.description } : {},
+        checkName: check.id,
+        readDeliverableNames: reads.names
+      });
       let thrownLocation;
       try {
         await check.fn(t, {
-          deliverables: args.deliverables,
+          deliverables: reads.proxied,
           files: args.files,
           task: args.task
         });
@@ -866,6 +922,26 @@ function proxyBrokenDeliverables(deliverables, brokenDeliverables) {
     }
   });
 }
+function readTaskMeta(task2) {
+  if (!task2 || typeof task2 !== "object") return {};
+  const record = task2;
+  return {
+    ...typeof record.id === "string" ? { id: record.id } : {},
+    ...typeof record.description === "string" ? { description: record.description } : {}
+  };
+}
+function trackDeliverableReads(deliverables) {
+  const read = /* @__PURE__ */ new Set();
+  const proxied = new Proxy(deliverables, {
+    get(target, property, receiver) {
+      if (typeof property === "string" && Object.prototype.hasOwnProperty.call(target, property)) {
+        read.add(property);
+      }
+      return Reflect.get(target, property, receiver);
+    }
+  });
+  return { proxied, names: () => [...read] };
+}
 
 // src/agent-task/task/defineTask.ts
 var TASK_ADAPTER_SYMBOL = /* @__PURE__ */ Symbol.for("agent-task.adapter-definition");
@@ -1553,6 +1629,8 @@ function observationTypeFor(t) {
       return "EMBEDDING";
     case "GUARDRAIL":
       return "GUARDRAIL";
+    case "SKILL":
+      return "SKILL";
     default:
       return "SPAN";
   }
@@ -2076,6 +2154,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
             summarize: (result2) => summarizeEvaluationResults(result2)
           },
           async () => {
+            const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
             if (!inlineChecks) {
               return loadAndRunFlowChecks(
                 checksPath,
@@ -2084,7 +2163,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
                   deliverables: collected,
                   files,
                   task: task2,
-                  judgeConfig: options?.judge
+                  ...judgeConfig ? { judgeConfig } : {}
                 },
                 validationResults.brokenDeliverables
               );
@@ -2097,7 +2176,7 @@ async function executeLoadedTask(loaded, options, rawTrace, traceRunId) {
               ),
               files,
               task: task2,
-              judgeConfig: options?.judge,
+              ...judgeConfig ? { judgeConfig } : {},
               moduleUrl,
               displayFile: evalFileName
             });
@@ -2277,12 +2356,13 @@ async function evaluate(loaded, options, phase1) {
     phase1.collected,
     validationResults.brokenDeliverables
   );
+  const judgeConfig = resolveJudgeConfig(options?.judge, task2.judge);
   const checksResults = await (inlineChecks ? runTraceChecks({
     snapshot: phase1.snapshot,
     deliverables,
     files,
     task: task2,
-    judgeConfig: options?.judge,
+    ...judgeConfig ? { judgeConfig } : {},
     moduleUrl,
     displayFile: evalFileName
   }) : loadAndRunFlowChecks(
@@ -2292,7 +2372,7 @@ async function evaluate(loaded, options, phase1) {
       deliverables,
       files,
       task: task2,
-      judgeConfig: options?.judge
+      ...judgeConfig ? { judgeConfig } : {}
     },
     validationResults.brokenDeliverables
   ));
@@ -2449,9 +2529,10 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
     const proxied = proxyBrokenDeliverables(deliverables, validation.brokenDeliverables);
     const passCounts = /* @__PURE__ */ new Map();
     let primary = [];
+    const judgeConfig = resolveJudgeConfig(options.judge, loaded.task.judge);
     for (let sample = 0; sample < samples; sample++) {
       if (samples > 1) progress(`judging sample ${sample + 1}/${samples}`);
-      const results = await runChecksOnce(loaded, proxied, snapshot, options.judge);
+      const results = await runChecksOnce(loaded, proxied, snapshot, judgeConfig);
       if (sample === 0) primary = results;
       for (const result of results) {
         passCounts.set(result.id, (passCounts.get(result.id) ?? 0) + (result.pass ? 1 : 0));
@@ -2468,9 +2549,9 @@ async function rejudgeTaskRun(runId, endpoints, options = {}) {
       taskId: detail.task_id,
       definitionRevisionId: revisionId,
       definitionRevisionIsPinned: revisionIsPinned,
-      judge: options.judge ? {
-        model: options.judge.model,
-        baseURL: options.judge.baseURL ?? process.env.OPENROUTER_BASE_URL
+      judge: judgeConfig ? {
+        model: judgeConfig.model,
+        baseURL: judgeConfig.baseURL ?? process.env.OPENROUTER_BASE_URL
       } : null,
       samples,
       checks: aggregate.checks,