@axiom-lattice/core 3.0.6 → 3.0.7

package/dist/index.mjs

@@ -26109,7 +26109,7 @@ File content: ${files[key4]}`
       {
         dimension: "correctness",
         weight: 100,
-        description: "\u6574\u4F53\u6B63\u786E\u6027\uFF0C\u662F\u5426\u7B26\u5408\u9884\u671F\u8F93\u51FA\u63CF\u8FF0\u3002"
+        description: "Overall correctness \u2014 whether the result matches the expected output description."
       }
     ];
     const evalRubrics = evalCase.eval.eval_rubrics && evalCase.eval.eval_rubrics.length > 0 ? evalCase.eval.eval_rubrics : defaultRubrics;
@@ -26123,53 +26123,53 @@ File content: ${files[key4]}`
 ${evalRubrics.map(
       (r) => `- **${r.dimension}**\uFF08\u6743\u91CD\uFF1A${r.weight}\uFF09\uFF1A${r.description}`
     ).join("\n")}`;
-    const testPrompt = `# \u89D2\u8272
-\u4F60\u662F\u4E00\u540D\u8D44\u6DF1\u7684 AI Agent \u8BC4\u4F30\u4E13\u5BB6\uFF0C\u8D1F\u8D23\u6839\u636E\u9884\u8BBE\u7684\u6307\u6807\uFF08Rubrics\uFF09\u5BF9 Agent \u7684\u6267\u884C\u8FC7\u7A0B\u4E0E\u7ED3\u679C\u8FDB\u884C"\u9ED1\u76D2\u6D4B\u8BD5"\u5224\u5B9A\u3002
+    const testPrompt = `# Role
+You are a senior AI Agent evaluation expert. Your job is to perform a "black-box test" judgment of the agent's execution process and results against the preset evaluation rubrics.
 
-# \u8F93\u5165\u4FE1\u606F
-\u6D4B\u8BD5\u6846\u67B6\u5C06\u4E3A\u4F60\u63D0\u4F9B\u4EE5\u4E0B\u4E94\u4E2A\u6838\u5FC3\u4E0A\u4E0B\u6587\uFF1A
+# Input Information
+The test framework provides you with the following five core contexts:
 
-1. **\u7528\u6237\u610F\u56FE\uFF08User Intent\uFF09**\uFF1A${evalCase.input.message}
+1. **User Intent**: ${evalCase.input.message}
 
-2. **\u8F93\u5165\u6587\u4EF6\uFF08Input Files\uFF09**\uFF1A${testCaseFilesContent || "\u65E0"}
+2. **Input Files**: ${testCaseFilesContent || "None"}
 
-3. **\u6267\u884C\u8FC7\u7A0B\uFF08Execution Trajectory\uFF0CAgent \u5168\u7A0B\u7684\u6D88\u606F/\u5DE5\u5177\u8C03\u7528\u8BB0\u5F55\uFF09**\uFF1A
+3. **Execution Trajectory** (the agent's full message/tool-call record):
 ${trajectory}
 
-4. **\u6700\u7EC8\u8F93\u51FA\uFF08Final Output\uFF0CAgent \u7684\u6700\u540E\u4E00\u6761\u56DE\u590D\uFF09**\uFF1A
+4. **Final Output** (the agent's last reply):
 ${finalOutput}
 
-5. **\u671F\u671B\u8F93\u51FA\u63CF\u8FF0\uFF08Expected Output Description\uFF09**\uFF1A${evalCase.eval.content_assertion}
+5. **Expected Output Description**: ${evalCase.eval.content_assertion}
 ${rubricsSection}
 
-# \u4EFB\u52A1
-\u4F60\u5FC5\u987B\u4E25\u683C\u5BF9\u7167"\u8BC4\u4F30\u6307\u6807\uFF08Evaluation Rubrics\uFF09"\u4E2D\u7684\u6BCF\u4E00\u9879\u6307\u6807\uFF0C\u7ED3\u5408"\u6267\u884C\u8FC7\u7A0B"\u4E0E"\u6700\u7EC8\u8F93\u51FA"\uFF0C\u5206\u6790 Agent \u662F\u5426\u8FBE\u6807\u3002\u8BC4\u4F30\u65F6\u65E2\u8981\u68C0\u67E5\u6700\u7EC8\u7ED3\u679C\uFF0C\u4E5F\u8981\u68C0\u67E5\u8FC7\u7A0B\u4E2D\u662F\u5426\u6B63\u786E\u5B8C\u6210\u4E86\u5FC5\u8981\u7684\u6B65\u9AA4\uFF08\u5982\u5DE5\u5177\u8C03\u7528\u3001\u4FE1\u606F\u68C0\u7D22\u7B49\uFF09\u3002
+# Task
+You must strictly evaluate the agent against every rubric in the "Evaluation Rubrics" section, using both the "Execution Trajectory" and the "Final Output". Evaluate the final result AND whether the process correctly performed the required intermediate steps (tool calls, information retrieval, etc.).
 
-# \u89C4\u5219
-1. **\u5BA2\u89C2\u6027**\uFF1A\u4EC5\u6839\u636E\u63D0\u4F9B\u7684\u4E0A\u4E0B\u6587\u5224\u5B9A\u3002\u5982\u679C\u6807\u51C6\u8981\u6C42"\u5305\u542B\u6570\u5B57"\uFF0C\u4F46\u8F93\u51FA\u53EA\u6709\u6587\u5B57\uFF0C\u5373\u4F7F\u8BED\u6C14\u518D\u597D\u4E5F\u5FC5\u987B\u6263\u5206\u3002
-2. **\u7ED3\u679C\u6821\u9A8C**\uFF1A\u5982\u679C"\u6700\u7EC8\u8F93\u51FA"\u4E2D\u7F3A\u5931\u9884\u671F\u7684\u5185\u5BB9\uFF0C\u6216\u5185\u5BB9\u4E0D\u7B26\u5408"\u8BC4\u4F30\u6307\u6807"\u4E2D\u7684\u6807\u51C6\uFF0C\u5BF9\u5E94\u7684\u6307\u6807\u5E94\u5224\u5B9A\u4E3A\u5931\u8D25\u3002
-3. **\u8FC7\u7A0B\u6821\u9A8C**\uFF1A\u5982\u679C"\u6267\u884C\u8FC7\u7A0B"\u663E\u793A Agent \u672A\u6267\u884C\u5FC5\u8981\u7684\u4E2D\u95F4\u6B65\u9AA4\uFF08\u5982\u5E94\u8C03\u7528\u5DE5\u5177\u800C\u672A\u8C03\u7528\uFF09\uFF0C\u5373\u4F7F\u6700\u7EC8\u8F93\u51FA\u770B\u4F3C\u5408\u7406\uFF0C\u4E5F\u5E94\u5728\u5BF9\u5E94\u6307\u6807\u4E0A\u6263\u5206\u3002
-4. **\u8BC1\u636E\u5BFC\u5411**\uFF1A\u5728\u7ED9\u51FA\u539F\u56E0\uFF08reason\uFF09\u65F6\uFF0C\u5FC5\u987B\u5F15\u7528\u6267\u884C\u8FC7\u7A0B\u6216\u6700\u7EC8\u8F93\u51FA\u4E2D\u7684\u5177\u4F53\u5185\u5BB9\u3002
-5. **\u52A0\u6743\u8BA1\u7B97**\uFF1A\u6700\u7EC8\u5206\u6570\u4E3A\u5404\u9879\u6307\u6807\u5F97\u5206\u4E0E\u5176\u6743\u91CD\u7684\u4E58\u79EF\u4E4B\u548C\uFF080-100\u5206\u5236\uFF09\u3002
-6. **HITL \u4E2D\u65AD\u5224\u5B9A**\uFF1A\u5982\u679C\u6267\u884C\u8FC7\u7A0B\u4E2D\u51FA\u73B0\u300CHITL \u6682\u505C\uFF1AAgent \u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u300D\u6761\u76EE\uFF0C\u8BF4\u660E Agent \u5728\u7B49\u5F85\u4EBA\u5DE5\u786E\u8BA4\u3002\u8BF7\u628A\u5B83\u5F53\u4F5C\u88AB\u6D4B\u4E1A\u52A1\u884C\u4E3A\u6765\u8BC4\u5224\uFF1A\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u81EA\u4E3B\u5B8C\u6210\uFF08\u5982"\u65E0\u9700\u786E\u8BA4\u81EA\u52A8\u6267\u884C"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u5E94\u5224\u5931\u8D25\uFF1B\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u5148\u83B7\u5F97\u786E\u8BA4\u6216\u6279\u51C6\uFF08\u5982"\u6267\u884C\u524D\u5FC5\u987B\u8BF7\u6C42\u6279\u51C6"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u662F\u6B63\u786E\u884C\u4E3A\uFF0C\u5E94\u7ED3\u5408\u5176\u65F6\u673A\u4E0E\u5185\u5BB9\u5224\u5B9A\u4E3A\u901A\u8FC7\u6216\u6309\u6307\u6807\u7ED9\u5206\u3002
-7. **HITL \u81EA\u52A8\u54CD\u5E94\u5224\u5B9A**\uFF1A\u5982\u679C\u300CHITL \u6682\u505C\u300D\u6761\u76EE\u4E4B\u540E\u51FA\u73B0\u300C\u5DF2\u81EA\u52A8\u54CD\u5E94\uFF08\u6D4B\u8BD5\u7B56\u7565 auto-approve/auto-reject/canned-response\uFF09\u300D\u6761\u76EE\uFF0C\u8BF4\u660E\u6D4B\u8BD5\u6846\u67B6\u6CE8\u5165\u4E86\u4EBA\u5DE5\u56DE\u590D\u3001\u6D41\u7A0B\u5DF2\u7EE7\u7EED\u2014\u2014\u8BF7\u6309**\u5B8C\u6574\u6D41\u7A0B**\u8BC4\u5224\u6682\u505C\u4E4B\u540E\u7684\u884C\u4E3A\uFF08\u5982\u6279\u51C6\u540E\u662F\u5426\u6B63\u786E\u6267\u884C\u4E86\u64CD\u4F5C\uFF09\uFF0C\u5E76\u6838\u5BF9\u81EA\u52A8\u54CD\u5E94\u5185\u5BB9\u662F\u5426\u7B26\u5408\u4EBA\u5DE5\u56DE\u590D\u7684\u5408\u7406\u9884\u671F\u3002
+# Rules
+1. **Objectivity**: Judge solely from the provided context. If the standard requires "contains a number" but the output has only text, points must be deducted even if the tone is good.
+2. **Result verification**: If the "Final Output" is missing expected content, or does not meet the criteria in the "Evaluation Rubrics", the corresponding rubric must be marked as failed.
+3. **Process verification**: If the "Execution Trajectory" shows the agent did not perform a necessary intermediate step (e.g., should have called a tool but did not), deduct points on the corresponding rubric even if the final output looks plausible.
+4. **Evidence-based**: When giving a reason, you must quote specific content from the execution trajectory or final output.
+5. **Weighted scoring**: The final score is the weighted sum of the rubric scores (on a 0-100 scale).
+6. **HITL interrupt judgment**: If the trajectory contains a "HITL pause: agent requested human input" entry, the agent is waiting for human confirmation. Treat this as the business behavior under test: if the expected output requires autonomous completion (e.g., "execute automatically without confirmation"), requesting human input should be judged a failure; if the expected output requires confirmation or approval first (e.g., "must request approval before executing"), requesting human input is correct behavior \u2014 judge its timing and content, passing or scoring according to the rubrics.
+7. **HITL auto-response judgment**: If a "HITL pause" entry is followed by an "auto-responded (test policy auto-approve/auto-reject/canned-response)" entry, the test framework injected a human reply and the flow continued \u2014 evaluate the behavior AFTER the pause as the complete flow (e.g., whether the operation was correctly executed after approval), and check whether the auto-response content matches a reasonable human reply.
 
-# \u8F93\u51FA\u683C\u5F0F\uFF08\u4EC5JSON\uFF09
-\u4F60\u5FC5\u987B\u4EC5\u4EE5 JSON \u683C\u5F0F\u56DE\u590D\uFF0C\u7ED3\u6784\u5982\u4E0B\uFF1A
+# Output Format (JSON only)
+You MUST reply with JSON only, using this structure:
 {
   "pass": true | false,
   "final_score": number,
   "dimension_results": [
     {
-      "name": "\u6307\u6807\u540D\u79F0",
+      "name": "rubric name",
       "score": number,
-      "reason": "\u5177\u4F53\u7684\u6263\u5206\u6216\u7ED9\u5206\u7406\u7531\uFF0C\u9700\u5F15\u7528\u8BC1\u636E"
+      "reason": "specific reason for deduction or credit, citing evidence"
     }
   ],
-  "summary": "\u5BF9 Agent \u8868\u73B0\u7684\u6574\u4F53\u8BC4\u4EF7"
+  "summary": "overall evaluation of the agent's performance"
 }
 
-\u6CE8\u610F\uFF1A\u5982\u679C final_score >= 80 \u4E14\u6CA1\u6709\u81F4\u547D\u6027\u9519\u8BEF\uFF0Cpass \u5E94\u4E3A true\uFF1B\u5426\u5219\u4E3A false\u3002`;
+Note: if final_score >= 80 and there are no fatal errors, pass should be true; otherwise false.`;
     this.lastTestPrompt = testPrompt;
     const judgeThreadId = v44();
     this.lastJudgeThreadId = judgeThreadId;
@@ -26552,6 +26552,32 @@ var LatticeEvalSuite = class {
 import { AgentType as AgentType6 } from "@axiom-lattice/protocols";
 import { HumanMessage as HumanMessage5 } from "@langchain/core/messages";
 import { v4 as uuidv46 } from "uuid";
+var DEFAULT_CALIBRATION_PROBES = [
+  {
+    id: "catch-up-correct",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "11:30",
+    expectedPass: true
+  },
+  {
+    id: "catch-up-wrong",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "12:30",
+    expectedPass: false
+  },
+  {
+    id: "discount-tax-correct",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "129.6",
+    expectedPass: true
+  },
+  {
+    id: "discount-tax-wrong",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "162",
+    expectedPass: false
+  }
+];
 var LatticeEvalProject = class {
   constructor(project, onCaseComplete) {
     this.suites = /* @__PURE__ */ new Map();
@@ -26659,21 +26685,22 @@ var LatticeEvalProject = class {
   }
   /**
    * Verify the judge agent can produce parseable, correct verdicts
-   * before committing to a full run. Uses two known-answer cases
-   * (one expected PASS, one expected FAIL) to catch broken judges.
+   * before committing to a full run. Uses known-answer probes (default
+   * multi-step reasoning set, overridable per project via
+   * judge_agent_config.calibration_cases) to catch broken judges.
+   * Checks both the pass verdict and the final_score direction.
    */
   async calibrateJudge() {
     const tenantId2 = this.project.lattice_server_config.tenant_id || "default";
     const judgeAgent = await getAgentClient(tenantId2, this.judgeAgentKey);
-    const cases = [
-      { output: "7", expected: "7", expectedPass: true },
-      { output: "7", expected: "999", expectedPass: false }
-    ];
-    for (const c of cases) {
-      const prompt = `\u4F60\u662F\u8BC4\u4F30\u4E13\u5BB6\u3002\u5224\u5B9A\u6700\u7EC8\u8F93\u51FA\u662F\u5426\u7B26\u5408\u671F\u671B\u3002
-\u6700\u7EC8\u8F93\u51FA\uFF1A${c.output}
-\u671F\u671B\u8F93\u51FA\uFF1A${c.expected}
-\u4EC5\u8F93\u51FA JSON\uFF1A{"pass": true|false, "final_score": 0-100, "summary": "\u7406\u7531"}`;
+    const probes = this.project.judge_agent_config.calibration_cases && this.project.judge_agent_config.calibration_cases.length > 0 ? this.project.judge_agent_config.calibration_cases : DEFAULT_CALIBRATION_PROBES;
+    for (const probe of probes) {
+      const trajectorySection = probe.trajectory ? `
+Trajectory: ${probe.trajectory}` : "";
+      const prompt = `You are an evaluation expert. Judge whether the final output meets the expected output description.
+Expected output description: ${probe.task}
+Final output: ${probe.finalOutput}${trajectorySection}
+Respond with JSON only: {"pass": true|false, "final_score": 0-100, "summary": "reason"}`;
       let raw = "";
       let invokeError = null;
       for (let attempt = 0; attempt < 2; attempt++) {
@@ -26698,13 +26725,24 @@ var LatticeEvalProject = class {
         return { ok: false, reason: `Calibration output unparseable: ${parsed.error}`, bypassed: true };
       }
       const actualPass = parsed.pass !== void 0 ? parsed.pass : (parsed.final_score ?? 0) >= 80;
-      if (actualPass !== c.expectedPass) {
+      if (actualPass !== probe.expectedPass) {
         return {
           ok: false,
-          reason: `Calibration mismatch: output="${c.output}" expected="${c.expected}" \u2014 judge said ${actualPass ? "PASS" : "FAIL"}, expected ${c.expectedPass ? "PASS" : "FAIL"}`,
+          reason: `Calibration mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "PASS" : "FAIL"}, judge said ${actualPass ? "PASS" : "FAIL"}`,
           bypassed: true
         };
       }
+      if (parsed.final_score !== void 0) {
+        const minScore = probe.expectedScoreMin ?? 80;
+        const scoreOk = probe.expectedPass ? parsed.final_score >= minScore : parsed.final_score < minScore;
+        if (!scoreOk) {
+          return {
+            ok: false,
+            reason: `Calibration score mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "final_score >= " + minScore : "final_score < " + minScore}, judge gave ${parsed.final_score}`,
+            bypassed: true
+          };
+        }
+      }
     }
     return { ok: true };
   }
@@ -31096,6 +31134,7 @@ export {
   ConsoleLoggerClient,
   CustomMetricsClient,
   CustomMiddlewareRegistry,
+  DEFAULT_CALIBRATION_PROBES,
   DaytonaInstance,
   DaytonaProvider,
   DefaultScheduleClient,