@axiom-lattice/core 3.0.5 → 3.0.7

package/dist/index.js

@@ -1623,6 +1623,7 @@ __export(index_exports, {
   ConsoleLoggerClient: () => ConsoleLoggerClient,
   CustomMetricsClient: () => CustomMetricsClient,
   CustomMiddlewareRegistry: () => CustomMiddlewareRegistry,
+  DEFAULT_CALIBRATION_PROBES: () => DEFAULT_CALIBRATION_PROBES,
   DaytonaInstance: () => DaytonaInstance,
   DaytonaProvider: () => DaytonaProvider,
   DefaultScheduleClient: () => DefaultScheduleClient,
@@ -12640,6 +12641,24 @@ cases, you MUST ask the user via ask_user_to_clarify \u2014 never proceed
 with an assumed expectation. A test case written against a guessed
 expectation validates the wrong thing. When in doubt, ask.
 
+## Knowledge in Skills (apply to EVERY sub-skill workflow)
+
+**Domain knowledge lives in SKILL.md files, not in prompts.** The
+deliverable's knowledge (rules, formats, decision logic, procedures) is
+authored as skills; the executable (agent prompt, workflow step) stays
+THIN \u2014 role/process only, loading knowledge via "Load [[skill-name]]
+and follow it". Never write domain knowledge directly into a system
+prompt or a workflow step's prompt.
+
+Why: knowledge in prompts cannot be reused, individually verified, or
+evolved. Knowledge in skills is shared (subSkills), regression-tested
+([[eval-verify]]), and improved without touching the executable.
+
+Applies to every construction path: [[learn-capability]] (skills are
+the primary output), [[agent-build]] (agent prompt thin, loads skill),
+[[design-workflow]] (steps reference [[skill-name]] or ref to
+skill-loading agents). All three follow this single principle.
+
 ## Goal-Driven Validation (apply to EVERY sub-skill workflow)
 
 The agent evaluates goal achievement ITSELF via multi-dimensional test
@@ -12659,6 +12678,29 @@ green = goal achieved (per [[completion-gate]] and [[eval-verify]]).
 The goal model is the acceptance standard \u2014 contentAssertion must
 encode the usable state, not just technical correctness.
 
+## Undefined Tasks (outside the skill map)
+
+If the request does not match any sub-skill workflow: do NOT guess, do
+NOT refuse, do NOT force-fit an existing flow. Follow the
+EXPLORE \u2192 PROPOSE \u2192 CONFIRM protocol:
+
+1. **Explore** \u2014 inventory before proposing anything:
+   \`list_agents\` / \`load_skills\` (existing assets), \`list_tools\` /
+   \`list_middleware_types\` (capabilities), \`list_connections\`
+   (data sources), eval projects (verification), docs at hand.
+   Goal: know what is reusable and what is missing.
+2. **Propose** \u2014 present 2-3 concrete options, each with: what it
+   does, cost, risk, and what it needs (new tools / new skills /
+   approvals).
+3. **Confirm** \u2014 the user picks an option or adjusts it. Never
+   execute without a chosen option.
+4. **New capability needed?** (new tool type, new skill, new
+   connection) \u2014 include creating it ([[create-skill]] / connection
+   setup) IN the proposed option; never silently proceed without it.
+5. **Boundary honesty** \u2014 state clearly what the architect cannot do
+   (e.g. deploy to production, monitor runtime, change frontend),
+   and give the alternative \u2014 never overpromise or silently refuse.
+
 ## Skill Map
 - [[learn-capability]] \u2014 Learn from any source material (user
   description, documents, API specs, conversations, spreadsheets) and
@@ -12767,6 +12809,9 @@ Do NOT create tasks for:
 **A task is a living record of the GOAL + ACCEPTANCE CRITERIA** \u2014 not a
 todo label. Every task's description must carry:
 
+- **Goal Model** \u2014 the full goal model ([[agent-architecture|Goal
+  Model]]): real goal, consumer (who uses the result), usable state
+  (what "done and usable" means concretely)
 - **Objective** \u2014 one measurable sentence: what result to achieve
 - **Acceptance Criteria** \u2014 checkboxes that define "done": when ALL
   are checked, the task is verifiably complete
@@ -12917,8 +12962,13 @@ When unsure, use \`show_widget\` for visual comparison.
 - **Follow [[agent-architecture|Goal Model]]** \u2014 establish the goal
   model (real goal / consumer / usable state) and design the agent to
   achieve it; verification is goal-driven ([[agent-architecture|Goal-Driven Validation]]).
-- **NEVER build before confirming.** Design \u2192 ask \u2192 wait for "yes" \u2192
-  only then build. No exceptions.
+- **Follow [[agent-architecture|Knowledge in Skills]]** \u2014 the prompt is
+  thin (role/behavior); domain knowledge lives in SKILL.md which the
+  agent loads ("Load [[skill-name]] and follow it"). Never write
+  domain knowledge directly into a system prompt.
+- **NEVER build before confirming.** Design \u2192 confirm via
+  \`ask_user_to_clarify\` \u2192 wait for approval \u2192 only then build.
+  No exceptions.
 - **Track with tasks once scope is clear.** After requirements are
   clarified, create the parent task ([[task-tracking]]) before starting
   design. Don't create tasks during clarification.
@@ -12931,18 +12981,23 @@ When unsure, use \`show_widget\` for visual comparison.
 
 ## REACT design steps
 
-1. Understand the goal (who uses it? inputs? outputs?)
+1. **Establish the goal model FIRST** \u2014 real goal / user expectation /
+   consumer / usable state ([[agent-architecture|Goal Model]]); record
+   it in the parent task. Design, build, and verification all derive
+   from it. Only then:
 2. Choose middleware \u2014 call \`list_tools\` and \`list_middleware_types\`
    first. MUST include \`ask_user_to_clarify\` if the agent needs
    confirmation or clarifying questions.
 3. Write the system prompt: role \u2192 workflow \u2192 constraints
 4. Present the design with \`show_widget\`
-5. Ask for explicit approval \u2014 do NOT build until confirmed
+5. Confirm via \`ask_user_to_clarify\` \u2014 do NOT build until approved
 6. Build with \`create_agent\`
 
 ## DEEP_AGENT design steps
 
-1. Domain analysis \u2014 explain why DEEP_AGENT is the right choice
+1. **Establish the goal model FIRST** \u2014 real goal / consumer / usable
+   state, recorded in the parent task ([[agent-architecture|Goal
+   Model]]); then explain why DEEP_AGENT is the right choice
 2. Capability mapping with \`show_widget\`
 3. System prompt emphasizes dynamic todo workflow (analyze \u2192 break
    into todos \u2192 work one at a time \u2192 refine). Middleware: code_eval,
@@ -13129,22 +13184,70 @@ subSkills:
   (hold-out, never run during fix loop)
 - 0.2 \u2460 \u2192 {skill}-api-verified \u2014 queryability assertion, single step
 
-## Layered verification (orchestrator + subAgents)
-
-When the design has a parent deep_agent with subAgents (learn-capability
-Phase 2), verification is layered:
-- **Each sub-agent**: its OWN eval project (eval-{sub-agent-id}) \u2014 the
-  sub capability is verified independently, with its own fix loop.
-- **The parent agent**: an integration eval project (eval-{parent-id}).
-  Integration cases: full end-to-end task input \u2192 parent invokes
-  sub-agents \u2192 final aggregated output \u2192 contentAssertion on the final
-  result. This verifies ORCHESTRATION (does the parent call the right
-  sub-agents in the right order and aggregate correctly).
-- **Parent trust upgrade** requires BOTH: all sub-agent evals pass AND
-  the parent's integration eval passes. The parent's metadata
-  (verified/source) records this dependency.
-- Independent agents (no parent) keep single-level eval \u2014 no integration
-  layer needed.
+## Layered verification (orchestrator + components)
+
+When the design delegates to other agents, verification is layered.
+"Orchestrator" = a parent deep_agent with subAgents (learn-capability
+Phase 2) OR a workflow with \`ref\` steps ([[design-workflow]]).
+"Components" = the subAgents / ref'd agents it calls. Order is MANDATORY:
+**component evals first, integration second** \u2014 never run the
+integration eval before every component eval passes.
+
+- **Each component**: its OWN eval project (eval-{sub-agent-id} /
+  eval-{ref-agent-id}) \u2014 the sub capability is verified independently,
+  with its own fix loop.
+- **The orchestrator**: an integration eval project (eval-{parent-id} /
+  eval-{workflow-id}). Integration cases: full end-to-end task input \u2192
+  orchestrator invokes components \u2192 final aggregated output \u2192
+  contentAssertion on the final result. This verifies ORCHESTRATION
+  (does the orchestrator call the right components in the right order
+  and aggregate correctly).
+- **Workflow integration cases** also include branch paths and \`ask\`
+  handling (see Workflow testing below) \u2014 but only AFTER the ref'd
+  agents are independently verified.
+- **Orchestrator trust upgrade** requires BOTH: all component evals pass
+  AND the orchestrator's integration eval passes. The orchestrator's
+  metadata (verified/source) records this dependency.
+- Independent agents (no parent, no ref) keep single-level eval \u2014 no
+  integration layer needed.
+
+## Workflow testing (WORKFLOW-type agents)
+
+Workflows compile to the same agent registry and run through the same
+eval path \u2014 same project naming (eval-{agent-id}), same case structure
+(inputMessage + steps + contentAssertion). Design differs because the
+pipeline is DETERMINISTIC:
+
+- **One case per branch path** \u2014 each if/map/parallel route gets a case
+  whose inputMessage drives it down that path; contentAssertion = the
+  exact output that path must produce (from the expected output spec,
+  Phase 1.5 in [[design-workflow]]).
+- **Goal-driven dimensions** \u2014 cases cover all four dimensions
+  ([[agent-architecture|Goal-Driven Validation]]), not just happy paths:
+  - Functional correctness \u2014 each branch path produces the right result
+  - Edge robustness \u2014 empty input, if-condition not met, map source
+    empty, malformed data: the pipeline must fail gracefully or take
+    the designed fallback, not crash
+  - Business usability \u2014 output reaches the usable state (usable-state
+    cases come from the confirmed spec, never invented)
+  - Consumer fit \u2014 exact fields/format for system consumers, readable
+    for human consumers
+- **Data contract cases** \u2014 intermediate \`{{refs}}\` handoffs and
+  \`map\` source shapes are contracts; a contract broken mid-pipeline
+  only surfaces at the end. One case per non-trivial handoff asserting
+  the intermediate output shape (source data + step output).
+- **\`ask\` steps** \u2014 case interruptPolicy controls them
+  (mode: stop | auto-approve | auto-reject | canned-response):
+  - \`stop\` \u2192 the run pauses at the ask; assert the partial output
+    BEFORE the interaction point
+  - auto-approve / auto-reject / canned-response \u2192 supply the response
+    and continue; assert the flow AFTER the interaction point
+  - \`value\` holds the response text (defaults "\u540C\u610F"/"\u62D2\u7EDD" for
+    approve/reject)
+- **Exact assertions** \u2014 deterministic pipeline means expected outputs
+  are precise; judge still scores semantics on top.
+- **Trust upgrade is the same gate** \u2014 [[completion-gate]] applies to
+  workflows: no eval \u2192 stays configured, never verified.
 
 ## Run
 
@@ -13210,11 +13313,39 @@ verified: unverified
 # Design Workflow \u2014 WORKFLOW Agent Design
 
 Use the WORKFLOW type when the process is fully known \u2014 a deterministic
-state machine with pre-defined paths.
+state machine with pre-defined paths. If the process is NOT fully known
+(open-ended, needs dynamic decomposition) \u2192 use [[learn-capability]] /
+[[agent-build]] (REACT / DEEP_AGENT) instead.
 Follow [[agent-architecture|User Interaction Rules]] and
 [[agent-architecture|Goal Model]] \u2014 establish the goal model (real
 goal / consumer / usable state) before designing, and design steps
 that achieve it. Acceptance = workflow outcome meets the usable state.
+Follow [[agent-architecture|Knowledge in Skills]]: workflow steps
+orchestrate; domain knowledge lives in SKILL.md. Never write domain
+knowledge directly into a step's prompt \u2014 load it via [[skill-name]]
+or delegate to an agent that loads the skill.
+
+## CRITICAL RULES
+- **NEVER build before confirming.** Design \u2192 present the flow as a
+  widget \u2192 discuss step-by-step with the user \u2192 confirm via
+  \`ask_user_to_clarify\` (blocking approval) \u2192 only then call
+  \`create_workflow\`. No exceptions.
+- **Always visualize the design** \u2014 present with \`show_widget\` as a
+  Flowchart (every step, branch, \`ask\` interaction point) \u2014 never a
+  bare text list (see Visual communication below).
+- **One decision at a time.** Each message asks exactly one question.
+- **Track with tasks once scope is clear.** Create the parent task
+  ([[task-tracking]]) before designing; record the expected output spec
+  (Phase 1.5) in it.
+
+## Visual communication
+
+Use \`show_widget\` for all structure explanations \u2014 never ASCII art.
+| Scenario | What |
+|----------|------|
+| Workflow flow | Flowchart (steps, branches, ask points) |
+| Step-level comparison | Comparison cards |
+| Data flow / {{refs}} | Flowchart |
 
 ## Phase 0: Load Skills
 
@@ -13224,20 +13355,122 @@ that achieve it. Acceptance = workflow outcome meets the usable state.
 
 ## Phase 1: Design
 
-1. Analyze the process. Map every step, branch, data dependency.
-2. Design using the YAML linear DSL (steps, parallel, map, if, ask).
-3. Present the design as a widget.
-4. Confirm with user before building.
-
-## Phase 2: Build
+1. **Establish the goal model FIRST** \u2014 real goal / user expectation /
+   consumer / usable state ([[agent-architecture|Goal Model]]); record
+   it in the parent task. It drives the expected output spec (Phase
+   1.5) and verification (Phase 4). Then analyze the process: map
+   every step, branch, data dependency.
+2. **Choose implementation mode per step \u2014 ASK the user** (present as
+   comparison cards). Each step's logic is either inline or \`ref\`:
+   - **inline prompt** \u2014 logic lives in the step's prompt. Fast, no
+     extra agents. Cost: not reusable, no own tools, verified ONLY via
+     the integration eval. OK for trivial one-off glue steps.
+   - **ref sub-agent** \u2014 the step delegates to a registered agent with
+     its own tools/model/skills (built via [[agent-build]], prompt =
+     "Load [[skill-name]] and follow it"). Reusable, independently
+     verified (Phase 2.6). Use when the step needs tools, non-trivial
+     or reusable logic, or independent verification.
+   Present the per-step choice with trade-offs and let the user
+   decide \u2014 NEVER silently pick inline or ref. When in doubt, ask.
+3. **Identify knowledge per step** \u2014 for each step, determine the domain
+   knowledge it needs:
+   - Existing skill covers it \u2192 reference [[skill-name]] in the step
+   - No skill yet, but the knowledge is reusable or non-trivial \u2192
+     plan to create it (Phase 1.5)
+   - Trivial one-off logic \u2192 may stay inline in the prompt (accept the
+     trade-off: it is not reusable or individually verifiable)
+4. Design using the YAML linear DSL (steps, parallel, map, if, ask).
+5. **Present the design as a Flowchart widget** (\`show_widget\`) \u2014 every
+   step, branch, and \`ask\` interaction point. Walk through it with the
+   user step-by-step (each step's responsibility, branch logic, ask
+   points). CONFIRM via \`ask_user_to_clarify\` \u2014 never build without
+   explicit user approval.
+
+## Phase 1.5: Expected Output Specification (mandatory \u2014 goal-driven)
+
+Define the workflow's EXPECTED OUTPUT SPEC from the goal model BEFORE
+writing skills or building: what the final outcome looks like, per
+consumer (0.1.5). This is the acceptance standard \u2014 [[eval-verify]]
+contentAssertion derives from it. HARD RULE: if the target/expected
+output is unclear, ask the user \u2014 never guess.
+Present the spec, confirm with the user, record in the parent task.
+
+## Phase 2: Create Skills (for missing knowledge)
+
+For each planned skill (Phase 1.2): write SKILL.md (frontmatter +
+body encoding the domain rules). Present each for user approval.
+When 3+ skills share a domain \u2192 create a MOC ([[domain-moc]]).
+If a ref step needs an agent \u2192 build it via [[agent-build]] (agent
+prompt = "Load [[skill-name]] and follow it" \u2014 thin, knowledge in
+skill). Order: sub-agents/skills first, then the workflow that
+references them.
+
+## Phase 2.6: Verify components FIRST (mandatory)
+
+Every agent referenced by a \`ref\` step is a component with its OWN
+independent eval (eval-{ref-agent-id}) \u2014 run it and pass it BEFORE
+building the integration eval. The workflow cannot be considered tested
+until: \u2460 each ref'd agent's eval passes independently, \u2461 then the
+workflow's integration eval (branch paths + ask handling) passes. See
+[[eval-verify|Layered verification]].
+
+## Phase 3: Build
+
+1. **Configure middleware & tools for the workflow itself** \u2014 inline
+   steps run on the workflow's own model/tools: call
+   \`list_middleware_types\` first; add what the workflow needs \u2014 skill
+   (if steps load [[skill-name]]), widget, ask_user_to_clarify, etc.
+   Tool filtering via \`allowedTools\`. \`ref\` steps use the ref'd
+   agent's own tools/model \u2014 nothing to configure here. Choose
+   \`modelKey\` only when a specific model is required (default
+   otherwise).
+2. Call \`create_workflow\` with \`skillLoaded: true\` \u2014 steps reference
+   [[skill-name]] or \`ref\` to skill-loading agents.
+3. Then \`validate_workflow(id)\`.
+
+## Phase 4: Test (mandatory \u2014 no eval, no trust tier)
+
+The authoritative verification is [[eval-verify]] \u2014 cases derive from
+the expected output spec (Phase 1.5). A workflow without a passing eval
+stays at "configured" forever \u2014 trust can never upgrade
+([[completion-gate]], no skip option).
+
+**Testing is managed through the eval project (eval-{workflow-id})
+and its cases \u2014 the same governance as agents.** Temporary or quick
+checks (ad-hoc runs, previewing behavior) may use [[review-agent]] as
+an interactive pre-check \u2014 but that is NOT the workflow's test suite:
+it never upgrades trust and never replaces the eval project. Only the
+eval project's cases passing determine "tested".
+
+**Test order \u2014 components first, then integration:**
+1. Each \`ref\`'d agent: its OWN eval (eval-{ref-agent-id}) must pass
+   independently (Phase 2.6) \u2014 fix it in isolation, not through the
+   workflow.
+2. Then the workflow's integration eval (eval-{workflow-id}): one case
+   per branch path (if/map/parallel); \`ask\` steps via case
+   interruptPolicy (auto-approve/canned-response to test the flow AFTER
+   the pause, stop to test up to the pause); assertions are exact \u2014
+   the pipeline is deterministic.
+
+Workflow trust upgrade requires BOTH layers passing.
+[[review-agent]] is an optional cheap pre-check only.
+
+## Editing workflows
+
+Get the current YAML \u2192 present the diff \u2192 confirm with the user \u2192
+\`update_workflow(id, ...)\`. Never re-create.
+After ANY change: verified resets to unverified and the eval is re-run
+([[eval-verify]]) \u2014 the change is not done until the eval passes again.
+Deleting: warn if any step \`ref\`s it \u2192 confirm \u2192 \`delete_agent\`.
 
-Call \`create_workflow\` with \`skillLoaded: true\`, then
-\`validate_workflow(id)\`.
-
-## Phase 3: Test
+## Metadata
 
-Ask user if they want to test \u2014 the authoritative verification is
-[[eval-verify]]. [[review-agent]] is an optional cheap pre-check only.
+Always set metadata on workflow creation. At minimum:
+- verified: "unverified" (upgraded after eval passes)
+- version: "1.0" (bump on each update)
+- source: the material name or "user-description"
+When trust upgrades, update BOTH the skill's verified frontmatter and
+the workflow's metadata.verified \u2014 they must stay in sync.
 
 ## No edges, state fields, or end step
 The engine auto-generates them. Steps execute top-to-bottom in written
@@ -23572,10 +23805,36 @@ TASK MANAGEMENT IS A CORE DUTY, not a per-skill option. Whenever the
 goal is clear and you know what to do, create a task FIRST (manage_task)
 before executing \u2014 for any multi-step work: learning, building,
 modifying, fixing, anything with an Objective and Acceptance Criteria.
-Subtasks per work item. Status must reflect reality. See [[task-tracking]].
+- **Check for duplicates BEFORE creating** \u2014 always manage_task
+  action: "list" first (filter ownerType: "agent"). If a task with the
+  same objective already exists (e.g. from an interrupted session),
+  RESUME it instead of creating a new one.
+- **Decompose into subtasks** \u2014 after the parent task, create a
+  subtask per work item / phase (e.g. design, build, eval), each with
+  its own Objective + Acceptance Criteria.
+- **Update on completion** \u2014 every finished subtask and the parent:
+  manage_task update(status: "completed", result: "what was done").
+  Use interrupted/failed with summary/failureReason when blocked or
+  unable. Status must always reflect reality \u2014 never leave a finished
+  task dangling in an in-progress state.
+See [[task-tracking]].
 The sub-skills below only ADD their own task details on top of this
 universal duty.
 
+BUILD GATES \u2014 hard behavioral requirements, no exceptions, no skipping:
+- Creating a WORKFLOW ([[design-workflow]]): \u2460 show the design as a
+  Flowchart widget (every step, branch, ask point) \u2461 walk through it
+  step-by-step with the user \u2462 ask inline-vs-ref per step \u2463 CONFIRM via
+  ask_user_to_clarify \u2014 only then call create_workflow.
+- Creating an AGENT ([[agent-build]]): \u2460 present the design with
+  show_widget \u2461 confirm via ask_user_to_clarify \u2014 only then call
+  create_agent.
+- Both: if the goal model (real goal / consumer / usable state) is
+  unclear, ask BEFORE designing \u2014 never guess.
+The skills document WHY and HOW; these gates are the unskippable
+minimum. If you cannot satisfy a gate (e.g. user says skip), record it
+and proceed only on the user's explicit instruction.
+
 Your sub-skills (accessible via the MOC or direct loading):
 - [[learn-capability]] \u2014 Learn from any source material \u2192 skills + agents
 - [[agent-build]] \u2014 Design and build single agents (REACT/DEEP_AGENT)
@@ -27695,7 +27954,7 @@ File content: ${files[key4]}`
       {
         dimension: "correctness",
         weight: 100,
-        description: "\u6574\u4F53\u6B63\u786E\u6027\uFF0C\u662F\u5426\u7B26\u5408\u9884\u671F\u8F93\u51FA\u63CF\u8FF0\u3002"
+        description: "Overall correctness \u2014 whether the result matches the expected output description."
       }
     ];
     const evalRubrics = evalCase.eval.eval_rubrics && evalCase.eval.eval_rubrics.length > 0 ? evalCase.eval.eval_rubrics : defaultRubrics;
@@ -27709,53 +27968,53 @@ File content: ${files[key4]}`
 ${evalRubrics.map(
       (r) => `- **${r.dimension}**\uFF08\u6743\u91CD\uFF1A${r.weight}\uFF09\uFF1A${r.description}`
     ).join("\n")}`;
-    const testPrompt = `# \u89D2\u8272
-\u4F60\u662F\u4E00\u540D\u8D44\u6DF1\u7684 AI Agent \u8BC4\u4F30\u4E13\u5BB6\uFF0C\u8D1F\u8D23\u6839\u636E\u9884\u8BBE\u7684\u6307\u6807\uFF08Rubrics\uFF09\u5BF9 Agent \u7684\u6267\u884C\u8FC7\u7A0B\u4E0E\u7ED3\u679C\u8FDB\u884C"\u9ED1\u76D2\u6D4B\u8BD5"\u5224\u5B9A\u3002
+    const testPrompt = `# Role
+You are a senior AI Agent evaluation expert. Your job is to perform a "black-box test" judgment of the agent's execution process and results against the preset evaluation rubrics.
 
-# \u8F93\u5165\u4FE1\u606F
-\u6D4B\u8BD5\u6846\u67B6\u5C06\u4E3A\u4F60\u63D0\u4F9B\u4EE5\u4E0B\u4E94\u4E2A\u6838\u5FC3\u4E0A\u4E0B\u6587\uFF1A
+# Input Information
+The test framework provides you with the following five core contexts:
 
-1. **\u7528\u6237\u610F\u56FE\uFF08User Intent\uFF09**\uFF1A${evalCase.input.message}
+1. **User Intent**: ${evalCase.input.message}
 
-2. **\u8F93\u5165\u6587\u4EF6\uFF08Input Files\uFF09**\uFF1A${testCaseFilesContent || "\u65E0"}
+2. **Input Files**: ${testCaseFilesContent || "None"}
 
-3. **\u6267\u884C\u8FC7\u7A0B\uFF08Execution Trajectory\uFF0CAgent \u5168\u7A0B\u7684\u6D88\u606F/\u5DE5\u5177\u8C03\u7528\u8BB0\u5F55\uFF09**\uFF1A
+3. **Execution Trajectory** (the agent's full message/tool-call record):
 ${trajectory}
 
-4. **\u6700\u7EC8\u8F93\u51FA\uFF08Final Output\uFF0CAgent \u7684\u6700\u540E\u4E00\u6761\u56DE\u590D\uFF09**\uFF1A
+4. **Final Output** (the agent's last reply):
 ${finalOutput}
 
-5. **\u671F\u671B\u8F93\u51FA\u63CF\u8FF0\uFF08Expected Output Description\uFF09**\uFF1A${evalCase.eval.content_assertion}
+5. **Expected Output Description**: ${evalCase.eval.content_assertion}
 ${rubricsSection}
 
-# \u4EFB\u52A1
-\u4F60\u5FC5\u987B\u4E25\u683C\u5BF9\u7167"\u8BC4\u4F30\u6307\u6807\uFF08Evaluation Rubrics\uFF09"\u4E2D\u7684\u6BCF\u4E00\u9879\u6307\u6807\uFF0C\u7ED3\u5408"\u6267\u884C\u8FC7\u7A0B"\u4E0E"\u6700\u7EC8\u8F93\u51FA"\uFF0C\u5206\u6790 Agent \u662F\u5426\u8FBE\u6807\u3002\u8BC4\u4F30\u65F6\u65E2\u8981\u68C0\u67E5\u6700\u7EC8\u7ED3\u679C\uFF0C\u4E5F\u8981\u68C0\u67E5\u8FC7\u7A0B\u4E2D\u662F\u5426\u6B63\u786E\u5B8C\u6210\u4E86\u5FC5\u8981\u7684\u6B65\u9AA4\uFF08\u5982\u5DE5\u5177\u8C03\u7528\u3001\u4FE1\u606F\u68C0\u7D22\u7B49\uFF09\u3002
+# Task
+You must strictly evaluate the agent against every rubric in the "Evaluation Rubrics" section, using both the "Execution Trajectory" and the "Final Output". Evaluate the final result AND whether the process correctly performed the required intermediate steps (tool calls, information retrieval, etc.).
 
-# \u89C4\u5219
-1. **\u5BA2\u89C2\u6027**\uFF1A\u4EC5\u6839\u636E\u63D0\u4F9B\u7684\u4E0A\u4E0B\u6587\u5224\u5B9A\u3002\u5982\u679C\u6807\u51C6\u8981\u6C42"\u5305\u542B\u6570\u5B57"\uFF0C\u4F46\u8F93\u51FA\u53EA\u6709\u6587\u5B57\uFF0C\u5373\u4F7F\u8BED\u6C14\u518D\u597D\u4E5F\u5FC5\u987B\u6263\u5206\u3002
-2. **\u7ED3\u679C\u6821\u9A8C**\uFF1A\u5982\u679C"\u6700\u7EC8\u8F93\u51FA"\u4E2D\u7F3A\u5931\u9884\u671F\u7684\u5185\u5BB9\uFF0C\u6216\u5185\u5BB9\u4E0D\u7B26\u5408"\u8BC4\u4F30\u6307\u6807"\u4E2D\u7684\u6807\u51C6\uFF0C\u5BF9\u5E94\u7684\u6307\u6807\u5E94\u5224\u5B9A\u4E3A\u5931\u8D25\u3002
-3. **\u8FC7\u7A0B\u6821\u9A8C**\uFF1A\u5982\u679C"\u6267\u884C\u8FC7\u7A0B"\u663E\u793A Agent \u672A\u6267\u884C\u5FC5\u8981\u7684\u4E2D\u95F4\u6B65\u9AA4\uFF08\u5982\u5E94\u8C03\u7528\u5DE5\u5177\u800C\u672A\u8C03\u7528\uFF09\uFF0C\u5373\u4F7F\u6700\u7EC8\u8F93\u51FA\u770B\u4F3C\u5408\u7406\uFF0C\u4E5F\u5E94\u5728\u5BF9\u5E94\u6307\u6807\u4E0A\u6263\u5206\u3002
-4. **\u8BC1\u636E\u5BFC\u5411**\uFF1A\u5728\u7ED9\u51FA\u539F\u56E0\uFF08reason\uFF09\u65F6\uFF0C\u5FC5\u987B\u5F15\u7528\u6267\u884C\u8FC7\u7A0B\u6216\u6700\u7EC8\u8F93\u51FA\u4E2D\u7684\u5177\u4F53\u5185\u5BB9\u3002
-5. **\u52A0\u6743\u8BA1\u7B97**\uFF1A\u6700\u7EC8\u5206\u6570\u4E3A\u5404\u9879\u6307\u6807\u5F97\u5206\u4E0E\u5176\u6743\u91CD\u7684\u4E58\u79EF\u4E4B\u548C\uFF080-100\u5206\u5236\uFF09\u3002
-6. **HITL \u4E2D\u65AD\u5224\u5B9A**\uFF1A\u5982\u679C\u6267\u884C\u8FC7\u7A0B\u4E2D\u51FA\u73B0\u300CHITL \u6682\u505C\uFF1AAgent \u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u300D\u6761\u76EE\uFF0C\u8BF4\u660E Agent \u5728\u7B49\u5F85\u4EBA\u5DE5\u786E\u8BA4\u3002\u8BF7\u628A\u5B83\u5F53\u4F5C\u88AB\u6D4B\u4E1A\u52A1\u884C\u4E3A\u6765\u8BC4\u5224\uFF1A\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u81EA\u4E3B\u5B8C\u6210\uFF08\u5982"\u65E0\u9700\u786E\u8BA4\u81EA\u52A8\u6267\u884C"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u5E94\u5224\u5931\u8D25\uFF1B\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u5148\u83B7\u5F97\u786E\u8BA4\u6216\u6279\u51C6\uFF08\u5982"\u6267\u884C\u524D\u5FC5\u987B\u8BF7\u6C42\u6279\u51C6"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u662F\u6B63\u786E\u884C\u4E3A\uFF0C\u5E94\u7ED3\u5408\u5176\u65F6\u673A\u4E0E\u5185\u5BB9\u5224\u5B9A\u4E3A\u901A\u8FC7\u6216\u6309\u6307\u6807\u7ED9\u5206\u3002
-7. **HITL \u81EA\u52A8\u54CD\u5E94\u5224\u5B9A**\uFF1A\u5982\u679C\u300CHITL \u6682\u505C\u300D\u6761\u76EE\u4E4B\u540E\u51FA\u73B0\u300C\u5DF2\u81EA\u52A8\u54CD\u5E94\uFF08\u6D4B\u8BD5\u7B56\u7565 auto-approve/auto-reject/canned-response\uFF09\u300D\u6761\u76EE\uFF0C\u8BF4\u660E\u6D4B\u8BD5\u6846\u67B6\u6CE8\u5165\u4E86\u4EBA\u5DE5\u56DE\u590D\u3001\u6D41\u7A0B\u5DF2\u7EE7\u7EED\u2014\u2014\u8BF7\u6309**\u5B8C\u6574\u6D41\u7A0B**\u8BC4\u5224\u6682\u505C\u4E4B\u540E\u7684\u884C\u4E3A\uFF08\u5982\u6279\u51C6\u540E\u662F\u5426\u6B63\u786E\u6267\u884C\u4E86\u64CD\u4F5C\uFF09\uFF0C\u5E76\u6838\u5BF9\u81EA\u52A8\u54CD\u5E94\u5185\u5BB9\u662F\u5426\u7B26\u5408\u4EBA\u5DE5\u56DE\u590D\u7684\u5408\u7406\u9884\u671F\u3002
+# Rules
+1. **Objectivity**: Judge solely from the provided context. If the standard requires "contains a number" but the output has only text, points must be deducted even if the tone is good.
+2. **Result verification**: If the "Final Output" is missing expected content, or does not meet the criteria in the "Evaluation Rubrics", the corresponding rubric must be marked as failed.
+3. **Process verification**: If the "Execution Trajectory" shows the agent did not perform a necessary intermediate step (e.g., should have called a tool but did not), deduct points on the corresponding rubric even if the final output looks plausible.
+4. **Evidence-based**: When giving a reason, you must quote specific content from the execution trajectory or final output.
+5. **Weighted scoring**: The final score is the weighted sum of the rubric scores (on a 0-100 scale).
+6. **HITL interrupt judgment**: If the trajectory contains a "HITL pause: agent requested human input" entry, the agent is waiting for human confirmation. Treat this as the business behavior under test: if the expected output requires autonomous completion (e.g., "execute automatically without confirmation"), requesting human input should be judged a failure; if the expected output requires confirmation or approval first (e.g., "must request approval before executing"), requesting human input is correct behavior \u2014 judge its timing and content, passing or scoring according to the rubrics.
+7. **HITL auto-response judgment**: If a "HITL pause" entry is followed by an "auto-responded (test policy auto-approve/auto-reject/canned-response)" entry, the test framework injected a human reply and the flow continued \u2014 evaluate the behavior AFTER the pause as the complete flow (e.g., whether the operation was correctly executed after approval), and check whether the auto-response content matches a reasonable human reply.
 
-# \u8F93\u51FA\u683C\u5F0F\uFF08\u4EC5JSON\uFF09
-\u4F60\u5FC5\u987B\u4EC5\u4EE5 JSON \u683C\u5F0F\u56DE\u590D\uFF0C\u7ED3\u6784\u5982\u4E0B\uFF1A
+# Output Format (JSON only)
+You MUST reply with JSON only, using this structure:
 {
   "pass": true | false,
   "final_score": number,
   "dimension_results": [
     {
-      "name": "\u6307\u6807\u540D\u79F0",
+      "name": "rubric name",
       "score": number,
-      "reason": "\u5177\u4F53\u7684\u6263\u5206\u6216\u7ED9\u5206\u7406\u7531\uFF0C\u9700\u5F15\u7528\u8BC1\u636E"
+      "reason": "specific reason for deduction or credit, citing evidence"
     }
   ],
-  "summary": "\u5BF9 Agent \u8868\u73B0\u7684\u6574\u4F53\u8BC4\u4EF7"
+  "summary": "overall evaluation of the agent's performance"
 }
 
-\u6CE8\u610F\uFF1A\u5982\u679C final_score >= 80 \u4E14\u6CA1\u6709\u81F4\u547D\u6027\u9519\u8BEF\uFF0Cpass \u5E94\u4E3A true\uFF1B\u5426\u5219\u4E3A false\u3002`;
+Note: if final_score >= 80 and there are no fatal errors, pass should be true; otherwise false.`;
     this.lastTestPrompt = testPrompt;
     const judgeThreadId = (0, import_uuid9.v4)();
     this.lastJudgeThreadId = judgeThreadId;
@@ -28138,6 +28397,32 @@ var LatticeEvalSuite = class {
 var import_protocols16 = require("@axiom-lattice/protocols");
 var import_messages7 = require("@langchain/core/messages");
 var import_uuid10 = require("uuid");
+var DEFAULT_CALIBRATION_PROBES = [
+  {
+    id: "catch-up-correct",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "11:30",
+    expectedPass: true
+  },
+  {
+    id: "catch-up-wrong",
+    task: "A train leaves Station A at 10:00 AM traveling at 60 mph. A second train leaves Station A at 10:30 AM traveling at 90 mph on the same route. The final answer must be the time (HH:MM) at which the second train catches up to the first.",
+    finalOutput: "12:30",
+    expectedPass: false
+  },
+  {
+    id: "discount-tax-correct",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "129.6",
+    expectedPass: true
+  },
+  {
+    id: "discount-tax-wrong",
+    task: "A store offers a 20% discount on an item priced at $150, then adds 8% sales tax to the discounted price. The final answer must be the total price in dollars.",
+    finalOutput: "162",
+    expectedPass: false
+  }
+];
 var LatticeEvalProject = class {
   constructor(project, onCaseComplete) {
     this.suites = /* @__PURE__ */ new Map();
@@ -28245,21 +28530,22 @@ var LatticeEvalProject = class {
   }
   /**
    * Verify the judge agent can produce parseable, correct verdicts
-   * before committing to a full run. Uses two known-answer cases
-   * (one expected PASS, one expected FAIL) to catch broken judges.
+   * before committing to a full run. Uses known-answer probes (default
+   * multi-step reasoning set, overridable per project via
+   * judge_agent_config.calibration_cases) to catch broken judges.
+   * Checks both the pass verdict and the final_score direction.
    */
   async calibrateJudge() {
     const tenantId2 = this.project.lattice_server_config.tenant_id || "default";
     const judgeAgent = await getAgentClient(tenantId2, this.judgeAgentKey);
-    const cases = [
-      { output: "7", expected: "7", expectedPass: true },
-      { output: "7", expected: "999", expectedPass: false }
-    ];
-    for (const c of cases) {
-      const prompt = `\u4F60\u662F\u8BC4\u4F30\u4E13\u5BB6\u3002\u5224\u5B9A\u6700\u7EC8\u8F93\u51FA\u662F\u5426\u7B26\u5408\u671F\u671B\u3002
-\u6700\u7EC8\u8F93\u51FA\uFF1A${c.output}
-\u671F\u671B\u8F93\u51FA\uFF1A${c.expected}
-\u4EC5\u8F93\u51FA JSON\uFF1A{"pass": true|false, "final_score": 0-100, "summary": "\u7406\u7531"}`;
+    const probes = this.project.judge_agent_config.calibration_cases && this.project.judge_agent_config.calibration_cases.length > 0 ? this.project.judge_agent_config.calibration_cases : DEFAULT_CALIBRATION_PROBES;
+    for (const probe of probes) {
+      const trajectorySection = probe.trajectory ? `
+Trajectory: ${probe.trajectory}` : "";
+      const prompt = `You are an evaluation expert. Judge whether the final output meets the expected output description.
+Expected output description: ${probe.task}
+Final output: ${probe.finalOutput}${trajectorySection}
+Respond with JSON only: {"pass": true|false, "final_score": 0-100, "summary": "reason"}`;
       let raw = "";
       let invokeError = null;
       for (let attempt = 0; attempt < 2; attempt++) {
@@ -28284,13 +28570,24 @@ var LatticeEvalProject = class {
         return { ok: false, reason: `Calibration output unparseable: ${parsed.error}`, bypassed: true };
       }
       const actualPass = parsed.pass !== void 0 ? parsed.pass : (parsed.final_score ?? 0) >= 80;
-      if (actualPass !== c.expectedPass) {
+      if (actualPass !== probe.expectedPass) {
         return {
           ok: false,
-          reason: `Calibration mismatch: output="${c.output}" expected="${c.expected}" \u2014 judge said ${actualPass ? "PASS" : "FAIL"}, expected ${c.expectedPass ? "PASS" : "FAIL"}`,
+          reason: `Calibration mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "PASS" : "FAIL"}, judge said ${actualPass ? "PASS" : "FAIL"}`,
           bypassed: true
         };
       }
+      if (parsed.final_score !== void 0) {
+        const minScore = probe.expectedScoreMin ?? 80;
+        const scoreOk = probe.expectedPass ? parsed.final_score >= minScore : parsed.final_score < minScore;
+        if (!scoreOk) {
+          return {
+            ok: false,
+            reason: `Calibration score mismatch (probe=${probe.id}): expected ${probe.expectedPass ? "final_score >= " + minScore : "final_score < " + minScore}, judge gave ${parsed.final_score}`,
+            bypassed: true
+          };
+        }
+      }
     }
     return { ok: true };
   }
@@ -32690,6 +32987,7 @@ registerBuiltinPlugins();
   ConsoleLoggerClient,
   CustomMetricsClient,
   CustomMiddlewareRegistry,
+  DEFAULT_CALIBRATION_PROBES,
   DaytonaInstance,
   DaytonaProvider,
   DefaultScheduleClient,