@axiom-lattice/core 3.0.3 → 3.0.5

package/dist/index.mjs

@@ -9394,8 +9394,11 @@ Please select a valid tool from the list above.`
      * The only place to access request.tools (all available tools).
      * Identifies unknown tools and stores error info in metadata.
      *
-     * Key: Preserve all tool_calls in original AIMessage, including unknown ones.
-     * This allows the model to see what it selected in the next round.
+     * Key: Strip valid tool_calls and only keep unknown ones in the returned
+     * AIMessage. Valid calls must NOT be preserved: afterModel jumps to "model"
+     * and skips ToolNode, so a preserved valid call would have no ToolMessage
+     * and the next model call would be rejected with a 400 dangling tool_calls
+     * error. Stripped valid calls are re-issued by the model in the next round.
      */
     wrapModelCall: async (request, handler) => {
       const availableTools = request.tools || [];
@@ -9425,10 +9428,11 @@ Please select a valid tool from the list above.`
         toolCallId: toolCall.id,
         errorMessage: errorMessageTemplate(toolCall.name, availableToolNames)
       }));
+      const unknownToolIds = new Set(unknownToolCalls.map((toolCall) => toolCall.id));
+      const remainingToolCalls = aiResponse.tool_calls.filter((toolCall) => unknownToolIds.has(toolCall.id));
       const modifiedResponse = new AIMessage({
         content: aiResponse.content,
-        tool_calls: aiResponse.tool_calls,
-        // Key: preserve all tool_calls, don't delete unknown
+        tool_calls: remainingToolCalls,
         response_metadata: {
           ...aiResponse.response_metadata,
           _unknownToolErrors: unknownToolErrors
@@ -9481,6 +9485,68 @@ Please select a valid tool from the list above.`
   });
 }
 
+// src/deep_agent_new/middleware/patch_tool_calls.ts
+import {
+  createMiddleware as createMiddleware9,
+  ToolMessage as ToolMessage3,
+  AIMessage as AIMessage2
+} from "langchain";
+function createPatchToolCallsMiddleware() {
+  return createMiddleware9({
+    name: "patchToolCallsMiddleware",
+    beforeAgent: async (state) => {
+      const messages = state.messages;
+      if (!messages || messages.length === 0) {
+        return;
+      }
+      const replacements = [];
+      for (let i = 0; i < messages.length; i++) {
+        const msg = messages[i];
+        if (AIMessage2.isInstance(msg) && msg.tool_calls != null) {
+          const respondedIds = /* @__PURE__ */ new Set();
+          for (const toolCall of msg.tool_calls) {
+            if (!toolCall.id) continue;
+            const correspondingToolMsg = messages.slice(i).find(
+              (m) => ToolMessage3.isInstance(m) && m.tool_call_id === toolCall.id
+            );
+            if (correspondingToolMsg) {
+              respondedIds.add(toolCall.id);
+            }
+          }
+          const remainingToolCalls = msg.tool_calls.filter(
+            (toolCall) => toolCall.id && respondedIds.has(toolCall.id)
+          );
+          if (remainingToolCalls.length === msg.tool_calls.length) {
+            continue;
+          }
+          const additionalKwargs = { ...msg.additional_kwargs };
+          delete additionalKwargs.tool_calls;
+          if (!msg.id) continue;
+          replacements.push(
+            new AIMessage2({
+              id: msg.id,
+              content: msg.content,
+              name: msg.name,
+              tool_calls: remainingToolCalls,
+              additional_kwargs: additionalKwargs,
+              response_metadata: msg.response_metadata
+            })
+          );
+        }
+      }
+      if (replacements.length === 0) {
+        return;
+      }
+      return {
+        messages: replacements
+      };
+    }
+  });
+}
+
+// src/agent_lattice/builders/commonMiddleware.ts
+import { summarizationMiddleware } from "langchain";
+
 // src/plugin/metaSerializer.ts
 function tryExtractTools(plugin) {
   if (!plugin.middleware) return [];
@@ -10717,6 +10783,12 @@ actually achieve, not just what to build:
 
 Record the goal model in the parent task's description ([[task-tracking]]).
 
+**HARD RULE \u2014 never guess the target.** If the goal, expected output,
+consumer, or usable state is unclear at ANY point before writing test
+cases, you MUST ask the user via ask_user_to_clarify \u2014 never proceed
+with an assumed expectation. A test case written against a guessed
+expectation validates the wrong thing. When in doubt, ask.
+
 ## Goal-Driven Validation (apply to EVERY sub-skill workflow)
 
 The agent evaluates goal achievement ITSELF via multi-dimensional test
@@ -10802,16 +10874,29 @@ Do NOT use reviewer as:
    verification. If findings show config errors, fix and re-check.`,
   "task-tracking": `---
 name: task-tracking
-description: Manage persistent tasks for agent creation workflows. Use
-  manage_task to create parent/subtasks, track progress, and resume
-  interrupted runs. Applies to ALL agent building processes.
+description: Manage persistent tasks with manage_task. Universal rule:
+  once the goal is clear and you know what to do, create the task FIRST
+  then execute. Track parent/subtasks, update status to reflect reality,
+  resume interrupted work. Applies to ANY multi-step agent work \u2014 not
+  just agent building.
 metadata:
   domain: agent-building
 verified: unverified
 ---
 # Task Tracking \u2014 manage_task for Agent Workflows
 
-Every agent creation workflow is a multi-step process \u2014 track it.
+**Task management is the ongoing record of a goal and its acceptance
+criteria** \u2014 it answers at any moment: what are we achieving, and what
+does "done" look like. Create a task when the goal is clear; keep its
+Objective and Acceptance Criteria current as work proceeds; change
+status only when the criteria are actually met.
+
+**Universal principle**: whenever the goal is understood and the work
+is about to start, create a task BEFORE executing. If you can write an
+Objective and Acceptance Criteria, it deserves a task. This is not
+optional and not limited to agent-building \u2014 it applies to any
+multi-step work (learning, building, modifying skills, fixing, anything
+with a clear goal).
 
 ## When to create (and when NOT)
 
@@ -10828,13 +10913,29 @@ Do NOT create tasks for:
 
 ## Setup
 
+**A task is a living record of the GOAL + ACCEPTANCE CRITERIA** \u2014 not a
+todo label. Every task's description must carry:
+
+- **Objective** \u2014 one measurable sentence: what result to achieve
+- **Acceptance Criteria** \u2014 checkboxes that define "done": when ALL
+  are checked, the task is verifiably complete
+
+The task is created when the goal is confirmed, and its description is
+CONTINUALLY UPDATED as the work progresses (spec evolves, criteria are
+met, new criteria emerge). Status changes only when the criteria are
+truly met \u2014 never as a workaround.
+
 - **Create the parent task when the scope is confirmed** \u2014 before
   starting the first real work phase (probe/design/build):
-  \`manage_task create(title: <goal>, description: <summary>, ownerType: "agent")\`
+  \`manage_task create(title: <goal>, description: <Objective + Acceptance Criteria>, ownerType: "agent")\`
   Record the returned parent task id.
 - **Create a subtask per phase** as you start each phase (probe /
   design / build / eval / retro):
-  \`manage_task create(title: <phase>, parentId: <parent>, ownerType: "agent")\`
+  \`manage_task create(title: <phase>, description: <Objective + Acceptance Criteria>, parentId: <parent>, ownerType: "agent")\`
+- **Update the description as work proceeds**: append progress, mark
+  criteria \`[x]\`, revise criteria when the goal model/spec changes.
+  The task tracks the target and its acceptance \u2014 read it to know what
+  "done" means, keep it current so it always reflects reality.
 
 ## Status discipline \u2014 MANDATORY
 
@@ -11023,6 +11124,10 @@ Always set metadata on agent creation. At minimum:
 - verified: "unverified" (upgraded after eval passes)
 - version: "1.0" (bump on each update_agent)
 - source: the material name or "user-description"
+- role: "orchestrator" | "sub-agent" \u2014 set when the agent is part of a
+  parent+subAgents structure (role clarity, User Interaction Rules).
+  The eval project is NOT recorded \u2014 it is derived by naming convention
+  (eval-{agent-id}, see [[eval-verify]] Setup).
 When trust upgrades, update both the skill's verified frontmatter and
 the agent's metadata.verified \u2014 they must stay in sync.
 
@@ -11138,16 +11243,32 @@ description: Run agent evaluations, interpret results, fix failures, and
 metadata:
   domain: agent-building
 verified: unverified
+subSkills:
+  - eval-design-tests
+  - eval-run-and-govern
 ---
 # Eval Verify \u2014 Run Evaluations and Upgrade Trust
 
 ## Setup
 
 0. Load [[eval-design-tests]] for case design guidance
-1. \`read_eval list_projects\` \u2192 find "eval-{domain}"
-   Exists \u2192 reuse projectId. New \u2192 create_project(name: "eval-{domain}")
+1. **One eval project per agent**, named \`eval-{agent-id}\`:
+   \`read_eval list_projects\` \u2192 find "eval-{agent-id}"
+   Exists \u2192 reuse projectId. New \u2192 create_project(name: "eval-{agent-id}")
+   - The agent-id is the eval project's subject. Observability: from an
+     agent's id you can find its eval project by naming convention.
+   - Orchestrator + subAgents \u2192 one eval project per sub-agent
+     (eval-{sub-agent-id}) PLUS one integration eval project for the
+     parent (eval-{parent-id}) \u2014 see Layered verification below.
 2. \`manage_eval create_case(suiteId, ...)\` per sample or scenario
    Required: inputMessage, steps=[{agent_id}], outputType, contentAssertion
+   **contentAssertion MUST come from the confirmed expected output spec**
+   (learn-capability Phase 2.6) \u2014 never invent expectations at
+   case-writing time. If a needed expectation is not in the spec, extend
+   the spec with user confirmation first.
+   **HARD RULE**: if the target/expected output is unclear at this
+   point, STOP and ask the user (ask_user_to_clarify) \u2014 do not write a
+   case with a guessed expectation.
 
 ## Suites per skill, by source
 
@@ -11491,10 +11612,20 @@ async function resolveConnections(type, connections, tenantId2) {
     throw err;
   }
 }
-async function createCommonMiddlewares(middlewareConfigs, filesystemBackend, fsIsExised, tenantId2) {
+async function createCommonMiddlewares(middlewareConfigs, filesystemBackend, fsIsExised, tenantId2, model) {
   const middlewares = [];
   middlewares.push(createUnknownToolHandlerMiddleware());
   middlewares.push(createModelSelectorMiddleware());
+  middlewares.push(createPatchToolCallsMiddleware());
+  if (model) {
+    middlewares.push(
+      summarizationMiddleware({
+        model,
+        trigger: { tokens: 17e4 },
+        keep: { messages: 6 }
+      })
+    );
+  }
   const filesystemConfig = middlewareConfigs.find((m) => m.type === "filesystem");
   const clawConfig = middlewareConfigs.find((m) => m.type === "claw");
   const needsFilesystemBackend = filesystemConfig?.enabled || clawConfig?.enabled;
@@ -11826,7 +11957,7 @@ var ReActAgentGraphBuilder = class {
     const stateSchema2 = createReactAgentSchema(params.stateSchema);
     const middlewareConfigs = params.middleware || [];
     const filesystemBackend = createFilesystemBackendFactory(middlewareConfigs);
-    const middlewares = await createCommonMiddlewares(middlewareConfigs, filesystemBackend, void 0, params.tenantId);
+    const middlewares = await createCommonMiddlewares(middlewareConfigs, filesystemBackend, void 0, params.tenantId, params.model);
     return createAgent({
       model: params.model,
       tools,
@@ -11844,17 +11975,16 @@ var ReActAgentGraphBuilder = class {
 import {
   createAgent as createAgent3,
   humanInTheLoopMiddleware as humanInTheLoopMiddleware2,
-  anthropicPromptCachingMiddleware,
-  summarizationMiddleware
+  anthropicPromptCachingMiddleware
 } from "langchain";
 
 // src/deep_agent_new/middleware/subagents.ts
 import { z as z42 } from "zod/v3";
 import {
-  createMiddleware as createMiddleware10,
+  createMiddleware as createMiddleware11,
   createAgent as createAgent2,
   tool as tool40,
-  ToolMessage as ToolMessage3,
+  ToolMessage as ToolMessage4,
   humanInTheLoopMiddleware
 } from "langchain";
 import { Command as Command3, getCurrentTaskInput as getCurrentTaskInput2, GraphInterrupt as GraphInterrupt3 } from "@langchain/langgraph";
@@ -13800,7 +13930,7 @@ var AgentInstanceManager = class _AgentInstanceManager {
 var agentInstanceManager = AgentInstanceManager.getInstance();
 
 // src/middlewares/taskMiddleware.ts
-import { createMiddleware as createMiddleware9, tool as tool39 } from "langchain";
+import { createMiddleware as createMiddleware10, tool as tool39 } from "langchain";
 import { z as z41 } from "zod";
 import { GraphInterrupt as GraphInterrupt2, interrupt as interrupt3 } from "@langchain/langgraph";
 function getRunConfig(config) {
@@ -14108,26 +14238,37 @@ function createTaskMiddleware() {
         });
     }
   };
-  return createMiddleware9({
+  return createMiddleware10({
     name: "TaskMiddleware",
     contextSchema,
     wrapModelCall: async (request, handler) => {
       const taskPrompt = `## Task Management
 
-You can use the \`manage_task\` tool to create persistent tasks for user-visible work tracking.
+You have the \`manage_task\` tool to track work. Task management is the
+ongoing record of a GOAL and its ACCEPTANCE CRITERIA.
 
-### When to create a task
+### When to create a task (universal rule)
+- The goal is clear and you are about to start real work \u2192 create the
+  parent task FIRST (with Objective + Acceptance Criteria in the
+  description), then execute. This is a core duty, not optional.
 - The user explicitly asks you to track, manage, or follow up on work
 - The work spans multiple sessions or might need resumption later
 - The user needs to review or approve output before it is considered done
 - There are multiple independent work items the user wants visibility into
 
 ### When NOT to create a task
+- Goal not yet clear (still clarifying) \u2014 clarify first, then create
 - One-shot lookups or simple Q&A ("what is X?", "search for Y")
 - Internal exploration steps you take to understand the problem (use \`write_todos\` for your execution plan instead)
 - Trivial single-step actions that complete in the same turn
 - Conversational or informational requests with no deliverable
 
+### Keep the task current
+A task is the living record of the goal + its acceptance criteria.
+Update the description as work proceeds: check off criteria as met,
+revise criteria when scope changes, append progress. Status changes
+only when the criteria are truly met.
+
 ### Ownership defaults
 - No params: ownerType defaults to "user" with current user's ID
 - ownerType="agent": auto-fills ownerId from current agent (subtask for yourself)
@@ -14181,6 +14322,29 @@ var taskPlugin = {
   skills: {
     "task-definition": `## Using manage_task
 
+### When to create a task (universal rule)
+
+Create a task BEFORE executing whenever the goal is clear and you know
+what to do \u2014 not just for long or complex work:
+
+- Goal is understood and you are about to start real work \u2192 create the
+  parent task FIRST, then execute. The task tracks the work.
+- Goal is NOT yet clear (still clarifying, gathering requirements) \u2192
+  do NOT create a task yet. Clarify first, create the task once scope
+  is defined.
+- One-shot lookups, trivial single-step actions, or internal reasoning
+  \u2192 no task needed.
+
+Rule of thumb: if you can write an Objective and Acceptance Criteria
+for it, create the task before doing it. Work without a task = work
+without a contract.
+
+**A task is the living record of the goal + its acceptance criteria.**
+Keep the description current as work proceeds: update the Objective
+when the target evolves, check off criteria as they are met, revise
+criteria when scope changes. Reading the task always tells you what
+"done" means; an outdated task is a broken contract.
+
 ### Task description format
 
 When creating a task with manage_task, write the description in this Markdown structure:
@@ -14382,7 +14546,7 @@ function returnCommandWithStateUpdate(result, toolCallId) {
     update: {
       ...stateUpdate,
       messages: [
-        new ToolMessage3({
+        new ToolMessage4({
           content: lastMessage?.content || "Task Failed to complete",
           tool_call_id: toolCallId,
           name: "task"
@@ -14575,7 +14739,7 @@ You are executing a persistent task (ID: ${input.taskId}). Use manage_task.updat
           return new Command3({
             update: {
               messages: [
-                new ToolMessage3({
+                new ToolMessage4({
                   content: `Async task started: ${subagent_thread_id}
 ${description}
 The result will be delivered as a notification when complete. Do not poll.`,
@@ -14609,7 +14773,7 @@ The result will be delivered as a notification when complete. Do not poll.`,
         return new Command3({
           update: {
             messages: [
-              new ToolMessage3({
+              new ToolMessage4({
                 content: error instanceof Error ? error.message : "Task Failed to complete",
                 tool_call_id: config.toolCall.id,
                 name: "task"
@@ -14846,7 +15010,7 @@ function createSubAgentMiddleware(options) {
     );
   }
   const effectiveSystemPrompt = allowAsync ? systemPrompt + getAsyncPromptText() : systemPrompt;
-  return createMiddleware10({
+  return createMiddleware11({
     name: "subAgentMiddleware",
     tools: allTools,
     wrapModelCall: async (request, handler) => {
@@ -14865,53 +15029,6 @@ ${effectiveSystemPrompt}` : effectiveSystemPrompt;
   });
 }
 
-// src/deep_agent_new/middleware/patch_tool_calls.ts
-import {
-  createMiddleware as createMiddleware11,
-  ToolMessage as ToolMessage4,
-  AIMessage as AIMessage2
-} from "langchain";
-function createPatchToolCallsMiddleware() {
-  return createMiddleware11({
-    name: "patchToolCallsMiddleware",
-    beforeAgent: async (state) => {
-      const messages = state.messages;
-      if (!messages || messages.length === 0) {
-        return;
-      }
-      const patchedMessages = [];
-      for (let i = 0; i < messages.length; i++) {
-        const msg = messages[i];
-        patchedMessages.push(msg);
-        if (AIMessage2.isInstance(msg) && msg.tool_calls != null) {
-          for (const toolCall of msg.tool_calls) {
-            const correspondingToolMsg = messages.slice(i).find(
-              (m) => ToolMessage4.isInstance(m) && m.tool_call_id === toolCall.id
-            );
-            if (!correspondingToolMsg) {
-              const toolMsg = `Tool call ${toolCall.name} with id ${toolCall.id} was cancelled - another message came in before it could be completed.`;
-              patchedMessages.push(
-                new ToolMessage4({
-                  content: toolMsg,
-                  name: toolCall.name,
-                  tool_call_id: toolCall.id
-                })
-              );
-            }
-          }
-        }
-      }
-      if (patchedMessages.length === messages.length) {
-        return;
-      }
-      return {
-        messages: patchedMessages.slice(messages.length)
-        // only the new ToolMessage patches
-      };
-    }
-  });
-}
-
 // src/deep_agent_new/middleware/date.ts
 import { createMiddleware as createMiddleware12, tool as tool41 } from "langchain";
 import { z as z43 } from "zod";
@@ -17824,36 +17941,20 @@ ${BASE_PROMPT}` : BASE_PROMPT;
         createFilesystemMiddleware({
           backend: filesystemBackend
         }),
-        // Subagent middleware: Automatic conversation summarization when token limits are approached
-        summarizationMiddleware({
-          model,
-          trigger: { tokens: 17e4 },
-          keep: { messages: 6 }
-        }),
         // Subagent middleware: Anthropic prompt caching for improved performance
         anthropicPromptCachingMiddleware({
           unsupportedModelBehavior: "ignore"
         }),
-        // Subagent middleware: Patches tool calls for compatibility
-        createPatchToolCallsMiddleware(),
         ...customMiddleware
       ],
       defaultInterruptOn: interruptOn,
       subagents,
       generalPurposeAgent: true
     }),
-    // Automatically summarizes conversation history when token limits are approached
-    summarizationMiddleware({
-      model,
-      trigger: { tokens: 17e4 },
-      keep: { messages: 6 }
-    }),
     // Enables Anthropic prompt caching for improved performance and reduced costs
     anthropicPromptCachingMiddleware({
       unsupportedModelBehavior: "ignore"
-    }),
-    // Patches tool calls to ensure compatibility across different model providers
-    createPatchToolCallsMiddleware()
+    })
   ];
   if (interruptOn) {
     middleware.push(humanInTheLoopMiddleware2({ interruptOn }));
@@ -17908,7 +18009,7 @@ var DeepAgentGraphBuilder = class {
     }));
     const middlewareConfigs = params.middleware || [];
     const filesystemBackend = createFilesystemBackendFactory(middlewareConfigs);
-    const middlewares = await createCommonMiddlewares(middlewareConfigs, filesystemBackend, true, params.tenantId);
+    const middlewares = await createCommonMiddlewares(middlewareConfigs, filesystemBackend, true, params.tenantId, params.model);
     const deepAgent = createDeepAgent({
       tools,
       model: params.model,
@@ -19524,7 +19625,7 @@ var TeamAgentGraphBuilder = class {
     });
     const middlewareConfigs = params.middleware || [];
     let filesystemBackend = createFilesystemBackendFactory(middlewareConfigs);
-    const middlewares = await createCommonMiddlewares(middlewareConfigs);
+    const middlewares = await createCommonMiddlewares(middlewareConfigs, void 0, void 0, void 0, params.model);
     if (!filesystemBackend) {
       filesystemBackend = async (config2) => {
         return new StateBackend(config2);
@@ -19941,7 +20042,7 @@ var WorkflowAgentGraphBuilder = class {
     const checkpointer = getCheckpointSaver("default");
     const tools = params.tools.map((t) => t.executor).filter(Boolean);
     const middlewareConfigs = params.middleware || [];
-    const middlewares = await createCommonMiddlewares(middlewareConfigs, void 0, false, params.tenantId);
+    const middlewares = await createCommonMiddlewares(middlewareConfigs, void 0, false, params.tenantId, params.model);
     const askMiddlewares = await createCommonMiddlewares([
       {
         id: "ask_user_to_clarify",
@@ -19951,7 +20052,7 @@ var WorkflowAgentGraphBuilder = class {
         enabled: true,
         config: {}
       }
-    ], void 0, false);
+    ], void 0, false, void 0, params.model);
     const noWrapMiddlewares = stripWrapModelCallHook(middlewares);
     const noWrapAskMiddlewares = stripWrapModelCallHook(askMiddlewares);
     console.log(`[WF BUILDER] building default agent | toolCount=${tools.length} | middlewareCount=${middlewares.length}`);
@@ -21628,10 +21729,19 @@ Call the \`skill\` tool with skill_name: "agent-architecture" to load the
 authoritative workflow. Never announce that you will follow a skill \u2014
 load it and follow its content. If the load fails, retry once, then report it.
 
+TASK MANAGEMENT IS A CORE DUTY, not a per-skill option. Whenever the
+goal is clear and you know what to do, create a task FIRST (manage_task)
+before executing \u2014 for any multi-step work: learning, building,
+modifying, fixing, anything with an Objective and Acceptance Criteria.
+Subtasks per work item. Status must reflect reality. See [[task-tracking]].
+The sub-skills below only ADD their own task details on top of this
+universal duty.
+
 Your sub-skills (accessible via the MOC or direct loading):
 - [[learn-capability]] \u2014 Learn from any source material \u2192 skills + agents
 - [[agent-build]] \u2014 Design and build single agents (REACT/DEEP_AGENT)
 - [[design-workflow]] \u2014 Design workflow agents
+- [[eval-verify]] \u2014 Run evaluations with fix loop, hold-out, trust upgrade
 - [[task-tracking]] \u2014 Manage persistent tasks (manage_task)
 - [[completion-gate]] \u2014 THE rule: no agent is "done" without eval
 - [[domain-moc]] \u2014 Create the domain MOC (mandatory per learning run)
@@ -21791,7 +21901,15 @@ var agentArchitectConfig = {
       id: "task",
       type: "task",
       name: "Task",
-      description: "Track learning processes and fix loops with approval gates",
+      description: "Core architect duty: once the goal is clear, create tasks to track every multi-step piece of work (learning, building, modifying, fixing). Subtasks per work item; status reflects reality.",
+      enabled: true,
+      config: {}
+    },
+    {
+      id: "code_eval",
+      type: "code_eval",
+      name: "Code Evaluation",
+      description: "Execute shell commands in the sandbox to support agent creation: unzip files, process data, run helper scripts. Distinct from the eval assessment system (run_eval).",
       enabled: true,
       config: {}
     },
@@ -25413,6 +25531,15 @@ function parseJudgeVerdict(raw) {
     return { error: error instanceof Error ? error.message : String(error) };
   }
 }
+var MAX_INTERRUPT_RESUMES = 5;
+function resolveInterruptResponse(policy, interrupt5) {
+  if (policy.mode === "auto-approve") return policy.value ?? "\u540C\u610F";
+  if (policy.mode === "auto-reject") return policy.value ?? "\u62D2\u7EDD";
+  return policy.value ?? "";
+}
+function interruptValueText(value) {
+  return typeof value === "string" ? value : JSON.stringify(value ?? "");
+}
 var _LatticeEval = class _LatticeEval {
   constructor(config = {}) {
     this.inMemoryLogs = [];
@@ -25479,7 +25606,8 @@ var _LatticeEval = class _LatticeEval {
       return acc;
     }, {});
   }
-  async executeAgentStep(step, threadId, inputMessage, files) {
+  async executeAgentStep(step, threadId, inputMessage, files, interruptPolicy) {
+    const hitlEvents = [];
     this.log("Executing agent step", {
       agent_id: step.agent_id,
       thread_id: threadId,
@@ -25497,19 +25625,74 @@ var _LatticeEval = class _LatticeEval {
     };
     const agent = agentInstanceManager.getAgent(agentParams);
     try {
-      const result = await agent.invoke({
-        input: {
-          message: step.override_message || inputMessage,
-          files: this.buildFileEntries(files)
+      const stepInput = {
+        message: step.override_message || inputMessage,
+        files: this.buildFileEntries(files)
+      };
+      let result = await agent.invokeWithState({ input: stepInput });
+      let resumeCount = 0;
+      let pendingInterrupt;
+      let interrupts = Array.isArray(result?.__interrupt__) ? result.__interrupt__ : [];
+      while (interrupts.length > 0) {
+        const interrupt5 = interrupts[0];
+        if (!interrupt5) break;
+        const policy = interruptPolicy;
+        if (!policy || policy.mode === "stop" || resumeCount >= MAX_INTERRUPT_RESUMES) {
+          pendingInterrupt = interrupt5;
+          break;
         }
-      });
+        const response = resolveInterruptResponse(policy, interrupt5);
+        hitlEvents.push({
+          type: "interrupt",
+          id: interrupt5.id,
+          value: interrupt5.value
+        });
+        this.log("Auto-resolving HITL interrupt", {
+          agent_id: step.agent_id,
+          thread_id: threadId,
+          mode: policy.mode,
+          interrupt_id: interrupt5.id,
+          response,
+          resume_count: resumeCount + 1
+        });
+        result = await agent.invokeWithState({ input: stepInput, command: { resume: response } });
+        hitlEvents.push({
+          type: "interrupt_response",
+          id: interrupt5.id,
+          mode: policy.mode,
+          response
+        });
+        resumeCount++;
+        interrupts = Array.isArray(result?.__interrupt__) ? result.__interrupt__ : [];
+      }
+      if (pendingInterrupt) {
+        hitlEvents.push({
+          type: "interrupt",
+          id: pendingInterrupt.id,
+          value: pendingInterrupt.value
+        });
+        this.log("Agent step interrupted by HITL (human input requested)", {
+          agent_id: step.agent_id,
+          thread_id: threadId,
+          interrupt_id: pendingInterrupt.id,
+          auto_resolved: resumeCount
+        });
+      } else {
+        this.log("Agent step completed", {
+          agent_id: step.agent_id,
+          thread_id: threadId,
+          response_keys: result ? Object.keys(result) : [],
+          auto_resolved: resumeCount
+        });
+      }
       const responseData = { success: true, ...result };
-      this.log("Agent step completed", {
-        agent_id: step.agent_id,
-        thread_id: threadId,
-        response_keys: result ? Object.keys(result) : []
-      });
-      return { threadId, responseData };
+      return {
+        threadId,
+        responseData,
+        interrupted: pendingInterrupt ? true : void 0,
+        interrupt: pendingInterrupt,
+        hitlEvents
+      };
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       this.log("Agent step failed", {
@@ -25572,15 +25755,32 @@ var _LatticeEval = class _LatticeEval {
     });
     let currentThreadId = threadId;
     let lastResponseData = null;
+    let interrupt5;
     for (const step of evalCase.steps) {
       const result = await this.executeAgentStep(
         step,
         currentThreadId,
         evalCase.input.message,
-        evalCase.input.files || {}
+        evalCase.input.files || {},
+        evalCase.interruptPolicy
       );
       currentThreadId = result.threadId;
       lastResponseData = result.responseData;
+      for (const evt of result.hitlEvents) {
+        if (evt.type === "interrupt") {
+          this.lastMessages.push({
+            role: "interrupt",
+            content: `HITL \u6682\u505C\uFF1AAgent \u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165 \u2014 ${interruptValueText(evt.value)}`,
+            id: evt.id
+          });
+        } else {
+          this.lastMessages.push({
+            role: "interrupt_response",
+            content: `\u5DF2\u81EA\u52A8\u54CD\u5E94\uFF08\u6D4B\u8BD5\u7B56\u7565 ${evt.mode}\uFF09\uFF1A${evt.response}`,
+            id: evt.id
+          });
+        }
+      }
       const existingIds = new Set(this.lastMessages.map((m) => m.id).filter(Boolean));
       if (result.responseData?.messages && Array.isArray(result.responseData.messages)) {
         for (const msg of result.responseData.messages) {
@@ -25602,6 +25802,13 @@ var _LatticeEval = class _LatticeEval {
             } else {
               content = String(msg.content || "");
             }
+            if (Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0) {
+              const toolCallStr = msg.tool_calls.map(
+                (tc) => `tool_call: ${tc.name}(${JSON.stringify(tc.args ?? {})})`
+              ).join("\n");
+              content = content ? `${content}
+${toolCallStr}` : toolCallStr;
+            }
             this.lastMessages.push({
               role,
               content,
@@ -25613,13 +25820,21 @@ var _LatticeEval = class _LatticeEval {
           }
         }
       }
+      if (result.interrupted) {
+        interrupt5 = result.interrupt;
+        this.log("Case paused for HITL \u2014 remaining steps skipped; judge will evaluate the pause", {
+          case_id: evalCase.caseId,
+          interrupt_id: interrupt5?.id
+        });
+        break;
+      }
     }
     this.log("All agent steps completed", {
       case_id: evalCase.caseId,
       final_thread_id: currentThreadId,
       message_count: this.lastMessages.length
     });
-    const finalOutput = this.extractFinalMessage(lastResponseData);
+    const finalOutput = interrupt5 ? typeof interrupt5.value === "string" ? interrupt5.value : JSON.stringify(interrupt5.value ?? "") : this.extractFinalMessage(lastResponseData);
     this.lastFinalOutput = finalOutput;
     const trajectory = this.buildTrajectory();
     this.log("Final output extracted", {
@@ -25678,6 +25893,8 @@ ${rubricsSection}
 3. **\u8FC7\u7A0B\u6821\u9A8C**\uFF1A\u5982\u679C"\u6267\u884C\u8FC7\u7A0B"\u663E\u793A Agent \u672A\u6267\u884C\u5FC5\u8981\u7684\u4E2D\u95F4\u6B65\u9AA4\uFF08\u5982\u5E94\u8C03\u7528\u5DE5\u5177\u800C\u672A\u8C03\u7528\uFF09\uFF0C\u5373\u4F7F\u6700\u7EC8\u8F93\u51FA\u770B\u4F3C\u5408\u7406\uFF0C\u4E5F\u5E94\u5728\u5BF9\u5E94\u6307\u6807\u4E0A\u6263\u5206\u3002
 4. **\u8BC1\u636E\u5BFC\u5411**\uFF1A\u5728\u7ED9\u51FA\u539F\u56E0\uFF08reason\uFF09\u65F6\uFF0C\u5FC5\u987B\u5F15\u7528\u6267\u884C\u8FC7\u7A0B\u6216\u6700\u7EC8\u8F93\u51FA\u4E2D\u7684\u5177\u4F53\u5185\u5BB9\u3002
 5. **\u52A0\u6743\u8BA1\u7B97**\uFF1A\u6700\u7EC8\u5206\u6570\u4E3A\u5404\u9879\u6307\u6807\u5F97\u5206\u4E0E\u5176\u6743\u91CD\u7684\u4E58\u79EF\u4E4B\u548C\uFF080-100\u5206\u5236\uFF09\u3002
+6. **HITL \u4E2D\u65AD\u5224\u5B9A**\uFF1A\u5982\u679C\u6267\u884C\u8FC7\u7A0B\u4E2D\u51FA\u73B0\u300CHITL \u6682\u505C\uFF1AAgent \u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u300D\u6761\u76EE\uFF0C\u8BF4\u660E Agent \u5728\u7B49\u5F85\u4EBA\u5DE5\u786E\u8BA4\u3002\u8BF7\u628A\u5B83\u5F53\u4F5C\u88AB\u6D4B\u4E1A\u52A1\u884C\u4E3A\u6765\u8BC4\u5224\uFF1A\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u81EA\u4E3B\u5B8C\u6210\uFF08\u5982"\u65E0\u9700\u786E\u8BA4\u81EA\u52A8\u6267\u884C"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u5E94\u5224\u5931\u8D25\uFF1B\u82E5\u671F\u671B\u8F93\u51FA\u8981\u6C42\u5148\u83B7\u5F97\u786E\u8BA4\u6216\u6279\u51C6\uFF08\u5982"\u6267\u884C\u524D\u5FC5\u987B\u8BF7\u6C42\u6279\u51C6"\uFF09\uFF0C\u8BF7\u6C42\u4EBA\u5DE5\u8F93\u5165\u662F\u6B63\u786E\u884C\u4E3A\uFF0C\u5E94\u7ED3\u5408\u5176\u65F6\u673A\u4E0E\u5185\u5BB9\u5224\u5B9A\u4E3A\u901A\u8FC7\u6216\u6309\u6307\u6807\u7ED9\u5206\u3002
+7. **HITL \u81EA\u52A8\u54CD\u5E94\u5224\u5B9A**\uFF1A\u5982\u679C\u300CHITL \u6682\u505C\u300D\u6761\u76EE\u4E4B\u540E\u51FA\u73B0\u300C\u5DF2\u81EA\u52A8\u54CD\u5E94\uFF08\u6D4B\u8BD5\u7B56\u7565 auto-approve/auto-reject/canned-response\uFF09\u300D\u6761\u76EE\uFF0C\u8BF4\u660E\u6D4B\u8BD5\u6846\u67B6\u6CE8\u5165\u4E86\u4EBA\u5DE5\u56DE\u590D\u3001\u6D41\u7A0B\u5DF2\u7EE7\u7EED\u2014\u2014\u8BF7\u6309**\u5B8C\u6574\u6D41\u7A0B**\u8BC4\u5224\u6682\u505C\u4E4B\u540E\u7684\u884C\u4E3A\uFF08\u5982\u6279\u51C6\u540E\u662F\u5426\u6B63\u786E\u6267\u884C\u4E86\u64CD\u4F5C\uFF09\uFF0C\u5E76\u6838\u5BF9\u81EA\u52A8\u54CD\u5E94\u5185\u5BB9\u662F\u5426\u7B26\u5408\u4EBA\u5DE5\u56DE\u590D\u7684\u5408\u7406\u9884\u671F\u3002
 
 # \u8F93\u51FA\u683C\u5F0F\uFF08\u4EC5JSON\uFF09
 \u4F60\u5FC5\u987B\u4EC5\u4EE5 JSON \u683C\u5F0F\u56DE\u590D\uFF0C\u7ED3\u6784\u5982\u4E0B\uFF1A
@@ -25832,7 +26049,9 @@ ${rubricsSection}
       pass,
       final_score: finalScore,
       dimension_results: dimensionResults,
-      summary: parsedResult.summary || testResultContent
+      summary: parsedResult.summary || testResultContent,
+      interrupted: interrupt5 ? true : void 0,
+      interrupt: interrupt5 ? { id: interrupt5.id, value: interrupt5.value } : void 0
     };
   }
 };
@@ -25847,6 +26066,8 @@ async function evaluateLatticeCaseWithLogs(evalCase, config) {
     return {
       caseId: evalCase.caseId,
       result,
+      interrupted: result?.interrupted,
+      interrupt: result?.interrupt,
       duration_ms: meta.duration_ms,
       thread_id: meta.thread_id,
       judge_thread_id: meta.judge_thread_id,
@@ -25929,7 +26150,8 @@ function resolveTemplateCase(templateCase, templates) {
     eval: {
       content_assertion: templateCase.eval.content_assertion,
       eval_rubrics: templateCase.eval.eval_rubrics || template.default_case.eval?.eval_rubrics
-    }
+    },
+    interruptPolicy: templateCase.interruptPolicy ?? template.default_case.interruptPolicy
   };
   return resolvedCase;
 }
@@ -25995,6 +26217,8 @@ var LatticeEvalSuite = class {
         result: run.result,
         error: run.error,
         error_stack: run.error_stack,
+        interrupted: run.interrupted,
+        interrupt: run.interrupt,
         duration_ms: run.duration_ms,
         thread_id: run.thread_id,
         judge_thread_id: run.judge_thread_id,
@@ -26027,6 +26251,8 @@ var LatticeEvalSuite = class {
           result: run.result,
           error: run.error,
           error_stack: run.error_stack,
+          interrupted: run.interrupted,
+          interrupt: run.interrupt,
           duration_ms: run.duration_ms,
           thread_id: run.thread_id,
           judge_thread_id: run.judge_thread_id,
@@ -26236,24 +26462,29 @@ Running batch: ${this.project.projectName} (${this.getSuiteNames().length} suite
     let total_cases = 0;
     let passed_cases = 0;
     let failed_cases = 0;
+    let interrupted_cases = 0;
     const suites = [];
     for (const [suiteName, caseResults] of results.entries()) {
       const suiteTotal = caseResults.length;
       const suitePassed = caseResults.filter((r) => r.result?.pass).length;
+      const suiteInterrupted = caseResults.filter((r) => r.interrupted).length;
       const suiteFailed = suiteTotal - suitePassed;
       total_cases += suiteTotal;
       passed_cases += suitePassed;
       failed_cases += suiteFailed;
+      interrupted_cases += suiteInterrupted;
       suites.push({
         suiteName,
         total_cases: suiteTotal,
         passed_cases: suitePassed,
         failed_cases: suiteFailed,
+        interrupted_cases: suiteInterrupted,
         cases: caseResults.map((r) => ({
           caseId: r.caseId,
           pass: r.result?.pass,
           final_score: r.result?.final_score,
-          error: r.error
+          error: r.error,
+          interrupted: r.interrupted
         }))
       });
     }
@@ -26271,13 +26502,14 @@ Running batch: ${this.project.projectName} (${this.getSuiteNames().length} suite
         total_cases,
         passed_cases,
         failed_cases,
+        interrupted_cases,
         pass_rate: total_cases > 0 ? passed_cases / total_cases : 0
       },
       suites
     };
     console.log(`
 === Summary ===`);
-    console.log(`Total: ${report.summary.total_cases} | Passed: ${report.summary.passed_cases} | Failed: ${report.summary.failed_cases} | Pass Rate: ${(report.summary.pass_rate * 100).toFixed(2)}%`);
+    console.log(`Total: ${report.summary.total_cases} | Passed: ${report.summary.passed_cases} | Failed: ${report.summary.failed_cases} | Interrupted: ${report.summary.interrupted_cases} | Pass Rate: ${(report.summary.pass_rate * 100).toFixed(2)}%`);
     return { batch_id, results, report };
   }
 };
@@ -28226,10 +28458,21 @@ Write assertions as objective, verifiable natural language:
 - steps: [{agent_id: "a"}, {agent_id: "b", override_message: "Based on..."}] for chain
 - outputType: "message_content" or "file_content"
 
+## Designing HITL Cases
+If the flow under test PAUSES for human input (HITL \u2014 the agent requests approval/confirmation), configure interruptPolicy on the case \u2014 otherwise the run stops at the pause and the flow AFTER the human input is never tested:
+
+- interruptPolicy: {mode: "auto-approve"} \u2014 inject "\u540C\u610F"; tests the full flow after approval (e.g. payment executed after approval). Use value to override the injected text.
+- interruptPolicy: {mode: "auto-reject"} \u2014 inject "\u62D2\u7EDD"; tests the rejection path.
+- interruptPolicy: {mode: "canned-response", value: "..."} \u2014 inject an exact human reply; tests behavior under a specific response.
+- Omit (default "stop") \u2014 the case pauses at the request; the judge evaluates whether pausing was the correct behavior (e.g. "must request approval" PASSES; "must be autonomous" FAILS).
+
+Choose per the assertion: if the assertion describes what happens AFTER the human input, you MUST set an auto-resolve policy.
+
 ## Checklist
 1. Check existing assets with read_eval to avoid duplication
 2. Start with 3-5 high-signal cases
-3. Confirm with user before calling manage_eval
+3. If the flow contains a human-approval step and the assertion covers what happens AFTER it, set interruptPolicy (auto-approve / auto-reject / canned-response) \u2014 never leave it unset in that case
+4. Confirm with user before calling manage_eval
 `,
   "eval-run-and-govern": `---
 name: eval-run-and-govern
@@ -28239,8 +28482,11 @@ description: Run agent evaluations, interpret results, diagnose failures, and re
 # Agent Governance Loop
 
 1. Discover project \u2192 read_eval list_projects
-2. Start evaluation \u2192 run_eval start(projectId) \u2014 ASYNC, may take minutes
-3. Poll status \u2192 run_eval status(runId) with backoff: 15s, 30s, 60s, max 120s
+2. Start evaluation \u2192 run_eval start(projectId) \u2014 SYNCHRONOUS by default:
+   blocks up to ~150s and returns the FINAL RESULTS in one call.
+   Hold-out (validation) runs return aggregates only.
+3. If still running (or use wait: false for fire-and-forget) \u2192 poll
+   run_eval status(runId, sleepMs) \u2014 pass sleepMs to pace (15s, 30s, 60s, max 120s)
 4. If runnerAlive=false and status=running \u2192 ORPHANED (gateway restart).
    resume(runId) marks it failed automatically \u2014 then start a new run.
 5. Get results \u2192 read_eval get_run_results(runId) for per-case dimension scores
@@ -28261,7 +28507,26 @@ Use read_eval get_run_results for multiple runs and present comparison.
 `
 };
 
+// src/tool_lattice/withToolTimeout.ts
+function withToolTimeout(executor, timeoutMs = 18e4) {
+  return async (input, exeConfig) => {
+    return new Promise((resolve4, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Tool execution timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+      executor(input, exeConfig).then((result) => {
+        clearTimeout(timer);
+        resolve4(result);
+      }).catch((err) => {
+        clearTimeout(timer);
+        reject(err);
+      });
+    });
+  };
+}
+
 // src/middlewares/evalMiddleware.ts
+var RUN_EVAL_SYNC_WAIT_MS = 15e4;
 function getStore() {
   return getStoreLattice("default", "eval").store;
 }
@@ -28288,6 +28553,25 @@ function sanitize(obj) {
   }
   return out;
 }
+function aggregateHoldoutResults(results) {
+  const passed = results.filter((r) => r.pass).length;
+  const interrupted = results.filter((r) => r.interrupted).length;
+  return {
+    holdout: true,
+    passedCases: passed,
+    failedCases: results.length - passed,
+    interruptedCases: interrupted,
+    passRate: results.length > 0 ? passed / results.length : 0,
+    totalCases: results.length
+  };
+}
+async function runWithResults(tid, store, svc, run, runnerAlive) {
+  const results = run.status === "completed" ? await store.getResultsByRun(tid, run.id) : void 0;
+  if (run.holdout && results) {
+    return { ...run, runnerAlive, results: aggregateHoldoutResults(results) };
+  }
+  return { ...run, runnerAlive, results };
+}
 function createReadEvalTool() {
   const schema6 = z66.object({
     action: z66.enum([
@@ -28347,13 +28631,8 @@ function createReadEvalTool() {
             if (!run) return JSON.stringify({ success: false, error: "Run not found" });
             const results = await store.getResultsByRun(tid, input.runId);
             if (run.holdout) {
-              const passed = results.filter((r) => r.pass).length;
               data = {
-                holdout: true,
-                passedCases: passed,
-                failedCases: results.length - passed,
-                passRate: results.length > 0 ? passed / results.length : 0,
-                totalCases: results.length,
+                ...aggregateHoldoutResults(results),
                 message: "Hold-out run \u2014 per-case results withheld. Only aggregates are available."
               };
             } else {
@@ -28388,6 +28667,7 @@ ACTIONS:
 - get_run_results(runId) \u2014 per-case results with dimension scores.
   For HOLD-OUT (validation-only) runs: returns AGGREGATES ONLY
   (passRate, counts) \u2014 per-case details are withheld by design.
+  Cases paused for human input (HITL) carry interrupted=true and are judged \u2014 the judge evaluates whether pausing was correct business behavior.
 - get_project_report(projectId) \u2014 aggregated stats across all runs`,
       schema: schema6
     }
@@ -28418,7 +28698,11 @@ function createManageEvalTool() {
     steps: z66.array(z66.object({ agent_id: z66.string(), override_message: z66.string().optional() })).optional(),
     outputType: z66.enum(["file_content", "message_content"]).optional(),
     contentAssertion: z66.string().optional(),
-    rubrics: z66.array(z66.object({ name: z66.string(), weight: z66.number(), description: z66.string() })).optional()
+    rubrics: z66.array(z66.object({ name: z66.string(), weight: z66.number(), description: z66.string() })).optional(),
+    interruptPolicy: z66.object({
+      mode: z66.enum(["stop", "auto-approve", "auto-reject", "canned-response"]).describe("stop=judge the pause; auto-approve/auto-reject/canned-response=resume the agent to test the flow after the human input"),
+      value: z66.string().optional().describe("Response to inject (defaults: \u540C\u610F / \u62D2\u7EDD; required for canned-response)")
+    }).optional().describe("Optional for create_case/update_case \u2014 how HITL interrupts are handled")
   });
   return tool62(
     async (input, exeConfig) => {
@@ -28476,7 +28760,8 @@ function createManageEvalTool() {
               steps: input.steps,
               outputType: input.outputType,
               contentAssertion: input.contentAssertion,
-              rubrics: input.rubrics
+              rubrics: input.rubrics,
+              interruptPolicy: input.interruptPolicy
             });
             break;
           case "update_case":
@@ -28484,7 +28769,8 @@ function createManageEvalTool() {
               inputMessage: input.inputMessage,
               contentAssertion: input.contentAssertion,
               steps: input.steps,
-              rubrics: input.rubrics
+              rubrics: input.rubrics,
+              interruptPolicy: input.interruptPolicy
             });
             break;
           case "delete_case":
@@ -28509,9 +28795,13 @@ Project: create_project(name, description?, judgeModelKey?, concurrency?) | upda
   **When creating a project from within a workspace, the workspace/project context is
   automatically bound \u2014 eval runs will execute in the same workspace.**
 Suite: create_suite(projectId, name) | update_suite | delete_suite
-Case: create_case(suiteId, inputMessage, steps, outputType, contentAssertion, inputFiles?, rubrics?)
+Case: create_case(suiteId, inputMessage, steps, outputType, contentAssertion, inputFiles?, rubrics?, interruptPolicy?)
   steps is [{agent_id, override_message?}]. outputType is "file_content" or "message_content".
-  rubrics is [{name, weight, description}]. | update_case | delete_case`,
+  rubrics is [{name, weight, description}]. | update_case | delete_case
+  interruptPolicy: {mode: "stop"|"auto-approve"|"auto-reject"|"canned-response", value?} \u2014 how HITL interrupts are handled:
+    stop (default): case pauses at the human-input request; the judge evaluates the pause as business behavior.
+    auto-approve / auto-reject: the runner injects approval/rejection and tests the FULL flow after the pause.
+    canned-response: injects the exact value (simulates a specific human reply).`,
       schema: schema6
     }
   );
@@ -28522,100 +28812,121 @@ function createRunEvalTool() {
     projectId: z66.string().optional().describe("Required for start"),
     suiteIds: z66.array(z66.string()).optional().describe("Optional for start \u2014 only run these suites (e.g. dev set only). Omit to run all."),
     caseIds: z66.array(z66.string()).optional().describe("Optional for start \u2014 only run these cases across the selected suites. Omit to run all cases in those suites."),
-    runId: z66.string().optional().describe("Required for status, resume, abort")
+    runId: z66.string().optional().describe("Required for status, resume, abort"),
+    sleepMs: z66.number().int().min(0).max(12e4).optional().describe("Optional for status \u2014 sleep this many ms BEFORE checking the run, to pace polling (e.g. 15000 \u2192 30000 \u2192 60000 \u2192 120000). Omit to check immediately."),
+    wait: z66.boolean().optional().describe("Optional for start \u2014 defaults to true: block synchronously (up to ~150s) and return final results in one call. Set false to return the runId immediately and poll.")
   });
   return tool62(
-    async (input, exeConfig) => {
-      const tid = tenantId(exeConfig);
-      if (!tid) {
-        return JSON.stringify({ success: false, error: "No tenant context. Agent must be invoked through gateway." });
-      }
-      try {
-        const store = getStore();
-        const svc = getEvalRunService();
-        let data;
-        switch (input.action) {
-          case "start": {
-            const ctx = workspaceContext(exeConfig);
-            const runId = await svc.startRun(tid, input.projectId, input.suiteIds, input.caseIds, {
-              workspaceId: ctx.workspaceId,
-              projectId: ctx.projectId
-            });
-            data = sanitize({ runId, message: "Run started. Poll with run_eval status (backoff: 15s\u219230s\u219260s\u2192max 120s)." });
-            break;
-          }
-          case "status": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            data = sanitize({ ...run, runnerAlive: svc.isRunning(input.runId) });
-            break;
-          }
-          case "resume": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            const runnerAlive = svc.isRunning(input.runId);
-            if (run.status === "running" && !runnerAlive) {
-              await store.updateRunStatus(tid, run.id, {
-                status: "failed",
-                error: "Gateway restarted \u2014 run orphaned",
-                completedAt: /* @__PURE__ */ new Date()
-              });
-              data = sanitize({
-                ...run,
-                status: "failed",
-                runnerAlive: false,
-                message: "Run was orphaned \u2014 marked failed. Start a new run."
-              });
+    withToolTimeout(
+      async (input, exeConfig) => {
+        const tid = tenantId(exeConfig);
+        if (!tid) {
+          return JSON.stringify({ success: false, error: "No tenant context. Agent must be invoked through gateway." });
+        }
+        try {
+          const store = getStore();
+          const svc = getEvalRunService();
+          let data;
+          switch (input.action) {
+            case "start": {
+              const ctx = workspaceContext(exeConfig);
+              const runId = await svc.startRun(tid, input.projectId, input.suiteIds, input.caseIds, ctx);
+              if (input.wait === false) {
+                data = sanitize({ runId, message: "Run started. Poll with run_eval status (backoff: 15s\u219230s\u219260s\u2192max 120s)." });
+                break;
+              }
+              let timer;
+              try {
+                await Promise.race([
+                  svc.waitForRun(runId).catch(() => {
+                  }),
+                  new Promise((resolve4) => {
+                    timer = setTimeout(resolve4, RUN_EVAL_SYNC_WAIT_MS);
+                  })
+                ]);
+              } finally {
+                if (timer) clearTimeout(timer);
+              }
+              const run = await store.getRunById(tid, runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              if (run.status === "running") {
+                data = sanitize({
+                  runId,
+                  status: "running",
+                  runnerAlive: svc.isRunning(runId),
+                  message: `Run not finished within ${RUN_EVAL_SYNC_WAIT_MS / 1e3}s \u2014 poll with run_eval status(runId, sleepMs) (e.g. 15000, doubling up to 120000), or abort with run_eval abort.`
+                });
+                break;
+              }
+              data = sanitize({ synced: true, ...await runWithResults(tid, store, svc, run, svc.isRunning(runId)) });
               break;
             }
-            const results = run.status === "completed" ? await store.getResultsByRun(tid, run.id) : void 0;
-            if (run.holdout && results) {
-              const passed = results.filter((r) => r.pass).length;
-              data = sanitize({
-                ...run,
-                runnerAlive,
-                results: {
-                  holdout: true,
-                  passedCases: passed,
-                  failedCases: results.length - passed,
-                  passRate: results.length > 0 ? passed / results.length : 0,
-                  totalCases: results.length
-                }
-              });
+            case "status": {
+              if (input.sleepMs && input.sleepMs > 0) {
+                await new Promise((resolve4) => setTimeout(resolve4, input.sleepMs));
+              }
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              data = sanitize({ ...run, runnerAlive: svc.isRunning(input.runId) });
               break;
             }
-            data = sanitize({ ...run, runnerAlive, results });
-            break;
-          }
-          case "abort": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            const ok = await svc.abortRun(input.runId);
-            data = sanitize({ aborted: ok });
-            break;
+            case "resume": {
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              const runnerAlive = svc.isRunning(input.runId);
+              if (run.status === "running" && !runnerAlive) {
+                await store.updateRunStatus(tid, run.id, {
+                  status: "failed",
+                  error: "Gateway restarted \u2014 run orphaned",
+                  completedAt: /* @__PURE__ */ new Date()
+                });
+                data = sanitize({
+                  ...run,
+                  status: "failed",
+                  runnerAlive: false,
+                  message: "Run was orphaned \u2014 marked failed. Start a new run."
+                });
+                break;
+              }
+              data = sanitize(await runWithResults(tid, store, svc, run, runnerAlive));
+              break;
+            }
+            case "abort": {
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              const ok = await svc.abortRun(input.runId);
+              data = sanitize({ aborted: ok });
+              break;
+            }
+            default:
+              return JSON.stringify({ success: false, error: `Unknown action: ${input.action}` });
           }
-          default:
-            return JSON.stringify({ success: false, error: `Unknown action: ${input.action}` });
+          return JSON.stringify({ success: true, data });
+        } catch (e) {
+          return JSON.stringify({ success: false, error: e.message });
         }
-        return JSON.stringify({ success: true, data });
-      } catch (e) {
-        return JSON.stringify({ success: false, error: e.message });
       }
-    },
+    ),
     {
       name: "run_eval",
-      description: `Execute and manage evaluation runs. ASYNCHRONOUS \u2014 may take minutes.
+      description: `Execute and manage evaluation runs.
 
 ACTIONS:
-- start(projectId, suiteIds?, caseIds?) \u2014 begin evaluation (optionally only the listed suites, and/or only the listed cases). Returns runId.
-- status(runId) \u2014 current status + runnerAlive flag:
-    \u2022 runnerAlive=true, status=running: keep polling
+- start(projectId, suiteIds?, caseIds?, wait?) \u2014 begin evaluation (optionally only the listed suites, and/or only the listed cases).
+    wait=true (DEFAULT): SYNCHRONOUS \u2014 blocks up to ~150s and returns the FINAL RESULTS in one call:
+      { status, results } with per-case pass/score (hold-out/validation runs: aggregates only \u2014 per-case details withheld by design).
+      Cases paused for human input (HITL) carry interrupted=true and ARE judged \u2014 the judge evaluates whether requesting
+      the human was the correct behavior (assertions like "must approve first" PASS; "must be autonomous" FAIL). interruptedCases counts these.
+      If the run exceeds 150s: returns { runId, status: "running" } \u2014 NOT an error \u2014 then poll with status/resume until completed.
+    wait=false: fire-and-forget \u2014 returns { runId } immediately; poll with status.
+- status(runId, sleepMs?) \u2014 sleep sleepMs first (to pace polling), then return current status + runnerAlive flag:
+    \u2022 runnerAlive=true, status=running: keep polling \u2014 call status(runId, sleepMs) with backoff 15s\u219230s\u219260s\u2192max 120s
     \u2022 runnerAlive=false, status=running: ORPHANED \u2014 resume marks it failed automatically; then start a new run
     \u2022 status=completed: get results with read_eval get_run_results or run_eval resume
 - resume(runId) \u2014 reconnect from new conversation. Returns status + results if completed.
 - abort(runId) \u2014 cancel running evaluation.
 
-Polling: start at 15s, double each time, max 120s between polls. Batch reports.`,
+Polling (only needed with wait=false or after a sync timeout): call status(runId, sleepMs) so the tool sleeps before checking; start at 15s, double each time, max 120s between polls. Batch reports.`,
       schema: schema6
     }
   );
@@ -28911,67 +29222,64 @@ verification choice, then start benchmarking.
 
 ## Task Tracking \u2014 see [[task-tracking]]
 
-**Create the parent task when the task is actually defined** \u2014 after
-Phase 0 clarification is complete and the user confirmed the path
-(fresh vs incremental). Do NOT create tasks during clarification:
-while asking questions (0.0-0.4) you don't know what the task is yet.
-Once the scope is clear (end of 0.5), that is the moment to create:
-manage_task create("Learn [material]", ownerType: "agent"). Then a
-subtask per phase as you start it. Update status to reflect reality \u2014
-never mark a subtask completed while eval fails. Resume interrupted
-runs with manage_task list.
+**Universal principle**: once the goal is clear and you know what to
+do, create the parent task BEFORE executing (manage_task create, see
+[[task-tracking]]). In this workflow: after Phase 0 clarification
+completes and the user confirmed the path (end of 0.5), create the
+parent task; then a subtask per phase as you start it. The parent task
+description carries the GOAL MODEL (0.1.5) as Objective + Acceptance
+Criteria; the expected output spec (2.6) updates the criteria. Update
+status to reflect reality \u2014 never mark a subtask completed while eval
+fails. Resume interrupted runs with manage_task list.
 
 Widgets: call \`load_guidelines\` ONCE before your first \`show_widget\`
 (show_widget hard-requires it), then reuse.
 
 ---
 
-## Phase 1: Probe (material-dependent)
-
-The probing strategy depends on the material type from Phase 0.0:
-
-**User-description material**: the requirements come from the
-conversation itself \u2014 no probing needed. Extract the agent's goal,
-inputs, outputs, and constraints from what the user described. Go
-straight to design.
-
-**Document material** (PDF / spec / manual):
-If the engine was chosen in Phase 0 (0.3 \u2460-\u2464): skip the comparison \u2014
-parse directly with \`parse_document\` using the chosen engine
-(file_path, engine, output_path per file).
-Otherwise: run the document-parser-benchmark subagent via \`task\` on each file.
-Collect engine scores, parsed output (via \`read_file\`), and feature signatures.
-
-**Engine selection IS distilled knowledge, not just setup.** For document
-material, the benchmark answers "which engine parses THIS document (or
-this class of document) best?" \u2014 that answer is knowledge that drives
-the whole rest of the run:
-- **Builds the agent**: the chosen engine's \`parse_document\` goes into
-  the production agent's middleware/tools.
-- **Designs the tests**: the chosen engine's parsed output becomes the
-  baseline input for eval cases \u2014 cases feed parsed output to the agent
-  and assert correct extraction from it.
-- **Seeds the skill**: the feature signature (tables? scans? mixed
-  zh/en?) plus the winning engine becomes a skill note ("for PO PDFs
-  with tables, use textin") reusable for future similar documents.
-So when the user wants an agent whose purpose is document PARSING (not
-extraction), the workflow is the same: benchmark to learn the engine
-choice, then build the agent around that engine and test against its
-output. Do NOT treat parsing as a pure tool-assembly task \u2014 the engine
-choice is unknown knowledge until probed.
-
-**API spec material**: read the spec directly \u2014 no parsing engine needed.
-Extract endpoints, schemas, request/response examples from the text.
-
-**Conversation material**: extract the workflow, decisions, and corrections
-from the conversation context \u2014 no parsing engine needed.
-
-**Spreadsheet material**: parse cells directly \u2014 structured data needs
-no engine comparison.
-If verification will happen (0.2 \u2460 or \u2461): concurrently, \`list_agents\` to
-discover existing agents with relevant capabilities (see \xA75).
-For \u2460, look for agents with data-access tools (SQL / API). For \u2461, look
-for agents with independence.
+## Phase 1: Explore (goal-driven path finding)
+
+The goal model (0.1.5) is set. Now EXPLORE how to achieve it \u2014 actively
+hunt for the path, do not passively read. Three exploration fronts:
+
+**A. Existing assets (reuse, don't rebuild):**
+- \`list_agents\` / \`load_skills\` \u2192 are there existing agents or skills
+  that already do part of this? Reuse them (update_agent if needed)
+  instead of building from scratch. This is a goal-relevant check, not
+  a materials step.
+- \`list_tools\` / \`list_middleware_types\` \u2192 what capabilities exist
+  that the goal needs (parsing, data access, browser...)?
+- \`list_connections\` \u2192 are the data sources the goal depends on
+  already connected?
+- If verification will happen (0.2 \u2460 or \u2461): concurrently discover
+  executor candidates (see \xA75). \u2460 \u2192 data-access tools; \u2461 \u2192 independence.
+
+**B. Material probing (by material type):**
+- **User-description**: the requirements come from the conversation.
+  Extract goal, inputs, outputs, constraints \u2014 then explore the
+  implementation path (A + feasibility): what assets exist, what tools
+  are needed, what blockers stand between the goal and its achievement.
+- **Document** (PDF / spec / manual): benchmark engines as needed \u2014
+  parse directly with the chosen engine (0.3 \u2460-\u2464) or run
+  document-parser-benchmark. Engine selection IS distilled knowledge:
+  it builds the agent (engine's parse_document into middleware), seeds
+  the skill (feature signature + winning engine), and designs the tests
+  (engine output as case baseline input).
+- **API spec**: read directly \u2014 endpoints, schemas, examples.
+- **Conversation**: extract workflow, decisions, corrections.
+- **Spreadsheet**: parse cells directly.
+
+**C. Feasibility (path blockers):**
+- What stands between the goal and achievement? Missing tools, missing
+  connections, data access, permission constraints, ambiguous
+  requirements.
+- Does the goal require orchestration (\u2192 Phase 2 split decision)?
+- Surface these in the recommendation (Phase 1.5) \u2014 the user decides
+  the path, informed by what exploration found.
+
+Exploration is COMPLETE when you can answer: what exists to reuse,
+what must be built, what tools/connections are needed, and what blocks
+the goal. Do not go to design without this map.
 
 ---
 
@@ -28986,17 +29294,21 @@ candidate and assess (Validation Agent Design \xA70) \u2014 state which are
 usable and which are not, with reasons. For \u2460, the executor needs data
 tools + independence. For \u2461, independence only. If no candidate fits,
 plan to build one via \xA75.
-Present probe results as widget, then MUST call
+Present the EXPLORATION map as widget \u2014 what exists to reuse, what
+must be built, tools/connections needed, blockers found \u2014 then
+recommend the IMPLEMENTATION PATH: reuse existing X, build new Y,
+split or single agent (Phase 2 input). MUST call
 \`ask_user_to_clarify\` NOW:
   {
     "questions": [{
-      "question": "Confirm the recommendation?",
+      "question": "Confirm the recommended path?",
       "options": ["Confirm", "Adjust"],
       "type": "single",
       "required": true
     }]
   }
-Skills planning belongs to Phase 2 \u2014 this phase presents data, not plans.
+Skills planning belongs to Phase 2 \u2014 this phase presents the path, not
+the detailed plan.
 
 ---
 
@@ -29064,9 +29376,53 @@ user-description material this IS the core phase; for material-based
 learning it designs the agent that runs the learned skill. Agent
 metadata (verified/version/source) must be set on creation.
 
+## Phase 2.6: Expected Output Specification (mandatory \u2014 goal-driven)
+
+**Expectations come FIRST, before writing the skill.** You cannot write
+a skill (or test cases) without a target. Define the expected output
+specification from the goal model (0.1.5: real goal / consumer / usable
+state) BEFORE Phase 3:
+
+**HARD RULE \u2014 never guess the target.** If the goal, the expected
+output, the consumer, or the usable state is unclear at ANY point
+before writing test cases, you MUST ask the user via
+\`ask_user_to_clarify\` \u2014 do NOT proceed with an assumed expectation.
+A test case written against a guessed expectation is worthless: it
+validates the wrong thing. When in doubt, ask.
+
+Per skill, define the EXPECTED OUTPUT SPEC (based on intent 0.1 and
+consumer 0.1.5):
+- Extract data \u2192 expected fields (names, types, formats), required vs
+  optional, output structure (JSON schema shape, table columns)
+- Validate rules \u2192 expected judgment outcomes (pass/fail conditions),
+  boundary values, and the reason format
+- Execute workflow \u2192 expected step sequence, decision points, final
+  outcome shape
+- Answer knowledge \u2192 expected answer form (with/without sources,
+  length, structure)
+
+This spec IS the acceptance standard. Phase 4 contentAssertion must be
+derived from it (not invented at case-writing time). Present the
+expected output spec to the user and MUST call \`ask_user_to_clarify\`
+NOW per skill:
+  {
+    "questions": [{
+      "question": "Confirm the expected output spec for {skill-name}?",
+      "options": ["Confirm", "Adjust"],
+      "type": "single",
+      "required": true,
+      "allowOther": true
+    }]
+  }
+Record the confirmed spec in the parent task description. This replaces
+guess-then-confirm: the skill is written TO MEET the spec, and test
+cases assert AGAINST the spec \u2014 no expectation is invented later.
+
 ## Phase 3: Create Skills
 
-Write SKILL.md to \`/root/.agents/skills/{name}/SKILL.md\` one at a time.
+Write SKILL.md to \`/root/.agents/skills/{name}/SKILL.md\` one at a time,
+designed TO MEET the expected output spec confirmed in Phase 2.6 \u2014 the
+skill encodes how to produce the spec's expected output.
 Show the skill content in text first, then MUST call
 \`ask_user_to_clarify\` NOW per skill:
   {
@@ -29295,7 +29651,8 @@ Both modes use the same agent type \u2014 skill only, no domain tools:
        verified: "unverified",      # upgraded after eval passes
        version: "1.0",              # bump on each update_agent
        source: "{material name}",   # provenance
-       skill: "skill-name"
+       skill: "skill-name",
+       role: "orchestrator" | "sub-agent"  # only for parent+subAgents structure
      }
    )
 
@@ -29359,7 +29716,8 @@ This learning loop adds its own scenario rules:
    - Business usability (output reaches the goal's "usable state")
    - Consumer fit (format/contract satisfies who uses the result)
    contentAssertion must encode the usable state from the goal model
-   (0.1.5), not just technical correctness.
+   (0.1.5), derived from the EXPECTED OUTPUT SPEC (Phase 2.6) \u2014 never
+   invented at case-writing time.
 
 1. One suite per skill per source: cases test "can this skill do it" \u2014
    never mix skills in one suite
@@ -29391,12 +29749,23 @@ Run evaluation, fix loop, hold-out validation, trust upgrade. See
 [[eval-verify]] for the full workflow. The eval-design-tests and
 eval-run-and-govern skills cover case design and run governance.
 
+**One eval project per agent**, named \`eval-{agent-id}\` \u2014 every agent
+built by this workflow gets its own eval project (see eval-verify
+Setup). Orchestrator + subAgents \u2192 one eval project per sub-agent plus
+one integration eval for the parent.
+
 Learning-specific suite guidance:
 - 0.2 \u2461 \u2192 {skill}-user-sample; samples \u22658 \u2192 also {skill}-validation
 - 0.2 \u2460 \u2192 {skill}-api-verified (single step, \xA74.1)
 - User-description material: {skill}-requirement-derived \u2014 cases from
   user's described requirements
 
+**Case expectations come from the confirmed spec** (Phase 2.6): the
+contentAssertion of every case must be derived from the expected output
+spec, NOT invented at case-writing time. If a case needs an expectation
+not in the spec, go back and extend the spec with user confirmation
+first \u2014 never guess expectations on the fly.
+
 [[completion-gate]] applies \u2014 eval must pass before declaring done.
 
 ## Phase 5: Retrospective
@@ -29429,11 +29798,12 @@ base is wanted (it is extra work beyond the skill).
 ## Fallback
 
 - All engines fail \u2192 suggest text version or different format.
-- No eval agent \u2192 judge-only scoring, or build a production agent via \xA75
-  (generic or API-verified variant, thin prompt) \u2014 never reuse an agent
-  whose knowledge derives from the learning material.
-  Judge-only scoring does NOT unlock machine-confirmed \u2014 trust caps at
-  human-reviewed; say so explicitly.
+- Eval runtime unavailable (no eval agent / service down) \u2192 still
+  DESIGN and CREATE the eval project with test cases (every agent MUST
+  have an eval \u2014 no skip). If the eval cannot RUN now, deliver with
+  trust capped at human-reviewed and state: "Test framework created;
+  run the evaluation once the eval service is available." Judge-only
+  scoring (when run) does NOT unlock machine-confirmed.
 - No test files \u2192 user-described scenarios as contentAssertion.
 - run_eval orphaned (resume shows runnerAlive=false) \u2192 \`run_eval resume(runId)\`
   marks it failed automatically; then \`run_eval start(projectId)\` to restart.