@axiom-lattice/core 3.0.3 → 3.0.4

package/dist/index.js

@@ -12878,6 +12878,10 @@ Always set metadata on agent creation. At minimum:
 - verified: "unverified" (upgraded after eval passes)
 - version: "1.0" (bump on each update_agent)
 - source: the material name or "user-description"
+- role: "orchestrator" | "sub-agent" \u2014 set when the agent is part of a
+  parent+subAgents structure (role clarity, User Interaction Rules).
+  The eval project is NOT recorded \u2014 it is derived by naming convention
+  (eval-{agent-id}, see [[eval-verify]] Setup).
 When trust upgrades, update both the skill's verified frontmatter and
 the agent's metadata.verified \u2014 they must stay in sync.
 
@@ -12999,8 +13003,14 @@ verified: unverified
 ## Setup
 
 0. Load [[eval-design-tests]] for case design guidance
-1. \`read_eval list_projects\` \u2192 find "eval-{domain}"
-   Exists \u2192 reuse projectId. New \u2192 create_project(name: "eval-{domain}")
+1. **One eval project per agent**, named \`eval-{agent-id}\`:
+   \`read_eval list_projects\` \u2192 find "eval-{agent-id}"
+   Exists \u2192 reuse projectId. New \u2192 create_project(name: "eval-{agent-id}")
+   - The agent-id is the eval project's subject. Observability: from an
+     agent's id you can find its eval project by naming convention.
+   - Orchestrator + subAgents \u2192 one eval project per sub-agent
+     (eval-{sub-agent-id}) PLUS one integration eval project for the
+     parent (eval-{parent-id}) \u2014 see Layered verification below.
 2. \`manage_eval create_case(suiteId, ...)\` per sample or scenario
    Required: inputMessage, steps=[{agent_id}], outputType, contentAssertion
 
@@ -23470,6 +23480,7 @@ Your sub-skills (accessible via the MOC or direct loading):
 - [[learn-capability]] \u2014 Learn from any source material \u2192 skills + agents
 - [[agent-build]] \u2014 Design and build single agents (REACT/DEEP_AGENT)
 - [[design-workflow]] \u2014 Design workflow agents
+- [[eval-verify]] \u2014 Run evaluations with fix loop, hold-out, trust upgrade
 - [[task-tracking]] \u2014 Manage persistent tasks (manage_task)
 - [[completion-gate]] \u2014 THE rule: no agent is "done" without eval
 - [[domain-moc]] \u2014 Create the domain MOC (mandatory per learning run)
@@ -30104,7 +30115,26 @@ Use read_eval get_run_results for multiple runs and present comparison.
 `
 };
 
+// src/tool_lattice/withToolTimeout.ts
+function withToolTimeout(executor, timeoutMs = 18e4) {
+  return async (input, exeConfig) => {
+    return new Promise((resolve4, reject) => {
+      const timer = setTimeout(() => {
+        reject(new Error(`Tool execution timed out after ${timeoutMs}ms`));
+      }, timeoutMs);
+      executor(input, exeConfig).then((result) => {
+        clearTimeout(timer);
+        resolve4(result);
+      }).catch((err) => {
+        clearTimeout(timer);
+        reject(err);
+      });
+    });
+  };
+}
+
 // src/middlewares/evalMiddleware.ts
+var RUN_EVAL_SYNC_WAIT_MS = 15e4;
 function getStore() {
   return getStoreLattice("default", "eval").store;
 }
@@ -30131,6 +30161,23 @@ function sanitize(obj) {
   }
   return out;
 }
+function aggregateHoldoutResults(results) {
+  const passed = results.filter((r) => r.pass).length;
+  return {
+    holdout: true,
+    passedCases: passed,
+    failedCases: results.length - passed,
+    passRate: results.length > 0 ? passed / results.length : 0,
+    totalCases: results.length
+  };
+}
+async function runWithResults(tid, store, svc, run, runnerAlive) {
+  const results = run.status === "completed" ? await store.getResultsByRun(tid, run.id) : void 0;
+  if (run.holdout && results) {
+    return { ...run, runnerAlive, results: aggregateHoldoutResults(results) };
+  }
+  return { ...run, runnerAlive, results };
+}
 function createReadEvalTool() {
   const schema6 = import_zod64.z.object({
     action: import_zod64.z.enum([
@@ -30365,100 +30412,119 @@ function createRunEvalTool() {
     projectId: import_zod64.z.string().optional().describe("Required for start"),
     suiteIds: import_zod64.z.array(import_zod64.z.string()).optional().describe("Optional for start \u2014 only run these suites (e.g. dev set only). Omit to run all."),
     caseIds: import_zod64.z.array(import_zod64.z.string()).optional().describe("Optional for start \u2014 only run these cases across the selected suites. Omit to run all cases in those suites."),
-    runId: import_zod64.z.string().optional().describe("Required for status, resume, abort")
+    runId: import_zod64.z.string().optional().describe("Required for status, resume, abort"),
+    sleepMs: import_zod64.z.number().int().min(0).max(12e4).optional().describe("Optional for status \u2014 sleep this many ms BEFORE checking the run, to pace polling (e.g. 15000 \u2192 30000 \u2192 60000 \u2192 120000). Omit to check immediately."),
+    wait: import_zod64.z.boolean().optional().describe("Optional for start \u2014 defaults to true: block synchronously (up to ~150s) and return final results in one call. Set false to return the runId immediately and poll.")
   });
   return (0, import_langchain78.tool)(
-    async (input, exeConfig) => {
-      const tid = tenantId(exeConfig);
-      if (!tid) {
-        return JSON.stringify({ success: false, error: "No tenant context. Agent must be invoked through gateway." });
-      }
-      try {
-        const store = getStore();
-        const svc = getEvalRunService();
-        let data;
-        switch (input.action) {
-          case "start": {
-            const ctx = workspaceContext(exeConfig);
-            const runId = await svc.startRun(tid, input.projectId, input.suiteIds, input.caseIds, {
-              workspaceId: ctx.workspaceId,
-              projectId: ctx.projectId
-            });
-            data = sanitize({ runId, message: "Run started. Poll with run_eval status (backoff: 15s\u219230s\u219260s\u2192max 120s)." });
-            break;
-          }
-          case "status": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            data = sanitize({ ...run, runnerAlive: svc.isRunning(input.runId) });
-            break;
-          }
-          case "resume": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            const runnerAlive = svc.isRunning(input.runId);
-            if (run.status === "running" && !runnerAlive) {
-              await store.updateRunStatus(tid, run.id, {
-                status: "failed",
-                error: "Gateway restarted \u2014 run orphaned",
-                completedAt: /* @__PURE__ */ new Date()
-              });
-              data = sanitize({
-                ...run,
-                status: "failed",
-                runnerAlive: false,
-                message: "Run was orphaned \u2014 marked failed. Start a new run."
-              });
+    withToolTimeout(
+      async (input, exeConfig) => {
+        const tid = tenantId(exeConfig);
+        if (!tid) {
+          return JSON.stringify({ success: false, error: "No tenant context. Agent must be invoked through gateway." });
+        }
+        try {
+          const store = getStore();
+          const svc = getEvalRunService();
+          let data;
+          switch (input.action) {
+            case "start": {
+              const ctx = workspaceContext(exeConfig);
+              const runId = await svc.startRun(tid, input.projectId, input.suiteIds, input.caseIds, ctx);
+              if (input.wait === false) {
+                data = sanitize({ runId, message: "Run started. Poll with run_eval status (backoff: 15s\u219230s\u219260s\u2192max 120s)." });
+                break;
+              }
+              let timer;
+              try {
+                await Promise.race([
+                  svc.waitForRun(runId).catch(() => {
+                  }),
+                  new Promise((resolve4) => {
+                    timer = setTimeout(resolve4, RUN_EVAL_SYNC_WAIT_MS);
+                  })
+                ]);
+              } finally {
+                if (timer) clearTimeout(timer);
+              }
+              const run = await store.getRunById(tid, runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              if (run.status === "running") {
+                data = sanitize({
+                  runId,
+                  status: "running",
+                  runnerAlive: svc.isRunning(runId),
+                  message: `Run not finished within ${RUN_EVAL_SYNC_WAIT_MS / 1e3}s \u2014 poll with run_eval status(runId, sleepMs) (e.g. 15000, doubling up to 120000), or abort with run_eval abort.`
+                });
+                break;
+              }
+              data = sanitize({ synced: true, ...await runWithResults(tid, store, svc, run, svc.isRunning(runId)) });
               break;
             }
-            const results = run.status === "completed" ? await store.getResultsByRun(tid, run.id) : void 0;
-            if (run.holdout && results) {
-              const passed = results.filter((r) => r.pass).length;
-              data = sanitize({
-                ...run,
-                runnerAlive,
-                results: {
-                  holdout: true,
-                  passedCases: passed,
-                  failedCases: results.length - passed,
-                  passRate: results.length > 0 ? passed / results.length : 0,
-                  totalCases: results.length
-                }
-              });
+            case "status": {
+              if (input.sleepMs && input.sleepMs > 0) {
+                await new Promise((resolve4) => setTimeout(resolve4, input.sleepMs));
+              }
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              data = sanitize({ ...run, runnerAlive: svc.isRunning(input.runId) });
               break;
             }
-            data = sanitize({ ...run, runnerAlive, results });
-            break;
-          }
-          case "abort": {
-            const run = await store.getRunById(tid, input.runId);
-            if (!run) return JSON.stringify({ success: false, error: "Run not found" });
-            const ok = await svc.abortRun(input.runId);
-            data = sanitize({ aborted: ok });
-            break;
+            case "resume": {
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              const runnerAlive = svc.isRunning(input.runId);
+              if (run.status === "running" && !runnerAlive) {
+                await store.updateRunStatus(tid, run.id, {
+                  status: "failed",
+                  error: "Gateway restarted \u2014 run orphaned",
+                  completedAt: /* @__PURE__ */ new Date()
+                });
+                data = sanitize({
+                  ...run,
+                  status: "failed",
+                  runnerAlive: false,
+                  message: "Run was orphaned \u2014 marked failed. Start a new run."
+                });
+                break;
+              }
+              data = sanitize(await runWithResults(tid, store, svc, run, runnerAlive));
+              break;
+            }
+            case "abort": {
+              const run = await store.getRunById(tid, input.runId);
+              if (!run) return JSON.stringify({ success: false, error: "Run not found" });
+              const ok = await svc.abortRun(input.runId);
+              data = sanitize({ aborted: ok });
+              break;
+            }
+            default:
+              return JSON.stringify({ success: false, error: `Unknown action: ${input.action}` });
           }
-          default:
-            return JSON.stringify({ success: false, error: `Unknown action: ${input.action}` });
+          return JSON.stringify({ success: true, data });
+        } catch (e) {
+          return JSON.stringify({ success: false, error: e.message });
         }
-        return JSON.stringify({ success: true, data });
-      } catch (e) {
-        return JSON.stringify({ success: false, error: e.message });
       }
-    },
+    ),
     {
       name: "run_eval",
-      description: `Execute and manage evaluation runs. ASYNCHRONOUS \u2014 may take minutes.
+      description: `Execute and manage evaluation runs.
 
 ACTIONS:
-- start(projectId, suiteIds?, caseIds?) \u2014 begin evaluation (optionally only the listed suites, and/or only the listed cases). Returns runId.
-- status(runId) \u2014 current status + runnerAlive flag:
-    \u2022 runnerAlive=true, status=running: keep polling
+- start(projectId, suiteIds?, caseIds?, wait?) \u2014 begin evaluation (optionally only the listed suites, and/or only the listed cases).
+    wait=true (DEFAULT): SYNCHRONOUS \u2014 blocks up to ~150s and returns the FINAL RESULTS in one call:
+      { status, results } with per-case pass/score (hold-out/validation runs: aggregates only \u2014 per-case details withheld by design).
+      If the run exceeds 150s: returns { runId, status: "running" } \u2014 NOT an error \u2014 then poll with status/resume until completed.
+    wait=false: fire-and-forget \u2014 returns { runId } immediately; poll with status.
+- status(runId, sleepMs?) \u2014 sleep sleepMs first (to pace polling), then return current status + runnerAlive flag:
+    \u2022 runnerAlive=true, status=running: keep polling \u2014 call status(runId, sleepMs) with backoff 15s\u219230s\u219260s\u2192max 120s
     \u2022 runnerAlive=false, status=running: ORPHANED \u2014 resume marks it failed automatically; then start a new run
     \u2022 status=completed: get results with read_eval get_run_results or run_eval resume
 - resume(runId) \u2014 reconnect from new conversation. Returns status + results if completed.
 - abort(runId) \u2014 cancel running evaluation.
 
-Polling: start at 15s, double each time, max 120s between polls. Batch reports.`,
+Polling (only needed with wait=false or after a sync timeout): call status(runId, sleepMs) so the tool sleeps before checking; start at 15s, double each time, max 120s between polls. Batch reports.`,
       schema: schema6
     }
   );
@@ -31138,7 +31204,8 @@ Both modes use the same agent type \u2014 skill only, no domain tools:
        verified: "unverified",      # upgraded after eval passes
        version: "1.0",              # bump on each update_agent
        source: "{material name}",   # provenance
-       skill: "skill-name"
+       skill: "skill-name",
+       role: "orchestrator" | "sub-agent"  # only for parent+subAgents structure
      }
    )
 
@@ -31234,6 +31301,11 @@ Run evaluation, fix loop, hold-out validation, trust upgrade. See
 [[eval-verify]] for the full workflow. The eval-design-tests and
 eval-run-and-govern skills cover case design and run governance.
 
+**One eval project per agent**, named \`eval-{agent-id}\` \u2014 every agent
+built by this workflow gets its own eval project (see eval-verify
+Setup). Orchestrator + subAgents \u2192 one eval project per sub-agent plus
+one integration eval for the parent.
+
 Learning-specific suite guidance:
 - 0.2 \u2461 \u2192 {skill}-user-sample; samples \u22658 \u2192 also {skill}-validation
 - 0.2 \u2460 \u2192 {skill}-api-verified (single step, \xA74.1)
@@ -31272,11 +31344,12 @@ base is wanted (it is extra work beyond the skill).
 ## Fallback
 
 - All engines fail \u2192 suggest text version or different format.
-- No eval agent \u2192 judge-only scoring, or build a production agent via \xA75
-  (generic or API-verified variant, thin prompt) \u2014 never reuse an agent
-  whose knowledge derives from the learning material.
-  Judge-only scoring does NOT unlock machine-confirmed \u2014 trust caps at
-  human-reviewed; say so explicitly.
+- Eval runtime unavailable (no eval agent / service down) \u2192 still
+  DESIGN and CREATE the eval project with test cases (every agent MUST
+  have an eval \u2014 no skip). If the eval cannot RUN now, deliver with
+  trust capped at human-reviewed and state: "Test framework created;
+  run the evaluation once the eval service is available." Judge-only
+  scoring (when run) does NOT unlock machine-confirmed.
 - No test files \u2192 user-described scenarios as contentAssertion.
 - run_eval orphaned (resume shows runnerAlive=false) \u2192 \`run_eval resume(runId)\`
   marks it failed automatically; then \`run_eval start(projectId)\` to restart.