opencode-agent-skill 14.2.0-beta.1 → 15.0.0

package/scripts/eval-pi.mjs

@@ -155,6 +155,35 @@ function parsePiTelemetry(stdout) {
     baseContextChars: 0,
   }
 
+  const consumeControllerDetails = (details) => {
+    const steps = Array.isArray(details?.steps) ? details.steps : []
+    for (const step of steps) {
+      const optimization = step?.optimizations
+      if (optimization && typeof optimization === "object") {
+        turbo.routedAgentSteps += 1
+        if (optimization.childRuntime === "rpc") turbo.rpcRuns += 1
+        if (optimization.childRuntime === "cli") turbo.cliRuns += 1
+        if (optimization.warmWorkerReused === true) turbo.warmWorkerReuses += 1
+        if (optimization.contextCacheHit === true) turbo.contextCacheHits += 1
+        if (optimization.microSkillCacheHit === true) turbo.microSkillCacheHits += 1
+        if (optimization.affectedTestCacheHit === true) turbo.affectedTestCacheHits += 1
+        turbo.reusableVerificationReceipts += Number(optimization.reusableVerificationReceipts || 0)
+        if (optimization.compactToolOutput === true) turbo.compactedToolOutputSteps += 1
+        turbo.adaptiveContextChars += Number(optimization.runtimeContextBudget || 0)
+        turbo.baseContextChars += Number(optimization.baseContextBudget || 0)
+      }
+      if (step?.usage) {
+        addUsage(childUsage, step.usage)
+        childUsageSamples += 1
+      }
+      childToolCalls += Number(step?.toolCalls || 0)
+      for (const name of step?.toolNames || []) {
+        const key = "child:" + String(name)
+        toolNames[key] = Number(toolNames[key] || 0) + 1
+      }
+    }
+  }
+
   for (const line of String(stdout || "").split(/\r?\n/)) {
     if (!line.trim()) continue
     let event
@@ -188,34 +217,19 @@ function parsePiTelemetry(stdout) {
 
     if (event.type === "tool_execution_end" && event.toolName === "ues_execute") {
       const details = event.result?.details
-      const steps = Array.isArray(details?.steps) ? details.steps : []
-      for (const step of steps) {
-        const optimization = step?.optimizations
-        if (optimization && typeof optimization === "object") {
-          turbo.routedAgentSteps += 1
-          if (optimization.childRuntime === "rpc") turbo.rpcRuns += 1
-          if (optimization.childRuntime === "cli") turbo.cliRuns += 1
-          if (optimization.warmWorkerReused === true) turbo.warmWorkerReuses += 1
-          if (optimization.contextCacheHit === true) turbo.contextCacheHits += 1
-          if (optimization.microSkillCacheHit === true) turbo.microSkillCacheHits += 1
-          if (optimization.affectedTestCacheHit === true) turbo.affectedTestCacheHits += 1
-          turbo.reusableVerificationReceipts += Number(optimization.reusableVerificationReceipts || 0)
-          if (optimization.compactToolOutput === true) turbo.compactedToolOutputSteps += 1
-          turbo.adaptiveContextChars += Number(optimization.runtimeContextBudget || 0)
-          turbo.baseContextChars += Number(optimization.baseContextBudget || 0)
-        }
-        if (step?.usage) {
-          addUsage(childUsage, step.usage)
-          childUsageSamples += 1
-        }
-        childToolCalls += Number(step?.toolCalls || 0)
-        for (const name of step?.toolNames || []) {
-          const key = "child:" + String(name)
-          toolNames[key] = Number(toolNames[key] || 0) + 1
-        }
-      }
+      consumeControllerDetails(details)
       if (event.isError !== true && details?.mode === "execute") controllerPass = true
     }
+
+    if (event.type === "ues_controller_progress") {
+      controllerUsed = true
+    }
+
+    if (event.type === "ues_controller_direct") {
+      controllerUsed = event.controllerUsed === true
+      controllerPass = event.controllerPass === true
+      consumeControllerDetails(event.details)
+    }
   }
 
   const totalUsage = zeroUsage()
@@ -334,6 +348,7 @@ try {
         const childEnv = {
           ...process.env,
           UES_CONFIG_DIR: uesConfigDir,
+          UES_EVAL_DIRECT_TELEMETRY: mode === "ues" ? "1" : "0",
           PI_SKIP_VERSION_CHECK: "true",
           NO_COLOR: "1",
         }
@@ -356,14 +371,10 @@ try {
         let prompt = task.prompt
         if (mode === "ues") {
           piArgs.push("--extension", path.join(root, "pi", "extensions", "ues.ts"))
-          prompt = [
-            "Use the UES runtime controller for this benchmark task.",
-            "You MUST call the ues_execute tool exactly once with the complete engineering task below and the current working directory.",
-            "Do not edit project files directly before that tool call. After the tool returns, report its verified result concisely.",
-            "",
-            "Engineering task:",
-            task.prompt,
-          ].join("\n")
+          // /ues-run is an extension command, resolved before prompt templates.
+          // The benchmark therefore exercises the controller deterministically
+          // instead of measuring whether a weak parent model remembers to call a tool.
+          prompt = "/ues-run " + task.prompt
         }
         piArgs.push(prompt)
 
@@ -371,6 +382,31 @@ try {
           "[" + mode + "] " + task.id + " trial " + trial + ": starting Pi " + piVersion +
           (providerExtensions.length ? " with provider extension isolation" : ""),
         )
+        const progressBuffers = { stdout: "", stderr: "" }
+        const consumeControllerProgress = (chunk, stream) => {
+          if (mode !== "ues") return
+          progressBuffers[stream] += String(chunk || "")
+          const lines = progressBuffers[stream].split(/\r?\n/)
+          progressBuffers[stream] = lines.pop() || ""
+          for (const line of lines) {
+            if (!line.trim()) continue
+            try {
+              const event = JSON.parse(line)
+              if (event?.type !== "ues_controller_progress") continue
+              console.log(
+                "[" + mode + "] " + task.id + " trial " + trial +
+                ": controller progress" +
+                (event.agent ? " agent=" + event.agent : "") +
+                (event.phase ? " phase=" + event.phase : "") +
+                " elapsed=" + Math.round(Number(event.elapsedMs || 0) / 1000) + "s" +
+                " idle=" + Math.round(Number(event.idleMs || 0) / 1000) + "s" +
+                " tools=" + Number(event.toolCalls || 0) +
+                (event.activeTool ? " activeTool=" + event.activeTool : "") +
+                (event.note ? " note=" + String(event.note).slice(0, 180) : ""),
+              )
+            } catch {}
+          }
+        }
         const agentRun = await runAsync(piCommand, piArgs, {
           cwd: workspace,
           env: childEnv,
@@ -379,6 +415,8 @@ try {
           timeoutMs,
           idleTimeoutMs,
           signal: abortController.signal,
+          onStdout: (chunk) => consumeControllerProgress(chunk, "stdout"),
+          onStderr: (chunk) => consumeControllerProgress(chunk, "stderr"),
           onHeartbeat: ({ elapsedMs, idleMs }) => {
             console.log(
               "[" + mode + "] " + task.id + " trial " + trial +
@@ -398,7 +436,9 @@ try {
 
         const afterSnapshot = await snapshotWorkspace(workspace)
         const changedFiles = diffWorkspaceSnapshots(beforeSnapshot, afterSnapshot)
-        const telemetry = parsePiTelemetry(agentRun.stdout)
+        const telemetry = parsePiTelemetry(
+          [agentRun.stdout, agentRun.stderr].filter(Boolean).join("\n"),
+        )
         const baselineIsolated = mode !== "baseline" || telemetry.controllerUsed === false
         const controllerValid =
           mode === "baseline"
@@ -437,7 +477,7 @@ try {
           "[" + mode + "] " + task.id + " trial " + trial + ": " +
           (passed ? "PASS" : "FAIL") +
           " (agent=" + agentRun.status + ", grader=" + graderRun.status +
-          ", controller=" + controllerValid +
+          ", controllerValid=" + controllerValid +
           (mode === "baseline" ? ", isolated=" + baselineIsolated : "") + ")",
         )
       }

package/scripts/smoke-pi-extension.mjs

@@ -132,7 +132,7 @@ try {
     0,
     promptResult.diagnostics.map((entry) => `${entry.path || entry.filePath || "prompt"}: ${entry.message || entry.error || JSON.stringify(entry)}`).join("\n"),
   )
-  assert.ok(promptResult.prompts.length >= 11, "Pi did not load the bundled UES prompts")
+  assert.equal(promptResult.prompts.length, 10, "Pi must load exactly 10 prompt templates; /ues-run is an extension command")
 
   console.log(
     `Pi package smoke passed via ${loaded.source}: ${result.extensions.length} extension(s), ${skillResult.skills.length} skill(s), ${promptResult.prompts.length} prompt(s) loaded`,

package/pi/prompts/ues-run.md

@@ -1,17 +0,0 @@
----
-description: Execute engineering work through the deterministic UES runtime controller on Pi, with adaptive context, model routing, retries, and evidence-gated verification.
----
-
-Run this task with UES on Pi: $@
-
-Preserve the user's requested outcome and approval boundaries.
-
-1. If the request is directly answerable and needs no repository evidence, edits, or machine-checkable verification, answer directly.
-2. Otherwise prefer the `ues_execute` tool for end-to-end engineering work. Pass a concise but complete task and the current repository cwd. The controller owns task policy, bounded context construction, model-tier routing, optional diagnosis/plan gating, implementation, retries, independent verification, integration verification, and performance telemetry.
-3. Do not manually reproduce the controller workflow unless `ues_execute` is unavailable or the user explicitly asks for plan-only/review-only/research-only work.
-4. Use `ues_cli` for deterministic inspection, durable `.ues-work/` state, task graphs, receipts, evidence, worktrees, browser/visual helpers, and explicit policy/config operations.
-5. Use `ues_dispatch` when you intentionally need one specialist, a custom chain, or bounded parallel read-only work. Routed dispatch automatically applies model policy and adaptive context.
-6. Parallel read-only agents may share the repository. Parallel writer agents must use explicit distinct isolated cwd/worktrees; otherwise run writers serially. Never bypass this guard.
-7. Preserve unrelated user changes. Do not force-clean a repository.
-8. Do not merge, push, publish, deploy, rewrite history, or perform destructive operations without explicit user approval.
-9. Final completion claims must come from a verified controller PASS or equivalent fresh command/test evidence plus the final diff/status.