agentv 2.17.3 → 2.18.2

package/README.md

@@ -119,21 +119,28 @@ See [AGENTS.md](AGENTS.md) for development guidelines and design principles.
 
 ### Releasing
 
-Stable release:
+Version bump:
 
 ```bash
 bun run release          # patch bump
 bun run release minor
 bun run release major
-bun run publish          # publish to npm `latest`
 ```
 
-Prerelease (`next`) channel:
+Canary rollout (recommended):
+
+```bash
+bun run publish:next         # publish current version to npm `next`
+bun run promote:latest       # promote same version to npm `latest`
+bun run tag:next 2.18.0      # point npm `next` to an explicit version
+bun run promote:latest 2.18.0 # point npm `latest` to an explicit version
+```
+
+Legacy prerelease flow (still available):
 
 ```bash
 bun run release:next         # bump/increment `-next.N`
 bun run release:next major   # start new major prerelease line
-bun run publish:next         # publish to npm `next`
 ```
 
 ## Core Concepts
@@ -347,7 +354,6 @@ targets:
 
   - name: vscode_dev
     provider: vscode
-    workspace_template: ${{ WORKSPACE_PATH }}
     judge_target: azure-base
 
   - name: local_agent
@@ -358,6 +364,8 @@ targets:
 
 Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`.
 
+Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`).
+
 Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields.
 
 ## Evaluation Features

package/dist/{chunk-JSOG3FT6.js → chunk-ID5SDIYE.js}

@@ -148,7 +148,7 @@ var require_dist = __commonJS({
   }
 });
 
-// ../../packages/core/dist/chunk-PSYFRPNT.js
+// ../../packages/core/dist/chunk-V42NUK73.js
 import { constants } from "node:fs";
 import { access, readFile } from "node:fs/promises";
 import path from "node:path";
@@ -4195,7 +4195,7 @@ var coerce = {
 };
 var NEVER = INVALID;
 
-// ../../packages/core/dist/chunk-PSYFRPNT.js
+// ../../packages/core/dist/chunk-V42NUK73.js
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
 var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
 var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
@@ -4603,6 +4603,11 @@ function resolveRetryConfig(target) {
 }
 function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
   const parsed = BASE_TARGET_SCHEMA.parse(definition);
+  if (parsed.workspace_template !== void 0 || parsed.workspaceTemplate !== void 0) {
+    throw new Error(
+      `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`
+    );
+  }
   const provider = parsed.provider.toLowerCase();
   const providerBatching = resolveOptionalBoolean(
     parsed.provider_batching ?? parsed.providerBatching
@@ -33961,7 +33966,7 @@ import { createServer } from "node:http";
 import fs2 from "node:fs/promises";
 import path31 from "node:path";
 import { createHash as createHash2, randomUUID as randomUUID7 } from "node:crypto";
-import { mkdir as mkdir12, stat as stat7 } from "node:fs/promises";
+import { mkdir as mkdir12, readdir as readdir6, stat as stat7 } from "node:fs/promises";
 import path39 from "node:path";
 import micromatch4 from "micromatch";
 import { readFileSync } from "node:fs";
@@ -37048,11 +37053,13 @@ function parseWorkspaceHookConfig(raw, evalFileDir) {
 function parseWorkspaceHooksConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
+  const enabled = typeof obj.enabled === "boolean" ? obj.enabled : void 0;
   const beforeAll = parseWorkspaceHookConfig(obj.before_all, evalFileDir);
   const beforeEach = parseWorkspaceHookConfig(obj.before_each, evalFileDir);
   const afterEach = parseWorkspaceHookConfig(obj.after_each, evalFileDir);
   const afterAll = parseWorkspaceHookConfig(obj.after_all, evalFileDir);
   const hooks = {
+    ...enabled !== void 0 && { enabled },
     ...beforeAll !== void 0 && { before_all: beforeAll },
     ...beforeEach !== void 0 && { before_each: beforeEach },
     ...afterEach !== void 0 && { after_each: afterEach },
@@ -37083,6 +37090,17 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
 function parseWorkspaceConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
+  if ("static_path" in obj) {
+    throw new Error(
+      "workspace.static_path has been removed. Use workspace.path with workspace.mode=static."
+    );
+  }
+  if ("pool" in obj) {
+    throw new Error("workspace.pool has been removed. Use workspace.mode='pooled' or 'temp'.");
+  }
+  if ("static" in obj) {
+    throw new Error("workspace.static has been removed. Use workspace.mode='static'.");
+  }
   let template = typeof obj.template === "string" ? obj.template : void 0;
   if (template && !path8.isAbsolute(template)) {
     template = path8.resolve(evalFileDir, template);
@@ -37090,19 +37108,17 @@ function parseWorkspaceConfig(raw, evalFileDir) {
   const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
   const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
   const hooks = parseWorkspaceHooksConfig(obj.hooks, evalFileDir);
-  const mode = obj.mode === "pooled" || obj.mode === "ephemeral" || obj.mode === "static" ? obj.mode : void 0;
-  const staticPath = typeof obj.static_path === "string" ? obj.static_path : void 0;
-  const pool = typeof obj.pool === "boolean" ? obj.pool : void 0;
-  if (!template && !isolation && !repos && !hooks && !mode && !staticPath && pool === void 0)
-    return void 0;
+  const explicitMode = obj.mode === "pooled" || obj.mode === "temp" || obj.mode === "static" ? obj.mode : void 0;
+  const workspacePath = typeof obj.path === "string" ? obj.path : void 0;
+  const mode = explicitMode ?? (workspacePath ? "static" : void 0);
+  if (!template && !isolation && !repos && !hooks && !mode && !workspacePath) return void 0;
   return {
     ...template !== void 0 && { template },
     ...isolation !== void 0 && { isolation },
     ...repos !== void 0 && { repos },
     ...hooks !== void 0 && { hooks },
     ...mode !== void 0 && { mode },
-    ...staticPath !== void 0 && { static_path: staticPath },
-    ...pool !== void 0 && { pool }
+    ...workspacePath !== void 0 && { path: workspacePath }
   };
 }
 function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
@@ -37116,21 +37132,22 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
       ...caseHook ?? {}
     };
   };
+  const mergedEnabled = caseLevel.hooks?.enabled ?? suiteLevel.hooks?.enabled;
   const mergedHooks = {
+    ...mergedEnabled !== void 0 && { enabled: mergedEnabled },
     before_all: mergeHook(suiteLevel.hooks?.before_all, caseLevel.hooks?.before_all),
     before_each: mergeHook(suiteLevel.hooks?.before_each, caseLevel.hooks?.before_each),
     after_each: mergeHook(suiteLevel.hooks?.after_each, caseLevel.hooks?.after_each),
     after_all: mergeHook(suiteLevel.hooks?.after_all, caseLevel.hooks?.after_all)
   };
-  const hasHooks = Object.values(mergedHooks).some((hook) => hook !== void 0);
+  const hasHooks = mergedEnabled !== void 0 || Object.values(mergedHooks).some((hook) => hook !== void 0 && typeof hook === "object");
   return {
     template: caseLevel.template ?? suiteLevel.template,
     isolation: caseLevel.isolation ?? suiteLevel.isolation,
     repos: caseLevel.repos ?? suiteLevel.repos,
     ...hasHooks && { hooks: mergedHooks },
     mode: caseLevel.mode ?? suiteLevel.mode,
-    static_path: caseLevel.static_path ?? suiteLevel.static_path,
-    pool: caseLevel.pool ?? suiteLevel.pool
+    path: caseLevel.path ?? suiteLevel.path
   };
 }
 function asString6(value) {
@@ -40000,14 +40017,18 @@ var PiAgentSdkProvider = class {
       }
     });
     try {
-      const timeoutMs = this.config.timeoutMs ?? 12e4;
-      const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(
-          () => reject(new Error(`Pi agent SDK timed out after ${timeoutMs}ms`)),
-          timeoutMs
-        );
-      });
-      await Promise.race([agent.prompt(request.question), timeoutPromise]);
+      if (this.config.timeoutMs) {
+        const timeoutMs = this.config.timeoutMs;
+        const timeoutPromise = new Promise((_, reject) => {
+          setTimeout(
+            () => reject(new Error(`Pi agent SDK timed out after ${timeoutMs}ms`)),
+            timeoutMs
+          );
+        });
+        await Promise.race([agent.prompt(request.question), timeoutPromise]);
+      } else {
+        await agent.prompt(request.question);
+      }
       await agent.waitForIdle();
       const agentMessages = agent.state.messages;
       for (const msg of agentMessages) {
@@ -47044,6 +47065,9 @@ function toScriptConfig(hook, hookName, context) {
 function hasHookCommand(hook) {
   return !!(hook?.command && hook.command.length > 0 || hook?.script && hook.script.length > 0);
 }
+function hooksEnabled(workspace) {
+  return workspace?.hooks?.enabled !== false;
+}
 function getWorkspaceTemplate(target) {
   const config2 = target.config;
   if ("workspaceTemplate" in config2 && typeof config2.workspaceTemplate === "string") {
@@ -47218,19 +47242,27 @@ async function runEvaluation(options) {
     }
   };
   const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
-  const configuredMode = suiteWorkspace?.mode ?? workspaceMode;
-  const configuredStaticPath = suiteWorkspace?.static_path ?? workspacePath ?? legacyWorkspacePath;
-  const useStaticWorkspace = configuredMode === "static" || !!configuredStaticPath && !configuredMode;
+  const cliWorkspacePath = workspacePath ?? legacyWorkspacePath;
+  const yamlWorkspacePath = suiteWorkspace?.path;
+  if (cliWorkspacePath && workspaceMode && workspaceMode !== "static") {
+    throw new Error("--workspace-path requires --workspace-mode static when both are provided");
+  }
+  const configuredMode = cliWorkspacePath ? "static" : workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? "static" : "pooled");
+  const configuredStaticPath = cliWorkspacePath ?? yamlWorkspacePath;
+  const useStaticWorkspace = configuredMode === "static";
   if (useStaticWorkspace && isPerTestIsolation) {
     throw new Error(
       "static workspace mode is incompatible with isolation: per_test. Use isolation: shared (default)."
     );
   }
   if (configuredMode === "static" && !configuredStaticPath) {
-    throw new Error("workspace.mode=static requires workspace.static_path or --workspace-path");
+    throw new Error("workspace.mode=static requires workspace.path or --workspace-path");
+  }
+  if (configuredMode !== "static" && configuredStaticPath) {
+    throw new Error("workspace.path requires workspace.mode=static");
   }
   const hasSharedWorkspace = !!(useStaticWorkspace || workspaceTemplate || suiteWorkspace?.hooks || suiteWorkspace?.repos?.length && !isPerTestIsolation);
-  const poolEnabled = configuredMode === "pooled" ? true : configuredMode === "ephemeral" || useStaticWorkspace ? false : suiteWorkspace?.pool ?? poolWorkspaces ?? true;
+  const poolEnabled = configuredMode === "pooled";
   const usePool = poolEnabled !== false && !!suiteWorkspace?.repos?.length && !isPerTestIsolation && !useStaticWorkspace;
   const resolvedRetainOnSuccess = retainOnSuccess ?? (keepWorkspaces ? "keep" : "cleanup");
   const resolvedRetainOnFailure = retainOnFailure ?? (cleanupWorkspaces ? "cleanup" : "keep");
@@ -47254,9 +47286,28 @@ async function runEvaluation(options) {
   const availablePoolSlots = [];
   const poolSlotBaselines = /* @__PURE__ */ new Map();
   const poolMaxSlots = Math.min(configPoolMaxSlots ?? 10, 50);
+  let staticMaterialised = false;
   if (useStaticWorkspace && configuredStaticPath) {
+    const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
+    const dirExists = await stat7(configuredStaticPath).then(
+      (s) => s.isDirectory(),
+      () => false
+    );
+    const isEmpty = dirExists ? (await readdir6(configuredStaticPath)).length === 0 : false;
+    if (isYamlConfiguredPath && (!dirExists || isEmpty)) {
+      if (!dirExists) {
+        await mkdir12(configuredStaticPath, { recursive: true });
+      }
+      if (workspaceTemplate) {
+        await copyDirectoryRecursive(workspaceTemplate, configuredStaticPath);
+        setupLog(`copied template into static workspace: ${configuredStaticPath}`);
+      }
+      staticMaterialised = true;
+      setupLog(`materialised static workspace at: ${configuredStaticPath}`);
+    } else {
+      setupLog(`reusing existing static workspace: ${configuredStaticPath}`);
+    }
     sharedWorkspacePath = configuredStaticPath;
-    setupLog(`using static workspace: ${configuredStaticPath}`);
   } else if (usePool && suiteWorkspace?.repos) {
     const slotsNeeded = workers;
     setupLog(`acquiring ${slotsNeeded} workspace pool slot(s) (pool capacity: ${poolMaxSlots})`);
@@ -47302,7 +47353,8 @@ async function runEvaluation(options) {
       } catch {
       }
     }
-    const repoManager = suiteWorkspace?.repos?.length && !usePool && !useStaticWorkspace ? new RepoManager(verbose) : void 0;
+    const needsRepoMaterialisation = !!suiteWorkspace?.repos?.length && !usePool && (!useStaticWorkspace || staticMaterialised);
+    const repoManager = needsRepoMaterialisation ? new RepoManager(verbose) : void 0;
     if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
       setupLog(
         `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
@@ -47319,8 +47371,9 @@ async function runEvaluation(options) {
         throw new Error(`Failed to materialize repos: ${message}`);
       }
     }
+    const suiteHooksEnabled = hooksEnabled(suiteWorkspace);
     const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
-    if (sharedWorkspacePath && hasHookCommand(suiteBeforeAllHook)) {
+    if (sharedWorkspacePath && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
       const beforeAllHook = suiteBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
       setupLog(
@@ -47347,7 +47400,7 @@ async function runEvaluation(options) {
         throw new Error(`before_all script failed: ${message}`);
       }
     }
-    if (availablePoolSlots.length > 0 && hasHookCommand(suiteBeforeAllHook)) {
+    if (availablePoolSlots.length > 0 && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
       const beforeAllHook = suiteBeforeAllHook;
       for (const slot of availablePoolSlots) {
         setupLog(`running before_all on pool slot ${slot.index}`);
@@ -47589,7 +47642,7 @@ async function runEvaluation(options) {
     }
     const afterAllWorkspaces = poolSlots.length > 1 ? poolSlots.map((s) => s.path) : sharedWorkspacePath ? [sharedWorkspacePath] : [];
     const suiteAfterAllHook = suiteWorkspace?.hooks?.after_all;
-    if (afterAllWorkspaces.length > 0 && hasHookCommand(suiteAfterAllHook)) {
+    if (afterAllWorkspaces.length > 0 && suiteHooksEnabled && hasHookCommand(suiteAfterAllHook)) {
       const afterAllHook = suiteAfterAllHook;
       for (const wsPath of afterAllWorkspaces) {
         const scriptContext = {
@@ -47837,6 +47890,7 @@ async function runEvalCase(options) {
   let afterEachOutput;
   const isSharedWorkspace = !!sharedWorkspacePath;
   let caseWorkspaceFile;
+  const caseHooksEnabled = hooksEnabled(evalCase.workspace);
   if (!workspacePath) {
     const rawCaseTemplate = evalCase.workspace?.template ?? getWorkspaceTemplate(target);
     const resolvedCaseTemplate = await resolveWorkspaceTemplate(rawCaseTemplate);
@@ -47898,7 +47952,7 @@ async function runEvalCase(options) {
       }
     }
     const caseBeforeAllHook = evalCase.workspace?.hooks?.before_all;
-    if (workspacePath && hasHookCommand(caseBeforeAllHook)) {
+    if (workspacePath && caseHooksEnabled && hasHookCommand(caseBeforeAllHook)) {
       const beforeAllHook = caseBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
       if (setupDebug) {
@@ -47942,7 +47996,7 @@ async function runEvalCase(options) {
     }
   }
   const caseBeforeEachHook = evalCase.workspace?.hooks?.before_each;
-  if (workspacePath && hasHookCommand(caseBeforeEachHook)) {
+  if (workspacePath && caseHooksEnabled && hasHookCommand(caseBeforeEachHook)) {
     const beforeEachHook = caseBeforeEachHook;
     const scriptContext = {
       workspacePath,
@@ -48071,7 +48125,7 @@ async function runEvalCase(options) {
     }
   }
   const providerError = extractProviderError(providerResponse);
-  if (repoManager && workspacePath && evalCase.workspace?.hooks?.after_each?.reset && evalCase.workspace.hooks.after_each.reset !== "none" && evalCase.workspace.repos) {
+  if (caseHooksEnabled && repoManager && workspacePath && evalCase.workspace?.hooks?.after_each?.reset && evalCase.workspace.hooks.after_each.reset !== "none" && evalCase.workspace.repos) {
     try {
       await repoManager.reset(
         evalCase.workspace.repos,
@@ -48082,7 +48136,7 @@ async function runEvalCase(options) {
     }
   }
   const caseAfterEachHook = evalCase.workspace?.hooks?.after_each;
-  if (workspacePath && hasHookCommand(caseAfterEachHook)) {
+  if (workspacePath && caseHooksEnabled && hasHookCommand(caseAfterEachHook)) {
     const afterEachHook = caseAfterEachHook;
     const scriptContext = {
       workspacePath,
@@ -48857,7 +48911,7 @@ async function evaluate(config2) {
     repoRoot,
     target: resolvedTarget,
     maxRetries: config2.maxRetries ?? 2,
-    agentTimeoutMs: config2.agentTimeoutMs ?? 12e4,
+    agentTimeoutMs: config2.agentTimeoutMs,
     verbose: config2.verbose,
     maxConcurrency: config2.workers ?? 3,
     filter: config2.filter,
@@ -48957,7 +49011,7 @@ var AgentVConfigSchema = external_exports.object({
     workers: external_exports.number().int().min(1).max(50).optional(),
     /** Maximum retries on failure (default: 2) */
     maxRetries: external_exports.number().int().min(0).optional(),
-    /** Agent timeout in milliseconds (default: 120000) */
+    /** Agent timeout in milliseconds. No timeout if not set. */
     agentTimeoutMs: external_exports.number().int().min(0).optional(),
     /** Enable verbose logging */
     verbose: external_exports.boolean().optional(),
@@ -49711,4 +49765,4 @@ export {
   OtelStreamingObserver,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-JSOG3FT6.js.map
+//# sourceMappingURL=chunk-ID5SDIYE.js.map