@agentv/core 2.18.1 → 2.18.2

package/dist/index.d.cts

@@ -528,6 +528,8 @@ type WorkspaceHookConfig = {
     readonly reset?: 'none' | 'fast' | 'strict';
 };
 type WorkspaceHooksConfig = {
+    /** Whether hooks are enabled (default: true). When false, all hooks are skipped. */
+    readonly enabled?: boolean;
     /** Runs once before first test in the workspace lifecycle */
     readonly before_all?: WorkspaceHookConfig;
     /** Runs before each test case */
@@ -2599,7 +2601,7 @@ interface EvalConfig {
     readonly workers?: number;
     /** Maximum retries on failure (default: 2) */
     readonly maxRetries?: number;
-    /** Agent timeout in milliseconds (default: 120000) */
+    /** Agent timeout in milliseconds. No timeout if not set. */
     readonly agentTimeoutMs?: number;
     /** Enable response caching */
     readonly cache?: boolean;
@@ -2706,7 +2708,7 @@ declare const AgentVConfigSchema: z.ZodObject<{
         workers: z.ZodOptional<z.ZodNumber>;
         /** Maximum retries on failure (default: 2) */
         maxRetries: z.ZodOptional<z.ZodNumber>;
-        /** Agent timeout in milliseconds (default: 120000) */
+        /** Agent timeout in milliseconds. No timeout if not set. */
         agentTimeoutMs: z.ZodOptional<z.ZodNumber>;
         /** Enable verbose logging */
         verbose: z.ZodOptional<z.ZodBoolean>;

package/dist/index.d.ts

@@ -528,6 +528,8 @@ type WorkspaceHookConfig = {
     readonly reset?: 'none' | 'fast' | 'strict';
 };
 type WorkspaceHooksConfig = {
+    /** Whether hooks are enabled (default: true). When false, all hooks are skipped. */
+    readonly enabled?: boolean;
     /** Runs once before first test in the workspace lifecycle */
     readonly before_all?: WorkspaceHookConfig;
     /** Runs before each test case */
@@ -2599,7 +2601,7 @@ interface EvalConfig {
     readonly workers?: number;
     /** Maximum retries on failure (default: 2) */
     readonly maxRetries?: number;
-    /** Agent timeout in milliseconds (default: 120000) */
+    /** Agent timeout in milliseconds. No timeout if not set. */
     readonly agentTimeoutMs?: number;
     /** Enable response caching */
     readonly cache?: boolean;
@@ -2706,7 +2708,7 @@ declare const AgentVConfigSchema: z.ZodObject<{
         workers: z.ZodOptional<z.ZodNumber>;
         /** Maximum retries on failure (default: 2) */
         maxRetries: z.ZodOptional<z.ZodNumber>;
-        /** Agent timeout in milliseconds (default: 120000) */
+        /** Agent timeout in milliseconds. No timeout if not set. */
         agentTimeoutMs: z.ZodOptional<z.ZodNumber>;
         /** Enable verbose logging */
         verbose: z.ZodOptional<z.ZodBoolean>;

package/dist/index.js

@@ -17,7 +17,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-I4VQY3XJ.js";
+} from "./chunk-V42NUK73.js";
 import {
   OtlpJsonFileExporter
 } from "./chunk-HFSYZHGF.js";
@@ -3152,11 +3152,13 @@ function parseWorkspaceHookConfig(raw, evalFileDir) {
 function parseWorkspaceHooksConfig(raw, evalFileDir) {
   if (!isJsonObject(raw)) return void 0;
   const obj = raw;
+  const enabled = typeof obj.enabled === "boolean" ? obj.enabled : void 0;
   const beforeAll = parseWorkspaceHookConfig(obj.before_all, evalFileDir);
   const beforeEach = parseWorkspaceHookConfig(obj.before_each, evalFileDir);
   const afterEach = parseWorkspaceHookConfig(obj.after_each, evalFileDir);
   const afterAll = parseWorkspaceHookConfig(obj.after_all, evalFileDir);
   const hooks = {
+    ...enabled !== void 0 && { enabled },
     ...beforeAll !== void 0 && { before_all: beforeAll },
     ...beforeEach !== void 0 && { before_each: beforeEach },
     ...afterEach !== void 0 && { after_each: afterEach },
@@ -3229,13 +3231,15 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
       ...caseHook ?? {}
     };
   };
+  const mergedEnabled = caseLevel.hooks?.enabled ?? suiteLevel.hooks?.enabled;
   const mergedHooks = {
+    ...mergedEnabled !== void 0 && { enabled: mergedEnabled },
     before_all: mergeHook(suiteLevel.hooks?.before_all, caseLevel.hooks?.before_all),
     before_each: mergeHook(suiteLevel.hooks?.before_each, caseLevel.hooks?.before_each),
     after_each: mergeHook(suiteLevel.hooks?.after_each, caseLevel.hooks?.after_each),
     after_all: mergeHook(suiteLevel.hooks?.after_all, caseLevel.hooks?.after_all)
   };
-  const hasHooks = Object.values(mergedHooks).some((hook) => hook !== void 0);
+  const hasHooks = mergedEnabled !== void 0 || Object.values(mergedHooks).some((hook) => hook !== void 0 && typeof hook === "object");
   return {
     template: caseLevel.template ?? suiteLevel.template,
     isolation: caseLevel.isolation ?? suiteLevel.isolation,
@@ -6181,14 +6185,18 @@ var PiAgentSdkProvider = class {
       }
     });
     try {
-      const timeoutMs = this.config.timeoutMs ?? 12e4;
-      const timeoutPromise = new Promise((_, reject) => {
-        setTimeout(
-          () => reject(new Error(`Pi agent SDK timed out after ${timeoutMs}ms`)),
-          timeoutMs
-        );
-      });
-      await Promise.race([agent.prompt(request.question), timeoutPromise]);
+      if (this.config.timeoutMs) {
+        const timeoutMs = this.config.timeoutMs;
+        const timeoutPromise = new Promise((_, reject) => {
+          setTimeout(
+            () => reject(new Error(`Pi agent SDK timed out after ${timeoutMs}ms`)),
+            timeoutMs
+          );
+        });
+        await Promise.race([agent.prompt(request.question), timeoutPromise]);
+      } else {
+        await agent.prompt(request.question);
+      }
       await agent.waitForIdle();
       const agentMessages = agent.state.messages;
       for (const msg of agentMessages) {
@@ -12028,7 +12036,7 @@ function runEqualsAssertion(output, value) {
 
 // src/evaluation/orchestrator.ts
 import { createHash as createHash2, randomUUID as randomUUID7 } from "node:crypto";
-import { mkdir as mkdir12, stat as stat7 } from "node:fs/promises";
+import { mkdir as mkdir12, readdir as readdir6, stat as stat7 } from "node:fs/promises";
 import path39 from "node:path";
 import micromatch4 from "micromatch";
 
@@ -13428,6 +13436,9 @@ function toScriptConfig(hook, hookName, context) {
 function hasHookCommand(hook) {
   return !!(hook?.command && hook.command.length > 0 || hook?.script && hook.script.length > 0);
 }
+function hooksEnabled(workspace) {
+  return workspace?.hooks?.enabled !== false;
+}
 function getWorkspaceTemplate(target) {
   const config = target.config;
   if ("workspaceTemplate" in config && typeof config.workspaceTemplate === "string") {
@@ -13646,9 +13657,28 @@ async function runEvaluation(options) {
   const availablePoolSlots = [];
   const poolSlotBaselines = /* @__PURE__ */ new Map();
   const poolMaxSlots = Math.min(configPoolMaxSlots ?? 10, 50);
+  let staticMaterialised = false;
   if (useStaticWorkspace && configuredStaticPath) {
+    const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
+    const dirExists = await stat7(configuredStaticPath).then(
+      (s) => s.isDirectory(),
+      () => false
+    );
+    const isEmpty = dirExists ? (await readdir6(configuredStaticPath)).length === 0 : false;
+    if (isYamlConfiguredPath && (!dirExists || isEmpty)) {
+      if (!dirExists) {
+        await mkdir12(configuredStaticPath, { recursive: true });
+      }
+      if (workspaceTemplate) {
+        await copyDirectoryRecursive(workspaceTemplate, configuredStaticPath);
+        setupLog(`copied template into static workspace: ${configuredStaticPath}`);
+      }
+      staticMaterialised = true;
+      setupLog(`materialised static workspace at: ${configuredStaticPath}`);
+    } else {
+      setupLog(`reusing existing static workspace: ${configuredStaticPath}`);
+    }
     sharedWorkspacePath = configuredStaticPath;
-    setupLog(`using static workspace: ${configuredStaticPath}`);
   } else if (usePool && suiteWorkspace?.repos) {
     const slotsNeeded = workers;
     setupLog(`acquiring ${slotsNeeded} workspace pool slot(s) (pool capacity: ${poolMaxSlots})`);
@@ -13694,7 +13724,8 @@ async function runEvaluation(options) {
       } catch {
       }
     }
-    const repoManager = suiteWorkspace?.repos?.length && !usePool && !useStaticWorkspace ? new RepoManager(verbose) : void 0;
+    const needsRepoMaterialisation = !!suiteWorkspace?.repos?.length && !usePool && (!useStaticWorkspace || staticMaterialised);
+    const repoManager = needsRepoMaterialisation ? new RepoManager(verbose) : void 0;
     if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
       setupLog(
         `materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`
@@ -13711,8 +13742,9 @@ async function runEvaluation(options) {
         throw new Error(`Failed to materialize repos: ${message}`);
       }
     }
+    const suiteHooksEnabled = hooksEnabled(suiteWorkspace);
     const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
-    if (sharedWorkspacePath && hasHookCommand(suiteBeforeAllHook)) {
+    if (sharedWorkspacePath && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
       const beforeAllHook = suiteBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
       setupLog(
@@ -13739,7 +13771,7 @@ async function runEvaluation(options) {
         throw new Error(`before_all script failed: ${message}`);
       }
     }
-    if (availablePoolSlots.length > 0 && hasHookCommand(suiteBeforeAllHook)) {
+    if (availablePoolSlots.length > 0 && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
       const beforeAllHook = suiteBeforeAllHook;
       for (const slot of availablePoolSlots) {
         setupLog(`running before_all on pool slot ${slot.index}`);
@@ -13981,7 +14013,7 @@ async function runEvaluation(options) {
     }
     const afterAllWorkspaces = poolSlots.length > 1 ? poolSlots.map((s) => s.path) : sharedWorkspacePath ? [sharedWorkspacePath] : [];
     const suiteAfterAllHook = suiteWorkspace?.hooks?.after_all;
-    if (afterAllWorkspaces.length > 0 && hasHookCommand(suiteAfterAllHook)) {
+    if (afterAllWorkspaces.length > 0 && suiteHooksEnabled && hasHookCommand(suiteAfterAllHook)) {
       const afterAllHook = suiteAfterAllHook;
       for (const wsPath of afterAllWorkspaces) {
         const scriptContext = {
@@ -14229,6 +14261,7 @@ async function runEvalCase(options) {
   let afterEachOutput;
   const isSharedWorkspace = !!sharedWorkspacePath;
   let caseWorkspaceFile;
+  const caseHooksEnabled = hooksEnabled(evalCase.workspace);
   if (!workspacePath) {
     const rawCaseTemplate = evalCase.workspace?.template ?? getWorkspaceTemplate(target);
     const resolvedCaseTemplate = await resolveWorkspaceTemplate(rawCaseTemplate);
@@ -14290,7 +14323,7 @@ async function runEvalCase(options) {
       }
     }
     const caseBeforeAllHook = evalCase.workspace?.hooks?.before_all;
-    if (workspacePath && hasHookCommand(caseBeforeAllHook)) {
+    if (workspacePath && caseHooksEnabled && hasHookCommand(caseBeforeAllHook)) {
       const beforeAllHook = caseBeforeAllHook;
       const beforeAllCommand = (beforeAllHook.command ?? beforeAllHook.script ?? []).join(" ");
       if (setupDebug) {
@@ -14334,7 +14367,7 @@ async function runEvalCase(options) {
     }
   }
   const caseBeforeEachHook = evalCase.workspace?.hooks?.before_each;
-  if (workspacePath && hasHookCommand(caseBeforeEachHook)) {
+  if (workspacePath && caseHooksEnabled && hasHookCommand(caseBeforeEachHook)) {
     const beforeEachHook = caseBeforeEachHook;
     const scriptContext = {
       workspacePath,
@@ -14463,7 +14496,7 @@ async function runEvalCase(options) {
     }
   }
   const providerError = extractProviderError(providerResponse);
-  if (repoManager && workspacePath && evalCase.workspace?.hooks?.after_each?.reset && evalCase.workspace.hooks.after_each.reset !== "none" && evalCase.workspace.repos) {
+  if (caseHooksEnabled && repoManager && workspacePath && evalCase.workspace?.hooks?.after_each?.reset && evalCase.workspace.hooks.after_each.reset !== "none" && evalCase.workspace.repos) {
     try {
       await repoManager.reset(
         evalCase.workspace.repos,
@@ -14474,7 +14507,7 @@ async function runEvalCase(options) {
     }
   }
   const caseAfterEachHook = evalCase.workspace?.hooks?.after_each;
-  if (workspacePath && hasHookCommand(caseAfterEachHook)) {
+  if (workspacePath && caseHooksEnabled && hasHookCommand(caseAfterEachHook)) {
     const afterEachHook = caseAfterEachHook;
     const scriptContext = {
       workspacePath,
@@ -15253,7 +15286,7 @@ async function evaluate(config) {
     repoRoot,
     target: resolvedTarget,
     maxRetries: config.maxRetries ?? 2,
-    agentTimeoutMs: config.agentTimeoutMs ?? 12e4,
+    agentTimeoutMs: config.agentTimeoutMs,
     verbose: config.verbose,
     maxConcurrency: config.workers ?? 3,
     filter: config.filter,
@@ -15356,7 +15389,7 @@ var AgentVConfigSchema = z5.object({
     workers: z5.number().int().min(1).max(50).optional(),
     /** Maximum retries on failure (default: 2) */
     maxRetries: z5.number().int().min(0).optional(),
-    /** Agent timeout in milliseconds (default: 120000) */
+    /** Agent timeout in milliseconds. No timeout if not set. */
     agentTimeoutMs: z5.number().int().min(0).optional(),
     /** Enable verbose logging */
     verbose: z5.boolean().optional(),