@agentv/core 1.2.0 → 1.3.1

package/dist/index.d.cts

@@ -105,7 +105,7 @@ type TestMessageRole = (typeof TEST_MESSAGE_ROLE_VALUES)[number];
 /**
  * Text or structured payload attached to a message.
  */
-type TestMessageContent = string | readonly JsonObject[];
+type TestMessageContent = string | JsonObject | readonly JsonObject[];
 /**
  * System-authored instruction message.
  */

package/dist/index.d.ts

@@ -105,7 +105,7 @@ type TestMessageRole = (typeof TEST_MESSAGE_ROLE_VALUES)[number];
 /**
  * Text or structured payload attached to a message.
  */
-type TestMessageContent = string | readonly JsonObject[];
+type TestMessageContent = string | JsonObject | readonly JsonObject[];
 /**
  * System-authored instruction message.
  */

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   readTextFile,
   resolveFileReference,
   resolveTargetDefinition
-} from "./chunk-V3JCB3HI.js";
+} from "./chunk-4A6L2F6L.js";
 
 // src/evaluation/types.ts
 var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
@@ -774,6 +774,17 @@ async function processMessages(options) {
       }
       continue;
     }
+    if (isJsonObject(content)) {
+      const rendered = JSON.stringify(content, null, 2);
+      segments.push({ type: "text", value: rendered });
+      if (textParts) {
+        textParts.push(rendered);
+      }
+      continue;
+    }
+    if (!Array.isArray(content)) {
+      continue;
+    }
     for (const rawSegment of content) {
       if (!isJsonObject(rawSegment)) {
         continue;
@@ -1000,6 +1011,11 @@ async function buildPromptInputs(testCase, mode = "lm") {
           }
         }
       }
+    } else if (isJsonObject(message.content)) {
+      const rendered = JSON.stringify(message.content, null, 2);
+      if (rendered.trim().length > 0) {
+        messageSegments.push({ type: "text", value: rendered });
+      }
     }
     segmentsByMessage.push(messageSegments);
   }
@@ -1733,7 +1749,7 @@ var CliProvider = class {
   id;
   kind = "cli";
   targetName;
-  supportsBatch = false;
+  supportsBatch = true;
   config;
   runCommand;
   verbose;
@@ -1753,6 +1769,11 @@ var CliProvider = class {
     const outputFilePath = generateOutputFilePath(request.evalCaseId);
     const templateValues = buildTemplateValues(request, this.config, outputFilePath);
     const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
     const result = await this.runCommand(renderedCommand, {
       cwd: this.config.cwd,
       env: process.env,
@@ -1787,6 +1808,114 @@ var CliProvider = class {
       }
     };
   }
+  async invokeBatch(requests) {
+    if (requests.length === 0) {
+      return [];
+    }
+    for (const request of requests) {
+      if (request.signal?.aborted) {
+        throw new Error("CLI provider batch request was aborted before execution");
+      }
+    }
+    const controller = new AbortController();
+    for (const request of requests) {
+      request.signal?.addEventListener("abort", () => controller.abort(), { once: true });
+    }
+    await this.ensureHealthy(controller.signal);
+    const outputFilePath = generateOutputFilePath("batch", ".jsonl");
+    const batchInputFiles = [];
+    for (const request of requests) {
+      if (request.inputFiles && request.inputFiles.length > 0) {
+        batchInputFiles.push(...request.inputFiles);
+      }
+    }
+    const templateValues = buildTemplateValues(
+      {
+        question: "",
+        guidelines: "",
+        inputFiles: batchInputFiles,
+        evalCaseId: "batch",
+        attempt: 0
+      },
+      this.config,
+      outputFilePath
+    );
+    const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
+    if (this.verbose) {
+      console.log(
+        `[cli-provider:${this.targetName}] (batch size=${requests.length}) cwd=${this.config.cwd ?? ""} command=${renderedCommand}`
+      );
+    }
+    const result = await this.runCommand(renderedCommand, {
+      cwd: this.config.cwd,
+      env: process.env,
+      timeoutMs: this.config.timeoutMs,
+      signal: controller.signal
+    });
+    if (result.failed || (result.exitCode ?? 0) !== 0) {
+      if (controller.signal.aborted) {
+        throw new Error("CLI provider request was aborted");
+      }
+      if (result.timedOut) {
+        throw new Error(
+          `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
+        );
+      }
+      const codeText = result.exitCode !== null ? result.exitCode : "unknown";
+      const detail = result.stderr.trim() || result.stdout.trim();
+      const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
+      throw new Error(message);
+    }
+    const responseContent = await this.readAndCleanupOutputFile(outputFilePath);
+    const recordsById = this.parseJsonlBatchOutput(responseContent);
+    const requestedIds = requests.map((request) => request.evalCaseId).filter((id) => typeof id === "string" && id.trim().length > 0);
+    const missingIds = requestedIds.filter((id) => !recordsById.has(id));
+    if (missingIds.length > 0) {
+      throw new Error(`CLI batch output missing ids: ${missingIds.join(", ")}`);
+    }
+    const responses = requests.map((request) => {
+      const evalCaseId = request.evalCaseId;
+      if (!evalCaseId) {
+        return {
+          text: "",
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      const parsed = recordsById.get(evalCaseId);
+      if (!parsed) {
+        return {
+          text: "",
+          raw: {
+            command: renderedCommand,
+            stderr: result.stderr,
+            exitCode: result.exitCode ?? 0,
+            cwd: this.config.cwd,
+            outputFile: outputFilePath
+          }
+        };
+      }
+      return {
+        text: parsed.text,
+        trace: parsed.trace,
+        traceRef: parsed.traceRef,
+        raw: {
+          command: renderedCommand,
+          stderr: result.stderr,
+          exitCode: result.exitCode ?? 0,
+          cwd: this.config.cwd,
+          outputFile: outputFilePath,
+          recordId: evalCaseId
+        }
+      };
+    });
+    return responses;
+  }
   /**
    * Parse output content from CLI.
    * If the content is valid JSON with a 'text' field, extract text and optional trace.
@@ -1812,6 +1941,38 @@ var CliProvider = class {
     const validEvents = trace.filter(isTraceEvent);
     return validEvents.length > 0 ? validEvents : void 0;
   }
+  parseJsonlBatchOutput(content) {
+    const records = /* @__PURE__ */ new Map();
+    const lines = content.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
+    for (const line of lines) {
+      let parsed;
+      try {
+        parsed = JSON.parse(line);
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`CLI batch output contains invalid JSONL line: ${reason}`);
+      }
+      if (typeof parsed !== "object" || parsed === null) {
+        throw new Error("CLI batch output JSONL line must be an object");
+      }
+      const obj = parsed;
+      const id = typeof obj.id === "string" ? obj.id : void 0;
+      if (!id || id.trim().length === 0) {
+        throw new Error("CLI batch output JSONL line missing required string field: id");
+      }
+      if (records.has(id)) {
+        throw new Error(`CLI batch output contains duplicate id: ${id}`);
+      }
+      const text = typeof obj.text === "string" ? obj.text : obj.text === void 0 ? "" : JSON.stringify(obj.text);
+      const traceRef = typeof obj.traceRef === "string" ? obj.traceRef : typeof obj.trace_ref === "string" ? obj.trace_ref : void 0;
+      records.set(id, {
+        text,
+        trace: this.parseTrace(obj.trace),
+        traceRef
+      });
+    }
+    return records;
+  }
   async readAndCleanupOutputFile(filePath) {
     try {
       const content = await readTextFile(filePath);
@@ -1873,7 +2034,7 @@ var CliProvider = class {
     );
     if (this.verbose) {
       console.log(
-        `[cli-provider:${this.targetName}] (healthcheck) CLI_EVALS_DIR=${process.env.CLI_EVALS_DIR ?? ""} cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
+        `[cli-provider:${this.targetName}] (healthcheck) cwd=${healthcheck.cwd ?? this.config.cwd ?? ""} command=${renderedCommand}`
       );
     }
     const result = await this.runCommand(renderedCommand, {
@@ -1941,11 +2102,11 @@ function shellEscape(value) {
   }
   return `'${value.replace(/'/g, `'"'"'`)}'`;
 }
-function generateOutputFilePath(evalCaseId) {
+function generateOutputFilePath(evalCaseId, extension = ".json") {
   const safeEvalId = evalCaseId || "unknown";
   const timestamp = Date.now();
   const random = Math.random().toString(36).substring(2, 9);
-  return path7.join(os.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}.json`);
+  return path7.join(os.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
 }
 function formatTimeoutSuffix(timeoutMs) {
   if (!timeoutMs || timeoutMs <= 0) {
@@ -3489,6 +3650,7 @@ var CodeEvaluator = class {
       {
         question: context.evalCase.question,
         expected_outcome: context.evalCase.expected_outcome,
+        expected_messages: context.evalCase.expected_messages,
         reference_answer: context.evalCase.reference_answer,
         candidate_answer: context.candidate,
         guideline_files: context.evalCase.guideline_paths,