agentv 5.3.1-next.1 → 5.3.3

package/dist/chunk-AQ5BIAXF.js

@@ -0,0 +1,604 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+import {
+  external_exports,
+  toCamelCaseDeep
+} from "./chunk-6262OKXM.js";
+
+// ../../packages/core/dist/script-grader.js
+import { readFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { copyFile, mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import nodePath from "node:path";
+var TokenUsageSchema = external_exports.object({
+  input: external_exports.number(),
+  output: external_exports.number(),
+  cached: external_exports.number().optional(),
+  reasoning: external_exports.number().optional()
+});
+var TraceSummarySchema = external_exports.object({
+  eventCount: external_exports.number(),
+  toolCalls: external_exports.record(external_exports.string(), external_exports.number()),
+  errorCount: external_exports.number(),
+  toolDurations: external_exports.record(external_exports.string(), external_exports.array(external_exports.number())).optional(),
+  llmCallCount: external_exports.number().optional()
+});
+var TRACE_SOURCE_KINDS = [
+  "agentv_run",
+  "otlp",
+  "phoenix",
+  "langfuse",
+  "pi_session",
+  "imported_transcript",
+  "compact_transcript"
+];
+var TRACE_EVENT_TYPES = [
+  "message",
+  "model_turn",
+  "tool_call",
+  "tool_result",
+  "final_response",
+  "error"
+];
+var TRACE_TOOL_STATUSES = ["ok", "error", "timeout", "cancelled", "unknown"];
+var TRACE_REDACTION_LEVELS = ["none", "partial", "full"];
+var MetadataSchema = external_exports.record(external_exports.string(), external_exports.unknown());
+var TraceRedactionStateSchema = external_exports.object({
+  level: external_exports.enum(TRACE_REDACTION_LEVELS),
+  fields: external_exports.array(external_exports.string()).optional(),
+  reason: external_exports.string().optional()
+});
+var TraceErrorSchema = external_exports.object({
+  message: external_exports.string(),
+  name: external_exports.string().optional(),
+  code: external_exports.string().optional(),
+  stack: external_exports.string().optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceSourceSchema = external_exports.object({
+  kind: external_exports.enum(TRACE_SOURCE_KINDS),
+  path: external_exports.string().optional(),
+  url: external_exports.string().optional(),
+  provider: external_exports.string().optional(),
+  format: external_exports.string().optional(),
+  version: external_exports.string().optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceSessionSchema = external_exports.object({
+  sessionId: external_exports.string().optional(),
+  conversationId: external_exports.string().optional(),
+  cwd: external_exports.string().optional(),
+  startedAt: external_exports.string().optional(),
+  endedAt: external_exports.string().optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceBranchSchema = external_exports.object({
+  selectedLeafId: external_exports.string().optional(),
+  selectedPathIds: external_exports.array(external_exports.string()).optional(),
+  includedEventIds: external_exports.array(external_exports.string()).optional(),
+  omittedEventIds: external_exports.array(external_exports.string()).optional(),
+  selectionReason: external_exports.string().optional()
+});
+var TraceSourceRefSchema = external_exports.object({
+  eventId: external_exports.string().optional(),
+  messageId: external_exports.string().optional(),
+  spanId: external_exports.string().optional(),
+  traceId: external_exports.string().optional(),
+  rawKind: external_exports.string().optional(),
+  path: external_exports.string().optional(),
+  line: external_exports.number().int().nonnegative().optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceRawEvidenceSchema = external_exports.object({
+  kind: external_exports.string(),
+  ref: external_exports.string().optional(),
+  mediaType: external_exports.string().optional(),
+  content: external_exports.unknown().optional(),
+  redacted: external_exports.boolean().optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceMessageSchema = external_exports.object({
+  role: external_exports.string(),
+  name: external_exports.string().optional(),
+  content: external_exports.unknown().optional(),
+  redaction: TraceRedactionStateSchema.optional(),
+  tokenUsage: TokenUsageSchema.optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceModelSchema = external_exports.object({
+  provider: external_exports.string().optional(),
+  name: external_exports.string().optional(),
+  invocationId: external_exports.string().optional(),
+  tokenUsage: TokenUsageSchema.optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceToolSchema = external_exports.object({
+  name: external_exports.string(),
+  callId: external_exports.string().optional(),
+  input: external_exports.unknown().optional(),
+  output: external_exports.unknown().optional(),
+  status: external_exports.enum(TRACE_TOOL_STATUSES).optional(),
+  error: TraceErrorSchema.optional(),
+  redaction: TraceRedactionStateSchema.optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceEventSchema = external_exports.object({
+  eventId: external_exports.string(),
+  parentEventId: external_exports.string().optional(),
+  ordinal: external_exports.number().int().nonnegative(),
+  type: external_exports.enum(TRACE_EVENT_TYPES),
+  timestamp: external_exports.string().optional(),
+  durationMs: external_exports.number().nonnegative().optional(),
+  durationInferred: external_exports.boolean().optional(),
+  turnIndex: external_exports.number().int().nonnegative().optional(),
+  message: TraceMessageSchema.optional(),
+  model: TraceModelSchema.optional(),
+  tool: TraceToolSchema.optional(),
+  error: TraceErrorSchema.optional(),
+  sourceRef: TraceSourceRefSchema.optional(),
+  rawEvidence: external_exports.array(TraceRawEvidenceSchema).optional(),
+  redaction: TraceRedactionStateSchema.optional(),
+  metadata: MetadataSchema.optional()
+});
+var TraceArtifactSchema = external_exports.object({
+  source: TraceSourceSchema,
+  session: TraceSessionSchema,
+  branch: TraceBranchSchema.optional(),
+  events: external_exports.array(TraceEventSchema),
+  tokenUsage: TokenUsageSchema.optional(),
+  costUsd: external_exports.number().optional(),
+  durationMs: external_exports.number().optional(),
+  startedAt: external_exports.string().optional(),
+  endedAt: external_exports.string().optional(),
+  metadata: MetadataSchema.optional()
+});
+var ToolCallSchema = external_exports.object({
+  tool: external_exports.string(),
+  input: external_exports.unknown().optional(),
+  output: external_exports.unknown().optional(),
+  id: external_exports.string().optional(),
+  startTime: external_exports.string().optional(),
+  endTime: external_exports.string().optional(),
+  durationMs: external_exports.number().optional()
+});
+var ContentTextSchema = external_exports.object({
+  type: external_exports.literal("text"),
+  text: external_exports.string()
+});
+var ContentImageSchema = external_exports.object({
+  type: external_exports.literal("image"),
+  media_type: external_exports.string(),
+  path: external_exports.string()
+});
+var ContentFileSchema = external_exports.object({
+  type: external_exports.literal("file"),
+  media_type: external_exports.string(),
+  path: external_exports.string()
+});
+var ContentSchema = external_exports.discriminatedUnion("type", [
+  ContentTextSchema,
+  ContentImageSchema,
+  ContentFileSchema
+]);
+var MessageContentBlockSchema = external_exports.union([ContentSchema, external_exports.record(external_exports.unknown())]);
+var MessageSchema = external_exports.object({
+  role: external_exports.enum(["assistant", "user", "system", "tool"]),
+  content: external_exports.union([external_exports.string(), external_exports.array(MessageContentBlockSchema), external_exports.record(external_exports.unknown())]).optional(),
+  toolCalls: external_exports.array(ToolCallSchema).optional(),
+  name: external_exports.string().optional(),
+  startTime: external_exports.string().optional(),
+  endTime: external_exports.string().optional(),
+  durationMs: external_exports.number().optional(),
+  metadata: external_exports.record(external_exports.unknown()).optional()
+});
+var TraceSchema = TraceSummarySchema.extend({
+  messages: external_exports.array(MessageSchema),
+  events: external_exports.array(TraceEventSchema),
+  tokenUsage: TokenUsageSchema.optional(),
+  costUsd: external_exports.number().optional(),
+  durationMs: external_exports.number().optional(),
+  startTime: external_exports.string().optional(),
+  endTime: external_exports.string().optional(),
+  metadata: MetadataSchema.optional()
+});
+var ScriptGraderInputSchema = external_exports.object({
+  criteria: external_exports.string(),
+  expectedOutput: external_exports.array(MessageSchema),
+  output: external_exports.string().nullable().optional(),
+  messages: external_exports.array(MessageSchema).optional().default([]),
+  /** Path to a temp file containing the output JSON (used for large payloads). */
+  outputPath: external_exports.string().optional(),
+  inputFiles: external_exports.array(external_exports.string()),
+  input: external_exports.array(MessageSchema),
+  metadata: external_exports.record(external_exports.unknown()).nullable().optional(),
+  trace: TraceSchema.nullable().optional(),
+  traceSummary: TraceSummarySchema.nullable().optional(),
+  tokenUsage: TokenUsageSchema.nullable().optional(),
+  costUsd: external_exports.number().nullable().optional(),
+  durationMs: external_exports.number().nullable().optional(),
+  startTime: external_exports.string().nullable().optional(),
+  endTime: external_exports.string().nullable().optional(),
+  fileChanges: external_exports.string().nullable().optional(),
+  workspacePath: external_exports.string().nullable().optional(),
+  config: external_exports.record(external_exports.unknown()).nullable().optional()
+});
+var ScriptGraderCheckSchema = external_exports.object({
+  id: external_exports.string().optional(),
+  text: external_exports.string(),
+  pass: external_exports.boolean(),
+  score: external_exports.number().min(0).max(1).optional(),
+  reason: external_exports.string(),
+  evidence: external_exports.string().optional()
+});
+var ScriptGraderResultSchema = external_exports.object({
+  pass: external_exports.boolean(),
+  score: external_exports.number().min(0).max(1),
+  reason: external_exports.string(),
+  checks: external_exports.array(ScriptGraderCheckSchema).optional().default([]),
+  /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */
+  details: external_exports.record(external_exports.unknown()).optional()
+});
+var PromptTemplateInputSchema = ScriptGraderInputSchema;
+var CodeGraderInputSchema = ScriptGraderInputSchema;
+var CodeGraderResultSchema = ScriptGraderResultSchema;
+function readStdin() {
+  return readFileSync(0, "utf8");
+}
+function clampScore(value) {
+  if (Number.isNaN(value) || !Number.isFinite(value)) {
+    return 0;
+  }
+  return Math.max(0, Math.min(1, value));
+}
+function formatError(error) {
+  if (error instanceof Error) {
+    return error.message;
+  }
+  return String(error);
+}
+async function runScriptGrader(handler) {
+  try {
+    const stdin = readStdin();
+    const rawInput = JSON.parse(stdin);
+    const camelInput = toCamelCaseDeep(rawInput);
+    const input = ScriptGraderInputSchema.parse(camelInput);
+    if (input.outputPath && (input.output === null || input.output === void 0)) {
+      let cachedOutput;
+      const filePath = input.outputPath;
+      Object.defineProperty(input, "output", {
+        get() {
+          if (cachedOutput === void 0) {
+            cachedOutput = JSON.parse(readFileSync(filePath, "utf8"));
+          }
+          return cachedOutput;
+        },
+        configurable: true,
+        enumerable: true
+      });
+    }
+    const rawResult = await handler(input);
+    const result = ScriptGraderResultSchema.parse({
+      ...rawResult,
+      score: clampScore(rawResult.score),
+      checks: rawResult.checks?.map((check) => ({
+        ...check,
+        ...check.score !== void 0 ? { score: clampScore(check.score) } : {}
+      }))
+    });
+    console.log(JSON.stringify(result, null, 2));
+  } catch (error) {
+    const errorMessage = formatError(error);
+    const errorResult = {
+      pass: false,
+      score: 0,
+      reason: `Evaluation failed: ${errorMessage}`,
+      checks: [
+        {
+          text: "Script grader execution",
+          pass: false,
+          reason: errorMessage
+        }
+      ]
+    };
+    console.log(JSON.stringify(errorResult, null, 2));
+    process.exit(1);
+  }
+}
+var runCodeGrader = runScriptGrader;
+function workspacePathFrom(input) {
+  const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH;
+  return workspacePath?.trim() ? workspacePath : void 0;
+}
+function truncate(value, maxLength = 2e3) {
+  if (value.length <= maxLength) {
+    return value;
+  }
+  return `${value.slice(0, maxLength)}
+...(truncated)`;
+}
+function resolveInsideWorkspace(workspacePath, relativePath, label, options = {}) {
+  if (!relativePath.trim()) {
+    throw new Error(`${label} must not be empty.`);
+  }
+  if (nodePath.isAbsolute(relativePath)) {
+    throw new Error(`${label} must be relative to the workspace: ${relativePath}`);
+  }
+  const root = nodePath.resolve(workspacePath);
+  const resolvedPath = nodePath.resolve(root, relativePath);
+  const relativeToRoot = nodePath.relative(root, resolvedPath);
+  if (relativeToRoot === "" && options.allowRoot === true) {
+    return resolvedPath;
+  }
+  if (relativeToRoot === "" || relativeToRoot.startsWith("..") || nodePath.isAbsolute(relativeToRoot)) {
+    throw new Error(`${label} must stay inside the workspace: ${relativePath}`);
+  }
+  return resolvedPath;
+}
+function normalizeTestFiles(testFile) {
+  if (testFile === void 0) {
+    return [];
+  }
+  return typeof testFile === "string" ? [testFile] : [...testFile];
+}
+function buildCommand(options, testFiles = normalizeTestFiles(options.testFile)) {
+  if (options.command && options.command.length > 0) {
+    return [...options.command];
+  }
+  const command = [...options.vitestCommand ?? ["bunx", "vitest", "run"]];
+  command.push(...testFiles);
+  return command;
+}
+function parseJsonObjectFromText(text) {
+  const trimmed = text.trim();
+  if (!trimmed) {
+    return void 0;
+  }
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const start = trimmed.indexOf("{");
+    const end = trimmed.lastIndexOf("}");
+    if (start >= 0 && end > start) {
+      return JSON.parse(trimmed.slice(start, end + 1));
+    }
+    throw new Error("Vitest output did not contain a JSON object.");
+  }
+}
+function isVitestJsonReport(value) {
+  return typeof value === "object" && value !== null && Array.isArray(value.testResults);
+}
+function assertionText(file, assertion) {
+  const title = assertion.fullName ?? [...assertion.ancestorTitles ?? [], assertion.title].filter(Boolean).join(" ");
+  return title || file.name || "Vitest assertion";
+}
+function vitestReportToScriptGraderResult(report, options = {}) {
+  const checks = (report.testResults ?? []).flatMap(
+    (file) => (file.assertionResults ?? []).map((item) => {
+      const pass2 = item.status === "passed";
+      const evidence = item.failureMessages && item.failureMessages.length > 0 ? truncate(item.failureMessages.join("\n\n")) : void 0;
+      return {
+        text: assertionText(file, item),
+        pass: pass2,
+        reason: pass2 ? "Vitest test passed." : "Vitest test failed.",
+        ...evidence !== void 0 ? { evidence } : {}
+      };
+    })
+  );
+  if (checks.length === 0) {
+    const pass2 = options.passWithNoTests === true;
+    return ScriptGraderResultSchema.parse({
+      pass: pass2,
+      score: pass2 ? 1 : 0,
+      reason: pass2 ? "Vitest reported no tests; configured to pass." : "Vitest reported no tests.",
+      checks: [
+        {
+          text: "Vitest reported no tests",
+          pass: pass2,
+          reason: pass2 ? "passWithNoTests is enabled." : "No Vitest tests were discovered."
+        }
+      ],
+      details: {
+        vitest_success: report.success ?? false,
+        num_total_tests: report.numTotalTests ?? 0,
+        num_passed_tests: report.numPassedTests ?? 0,
+        num_failed_tests: report.numFailedTests ?? 0,
+        num_pending_tests: report.numPendingTests ?? 0,
+        num_todo_tests: report.numTodoTests ?? 0
+      }
+    });
+  }
+  const passedCount = checks.filter((item) => item.pass).length;
+  const pass = passedCount === checks.length;
+  return ScriptGraderResultSchema.parse({
+    pass,
+    score: passedCount / checks.length,
+    reason: `${passedCount}/${checks.length} Vitest tests passed.`,
+    checks,
+    details: {
+      vitest_success: report.success ?? pass,
+      num_total_tests: report.numTotalTests ?? checks.length,
+      num_passed_tests: report.numPassedTests ?? passedCount,
+      num_failed_tests: report.numFailedTests ?? checks.length - passedCount,
+      num_pending_tests: report.numPendingTests ?? 0,
+      num_todo_tests: report.numTodoTests ?? 0
+    }
+  });
+}
+var vitestReportToCodeGraderResult = vitestReportToScriptGraderResult;
+function runCommand(command, options) {
+  if (command.length === 0) {
+    return Promise.reject(new Error("Vitest command must not be empty."));
+  }
+  return new Promise((resolve, reject) => {
+    const child = spawn(command[0], command.slice(1), {
+      cwd: options.cwd,
+      env: { ...process.env, ...options.env },
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const timeout = options.timeoutMs !== void 0 ? setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+    }, options.timeoutMs) : void 0;
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+      stderr += chunk;
+    });
+    child.on("error", reject);
+    child.on("close", (exitCode) => {
+      if (timeout) {
+        clearTimeout(timeout);
+      }
+      if (timedOut) {
+        reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`));
+        return;
+      }
+      resolve({ exitCode, stdout, stderr });
+    });
+  });
+}
+function resolveSourceTestFile(testFileRoot, testFile) {
+  if (!testFile.trim()) {
+    throw new Error("testFile entries must not be empty.");
+  }
+  return nodePath.isAbsolute(testFile) ? nodePath.resolve(testFile) : nodePath.resolve(testFileRoot, testFile);
+}
+async function copyTestFilesIntoWorkspace(testFiles, options, cwd) {
+  if (testFiles.length === 0) {
+    return { testFiles };
+  }
+  const tempDir = await mkdtemp(nodePath.join(cwd, ".agentv-vitest-"));
+  const testFileRoot = nodePath.resolve(options.testFileRoot ?? process.cwd());
+  const copiedFiles = await Promise.all(
+    testFiles.map(async (testFile, index) => {
+      const sourcePath = resolveSourceTestFile(testFileRoot, testFile);
+      const destinationPath = nodePath.join(tempDir, `${index}-${nodePath.basename(testFile)}`);
+      await copyFile(sourcePath, destinationPath);
+      return nodePath.relative(cwd, destinationPath);
+    })
+  );
+  return { testFiles: copiedFiles, tempDir };
+}
+async function readVitestReport(commandResult, outputFile) {
+  const rawReport = outputFile !== void 0 ? await readFile(outputFile, "utf8") : commandResult.stdout;
+  const parsed = parseJsonObjectFromText(rawReport);
+  if (!isVitestJsonReport(parsed)) {
+    throw new Error("Vitest JSON report did not include testResults[].");
+  }
+  return parsed;
+}
+async function runVitestWorkspaceGrader(options, input) {
+  const workspacePath = workspacePathFrom(input);
+  if (!workspacePath) {
+    return {
+      pass: false,
+      score: 0,
+      reason: "Vitest workspace verifier requires workspace_path.",
+      checks: [
+        {
+          text: "Vitest workspace verifier requires workspace_path",
+          pass: false,
+          reason: "Configure an environment recipe in the eval YAML so AgentV can pass workspace_path."
+        }
+      ]
+    };
+  }
+  const tempDirs = [];
+  try {
+    const cwd = options.cwd ? resolveInsideWorkspace(workspacePath, options.cwd, "cwd", { allowRoot: true }) : workspacePath;
+    const testFiles = normalizeTestFiles(options.testFile);
+    const preparedTestFiles = options.copyTestFilesToWorkspace === true ? await copyTestFilesIntoWorkspace(testFiles, options, cwd) : { testFiles };
+    if (preparedTestFiles.tempDir) {
+      tempDirs.push(preparedTestFiles.tempDir);
+    }
+    const command = buildCommand(options, preparedTestFiles.testFiles);
+    const appendReporterArgs = options.appendReporterArgs ?? options.command === void 0;
+    let outputFile = options.outputFile ? resolveInsideWorkspace(cwd, options.outputFile, "outputFile") : void 0;
+    if (appendReporterArgs) {
+      const tempDir = await mkdtemp(nodePath.join(tmpdir(), "agentv-vitest-"));
+      tempDirs.push(tempDir);
+      outputFile = nodePath.join(tempDir, "results.json");
+      command.push("--reporter=json", `--outputFile=${outputFile}`);
+    }
+    const result = await runCommand(command, {
+      cwd,
+      timeoutMs: options.timeoutMs,
+      env: {
+        AGENTV_WORKSPACE_PATH: workspacePath,
+        ...outputFile !== void 0 ? { AGENTV_VITEST_JSON_PATH: outputFile } : {},
+        ...options.env
+      }
+    });
+    const report = await readVitestReport(result, outputFile);
+    return vitestReportToScriptGraderResult(report, options);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    return {
+      pass: false,
+      score: 0,
+      reason: "Vitest workspace verifier failed to run.",
+      checks: [
+        {
+          text: "Vitest workspace verifier failed to run",
+          pass: false,
+          reason
+        }
+      ]
+    };
+  } finally {
+    for (const tempDir of tempDirs.reverse()) {
+      await rm(tempDir, { recursive: true, force: true }).catch(() => {
+      });
+    }
+  }
+}
+function defineVitestWorkspaceGrader(options) {
+  runScriptGrader((input) => runVitestWorkspaceGrader(options, input));
+}
+
+export {
+  TokenUsageSchema,
+  TraceSummarySchema,
+  TRACE_SOURCE_KINDS,
+  TRACE_EVENT_TYPES,
+  TRACE_TOOL_STATUSES,
+  TRACE_REDACTION_LEVELS,
+  TraceRedactionStateSchema,
+  TraceErrorSchema,
+  TraceSourceSchema,
+  TraceSessionSchema,
+  TraceBranchSchema,
+  TraceSourceRefSchema,
+  TraceRawEvidenceSchema,
+  TraceMessageSchema,
+  TraceModelSchema,
+  TraceToolSchema,
+  TraceEventSchema,
+  TraceArtifactSchema,
+  ToolCallSchema,
+  ContentTextSchema,
+  ContentImageSchema,
+  ContentFileSchema,
+  ContentSchema,
+  MessageSchema,
+  TraceSchema,
+  ScriptGraderInputSchema,
+  ScriptGraderCheckSchema,
+  ScriptGraderResultSchema,
+  PromptTemplateInputSchema,
+  CodeGraderInputSchema,
+  CodeGraderResultSchema,
+  runScriptGrader,
+  runCodeGrader,
+  vitestReportToScriptGraderResult,
+  vitestReportToCodeGraderResult,
+  runVitestWorkspaceGrader,
+  defineVitestWorkspaceGrader
+};
+//# sourceMappingURL=chunk-AQ5BIAXF.js.map