agentv 5.3.1-next.1 → 5.3.3

package/dist/chunk-AQ5BIAXF.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../packages/core/src/evaluation/script-grader-runtime.ts","../../../packages/core/src/evaluation/script-grader-schemas.ts","../../../packages/core/src/evaluation/vitest-workspace-grader.ts"],"sourcesContent":["/**\n * Runtime for script grader evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\nimport { toCamelCaseDeep } from './case-conversion.js';\n\nimport {\n  type ScriptGraderInput,\n  ScriptGraderInputSchema,\n  type ScriptGraderResult,\n  ScriptGraderResultSchema,\n} from './script-grader-schemas.js';\n\n/**\n * Handler function type for script graders.\n */\nexport type ScriptGraderHandler = (\n  input: ScriptGraderInput,\n) => ScriptGraderResult | Promise<ScriptGraderResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n  return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n  if (Number.isNaN(value) || !Number.isFinite(value)) {\n    return 0;\n  }\n  return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n  if (error instanceof Error) {\n    return error.message;\n  }\n  return String(error);\n}\n\n/**\n * Run a script grader handler with full stdin/stdout handling.\n * This is the internal implementation called by defineScriptGrader.\n */\nexport async function runScriptGrader(handler: ScriptGraderHandler): Promise<void> {\n  try {\n    // 1. Read stdin\n    const stdin = readStdin();\n\n    // 2. Parse JSON\n    const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n    // 3. Convert snake_case to camelCase\n    const camelInput = toCamelCaseDeep(rawInput);\n\n    // 4. Validate input with Zod\n    const input = ScriptGraderInputSchema.parse(camelInput);\n\n    // 5. Set up lazy file-backed output loading if applicable\n    if (input.outputPath && (input.output === null || input.output === undefined)) {\n      let cachedOutput: ScriptGraderInput['output'] | undefined;\n      const filePath = input.outputPath;\n      Object.defineProperty(input, 'output', {\n        get() {\n          if (cachedOutput === undefined) {\n            cachedOutput = JSON.parse(readFileSync(filePath, 'utf8'));\n          }\n          return cachedOutput;\n        },\n        configurable: true,\n        enumerable: true,\n      });\n    }\n\n    // 6. Run handler\n    const rawResult = await handler(input);\n\n    // 7. Validate and normalize output\n    const result = ScriptGraderResultSchema.parse({\n      ...rawResult,\n      score: clampScore(rawResult.score),\n      checks: rawResult.checks?.map((check) => ({\n        ...check,\n        ...(check.score !== undefined ? { score: clampScore(check.score) } : {}),\n      })),\n    });\n\n    // 8. Output JSON\n    console.log(JSON.stringify(result, null, 2));\n  } catch (error) {\n    // Output failure result\n    const errorMessage = formatError(error);\n    const errorResult: ScriptGraderResult = {\n      pass: false,\n      score: 0,\n      reason: `Evaluation failed: ${errorMessage}`,\n      checks: [\n        {\n          text: 'Script grader execution',\n          pass: false,\n          reason: errorMessage,\n        },\n      ],\n    };\n    console.log(JSON.stringify(errorResult, null, 2));\n    process.exit(1);\n  }\n}\n\n/** @deprecated Use ScriptGraderHandler. */\nexport type CodeGraderHandler = ScriptGraderHandler;\n\n/** @deprecated Use runScriptGrader. */\nexport const runCodeGrader = runScriptGrader;\n","/**\n * Zod schemas for script grader input/output validation.\n * Provides both compile-time types and runtime validation.\n *\n * ## Content model\n *\n * `Message.content` accepts `string | object[] | object`:\n * - `string` — backward-compatible plain text (most common case)\n * - `object[]` — typed content blocks for multimodal messages, plus AgentV\n *   eval input blocks such as `{ type: \"file\", value, path, text }`\n * - `object` — structured YAML/JSON content such as expected outputs\n *\n * Content variants:\n * - `ContentText`  — `{ type: 'text', text: string }`\n * - `ContentImage` — `{ type: 'image', media_type: string, path: string }` (file path, not base64)\n * - `ContentFile`  — `{ type: 'file', media_type: string, path: string }`\n *\n * To add a new content variant:\n * 1. Define a new Zod schema with a unique `type` literal\n * 2. Add it to `ContentSchema` discriminated union\n * 3. Re-export from `index.ts`\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n  input: z.number(),\n  output: z.number(),\n  cached: z.number().optional(),\n  reasoning: z.number().optional(),\n});\n\n/**\n * Derived trace summary schema (camelCase for TypeScript ergonomics).\n *\n * This is a compact read model for metric-style graders. Full transcript/tool\n * evidence lives in the canonical `Trace` under `messages` and `events`.\n */\nexport const TraceSummarySchema = z.object({\n  eventCount: z.number(),\n  toolCalls: z.record(z.string(), z.number()),\n  errorCount: z.number(),\n  toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n  llmCallCount: z.number().optional(),\n});\n\nexport const TRACE_SOURCE_KINDS = [\n  'agentv_run',\n  'otlp',\n  'phoenix',\n  'langfuse',\n  'pi_session',\n  'imported_transcript',\n  'compact_transcript',\n] as const;\n\nexport const TRACE_EVENT_TYPES = [\n  'message',\n  'model_turn',\n  'tool_call',\n  'tool_result',\n  'final_response',\n  'error',\n] as const;\n\nexport const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const;\n\nexport const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const;\n\nconst MetadataSchema = z.record(z.string(), z.unknown());\n\nexport const TraceRedactionStateSchema = z.object({\n  level: z.enum(TRACE_REDACTION_LEVELS),\n  fields: z.array(z.string()).optional(),\n  reason: z.string().optional(),\n});\n\nexport const TraceErrorSchema = z.object({\n  message: z.string(),\n  name: z.string().optional(),\n  code: z.string().optional(),\n  stack: z.string().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceSourceSchema = z.object({\n  kind: z.enum(TRACE_SOURCE_KINDS),\n  path: z.string().optional(),\n  url: z.string().optional(),\n  provider: z.string().optional(),\n  format: z.string().optional(),\n  version: z.string().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceSessionSchema = z.object({\n  sessionId: z.string().optional(),\n  conversationId: z.string().optional(),\n  cwd: z.string().optional(),\n  startedAt: z.string().optional(),\n  endedAt: z.string().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceBranchSchema = z.object({\n  selectedLeafId: z.string().optional(),\n  selectedPathIds: z.array(z.string()).optional(),\n  includedEventIds: z.array(z.string()).optional(),\n  omittedEventIds: z.array(z.string()).optional(),\n  selectionReason: z.string().optional(),\n});\n\nexport const TraceSourceRefSchema = z.object({\n  eventId: z.string().optional(),\n  messageId: z.string().optional(),\n  spanId: z.string().optional(),\n  traceId: z.string().optional(),\n  rawKind: z.string().optional(),\n  path: z.string().optional(),\n  line: z.number().int().nonnegative().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceRawEvidenceSchema = z.object({\n  kind: z.string(),\n  ref: z.string().optional(),\n  mediaType: z.string().optional(),\n  content: z.unknown().optional(),\n  redacted: z.boolean().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceMessageSchema = z.object({\n  role: z.string(),\n  name: z.string().optional(),\n  content: z.unknown().optional(),\n  redaction: TraceRedactionStateSchema.optional(),\n  tokenUsage: TokenUsageSchema.optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceModelSchema = z.object({\n  provider: z.string().optional(),\n  name: z.string().optional(),\n  invocationId: z.string().optional(),\n  tokenUsage: TokenUsageSchema.optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceToolSchema = z.object({\n  name: z.string(),\n  callId: z.string().optional(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  status: z.enum(TRACE_TOOL_STATUSES).optional(),\n  error: TraceErrorSchema.optional(),\n  redaction: TraceRedactionStateSchema.optional(),\n  metadata: MetadataSchema.optional(),\n});\n\nexport const TraceEventSchema = z.object({\n  eventId: z.string(),\n  parentEventId: z.string().optional(),\n  ordinal: z.number().int().nonnegative(),\n  type: z.enum(TRACE_EVENT_TYPES),\n  timestamp: z.string().optional(),\n  durationMs: z.number().nonnegative().optional(),\n  durationInferred: z.boolean().optional(),\n  turnIndex: z.number().int().nonnegative().optional(),\n  message: TraceMessageSchema.optional(),\n  model: TraceModelSchema.optional(),\n  tool: TraceToolSchema.optional(),\n  error: TraceErrorSchema.optional(),\n  sourceRef: TraceSourceRefSchema.optional(),\n  rawEvidence: z.array(TraceRawEvidenceSchema).optional(),\n  redaction: TraceRedactionStateSchema.optional(),\n  metadata: MetadataSchema.optional(),\n});\n\n/**\n * Derived trace artifact shape used by import/replay helpers.\n *\n * This is not a persisted public trace contract. Grader authors normally\n * receive the result-local `Trace` shape below.\n */\nexport const TraceArtifactSchema = z.object({\n  source: TraceSourceSchema,\n  session: TraceSessionSchema,\n  branch: TraceBranchSchema.optional(),\n  events: z.array(TraceEventSchema),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  startedAt: z.string().optional(),\n  endedAt: z.string().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\n/**\n * Tool call schema.\n */\nexport const ToolCallSchema = z.object({\n  tool: z.string(),\n  input: z.unknown().optional(),\n  output: z.unknown().optional(),\n  id: z.string().optional(),\n  startTime: z.string().optional(),\n  endTime: z.string().optional(),\n  durationMs: z.number().optional(),\n});\n\n// ---------------------------------------------------------------------------\n// Content block schemas (discriminated union on `type`)\n// ---------------------------------------------------------------------------\n\n/** Text content block. */\nexport const ContentTextSchema = z.object({\n  type: z.literal('text'),\n  text: z.string(),\n});\n\n/**\n * Image content block.\n * `path` is a filesystem path — never inline base64.\n */\nexport const ContentImageSchema = z.object({\n  type: z.literal('image'),\n  media_type: z.string(),\n  path: z.string(),\n});\n\n/** File content block. */\nexport const ContentFileSchema = z.object({\n  type: z.literal('file'),\n  media_type: z.string(),\n  path: z.string(),\n});\n\n/** Discriminated union of all content block types. */\nexport const ContentSchema = z.discriminatedUnion('type', [\n  ContentTextSchema,\n  ContentImageSchema,\n  ContentFileSchema,\n]);\n\nconst MessageContentBlockSchema = z.union([ContentSchema, z.record(z.unknown())]);\n\n/**\n * Unified message schema for input, expected, and output messages.\n *\n * `content` is a plain string, an array of structured blocks, or a\n * structured object from YAML/JSON eval files. Use `getTextContent()` from\n * `agentv/contracts` to extract plain text when the content is textual.\n */\nexport const MessageSchema = z.object({\n  role: z.enum(['assistant', 'user', 'system', 'tool']),\n  content: z\n    .union([z.string(), z.array(MessageContentBlockSchema), z.record(z.unknown())])\n    .optional(),\n  toolCalls: z.array(ToolCallSchema).optional(),\n  name: z.string().optional(),\n  startTime: z.string().optional(),\n  endTime: z.string().optional(),\n  durationMs: z.number().optional(),\n  metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Derived evaluation trace read model exposed to custom assertions and script graders.\n *\n * Top-level summary fields (`eventCount`, `toolCalls`, `errorCount`) remain\n * available for existing metric graders; full transcript/tool evidence is under\n * `messages` and structured execution events under `events`.\n */\nexport const TraceSchema = TraceSummarySchema.extend({\n  messages: z.array(MessageSchema),\n  events: z.array(TraceEventSchema),\n  tokenUsage: TokenUsageSchema.optional(),\n  costUsd: z.number().optional(),\n  durationMs: z.number().optional(),\n  startTime: z.string().optional(),\n  endTime: z.string().optional(),\n  metadata: MetadataSchema.optional(),\n});\n\n/**\n * Script grader input schema (camelCase, converted from snake_case wire format).\n *\n * `output` is the final answer/scored result only. Transcript-aware graders\n * should inspect `messages`, `trace.messages`, or `trace.events`.\n */\nexport const ScriptGraderInputSchema = z.object({\n  criteria: z.string(),\n  expectedOutput: z.array(MessageSchema),\n  output: z.string().nullable().optional(),\n  messages: z.array(MessageSchema).optional().default([]),\n  /** Path to a temp file containing the output JSON (used for large payloads). */\n  outputPath: z.string().optional(),\n  inputFiles: z.array(z.string()),\n  input: z.array(MessageSchema),\n  metadata: z.record(z.unknown()).nullable().optional(),\n  trace: TraceSchema.nullable().optional(),\n  traceSummary: TraceSummarySchema.nullable().optional(),\n  tokenUsage: TokenUsageSchema.nullable().optional(),\n  costUsd: z.number().nullable().optional(),\n  durationMs: z.number().nullable().optional(),\n  startTime: z.string().nullable().optional(),\n  endTime: z.string().nullable().optional(),\n  fileChanges: z.string().nullable().optional(),\n  workspacePath: z.string().nullable().optional(),\n  config: z.record(z.unknown()).nullable().optional(),\n});\n\nexport const ScriptGraderCheckSchema = z.object({\n  id: z.string().optional(),\n  text: z.string(),\n  pass: z.boolean(),\n  score: z.number().min(0).max(1).optional(),\n  reason: z.string(),\n  evidence: z.string().optional(),\n});\n\n/**\n * Script grader result schema (validated before output).\n */\nexport const ScriptGraderResultSchema = z.object({\n  pass: z.boolean(),\n  score: z.number().min(0).max(1),\n  reason: z.string(),\n  checks: z.array(ScriptGraderCheckSchema).optional().default([]),\n  /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */\n  details: z.record(z.unknown()).optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type ScriptGraderInput = z.infer<typeof ScriptGraderInputSchema>;\nexport type ScriptGraderCheck = z.infer<typeof ScriptGraderCheckSchema>;\nexport type ScriptGraderResult = z.infer<typeof ScriptGraderResultSchema>;\n\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type Trace = z.infer<typeof TraceSchema>;\nexport type TraceArtifact = z.infer<typeof TraceArtifactSchema>;\nexport type TraceSource = z.infer<typeof TraceSourceSchema>;\nexport type TraceSession = z.infer<typeof TraceSessionSchema>;\nexport type TraceBranch = z.infer<typeof TraceBranchSchema>;\nexport type TraceEvent = z.infer<typeof TraceEventSchema>;\nexport type TraceMessage = z.infer<typeof TraceMessageSchema>;\nexport type TraceModel = z.infer<typeof TraceModelSchema>;\nexport type TraceTool = z.infer<typeof TraceToolSchema>;\nexport type TraceError = z.infer<typeof TraceErrorSchema>;\nexport type TraceSourceRef = z.infer<typeof TraceSourceRefSchema>;\nexport type TraceRawEvidence = z.infer<typeof TraceRawEvidenceSchema>;\nexport type TraceRedactionState = z.infer<typeof TraceRedactionStateSchema>;\nexport type Message = z.infer<typeof MessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n\nexport type ContentText = z.infer<typeof ContentTextSchema>;\nexport type ContentImage = z.infer<typeof ContentImageSchema>;\nexport type ContentFile = z.infer<typeof ContentFileSchema>;\nexport type Content = z.infer<typeof ContentSchema>;\n\n/**\n * Prompt template input schema (camelCase, converted from snake_case wire format).\n * Uses the same schema as ScriptGraderInput since the orchestrator sends identical payloads.\n */\nexport const PromptTemplateInputSchema = ScriptGraderInputSchema;\n\nexport type PromptTemplateInput = ScriptGraderInput;\n\n// ── Backward-compat aliases (deprecated) ────────────────────────────────────────\n/** @deprecated Use ScriptGraderInputSchema */\nexport const CodeGraderInputSchema = ScriptGraderInputSchema;\n/** @deprecated Use ScriptGraderResultSchema */\nexport const CodeGraderResultSchema = ScriptGraderResultSchema;\n/** @deprecated Use ScriptGraderInput */\nexport type CodeGraderInput = ScriptGraderInput;\n/** @deprecated Use ScriptGraderResult */\nexport type CodeGraderResult = ScriptGraderResult;\n/** @deprecated Use ScriptGraderInputSchema */\nexport const CodeJudgeInputSchema = ScriptGraderInputSchema;\n/** @deprecated Use ScriptGraderResultSchema */\nexport const CodeJudgeResultSchema = ScriptGraderResultSchema;\n/** @deprecated Use ScriptGraderInput */\nexport type CodeJudgeInput = ScriptGraderInput;\n/** @deprecated Use ScriptGraderResult */\nexport type CodeJudgeResult = ScriptGraderResult;\n","/**\n * Vitest workspace verifier adapter.\n *\n * This module keeps deterministic workspace verification in familiar Vitest\n * tests while translating the JSON reporter output into AgentV's script-grader\n * result contract.\n */\nimport { spawn } from 'node:child_process';\nimport { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises';\nimport { tmpdir } from 'node:os';\nimport nodePath from 'node:path';\n\nimport { runScriptGrader } from './script-grader-runtime.js';\nimport {\n  type ScriptGraderInput,\n  type ScriptGraderResult,\n  ScriptGraderResultSchema,\n} from './script-grader-schemas.js';\n\nexport interface VitestWorkspaceGraderOptions {\n  /**\n   * Vitest verifier file(s). By default these are relative to the prepared\n   * workspace. When `copyTestFilesToWorkspace` is true, relative paths resolve\n   * from `testFileRoot` instead.\n   *\n   * When provided without `command`, the adapter runs\n   * `bunx vitest run <testFile> --reporter=json --outputFile <tmp>`.\n   */\n  readonly testFile?: string | readonly string[];\n  /**\n   * Copy `testFile` entries into a temporary directory inside the workspace\n   * before running Vitest. Use this for hidden verifier files that live beside\n   * the eval instead of inside the prepared workspace.\n   */\n  readonly copyTestFilesToWorkspace?: boolean;\n  /**\n   * Base directory for copied `testFile` entries. Defaults to `process.cwd()`.\n   */\n  readonly testFileRoot?: string;\n  /**\n   * Full command to run. Use this for package scripts such as\n   * `[\"bun\", \"run\", \"verify:workspace\"]`.\n   */\n  readonly command?: readonly string[];\n  /**\n   * Base Vitest command used with `testFile`. Defaults to\n   * `[\"bunx\", \"vitest\", \"run\"]`.\n   */\n  readonly vitestCommand?: readonly string[];\n  /** Workspace-relative directory to run the command in. Defaults to workspace root. */\n  readonly cwd?: string;\n  /** Append `--reporter=json --outputFile <tmp>` to the command. Defaults to true for testFile mode. */\n  readonly appendReporterArgs?: boolean;\n  /** Read Vitest JSON from this path instead of stdout. Relative paths resolve under `cwd`. */\n  readonly outputFile?: string;\n  readonly timeoutMs?: number;\n  readonly env?: Readonly<Record<string, string>>;\n  readonly passWithNoTests?: boolean;\n}\n\ninterface CommandResult {\n  readonly exitCode: number | null;\n  readonly stdout: string;\n  readonly stderr: string;\n}\n\ninterface VitestAssertionResult {\n  readonly ancestorTitles?: readonly string[];\n  readonly fullName?: string;\n  readonly title?: string;\n  readonly status?: string;\n  readonly failureMessages?: readonly string[];\n  readonly duration?: number;\n}\n\ninterface VitestFileResult {\n  readonly name?: string;\n  readonly assertionResults?: readonly VitestAssertionResult[];\n}\n\ninterface VitestJsonReport {\n  readonly success?: boolean;\n  readonly numTotalTests?: number;\n  readonly numPassedTests?: number;\n  readonly numFailedTests?: number;\n  readonly numPendingTests?: number;\n  readonly numTodoTests?: number;\n  readonly testResults?: readonly VitestFileResult[];\n}\n\nfunction workspacePathFrom(input: ScriptGraderInput): string | undefined {\n  const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH;\n  return workspacePath?.trim() ? workspacePath : undefined;\n}\n\nfunction truncate(value: string, maxLength = 2000): string {\n  if (value.length <= maxLength) {\n    return value;\n  }\n  return `${value.slice(0, maxLength)}\\n...(truncated)`;\n}\n\nfunction resolveInsideWorkspace(\n  workspacePath: string,\n  relativePath: string,\n  label: string,\n  options: { readonly allowRoot?: boolean } = {},\n): string {\n  if (!relativePath.trim()) {\n    throw new Error(`${label} must not be empty.`);\n  }\n\n  if (nodePath.isAbsolute(relativePath)) {\n    throw new Error(`${label} must be relative to the workspace: ${relativePath}`);\n  }\n\n  const root = nodePath.resolve(workspacePath);\n  const resolvedPath = nodePath.resolve(root, relativePath);\n  const relativeToRoot = nodePath.relative(root, resolvedPath);\n  if (relativeToRoot === '' && options.allowRoot === true) {\n    return resolvedPath;\n  }\n\n  if (\n    relativeToRoot === '' ||\n    relativeToRoot.startsWith('..') ||\n    nodePath.isAbsolute(relativeToRoot)\n  ) {\n    throw new Error(`${label} must stay inside the workspace: ${relativePath}`);\n  }\n\n  return resolvedPath;\n}\n\nfunction normalizeTestFiles(testFile: string | readonly string[] | undefined): readonly string[] {\n  if (testFile === undefined) {\n    return [];\n  }\n  return typeof testFile === 'string' ? [testFile] : [...testFile];\n}\n\nfunction buildCommand(\n  options: VitestWorkspaceGraderOptions,\n  testFiles: readonly string[] = normalizeTestFiles(options.testFile),\n) {\n  if (options.command && options.command.length > 0) {\n    return [...options.command];\n  }\n\n  const command = [...(options.vitestCommand ?? ['bunx', 'vitest', 'run'])];\n  command.push(...testFiles);\n  return command;\n}\n\nfunction parseJsonObjectFromText(text: string): unknown {\n  const trimmed = text.trim();\n  if (!trimmed) {\n    return undefined;\n  }\n\n  try {\n    return JSON.parse(trimmed);\n  } catch {\n    const start = trimmed.indexOf('{');\n    const end = trimmed.lastIndexOf('}');\n    if (start >= 0 && end > start) {\n      return JSON.parse(trimmed.slice(start, end + 1));\n    }\n    throw new Error('Vitest output did not contain a JSON object.');\n  }\n}\n\nfunction isVitestJsonReport(value: unknown): value is VitestJsonReport {\n  return (\n    typeof value === 'object' &&\n    value !== null &&\n    Array.isArray((value as VitestJsonReport).testResults)\n  );\n}\n\nfunction assertionText(file: VitestFileResult, assertion: VitestAssertionResult): string {\n  const title =\n    assertion.fullName ??\n    [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean).join(' ');\n  return title || file.name || 'Vitest assertion';\n}\n\nexport function vitestReportToScriptGraderResult(\n  report: VitestJsonReport,\n  options: Pick<VitestWorkspaceGraderOptions, 'passWithNoTests'> = {},\n): ScriptGraderResult {\n  const checks = (report.testResults ?? []).flatMap((file) =>\n    (file.assertionResults ?? []).map((item) => {\n      const pass = item.status === 'passed';\n      const evidence =\n        item.failureMessages && item.failureMessages.length > 0\n          ? truncate(item.failureMessages.join('\\n\\n'))\n          : undefined;\n      return {\n        text: assertionText(file, item),\n        pass,\n        reason: pass ? 'Vitest test passed.' : 'Vitest test failed.',\n        ...(evidence !== undefined ? { evidence } : {}),\n      };\n    }),\n  );\n\n  if (checks.length === 0) {\n    const pass = options.passWithNoTests === true;\n    return ScriptGraderResultSchema.parse({\n      pass,\n      score: pass ? 1 : 0,\n      reason: pass ? 'Vitest reported no tests; configured to pass.' : 'Vitest reported no tests.',\n      checks: [\n        {\n          text: 'Vitest reported no tests',\n          pass,\n          reason: pass ? 'passWithNoTests is enabled.' : 'No Vitest tests were discovered.',\n        },\n      ],\n      details: {\n        vitest_success: report.success ?? false,\n        num_total_tests: report.numTotalTests ?? 0,\n        num_passed_tests: report.numPassedTests ?? 0,\n        num_failed_tests: report.numFailedTests ?? 0,\n        num_pending_tests: report.numPendingTests ?? 0,\n        num_todo_tests: report.numTodoTests ?? 0,\n      },\n    });\n  }\n\n  const passedCount = checks.filter((item) => item.pass).length;\n  const pass = passedCount === checks.length;\n  return ScriptGraderResultSchema.parse({\n    pass,\n    score: passedCount / checks.length,\n    reason: `${passedCount}/${checks.length} Vitest tests passed.`,\n    checks,\n    details: {\n      vitest_success: report.success ?? pass,\n      num_total_tests: report.numTotalTests ?? checks.length,\n      num_passed_tests: report.numPassedTests ?? passedCount,\n      num_failed_tests: report.numFailedTests ?? checks.length - passedCount,\n      num_pending_tests: report.numPendingTests ?? 0,\n      num_todo_tests: report.numTodoTests ?? 0,\n    },\n  });\n}\n\n/** @deprecated Use vitestReportToScriptGraderResult. */\nexport const vitestReportToCodeGraderResult = vitestReportToScriptGraderResult;\n\nfunction runCommand(\n  command: readonly string[],\n  options: {\n    readonly cwd: string;\n    readonly timeoutMs?: number;\n    readonly env?: Readonly<Record<string, string>>;\n  },\n): Promise<CommandResult> {\n  if (command.length === 0) {\n    return Promise.reject(new Error('Vitest command must not be empty.'));\n  }\n\n  return new Promise((resolve, reject) => {\n    const child = spawn(command[0], command.slice(1), {\n      cwd: options.cwd,\n      env: { ...process.env, ...options.env },\n      stdio: ['ignore', 'pipe', 'pipe'],\n    });\n\n    let stdout = '';\n    let stderr = '';\n    let timedOut = false;\n    const timeout =\n      options.timeoutMs !== undefined\n        ? setTimeout(() => {\n            timedOut = true;\n            child.kill('SIGTERM');\n          }, options.timeoutMs)\n        : undefined;\n\n    child.stdout.setEncoding('utf8');\n    child.stderr.setEncoding('utf8');\n    child.stdout.on('data', (chunk) => {\n      stdout += chunk;\n    });\n    child.stderr.on('data', (chunk) => {\n      stderr += chunk;\n    });\n    child.on('error', reject);\n    child.on('close', (exitCode) => {\n      if (timeout) {\n        clearTimeout(timeout);\n      }\n      if (timedOut) {\n        reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`));\n        return;\n      }\n      resolve({ exitCode, stdout, stderr });\n    });\n  });\n}\n\nfunction resolveSourceTestFile(testFileRoot: string, testFile: string): string {\n  if (!testFile.trim()) {\n    throw new Error('testFile entries must not be empty.');\n  }\n\n  return nodePath.isAbsolute(testFile)\n    ? nodePath.resolve(testFile)\n    : nodePath.resolve(testFileRoot, testFile);\n}\n\nasync function copyTestFilesIntoWorkspace(\n  testFiles: readonly string[],\n  options: Pick<VitestWorkspaceGraderOptions, 'testFileRoot'>,\n  cwd: string,\n): Promise<{ readonly testFiles: readonly string[]; readonly tempDir?: string }> {\n  if (testFiles.length === 0) {\n    return { testFiles };\n  }\n\n  const tempDir = await mkdtemp(nodePath.join(cwd, '.agentv-vitest-'));\n  const testFileRoot = nodePath.resolve(options.testFileRoot ?? process.cwd());\n  const copiedFiles = await Promise.all(\n    testFiles.map(async (testFile, index) => {\n      const sourcePath = resolveSourceTestFile(testFileRoot, testFile);\n      const destinationPath = nodePath.join(tempDir, `${index}-${nodePath.basename(testFile)}`);\n      await copyFile(sourcePath, destinationPath);\n      return nodePath.relative(cwd, destinationPath);\n    }),\n  );\n  return { testFiles: copiedFiles, tempDir };\n}\n\nasync function readVitestReport(\n  commandResult: CommandResult,\n  outputFile: string | undefined,\n): Promise<VitestJsonReport> {\n  const rawReport =\n    outputFile !== undefined ? await readFile(outputFile, 'utf8') : commandResult.stdout;\n  const parsed = parseJsonObjectFromText(rawReport);\n  if (!isVitestJsonReport(parsed)) {\n    throw new Error('Vitest JSON report did not include testResults[].');\n  }\n  return parsed;\n}\n\nexport async function runVitestWorkspaceGrader(\n  options: VitestWorkspaceGraderOptions,\n  input: ScriptGraderInput,\n): Promise<ScriptGraderResult> {\n  const workspacePath = workspacePathFrom(input);\n  if (!workspacePath) {\n    return {\n      pass: false,\n      score: 0,\n      reason: 'Vitest workspace verifier requires workspace_path.',\n      checks: [\n        {\n          text: 'Vitest workspace verifier requires workspace_path',\n          pass: false,\n          reason:\n            'Configure an environment recipe in the eval YAML so AgentV can pass workspace_path.',\n        },\n      ],\n    };\n  }\n\n  const tempDirs: string[] = [];\n  try {\n    const cwd = options.cwd\n      ? resolveInsideWorkspace(workspacePath, options.cwd, 'cwd', { allowRoot: true })\n      : workspacePath;\n    const testFiles = normalizeTestFiles(options.testFile);\n    const preparedTestFiles =\n      options.copyTestFilesToWorkspace === true\n        ? await copyTestFilesIntoWorkspace(testFiles, options, cwd)\n        : { testFiles };\n    if (preparedTestFiles.tempDir) {\n      tempDirs.push(preparedTestFiles.tempDir);\n    }\n\n    const command = buildCommand(options, preparedTestFiles.testFiles);\n    const appendReporterArgs = options.appendReporterArgs ?? options.command === undefined;\n    let outputFile = options.outputFile\n      ? resolveInsideWorkspace(cwd, options.outputFile, 'outputFile')\n      : undefined;\n\n    if (appendReporterArgs) {\n      const tempDir = await mkdtemp(nodePath.join(tmpdir(), 'agentv-vitest-'));\n      tempDirs.push(tempDir);\n      outputFile = nodePath.join(tempDir, 'results.json');\n      command.push('--reporter=json', `--outputFile=${outputFile}`);\n    }\n\n    const result = await runCommand(command, {\n      cwd,\n      timeoutMs: options.timeoutMs,\n      env: {\n        AGENTV_WORKSPACE_PATH: workspacePath,\n        ...(outputFile !== undefined ? { AGENTV_VITEST_JSON_PATH: outputFile } : {}),\n        ...options.env,\n      },\n    });\n\n    const report = await readVitestReport(result, outputFile);\n    return vitestReportToScriptGraderResult(report, options);\n  } catch (error) {\n    const reason = error instanceof Error ? error.message : String(error);\n    return {\n      pass: false,\n      score: 0,\n      reason: 'Vitest workspace verifier failed to run.',\n      checks: [\n        {\n          text: 'Vitest workspace verifier failed to run',\n          pass: false,\n          reason,\n        },\n      ],\n    };\n  } finally {\n    for (const tempDir of tempDirs.reverse()) {\n      await rm(tempDir, { recursive: true, force: true }).catch(() => {});\n    }\n  }\n}\n\nexport function defineVitestWorkspaceGrader(options: VitestWorkspaceGraderOptions): void {\n  runScriptGrader((input) => runVitestWorkspaceGrader(options, input));\n}\n"],"mappings":";;;;;;;AAIA,SAAS,oBAAoB;AEG7B,SAAS,aAAa;AACtB,SAAS,UAAU,SAAS,UAAU,UAAU;AAChD,SAAS,cAAc;AACvB,OAAO,cAAc;ADiBd,IAAM,mBAAmB,iBAAE,OAAO;EACvC,OAAO,iBAAE,OAAO;EAChB,QAAQ,iBAAE,OAAO;EACjB,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,WAAW,iBAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAQM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,YAAY,iBAAE,OAAO;EACrB,WAAW,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,OAAO,CAAC;EAC1C,YAAY,iBAAE,OAAO;EACrB,eAAe,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,MAAM,iBAAE,OAAO,CAAC,CAAC,EAAE,SAAS;EAClE,cAAc,iBAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB;EAChC;EACA;EACA;EACA;EACA;EACA;EACA;AACF;AAEO,IAAM,oBAAoB;EAC/B;EACA;EACA;EACA;EACA;EACA;AACF;AAEO,IAAM,sBAAsB,CAAC,MAAM,SAAS,WAAW,aAAa,SAAS;AAE7E,IAAM,yBAAyB,CAAC,QAAQ,WAAW,MAAM;AAEhE,IAAM,iBAAiB,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,QAAQ,CAAC;AAEhD,IAAM,4BAA4B,iBAAE,OAAO;EAChD,OAAO,iBAAE,KAAK,sBAAsB;EACpC,QAAQ,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EACrC,QAAQ,iBAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,SAAS,iBAAE,OAAO;EAClB,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,OAAO,iBAAE,OAAO,EAAE,SAAS;EAC3B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,KAAK,kBAAkB;EAC/B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,UAAU,iBAAE,OAAO,EAAE,SAAS;EAC9B,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,gBAAgB,iBAAE,OAAO,EAAE,SAAS;EACpC,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,gBAAgB,iBAAE,OAAO,EAAE,SAAS;EACpC,iBAAiB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC9C,kBAAkB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC/C,iBAAiB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC9C,iBAAiB,iBAAE,OAAO,EAAE,SAAS;AACvC,CAAC;AAEM,IAAM,uBAAuB,iBAAE,OAAO;EAC3C,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,MAAM,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,yBAAyB,iBAAE,OAAO;EAC7C,MAAM,iBAAE,OAAO;EACf,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,QAAQ,EAAE,SAAS;EAC9B,UAAU,iBAAE,QAAQ,EAAE,SAAS;EAC/B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,MAAM,iBAAE,OAAO;EACf,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,SAAS,iBAAE,QAAQ,EAAE,SAAS;EAC9B,WAAW,0BAA0B,SAAS;EAC9C,YAAY,iBAAiB,SAAS;EACtC,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,UAAU,iBAAE,OAAO,EAAE,SAAS;EAC9B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,cAAc,iBAAE,OAAO,EAAE,SAAS;EAClC,YAAY,iBAAiB,SAAS;EACtC,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,kBAAkB,iBAAE,OAAO;EACtC,MAAM,iBAAE,OAAO;EACf,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,OAAO,iBAAE,QAAQ,EAAE,SAAS;EAC5B,QAAQ,iBAAE,QAAQ,EAAE,SAAS;EAC7B,QAAQ,iBAAE,KAAK,mBAAmB,EAAE,SAAS;EAC7C,OAAO,iBAAiB,SAAS;EACjC,WAAW,0BAA0B,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,SAAS,iBAAE,OAAO;EAClB,eAAe,iBAAE,OAAO,EAAE,SAAS;EACnC,SAAS,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY;EACtC,MAAM,iBAAE,KAAK,iBAAiB;EAC9B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,YAAY,iBAAE,OAAO,EAAE,YAAY,EAAE,SAAS;EAC9C,kBAAkB,iBAAE,QAAQ,EAAE,SAAS;EACvC,WAAW,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;EACnD,SAAS,mBAAmB,SAAS;EACrC,OAAO,iBAAiB,SAAS;EACjC,MAAM,gBAAgB,SAAS;EAC/B,OAAO,iBAAiB,SAAS;EACjC,WAAW,qBAAqB,SAAS;EACzC,aAAa,iBAAE,MAAM,sBAAsB,EAAE,SAAS;EACtD,WAAW,0BAA0B,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAQM,IAAM,sBAAsB,iBAAE,OAAO;EAC1C,QAAQ;EACR,SAAS;EACT,QAAQ,kBAAkB,SAAS;EACnC,QAAQ,iBAAE,MAAM,gBAAgB;EAChC,YAAY,iBAAiB,SAAS;EACtC,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAKM,IAAM,iBAAiB,iBAAE,OAAO;EACrC,MAAM,iBAAE,OAAO;EACf,OAAO,iBAAE,QAAQ,EAAE,SAAS;EAC5B,QAAQ,iBAAE,QAAQ,EAAE,SAAS;EAC7B,IAAI,iBAAE,OAAO,EAAE,SAAS;EACxB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAOM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,QAAQ,MAAM;EACtB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAMM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,MAAM,iBAAE,QAAQ,OAAO;EACvB,YAAY,iBAAE,OAAO;EACrB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAGM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,QAAQ,MAAM;EACtB,YAAY,iBAAE,OAAO;EACrB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAGM,IAAM,gBAAgB,iBAAE,mBAAmB,QAAQ;EACxD;EACA;EACA;AACF,CAAC;AAED,IAAM,4BAA4B,iBAAE,MAAM,CAAC,eAAe,iBAAE,OAAO,iBAAE,QAAQ,CAAC,CAAC,CAAC;AASzE,IAAM,gBAAgB,iBAAE,OAAO;EACpC,MAAM,iBAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;EACpD,SAAS,iBACN,MAAM,CAAC,iBAAE,OAAO,GAAG,iBAAE,MAAM,yBAAyB,GAAG,iBAAE,OAAO,iBAAE,QAAQ,CAAC,CAAC,CAAC,EAC7E,SAAS;EACZ,WAAW,iBAAE,MAAM,cAAc,EAAE,SAAS;EAC5C,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,UAAU,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AASM,IAAM,cAAc,mBAAmB,OAAO;EACnD,UAAU,iBAAE,MAAM,aAAa;EAC/B,QAAQ,iBAAE,MAAM,gBAAgB;EAChC,YAAY,iBAAiB,SAAS;EACtC,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAQM,IAAM,0BAA0B,iBAAE,OAAO;EAC9C,UAAU,iBAAE,OAAO;EACnB,gBAAgB,iBAAE,MAAM,aAAa;EACrC,QAAQ,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACvC,UAAU,iBAAE,MAAM,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;;EAEtD,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,YAAY,iBAAE,MAAM,iBAAE,OAAO,CAAC;EAC9B,OAAO,iBAAE,MAAM,aAAa;EAC5B,UAAU,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;EACpD,OAAO,YAAY,SAAS,EAAE,SAAS;EACvC,cAAc,mBAAmB,SAAS,EAAE,SAAS;EACrD,YAAY,iBAAiB,SAAS,EAAE,SAAS;EACjD,SAAS,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACxC,YAAY,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC3C,WAAW,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC1C,SAAS,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACxC,aAAa,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC5C,eAAe,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC9C,QAAQ,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAEM,IAAM,0BAA0B,iBAAE,OAAO;EAC9C,IAAI,iBAAE,OAAO,EAAE,SAAS;EACxB,MAAM,iBAAE,OAAO;EACf,MAAM,iBAAE,QAAQ;EAChB,OAAO,iBAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS;EACzC,QAAQ,iBAAE,OAAO;EACjB,UAAU,iBAAE,OAAO,EAAE,SAAS;AAChC,CAAC;AAKM,IAAM,2BAA2B,iBAAE,OAAO;EAC/C,MAAM,iBAAE,QAAQ;EAChB,OAAO,iBAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;EAC9B,QAAQ,iBAAE,OAAO;EACjB,QAAQ,iBAAE,MAAM,uBAAuB,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;;EAE9D,SAAS,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS;AAC1C,CAAC;AAoCM,IAAM,4BAA4B;AAMlC,IAAM,wBAAwB;AAE9B,IAAM,yBAAyB;ADlWtC,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,gBAAgB,SAA6C;AACjF,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,wBAAwB,MAAM,UAAU;AAGtD,QAAI,MAAM,eAAe,MAAM,WAAW,QAAQ,MAAM,WAAW,SAAY;AAC7E,UAAI;AACJ,YAAM,WAAW,MAAM;AACvB,aAAO,eAAe,OAAO,UAAU;QACrC,MAAM;AACJ,cAAI,iBAAiB,QAAW;AAC9B,2BAAe,KAAK,MAAM,aAAa,UAAU,MAAM,CAAC;UAC1D;AACA,iBAAO;QACT;QACA,cAAc;QACd,YAAY;MACd,CAAC;IACH;AAGA,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,yBAAyB,MAAM;MAC5C,GAAG;MACH,OAAO,WAAW,UAAU,KAAK;MACjC,QAAQ,UAAU,QAAQ,IAAI,CAAC,WAAW;QACxC,GAAG;QACH,GAAI,MAAM,UAAU,SAAY,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;MACxE,EAAE;IACJ,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAAkC;MACtC,MAAM;MACN,OAAO;MACP,QAAQ,sBAAsB,YAAY;MAC1C,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN,QAAQ;QACV;MACF;IACF;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;EAChB;AACF;AAMO,IAAM,gBAAgB;AE/B7B,SAAS,kBAAkB,OAA8C;AACvE,QAAM,gBAAgB,MAAM,iBAAiB,QAAQ,IAAI;AACzD,SAAO,eAAe,KAAK,IAAI,gBAAgB;AACjD;AAEA,SAAS,SAAS,OAAe,YAAY,KAAc;AACzD,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,SAAS,CAAC;;AACrC;AAEA,SAAS,uBACP,eACA,cACA,OACA,UAA4C,CAAC,GACrC;AACR,MAAI,CAAC,aAAa,KAAK,GAAG;AACxB,UAAM,IAAI,MAAM,GAAG,KAAK,qBAAqB;EAC/C;AAEA,MAAI,SAAS,WAAW,YAAY,GAAG;AACrC,UAAM,IAAI,MAAM,GAAG,KAAK,uCAAuC,YAAY,EAAE;EAC/E;AAEA,QAAM,OAAO,SAAS,QAAQ,aAAa;AAC3C,QAAM,eAAe,SAAS,QAAQ,MAAM,YAAY;AACxD,QAAM,iBAAiB,SAAS,SAAS,MAAM,YAAY;AAC3D,MAAI,mBAAmB,MAAM,QAAQ,cAAc,MAAM;AACvD,WAAO;EACT;AAEA,MACE,mBAAmB,MACnB,eAAe,WAAW,IAAI,KAC9B,SAAS,WAAW,cAAc,GAClC;AACA,UAAM,IAAI,MAAM,GAAG,KAAK,oCAAoC,YAAY,EAAE;EAC5E;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,UAAqE;AAC/F,MAAI,aAAa,QAAW;AAC1B,WAAO,CAAC;EACV;AACA,SAAO,OAAO,aAAa,WAAW,CAAC,QAAQ,IAAI,CAAC,GAAG,QAAQ;AACjE;AAEA,SAAS,aACP,SACA,YAA+B,mBAAmB,QAAQ,QAAQ,GAClE;AACA,MAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAAG;AACjD,WAAO,CAAC,GAAG,QAAQ,OAAO;EAC5B;AAEA,QAAM,UAAU,CAAC,GAAI,QAAQ,iBAAiB,CAAC,QAAQ,UAAU,KAAK,CAAE;AACxE,UAAQ,KAAK,GAAG,SAAS;AACzB,SAAO;AACT;AAEA,SAAS,wBAAwB,MAAuB;AACtD,QAAM,UAAU,KAAK,KAAK;AAC1B,MAAI,CAAC,SAAS;AACZ,WAAO;EACT;AAEA,MAAI;AACF,WAAO,KAAK,MAAM,OAAO;EAC3B,QAAQ;AACN,UAAM,QAAQ,QAAQ,QAAQ,GAAG;AACjC,UAAM,MAAM,QAAQ,YAAY,GAAG;AACnC,QAAI,SAAS,KAAK,MAAM,OAAO;AAC7B,aAAO,KAAK,MAAM,QAAQ,MAAM,OAAO,MAAM,CAAC,CAAC;IACjD;AACA,UAAM,IAAI,MAAM,8CAA8C;EAChE;AACF;AAEA,SAAS,mBAAmB,OAA2C;AACrE,SACE,OAAO,UAAU,YACjB,UAAU,QACV,MAAM,QAAS,MAA2B,WAAW;AAEzD;AAEA,SAAS,cAAc,MAAwB,WAA0C;AACvF,QAAM,QACJ,UAAU,YACV,CAAC,GAAI,UAAU,kBAAkB,CAAC,GAAI,UAAU,KAAK,EAAE,OAAO,OAAO,EAAE,KAAK,GAAG;AACjF,SAAO,SAAS,KAAK,QAAQ;AAC/B;AAEO,SAAS,iCACd,QACA,UAAiE,CAAC,GAC9C;AACpB,QAAM,UAAU,OAAO,eAAe,CAAC,GAAG;IAAQ,CAAC,UAChD,KAAK,oBAAoB,CAAC,GAAG,IAAI,CAAC,SAAS;AAC1C,YAAMA,QAAO,KAAK,WAAW;AAC7B,YAAM,WACJ,KAAK,mBAAmB,KAAK,gBAAgB,SAAS,IAClD,SAAS,KAAK,gBAAgB,KAAK,MAAM,CAAC,IAC1C;AACN,aAAO;QACL,MAAM,cAAc,MAAM,IAAI;QAC9B,MAAAA;QACA,QAAQA,QAAO,wBAAwB;QACvC,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;MAC/C;IACF,CAAC;EACH;AAEA,MAAI,OAAO,WAAW,GAAG;AACvB,UAAMA,QAAO,QAAQ,oBAAoB;AACzC,WAAO,yBAAyB,MAAM;MACpC,MAAAA;MACA,OAAOA,QAAO,IAAI;MAClB,QAAQA,QAAO,kDAAkD;MACjE,QAAQ;QACN;UACE,MAAM;UACN,MAAAA;UACA,QAAQA,QAAO,gCAAgC;QACjD;MACF;MACA,SAAS;QACP,gBAAgB,OAAO,WAAW;QAClC,iBAAiB,OAAO,iBAAiB;QACzC,kBAAkB,OAAO,kBAAkB;QAC3C,kBAAkB,OAAO,kBAAkB;QAC3C,mBAAmB,OAAO,mBAAmB;QAC7C,gBAAgB,OAAO,gBAAgB;MACzC;IACF,CAAC;EACH;AAEA,QAAM,cAAc,OAAO,OAAO,CAAC,SAAS,KAAK,IAAI,EAAE;AACvD,QAAM,OAAO,gBAAgB,OAAO;AACpC,SAAO,yBAAyB,MAAM;IACpC;IACA,OAAO,cAAc,OAAO;IAC5B,QAAQ,GAAG,WAAW,IAAI,OAAO,MAAM;IACvC;IACA,SAAS;MACP,gBAAgB,OAAO,WAAW;MAClC,iBAAiB,OAAO,iBAAiB,OAAO;MAChD,kBAAkB,OAAO,kBAAkB;MAC3C,kBAAkB,OAAO,kBAAkB,OAAO,SAAS;MAC3D,mBAAmB,OAAO,mBAAmB;MAC7C,gBAAgB,OAAO,gBAAgB;IACzC;EACF,CAAC;AACH;AAGO,IAAM,iCAAiC;AAE9C,SAAS,WACP,SACA,SAKwB;AACxB,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,QAAQ,OAAO,IAAI,MAAM,mCAAmC,CAAC;EACtE;AAEA,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,QAAQ,MAAM,QAAQ,CAAC,GAAG,QAAQ,MAAM,CAAC,GAAG;MAChD,KAAK,QAAQ;MACb,KAAK,EAAE,GAAG,QAAQ,KAAK,GAAG,QAAQ,IAAI;MACtC,OAAO,CAAC,UAAU,QAAQ,MAAM;IAClC,CAAC;AAED,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,WAAW;AACf,UAAM,UACJ,QAAQ,cAAc,SAClB,WAAW,MAAM;AACf,iBAAW;AACX,YAAM,KAAK,SAAS;IACtB,GAAG,QAAQ,SAAS,IACpB;AAEN,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;IACZ,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;IACZ,CAAC;AACD,UAAM,GAAG,SAAS,MAAM;AACxB,UAAM,GAAG,SAAS,CAAC,aAAa;AAC9B,UAAI,SAAS;AACX,qBAAa,OAAO;MACtB;AACA,UAAI,UAAU;AACZ,eAAO,IAAI,MAAM,kCAAkC,QAAQ,SAAS,KAAK,CAAC;AAC1E;MACF;AACA,cAAQ,EAAE,UAAU,QAAQ,OAAO,CAAC;IACtC,CAAC;EACH,CAAC;AACH;AAEA,SAAS,sBAAsB,cAAsB,UAA0B;AAC7E,MAAI,CAAC,SAAS,KAAK,GAAG;AACpB,UAAM,IAAI,MAAM,qCAAqC;EACvD;AAEA,SAAO,SAAS,WAAW,QAAQ,IAC/B,SAAS,QAAQ,QAAQ,IACzB,SAAS,QAAQ,cAAc,QAAQ;AAC7C;AAEA,eAAe,2BACb,WACA,SACA,KAC+E;AAC/E,MAAI,UAAU,WAAW,GAAG;AAC1B,WAAO,EAAE,UAAU;EACrB;AAEA,QAAM,UAAU,MAAM,QAAQ,SAAS,KAAK,KAAK,iBAAiB,CAAC;AACnE,QAAM,eAAe,SAAS,QAAQ,QAAQ,gBAAgB,QAAQ,IAAI,CAAC;AAC3E,QAAM,cAAc,MAAM,QAAQ;IAChC,UAAU,IAAI,OAAO,UAAU,UAAU;AACvC,YAAM,aAAa,sBAAsB,cAAc,QAAQ;AAC/D,YAAM,kBAAkB,SAAS,KAAK,SAAS,GAAG,KAAK,IAAI,SAAS,SAAS,QAAQ,CAAC,EAAE;AACxF,YAAM,SAAS,YAAY,eAAe;AAC1C,aAAO,SAAS,SAAS,KAAK,eAAe;IAC/C,CAAC;EACH;AACA,SAAO,EAAE,WAAW,aAAa,QAAQ;AAC3C;AAEA,eAAe,iBACb,eACA,YAC2B;AAC3B,QAAM,YACJ,eAAe,SAAY,MAAM,SAAS,YAAY,MAAM,IAAI,cAAc;AAChF,QAAM,SAAS,wBAAwB,SAAS;AAChD,MAAI,CAAC,mBAAmB,MAAM,GAAG;AAC/B,UAAM,IAAI,MAAM,mDAAmD;EACrE;AACA,SAAO;AACT;AAEA,eAAsB,yBACpB,SACA,OAC6B;AAC7B,QAAM,gBAAgB,kBAAkB,KAAK;AAC7C,MAAI,CAAC,eAAe;AAClB,WAAO;MACL,MAAM;MACN,OAAO;MACP,QAAQ;MACR,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN,QACE;QACJ;MACF;IACF;EACF;AAEA,QAAM,WAAqB,CAAC;AAC5B,MAAI;AACF,UAAM,MAAM,QAAQ,MAChB,uBAAuB,eAAe,QAAQ,KAAK,OAAO,EAAE,WAAW,KAAK,CAAC,IAC7E;AACJ,UAAM,YAAY,mBAAmB,QAAQ,QAAQ;AACrD,UAAM,oBACJ,QAAQ,6BAA6B,OACjC,MAAM,2BAA2B,WAAW,SAAS,GAAG,IACxD,EAAE,UAAU;AAClB,QAAI,kBAAkB,SAAS;AAC7B,eAAS,KAAK,kBAAkB,OAAO;IACzC;AAEA,UAAM,UAAU,aAAa,SAAS,kBAAkB,SAAS;AACjE,UAAM,qBAAqB,QAAQ,sBAAsB,QAAQ,YAAY;AAC7E,QAAI,aAAa,QAAQ,aACrB,uBAAuB,KAAK,QAAQ,YAAY,YAAY,IAC5D;AAEJ,QAAI,oBAAoB;AACtB,YAAM,UAAU,MAAM,QAAQ,SAAS,KAAK,OAAO,GAAG,gBAAgB,CAAC;AACvE,eAAS,KAAK,OAAO;AACrB,mBAAa,SAAS,KAAK,SAAS,cAAc;AAClD,cAAQ,KAAK,mBAAmB,gBAAgB,UAAU,EAAE;IAC9D;AAEA,UAAM,SAAS,MAAM,WAAW,SAAS;MACvC;MACA,WAAW,QAAQ;MACnB,KAAK;QACH,uBAAuB;QACvB,GAAI,eAAe,SAAY,EAAE,yBAAyB,WAAW,IAAI,CAAC;QAC1E,GAAG,QAAQ;MACb;IACF,CAAC;AAED,UAAM,SAAS,MAAM,iBAAiB,QAAQ,UAAU;AACxD,WAAO,iCAAiC,QAAQ,OAAO;EACzD,SAAS,OAAO;AACd,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,WAAO;MACL,MAAM;MACN,OAAO;MACP,QAAQ;MACR,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN;QACF;MACF;IACF;EACF,UAAA;AACE,eAAW,WAAW,SAAS,QAAQ,GAAG;AACxC,YAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;MAAC,CAAC;IACpE;EACF;AACF;AAEO,SAAS,4BAA4B,SAA6C;AACvF,kBAAgB,CAAC,UAAU,yBAAyB,SAAS,KAAK,CAAC;AACrE;","names":["pass"]}

package/dist/chunk-BV5VQLI2.js

@@ -0,0 +1,2 @@
+import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
+//# sourceMappingURL=chunk-BV5VQLI2.js.map

package/dist/{chunk-LXBI3SPX.js → chunk-JGSRUJZQ.js}

@@ -16,7 +16,7 @@ import {
   prepareSharedWorkspaceSetup,
   readTranscriptJsonl,
   releaseSharedWorkspaceSetup
-} from "./chunk-ELCJ23K4.js";
+} from "./chunk-6262OKXM.js";
 
 // ../../packages/core/dist/index.js
 import { readFileSync } from "node:fs";
@@ -1492,6 +1492,9 @@ function assertionToNaturalLanguage(entry) {
   const type = entry.type;
   switch (type) {
     case "skill-trigger":
+      throw new Error(staleSkillTriggerMessage(entry));
+    case "skill-used":
+    case "not-skill-used":
       return null;
     case "llm-rubric":
       return typeof entry.value === "string" ? entry.value : null;
@@ -1518,10 +1521,9 @@ function assertionToNaturalLanguage(entry) {
     case "ends-with":
       return `Output ends with '${entry.value}'`;
     case "llm-grader":
-      if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
-        return null;
-      }
-      return typeof entry.prompt === "string" ? entry.prompt : null;
+      throw new Error(
+        "Authored assertion type 'llm-grader' has been removed. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks."
+      );
     case "tool-trajectory": {
       const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
       const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
@@ -1565,16 +1567,42 @@ function assertionToNaturalLanguageList(entry) {
       }).filter((value) => typeof value === "string");
     }
   }
-  if (entry.type === "llm-grader") {
-    if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
-      return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
-    }
-  }
   const nl = assertionToNaturalLanguage(entry);
   return nl !== null ? [nl] : [];
 }
-function extractTriggerAssertions(assertions) {
-  return assertions.filter((a) => a.type === "skill-trigger");
+function staleSkillTriggerMessage(entry) {
+  const skill = typeof entry.skill === "string" ? entry.skill.trim() : "";
+  const shouldTrigger = entry.should_trigger !== false;
+  if (!skill) {
+    return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
+  }
+  const replacementType = shouldTrigger ? "skill-used" : "not-skill-used";
+  return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skill} with type: ${replacementType}, value: ${skill}.`;
+}
+function skillNameFromValue(value) {
+  if (typeof value === "string" && value.trim()) {
+    return value.trim();
+  }
+  if (value && typeof value === "object" && !Array.isArray(value)) {
+    const name = value.name;
+    return typeof name === "string" && name.trim() ? name.trim() : void 0;
+  }
+  return void 0;
+}
+function extractSkillUseAssertions(assertions) {
+  return assertions.flatMap((entry) => {
+    if (entry.type === "skill-trigger") {
+      throw new Error(staleSkillTriggerMessage(entry));
+    }
+    if (entry.type !== "skill-used" && entry.type !== "not-skill-used") {
+      return [];
+    }
+    const skill = skillNameFromValue(entry.value);
+    if (!skill) {
+      return [];
+    }
+    return [{ skill, shouldTrigger: entry.type === "skill-used" }];
+  });
 }
 function extractInput(rawCase) {
   const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
@@ -1622,7 +1650,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
     throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
   }
   const suiteAssertions = rawSuite.assert ?? [];
-  const suiteNlAssertions = suiteAssertions.filter((a) => a.type !== "skill-trigger").flatMap(assertionToNaturalLanguageList);
+  const suiteNlAssertions = suiteAssertions.flatMap(assertionToNaturalLanguageList);
   function getSkillFile(skillName) {
     const existing = files.get(skillName);
     if (existing) return existing;
@@ -1644,12 +1672,12 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
       nlAssertions.push(rawCase.criteria.trim());
     }
     for (const entry of caseAssertions) {
-      if (entry.type !== "skill-trigger") {
+      if (entry.type !== "skill-used" && entry.type !== "not-skill-used") {
         nlAssertions.push(...assertionToNaturalLanguageList(entry));
       }
     }
     nlAssertions.push(...suiteNlAssertions);
-    const triggerJudges = extractTriggerAssertions(caseAssertions);
+    const triggerJudges = extractSkillUseAssertions(caseAssertions);
     const { prompt, files: inputFiles } = extractInput(rawCase);
     const expectedOutput = extractExpectedOutput(rawCase.expected_output);
     const numericId = idx + 1;
@@ -1665,10 +1693,8 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
       noSkillFile2.evals.push({ ...baseCase });
     } else {
       for (const tj of triggerJudges) {
-        const skillName = typeof tj.skill === "string" ? tj.skill : "_no-skill";
-        const shouldTrigger = tj.should_trigger !== false;
-        const skillFile = getSkillFile(skillName);
-        skillFile.evals.push({ ...baseCase, should_trigger: shouldTrigger });
+        const skillFile = getSkillFile(tj.skill);
+        skillFile.evals.push({ ...baseCase, should_trigger: tj.shouldTrigger });
       }
     }
   }
@@ -6217,4 +6243,4 @@ export {
   parseCopilotEvents,
   createAgentKernel
 };
-//# sourceMappingURL=chunk-LXBI3SPX.js.map
+//# sourceMappingURL=chunk-JGSRUJZQ.js.map