agentv 5.3.0-next.1 → 5.3.2-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -60
- package/dist/{artifact-writer-JFNIPMKW.js → artifact-writer-KJEOROKQ.js} +5 -5
- package/dist/{chunk-6ZZCDZPD.js → chunk-6262OKXM.js} +21774 -21034
- package/dist/chunk-6262OKXM.js.map +1 -0
- package/dist/chunk-AQ5BIAXF.js +604 -0
- package/dist/chunk-AQ5BIAXF.js.map +1 -0
- package/dist/chunk-BV5VQLI2.js +2 -0
- package/dist/{chunk-V52ATPTT.js → chunk-JGSRUJZQ.js} +186 -32
- package/dist/chunk-JGSRUJZQ.js.map +1 -0
- package/dist/chunk-MMDLXYBX.js +721 -0
- package/dist/chunk-MMDLXYBX.js.map +1 -0
- package/dist/{chunk-FVR4RQFK.js → chunk-QSK3PC44.js} +1131 -580
- package/dist/chunk-QSK3PC44.js.map +1 -0
- package/dist/chunk-TEEXVJWM.js +2 -0
- package/dist/{chunk-BHKQHG26.js → chunk-WIAJ7JAV.js} +186 -64
- package/dist/chunk-WIAJ7JAV.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +17517 -10
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +2 -0
- package/dist/{ts-eval-loader-DQDYRULE-V3377FOL.js → config.js} +7 -7
- package/dist/config.js.map +1 -0
- package/dist/contracts-DsZmZLl8.d.ts +742 -0
- package/dist/contracts.d.ts +2 -0
- package/dist/contracts.js +28 -0
- package/dist/contracts.js.map +1 -0
- package/dist/dashboard/assets/{index-DNgf3qJ2.js → index-BfqOlLOF.js} +1 -1
- package/dist/dashboard/assets/index-Bh8lpGce.css +1 -0
- package/dist/dashboard/assets/index-oPR3ywQb.js +121 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-6Z7U473R.js → dist-A3SGR7TW.js} +30 -18
- package/dist/dist-A3SGR7TW.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +137 -18
- package/dist/{interactive-RUY3OCBI.js → interactive-DL7N2C7K.js} +24 -24
- package/dist/interactive-DL7N2C7K.js.map +1 -0
- package/dist/provider.d.ts +2 -0
- package/dist/provider.js +24 -0
- package/dist/provider.js.map +1 -0
- package/dist/sdk.d.ts +802 -0
- package/dist/sdk.js +145 -0
- package/dist/sdk.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +14 -13
- package/dist/skills/agentv-bench/agents/analyzer.md +1 -1
- package/dist/skills/agentv-bench/agents/executor.md +1 -1
- package/dist/skills/agentv-bench/references/autoresearch.md +9 -9
- package/dist/skills/agentv-bench/references/environment-adaptation.md +4 -4
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +30 -47
- package/dist/skills/agentv-bench/references/schemas.md +44 -60
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +20 -18
- package/dist/skills/agentv-eval-migrations/SKILL.md +13 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +56 -39
- package/dist/skills/agentv-eval-writer/SKILL.md +101 -60
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +15 -10
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +4543 -5189
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +2 -2
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +20 -3
- package/dist/templates/.agentv/providers.yaml +42 -0
- package/dist/templates/.env.example +2 -2
- package/dist/ts-eval-loader-3G5GEAEC-6F52JLPP.js +18 -0
- package/dist/ts-eval-loader-3G5GEAEC-6F52JLPP.js.map +1 -0
- package/package.json +29 -4
- package/dist/chunk-6ZZCDZPD.js.map +0 -1
- package/dist/chunk-BHKQHG26.js.map +0 -1
- package/dist/chunk-FVR4RQFK.js.map +0 -1
- package/dist/chunk-T32NL3E6.js +0 -17973
- package/dist/chunk-T32NL3E6.js.map +0 -1
- package/dist/chunk-V52ATPTT.js.map +0 -1
- package/dist/dashboard/assets/index-D_bokML8.css +0 -1
- package/dist/dashboard/assets/index-r_jSJmlw.js +0 -121
- package/dist/interactive-RUY3OCBI.js.map +0 -1
- package/dist/templates/.agentv/targets.yaml +0 -97
- /package/dist/{artifact-writer-JFNIPMKW.js.map → artifact-writer-KJEOROKQ.js.map} +0 -0
- /package/dist/{dist-6Z7U473R.js.map → chunk-BV5VQLI2.js.map} +0 -0
- /package/dist/{ts-eval-loader-DQDYRULE-V3377FOL.js.map → chunk-TEEXVJWM.js.map} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../../packages/core/src/evaluation/script-grader-runtime.ts","../../../packages/core/src/evaluation/script-grader-schemas.ts","../../../packages/core/src/evaluation/vitest-workspace-grader.ts"],"sourcesContent":["/**\n * Runtime for script grader evaluators.\n * Handles stdin parsing, validation, error handling, and output formatting.\n */\nimport { readFileSync } from 'node:fs';\nimport { toCamelCaseDeep } from './case-conversion.js';\n\nimport {\n type ScriptGraderInput,\n ScriptGraderInputSchema,\n type ScriptGraderResult,\n ScriptGraderResultSchema,\n} from './script-grader-schemas.js';\n\n/**\n * Handler function type for script graders.\n */\nexport type ScriptGraderHandler = (\n input: ScriptGraderInput,\n) => ScriptGraderResult | Promise<ScriptGraderResult>;\n\n/**\n * Read stdin synchronously (works in both Node.js and Bun).\n */\nfunction readStdin(): string {\n return readFileSync(0, 'utf8');\n}\n\n/**\n * Clamp a value to the range [0, 1].\n */\nfunction clampScore(value: number): number {\n if (Number.isNaN(value) || !Number.isFinite(value)) {\n return 0;\n }\n return Math.max(0, Math.min(1, value));\n}\n\n/**\n * Format an error for output.\n */\nfunction formatError(error: unknown): string {\n if (error instanceof Error) {\n return error.message;\n }\n return String(error);\n}\n\n/**\n * Run a script grader handler with full stdin/stdout handling.\n * This is the internal implementation called by defineScriptGrader.\n */\nexport async function runScriptGrader(handler: ScriptGraderHandler): Promise<void> {\n try {\n // 1. Read stdin\n const stdin = readStdin();\n\n // 2. Parse JSON\n const rawInput = JSON.parse(stdin) as Record<string, unknown>;\n\n // 3. Convert snake_case to camelCase\n const camelInput = toCamelCaseDeep(rawInput);\n\n // 4. Validate input with Zod\n const input = ScriptGraderInputSchema.parse(camelInput);\n\n // 5. Set up lazy file-backed output loading if applicable\n if (input.outputPath && (input.output === null || input.output === undefined)) {\n let cachedOutput: ScriptGraderInput['output'] | undefined;\n const filePath = input.outputPath;\n Object.defineProperty(input, 'output', {\n get() {\n if (cachedOutput === undefined) {\n cachedOutput = JSON.parse(readFileSync(filePath, 'utf8'));\n }\n return cachedOutput;\n },\n configurable: true,\n enumerable: true,\n });\n }\n\n // 6. Run handler\n const rawResult = await handler(input);\n\n // 7. Validate and normalize output\n const result = ScriptGraderResultSchema.parse({\n ...rawResult,\n score: clampScore(rawResult.score),\n checks: rawResult.checks?.map((check) => ({\n ...check,\n ...(check.score !== undefined ? { score: clampScore(check.score) } : {}),\n })),\n });\n\n // 8. Output JSON\n console.log(JSON.stringify(result, null, 2));\n } catch (error) {\n // Output failure result\n const errorMessage = formatError(error);\n const errorResult: ScriptGraderResult = {\n pass: false,\n score: 0,\n reason: `Evaluation failed: ${errorMessage}`,\n checks: [\n {\n text: 'Script grader execution',\n pass: false,\n reason: errorMessage,\n },\n ],\n };\n console.log(JSON.stringify(errorResult, null, 2));\n process.exit(1);\n }\n}\n\n/** @deprecated Use ScriptGraderHandler. */\nexport type CodeGraderHandler = ScriptGraderHandler;\n\n/** @deprecated Use runScriptGrader. */\nexport const runCodeGrader = runScriptGrader;\n","/**\n * Zod schemas for script grader input/output validation.\n * Provides both compile-time types and runtime validation.\n *\n * ## Content model\n *\n * `Message.content` accepts `string | object[] | object`:\n * - `string` — backward-compatible plain text (most common case)\n * - `object[]` — typed content blocks for multimodal messages, plus AgentV\n * eval input blocks such as `{ type: \"file\", value, path, text }`\n * - `object` — structured YAML/JSON content such as expected outputs\n *\n * Content variants:\n * - `ContentText` — `{ type: 'text', text: string }`\n * - `ContentImage` — `{ type: 'image', media_type: string, path: string }` (file path, not base64)\n * - `ContentFile` — `{ type: 'file', media_type: string, path: string }`\n *\n * To add a new content variant:\n * 1. Define a new Zod schema with a unique `type` literal\n * 2. Add it to `ContentSchema` discriminated union\n * 3. Re-export from `index.ts`\n */\nimport { z } from 'zod';\n\n/**\n * Token usage metrics schema.\n */\nexport const TokenUsageSchema = z.object({\n input: z.number(),\n output: z.number(),\n cached: z.number().optional(),\n reasoning: z.number().optional(),\n});\n\n/**\n * Derived trace summary schema (camelCase for TypeScript ergonomics).\n *\n * This is a compact read model for metric-style graders. Full transcript/tool\n * evidence lives in the canonical `Trace` under `messages` and `events`.\n */\nexport const TraceSummarySchema = z.object({\n eventCount: z.number(),\n toolCalls: z.record(z.string(), z.number()),\n errorCount: z.number(),\n toolDurations: z.record(z.string(), z.array(z.number())).optional(),\n llmCallCount: z.number().optional(),\n});\n\nexport const TRACE_SOURCE_KINDS = [\n 'agentv_run',\n 'otlp',\n 'phoenix',\n 'langfuse',\n 'pi_session',\n 'imported_transcript',\n 'compact_transcript',\n] as const;\n\nexport const TRACE_EVENT_TYPES = [\n 'message',\n 'model_turn',\n 'tool_call',\n 'tool_result',\n 'final_response',\n 'error',\n] as const;\n\nexport const TRACE_TOOL_STATUSES = ['ok', 'error', 'timeout', 'cancelled', 'unknown'] as const;\n\nexport const TRACE_REDACTION_LEVELS = ['none', 'partial', 'full'] as const;\n\nconst MetadataSchema = z.record(z.string(), z.unknown());\n\nexport const TraceRedactionStateSchema = z.object({\n level: z.enum(TRACE_REDACTION_LEVELS),\n fields: z.array(z.string()).optional(),\n reason: z.string().optional(),\n});\n\nexport const TraceErrorSchema = z.object({\n message: z.string(),\n name: z.string().optional(),\n code: z.string().optional(),\n stack: z.string().optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceSourceSchema = z.object({\n kind: z.enum(TRACE_SOURCE_KINDS),\n path: z.string().optional(),\n url: z.string().optional(),\n provider: z.string().optional(),\n format: z.string().optional(),\n version: z.string().optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceSessionSchema = z.object({\n sessionId: z.string().optional(),\n conversationId: z.string().optional(),\n cwd: z.string().optional(),\n startedAt: z.string().optional(),\n endedAt: z.string().optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceBranchSchema = z.object({\n selectedLeafId: z.string().optional(),\n selectedPathIds: z.array(z.string()).optional(),\n includedEventIds: z.array(z.string()).optional(),\n omittedEventIds: z.array(z.string()).optional(),\n selectionReason: z.string().optional(),\n});\n\nexport const TraceSourceRefSchema = z.object({\n eventId: z.string().optional(),\n messageId: z.string().optional(),\n spanId: z.string().optional(),\n traceId: z.string().optional(),\n rawKind: z.string().optional(),\n path: z.string().optional(),\n line: z.number().int().nonnegative().optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceRawEvidenceSchema = z.object({\n kind: z.string(),\n ref: z.string().optional(),\n mediaType: z.string().optional(),\n content: z.unknown().optional(),\n redacted: z.boolean().optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceMessageSchema = z.object({\n role: z.string(),\n name: z.string().optional(),\n content: z.unknown().optional(),\n redaction: TraceRedactionStateSchema.optional(),\n tokenUsage: TokenUsageSchema.optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceModelSchema = z.object({\n provider: z.string().optional(),\n name: z.string().optional(),\n invocationId: z.string().optional(),\n tokenUsage: TokenUsageSchema.optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceToolSchema = z.object({\n name: z.string(),\n callId: z.string().optional(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n status: z.enum(TRACE_TOOL_STATUSES).optional(),\n error: TraceErrorSchema.optional(),\n redaction: TraceRedactionStateSchema.optional(),\n metadata: MetadataSchema.optional(),\n});\n\nexport const TraceEventSchema = z.object({\n eventId: z.string(),\n parentEventId: z.string().optional(),\n ordinal: z.number().int().nonnegative(),\n type: z.enum(TRACE_EVENT_TYPES),\n timestamp: z.string().optional(),\n durationMs: z.number().nonnegative().optional(),\n durationInferred: z.boolean().optional(),\n turnIndex: z.number().int().nonnegative().optional(),\n message: TraceMessageSchema.optional(),\n model: TraceModelSchema.optional(),\n tool: TraceToolSchema.optional(),\n error: TraceErrorSchema.optional(),\n sourceRef: TraceSourceRefSchema.optional(),\n rawEvidence: z.array(TraceRawEvidenceSchema).optional(),\n redaction: TraceRedactionStateSchema.optional(),\n metadata: MetadataSchema.optional(),\n});\n\n/**\n * Derived trace artifact shape used by import/replay helpers.\n *\n * This is not a persisted public trace contract. Grader authors normally\n * receive the result-local `Trace` shape below.\n */\nexport const TraceArtifactSchema = z.object({\n source: TraceSourceSchema,\n session: TraceSessionSchema,\n branch: TraceBranchSchema.optional(),\n events: z.array(TraceEventSchema),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n startedAt: z.string().optional(),\n endedAt: z.string().optional(),\n metadata: MetadataSchema.optional(),\n});\n\n/**\n * Tool call schema.\n */\nexport const ToolCallSchema = z.object({\n tool: z.string(),\n input: z.unknown().optional(),\n output: z.unknown().optional(),\n id: z.string().optional(),\n startTime: z.string().optional(),\n endTime: z.string().optional(),\n durationMs: z.number().optional(),\n});\n\n// ---------------------------------------------------------------------------\n// Content block schemas (discriminated union on `type`)\n// ---------------------------------------------------------------------------\n\n/** Text content block. */\nexport const ContentTextSchema = z.object({\n type: z.literal('text'),\n text: z.string(),\n});\n\n/**\n * Image content block.\n * `path` is a filesystem path — never inline base64.\n */\nexport const ContentImageSchema = z.object({\n type: z.literal('image'),\n media_type: z.string(),\n path: z.string(),\n});\n\n/** File content block. */\nexport const ContentFileSchema = z.object({\n type: z.literal('file'),\n media_type: z.string(),\n path: z.string(),\n});\n\n/** Discriminated union of all content block types. */\nexport const ContentSchema = z.discriminatedUnion('type', [\n ContentTextSchema,\n ContentImageSchema,\n ContentFileSchema,\n]);\n\nconst MessageContentBlockSchema = z.union([ContentSchema, z.record(z.unknown())]);\n\n/**\n * Unified message schema for input, expected, and output messages.\n *\n * `content` is a plain string, an array of structured blocks, or a\n * structured object from YAML/JSON eval files. Use `getTextContent()` from\n * `agentv/contracts` to extract plain text when the content is textual.\n */\nexport const MessageSchema = z.object({\n role: z.enum(['assistant', 'user', 'system', 'tool']),\n content: z\n .union([z.string(), z.array(MessageContentBlockSchema), z.record(z.unknown())])\n .optional(),\n toolCalls: z.array(ToolCallSchema).optional(),\n name: z.string().optional(),\n startTime: z.string().optional(),\n endTime: z.string().optional(),\n durationMs: z.number().optional(),\n metadata: z.record(z.unknown()).optional(),\n});\n\n/**\n * Derived evaluation trace read model exposed to custom assertions and script graders.\n *\n * Top-level summary fields (`eventCount`, `toolCalls`, `errorCount`) remain\n * available for existing metric graders; full transcript/tool evidence is under\n * `messages` and structured execution events under `events`.\n */\nexport const TraceSchema = TraceSummarySchema.extend({\n messages: z.array(MessageSchema),\n events: z.array(TraceEventSchema),\n tokenUsage: TokenUsageSchema.optional(),\n costUsd: z.number().optional(),\n durationMs: z.number().optional(),\n startTime: z.string().optional(),\n endTime: z.string().optional(),\n metadata: MetadataSchema.optional(),\n});\n\n/**\n * Script grader input schema (camelCase, converted from snake_case wire format).\n *\n * `output` is the final answer/scored result only. Transcript-aware graders\n * should inspect `messages`, `trace.messages`, or `trace.events`.\n */\nexport const ScriptGraderInputSchema = z.object({\n criteria: z.string(),\n expectedOutput: z.array(MessageSchema),\n output: z.string().nullable().optional(),\n messages: z.array(MessageSchema).optional().default([]),\n /** Path to a temp file containing the output JSON (used for large payloads). */\n outputPath: z.string().optional(),\n inputFiles: z.array(z.string()),\n input: z.array(MessageSchema),\n metadata: z.record(z.unknown()).nullable().optional(),\n trace: TraceSchema.nullable().optional(),\n traceSummary: TraceSummarySchema.nullable().optional(),\n tokenUsage: TokenUsageSchema.nullable().optional(),\n costUsd: z.number().nullable().optional(),\n durationMs: z.number().nullable().optional(),\n startTime: z.string().nullable().optional(),\n endTime: z.string().nullable().optional(),\n fileChanges: z.string().nullable().optional(),\n workspacePath: z.string().nullable().optional(),\n config: z.record(z.unknown()).nullable().optional(),\n});\n\nexport const ScriptGraderCheckSchema = z.object({\n id: z.string().optional(),\n text: z.string(),\n pass: z.boolean(),\n score: z.number().min(0).max(1).optional(),\n reason: z.string(),\n evidence: z.string().optional(),\n});\n\n/**\n * Script grader result schema (validated before output).\n */\nexport const ScriptGraderResultSchema = z.object({\n pass: z.boolean(),\n score: z.number().min(0).max(1),\n reason: z.string(),\n checks: z.array(ScriptGraderCheckSchema).optional().default([]),\n /** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */\n details: z.record(z.unknown()).optional(),\n});\n\n/**\n * Inferred types from schemas.\n */\nexport type ScriptGraderInput = z.infer<typeof ScriptGraderInputSchema>;\nexport type ScriptGraderCheck = z.infer<typeof ScriptGraderCheckSchema>;\nexport type ScriptGraderResult = z.infer<typeof ScriptGraderResultSchema>;\n\nexport type TraceSummary = z.infer<typeof TraceSummarySchema>;\nexport type Trace = z.infer<typeof TraceSchema>;\nexport type TraceArtifact = z.infer<typeof TraceArtifactSchema>;\nexport type TraceSource = z.infer<typeof TraceSourceSchema>;\nexport type TraceSession = z.infer<typeof TraceSessionSchema>;\nexport type TraceBranch = z.infer<typeof TraceBranchSchema>;\nexport type TraceEvent = z.infer<typeof TraceEventSchema>;\nexport type TraceMessage = z.infer<typeof TraceMessageSchema>;\nexport type TraceModel = z.infer<typeof TraceModelSchema>;\nexport type TraceTool = z.infer<typeof TraceToolSchema>;\nexport type TraceError = z.infer<typeof TraceErrorSchema>;\nexport type TraceSourceRef = z.infer<typeof TraceSourceRefSchema>;\nexport type TraceRawEvidence = z.infer<typeof TraceRawEvidenceSchema>;\nexport type TraceRedactionState = z.infer<typeof TraceRedactionStateSchema>;\nexport type Message = z.infer<typeof MessageSchema>;\nexport type ToolCall = z.infer<typeof ToolCallSchema>;\nexport type TokenUsage = z.infer<typeof TokenUsageSchema>;\n\nexport type ContentText = z.infer<typeof ContentTextSchema>;\nexport type ContentImage = z.infer<typeof ContentImageSchema>;\nexport type ContentFile = z.infer<typeof ContentFileSchema>;\nexport type Content = z.infer<typeof ContentSchema>;\n\n/**\n * Prompt template input schema (camelCase, converted from snake_case wire format).\n * Uses the same schema as ScriptGraderInput since the orchestrator sends identical payloads.\n */\nexport const PromptTemplateInputSchema = ScriptGraderInputSchema;\n\nexport type PromptTemplateInput = ScriptGraderInput;\n\n// ── Backward-compat aliases (deprecated) ────────────────────────────────────────\n/** @deprecated Use ScriptGraderInputSchema */\nexport const CodeGraderInputSchema = ScriptGraderInputSchema;\n/** @deprecated Use ScriptGraderResultSchema */\nexport const CodeGraderResultSchema = ScriptGraderResultSchema;\n/** @deprecated Use ScriptGraderInput */\nexport type CodeGraderInput = ScriptGraderInput;\n/** @deprecated Use ScriptGraderResult */\nexport type CodeGraderResult = ScriptGraderResult;\n/** @deprecated Use ScriptGraderInputSchema */\nexport const CodeJudgeInputSchema = ScriptGraderInputSchema;\n/** @deprecated Use ScriptGraderResultSchema */\nexport const CodeJudgeResultSchema = ScriptGraderResultSchema;\n/** @deprecated Use ScriptGraderInput */\nexport type CodeJudgeInput = ScriptGraderInput;\n/** @deprecated Use ScriptGraderResult */\nexport type CodeJudgeResult = ScriptGraderResult;\n","/**\n * Vitest workspace verifier adapter.\n *\n * This module keeps deterministic workspace verification in familiar Vitest\n * tests while translating the JSON reporter output into AgentV's script-grader\n * result contract.\n */\nimport { spawn } from 'node:child_process';\nimport { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises';\nimport { tmpdir } from 'node:os';\nimport nodePath from 'node:path';\n\nimport { runScriptGrader } from './script-grader-runtime.js';\nimport {\n type ScriptGraderInput,\n type ScriptGraderResult,\n ScriptGraderResultSchema,\n} from './script-grader-schemas.js';\n\nexport interface VitestWorkspaceGraderOptions {\n /**\n * Vitest verifier file(s). By default these are relative to the prepared\n * workspace. When `copyTestFilesToWorkspace` is true, relative paths resolve\n * from `testFileRoot` instead.\n *\n * When provided without `command`, the adapter runs\n * `bunx vitest run <testFile> --reporter=json --outputFile <tmp>`.\n */\n readonly testFile?: string | readonly string[];\n /**\n * Copy `testFile` entries into a temporary directory inside the workspace\n * before running Vitest. Use this for hidden verifier files that live beside\n * the eval instead of inside the prepared workspace.\n */\n readonly copyTestFilesToWorkspace?: boolean;\n /**\n * Base directory for copied `testFile` entries. Defaults to `process.cwd()`.\n */\n readonly testFileRoot?: string;\n /**\n * Full command to run. Use this for package scripts such as\n * `[\"bun\", \"run\", \"verify:workspace\"]`.\n */\n readonly command?: readonly string[];\n /**\n * Base Vitest command used with `testFile`. Defaults to\n * `[\"bunx\", \"vitest\", \"run\"]`.\n */\n readonly vitestCommand?: readonly string[];\n /** Workspace-relative directory to run the command in. Defaults to workspace root. */\n readonly cwd?: string;\n /** Append `--reporter=json --outputFile <tmp>` to the command. Defaults to true for testFile mode. */\n readonly appendReporterArgs?: boolean;\n /** Read Vitest JSON from this path instead of stdout. Relative paths resolve under `cwd`. */\n readonly outputFile?: string;\n readonly timeoutMs?: number;\n readonly env?: Readonly<Record<string, string>>;\n readonly passWithNoTests?: boolean;\n}\n\ninterface CommandResult {\n readonly exitCode: number | null;\n readonly stdout: string;\n readonly stderr: string;\n}\n\ninterface VitestAssertionResult {\n readonly ancestorTitles?: readonly string[];\n readonly fullName?: string;\n readonly title?: string;\n readonly status?: string;\n readonly failureMessages?: readonly string[];\n readonly duration?: number;\n}\n\ninterface VitestFileResult {\n readonly name?: string;\n readonly assertionResults?: readonly VitestAssertionResult[];\n}\n\ninterface VitestJsonReport {\n readonly success?: boolean;\n readonly numTotalTests?: number;\n readonly numPassedTests?: number;\n readonly numFailedTests?: number;\n readonly numPendingTests?: number;\n readonly numTodoTests?: number;\n readonly testResults?: readonly VitestFileResult[];\n}\n\nfunction workspacePathFrom(input: ScriptGraderInput): string | undefined {\n const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH;\n return workspacePath?.trim() ? workspacePath : undefined;\n}\n\nfunction truncate(value: string, maxLength = 2000): string {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}\\n...(truncated)`;\n}\n\nfunction resolveInsideWorkspace(\n workspacePath: string,\n relativePath: string,\n label: string,\n options: { readonly allowRoot?: boolean } = {},\n): string {\n if (!relativePath.trim()) {\n throw new Error(`${label} must not be empty.`);\n }\n\n if (nodePath.isAbsolute(relativePath)) {\n throw new Error(`${label} must be relative to the workspace: ${relativePath}`);\n }\n\n const root = nodePath.resolve(workspacePath);\n const resolvedPath = nodePath.resolve(root, relativePath);\n const relativeToRoot = nodePath.relative(root, resolvedPath);\n if (relativeToRoot === '' && options.allowRoot === true) {\n return resolvedPath;\n }\n\n if (\n relativeToRoot === '' ||\n relativeToRoot.startsWith('..') ||\n nodePath.isAbsolute(relativeToRoot)\n ) {\n throw new Error(`${label} must stay inside the workspace: ${relativePath}`);\n }\n\n return resolvedPath;\n}\n\nfunction normalizeTestFiles(testFile: string | readonly string[] | undefined): readonly string[] {\n if (testFile === undefined) {\n return [];\n }\n return typeof testFile === 'string' ? [testFile] : [...testFile];\n}\n\nfunction buildCommand(\n options: VitestWorkspaceGraderOptions,\n testFiles: readonly string[] = normalizeTestFiles(options.testFile),\n) {\n if (options.command && options.command.length > 0) {\n return [...options.command];\n }\n\n const command = [...(options.vitestCommand ?? ['bunx', 'vitest', 'run'])];\n command.push(...testFiles);\n return command;\n}\n\nfunction parseJsonObjectFromText(text: string): unknown {\n const trimmed = text.trim();\n if (!trimmed) {\n return undefined;\n }\n\n try {\n return JSON.parse(trimmed);\n } catch {\n const start = trimmed.indexOf('{');\n const end = trimmed.lastIndexOf('}');\n if (start >= 0 && end > start) {\n return JSON.parse(trimmed.slice(start, end + 1));\n }\n throw new Error('Vitest output did not contain a JSON object.');\n }\n}\n\nfunction isVitestJsonReport(value: unknown): value is VitestJsonReport {\n return (\n typeof value === 'object' &&\n value !== null &&\n Array.isArray((value as VitestJsonReport).testResults)\n );\n}\n\nfunction assertionText(file: VitestFileResult, assertion: VitestAssertionResult): string {\n const title =\n assertion.fullName ??\n [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean).join(' ');\n return title || file.name || 'Vitest assertion';\n}\n\nexport function vitestReportToScriptGraderResult(\n report: VitestJsonReport,\n options: Pick<VitestWorkspaceGraderOptions, 'passWithNoTests'> = {},\n): ScriptGraderResult {\n const checks = (report.testResults ?? []).flatMap((file) =>\n (file.assertionResults ?? []).map((item) => {\n const pass = item.status === 'passed';\n const evidence =\n item.failureMessages && item.failureMessages.length > 0\n ? truncate(item.failureMessages.join('\\n\\n'))\n : undefined;\n return {\n text: assertionText(file, item),\n pass,\n reason: pass ? 'Vitest test passed.' : 'Vitest test failed.',\n ...(evidence !== undefined ? { evidence } : {}),\n };\n }),\n );\n\n if (checks.length === 0) {\n const pass = options.passWithNoTests === true;\n return ScriptGraderResultSchema.parse({\n pass,\n score: pass ? 1 : 0,\n reason: pass ? 'Vitest reported no tests; configured to pass.' : 'Vitest reported no tests.',\n checks: [\n {\n text: 'Vitest reported no tests',\n pass,\n reason: pass ? 'passWithNoTests is enabled.' : 'No Vitest tests were discovered.',\n },\n ],\n details: {\n vitest_success: report.success ?? false,\n num_total_tests: report.numTotalTests ?? 0,\n num_passed_tests: report.numPassedTests ?? 0,\n num_failed_tests: report.numFailedTests ?? 0,\n num_pending_tests: report.numPendingTests ?? 0,\n num_todo_tests: report.numTodoTests ?? 0,\n },\n });\n }\n\n const passedCount = checks.filter((item) => item.pass).length;\n const pass = passedCount === checks.length;\n return ScriptGraderResultSchema.parse({\n pass,\n score: passedCount / checks.length,\n reason: `${passedCount}/${checks.length} Vitest tests passed.`,\n checks,\n details: {\n vitest_success: report.success ?? pass,\n num_total_tests: report.numTotalTests ?? checks.length,\n num_passed_tests: report.numPassedTests ?? passedCount,\n num_failed_tests: report.numFailedTests ?? checks.length - passedCount,\n num_pending_tests: report.numPendingTests ?? 0,\n num_todo_tests: report.numTodoTests ?? 0,\n },\n });\n}\n\n/** @deprecated Use vitestReportToScriptGraderResult. */\nexport const vitestReportToCodeGraderResult = vitestReportToScriptGraderResult;\n\nfunction runCommand(\n command: readonly string[],\n options: {\n readonly cwd: string;\n readonly timeoutMs?: number;\n readonly env?: Readonly<Record<string, string>>;\n },\n): Promise<CommandResult> {\n if (command.length === 0) {\n return Promise.reject(new Error('Vitest command must not be empty.'));\n }\n\n return new Promise((resolve, reject) => {\n const child = spawn(command[0], command.slice(1), {\n cwd: options.cwd,\n env: { ...process.env, ...options.env },\n stdio: ['ignore', 'pipe', 'pipe'],\n });\n\n let stdout = '';\n let stderr = '';\n let timedOut = false;\n const timeout =\n options.timeoutMs !== undefined\n ? setTimeout(() => {\n timedOut = true;\n child.kill('SIGTERM');\n }, options.timeoutMs)\n : undefined;\n\n child.stdout.setEncoding('utf8');\n child.stderr.setEncoding('utf8');\n child.stdout.on('data', (chunk) => {\n stdout += chunk;\n });\n child.stderr.on('data', (chunk) => {\n stderr += chunk;\n });\n child.on('error', reject);\n child.on('close', (exitCode) => {\n if (timeout) {\n clearTimeout(timeout);\n }\n if (timedOut) {\n reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`));\n return;\n }\n resolve({ exitCode, stdout, stderr });\n });\n });\n}\n\nfunction resolveSourceTestFile(testFileRoot: string, testFile: string): string {\n if (!testFile.trim()) {\n throw new Error('testFile entries must not be empty.');\n }\n\n return nodePath.isAbsolute(testFile)\n ? nodePath.resolve(testFile)\n : nodePath.resolve(testFileRoot, testFile);\n}\n\nasync function copyTestFilesIntoWorkspace(\n testFiles: readonly string[],\n options: Pick<VitestWorkspaceGraderOptions, 'testFileRoot'>,\n cwd: string,\n): Promise<{ readonly testFiles: readonly string[]; readonly tempDir?: string }> {\n if (testFiles.length === 0) {\n return { testFiles };\n }\n\n const tempDir = await mkdtemp(nodePath.join(cwd, '.agentv-vitest-'));\n const testFileRoot = nodePath.resolve(options.testFileRoot ?? process.cwd());\n const copiedFiles = await Promise.all(\n testFiles.map(async (testFile, index) => {\n const sourcePath = resolveSourceTestFile(testFileRoot, testFile);\n const destinationPath = nodePath.join(tempDir, `${index}-${nodePath.basename(testFile)}`);\n await copyFile(sourcePath, destinationPath);\n return nodePath.relative(cwd, destinationPath);\n }),\n );\n return { testFiles: copiedFiles, tempDir };\n}\n\nasync function readVitestReport(\n commandResult: CommandResult,\n outputFile: string | undefined,\n): Promise<VitestJsonReport> {\n const rawReport =\n outputFile !== undefined ? await readFile(outputFile, 'utf8') : commandResult.stdout;\n const parsed = parseJsonObjectFromText(rawReport);\n if (!isVitestJsonReport(parsed)) {\n throw new Error('Vitest JSON report did not include testResults[].');\n }\n return parsed;\n}\n\nexport async function runVitestWorkspaceGrader(\n options: VitestWorkspaceGraderOptions,\n input: ScriptGraderInput,\n): Promise<ScriptGraderResult> {\n const workspacePath = workspacePathFrom(input);\n if (!workspacePath) {\n return {\n pass: false,\n score: 0,\n reason: 'Vitest workspace verifier requires workspace_path.',\n checks: [\n {\n text: 'Vitest workspace verifier requires workspace_path',\n pass: false,\n reason:\n 'Configure an environment recipe in the eval YAML so AgentV can pass workspace_path.',\n },\n ],\n };\n }\n\n const tempDirs: string[] = [];\n try {\n const cwd = options.cwd\n ? resolveInsideWorkspace(workspacePath, options.cwd, 'cwd', { allowRoot: true })\n : workspacePath;\n const testFiles = normalizeTestFiles(options.testFile);\n const preparedTestFiles =\n options.copyTestFilesToWorkspace === true\n ? await copyTestFilesIntoWorkspace(testFiles, options, cwd)\n : { testFiles };\n if (preparedTestFiles.tempDir) {\n tempDirs.push(preparedTestFiles.tempDir);\n }\n\n const command = buildCommand(options, preparedTestFiles.testFiles);\n const appendReporterArgs = options.appendReporterArgs ?? options.command === undefined;\n let outputFile = options.outputFile\n ? resolveInsideWorkspace(cwd, options.outputFile, 'outputFile')\n : undefined;\n\n if (appendReporterArgs) {\n const tempDir = await mkdtemp(nodePath.join(tmpdir(), 'agentv-vitest-'));\n tempDirs.push(tempDir);\n outputFile = nodePath.join(tempDir, 'results.json');\n command.push('--reporter=json', `--outputFile=${outputFile}`);\n }\n\n const result = await runCommand(command, {\n cwd,\n timeoutMs: options.timeoutMs,\n env: {\n AGENTV_WORKSPACE_PATH: workspacePath,\n ...(outputFile !== undefined ? { AGENTV_VITEST_JSON_PATH: outputFile } : {}),\n ...options.env,\n },\n });\n\n const report = await readVitestReport(result, outputFile);\n return vitestReportToScriptGraderResult(report, options);\n } catch (error) {\n const reason = error instanceof Error ? error.message : String(error);\n return {\n pass: false,\n score: 0,\n reason: 'Vitest workspace verifier failed to run.',\n checks: [\n {\n text: 'Vitest workspace verifier failed to run',\n pass: false,\n reason,\n },\n ],\n };\n } finally {\n for (const tempDir of tempDirs.reverse()) {\n await rm(tempDir, { recursive: true, force: true }).catch(() => {});\n }\n }\n}\n\nexport function defineVitestWorkspaceGrader(options: VitestWorkspaceGraderOptions): void {\n runScriptGrader((input) => runVitestWorkspaceGrader(options, input));\n}\n"],"mappings":";;;;;;;AAIA,SAAS,oBAAoB;AEG7B,SAAS,aAAa;AACtB,SAAS,UAAU,SAAS,UAAU,UAAU;AAChD,SAAS,cAAc;AACvB,OAAO,cAAc;ADiBd,IAAM,mBAAmB,iBAAE,OAAO;EACvC,OAAO,iBAAE,OAAO;EAChB,QAAQ,iBAAE,OAAO;EACjB,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,WAAW,iBAAE,OAAO,EAAE,SAAS;AACjC,CAAC;AAQM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,YAAY,iBAAE,OAAO;EACrB,WAAW,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,OAAO,CAAC;EAC1C,YAAY,iBAAE,OAAO;EACrB,eAAe,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,MAAM,iBAAE,OAAO,CAAC,CAAC,EAAE,SAAS;EAClE,cAAc,iBAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB;EAChC;EACA;EACA;EACA;EACA;EACA;EACA;AACF;AAEO,IAAM,oBAAoB;EAC/B;EACA;EACA;EACA;EACA;EACA;AACF;AAEO,IAAM,sBAAsB,CAAC,MAAM,SAAS,WAAW,aAAa,SAAS;AAE7E,IAAM,yBAAyB,CAAC,QAAQ,WAAW,MAAM;AAEhE,IAAM,iBAAiB,iBAAE,OAAO,iBAAE,OAAO,GAAG,iBAAE,QAAQ,CAAC;AAEhD,IAAM,4BAA4B,iBAAE,OAAO;EAChD,OAAO,iBAAE,KAAK,sBAAsB;EACpC,QAAQ,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EACrC,QAAQ,iBAAE,OAAO,EAAE,SAAS;AAC9B,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,SAAS,iBAAE,OAAO;EAClB,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,OAAO,iBAAE,OAAO,EAAE,SAAS;EAC3B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,KAAK,kBAAkB;EAC/B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,UAAU,iBAAE,OAAO,EAAE,SAAS;EAC9B,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,gBAAgB,iBAAE,OAAO,EAAE,SAAS;EACpC,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,gBAAgB,iBAAE,OAAO,EAAE,SAAS;EACpC,iBAAiB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC9C,kBAAkB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC/C,iBAAiB,iBAAE,MAAM,iBAAE,OAAO,CAAC,EAAE,SAAS;EAC9C,iBAAiB,iBAAE,OAAO,EAAE,SAAS;AACvC,CAAC;AAEM,IAAM,uBAAuB,iBAAE,OAAO;EAC3C,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,MAAM,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,yBAAyB,iBAAE,OAAO;EAC7C,MAAM,iBAAE,OAAO;EACf,KAAK,iBAAE,OAAO,EAAE,SAAS;EACzB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,QAAQ,EAAE,SAAS;EAC9B,UAAU,iBAAE,QAAQ,EAAE,SAAS;EAC/B,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,MAAM,iBAAE,OAAO;EACf,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,SAAS,iBAAE,QAAQ,EAAE,SAAS;EAC9B,WAAW,0BAA0B,SAAS;EAC9C,YAAY,iBAAiB,SAAS;EACtC,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,UAAU,iBAAE,OAAO,EAAE,SAAS;EAC9B,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,cAAc,iBAAE,OAAO,EAAE,SAAS;EAClC,YAAY,iBAAiB,SAAS;EACtC,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,kBAAkB,iBAAE,OAAO;EACtC,MAAM,iBAAE,OAAO;EACf,QAAQ,iBAAE,OAAO,EAAE,SAAS;EAC5B,OAAO,iBAAE,QAAQ,EAAE,SAAS;EAC5B,QAAQ,iBAAE,QAAQ,EAAE,SAAS;EAC7B,QAAQ,iBAAE,KAAK,mBAAmB,EAAE,SAAS;EAC7C,OAAO,iBAAiB,SAAS;EACjC,WAAW,0BAA0B,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAEM,IAAM,mBAAmB,iBAAE,OAAO;EACvC,SAAS,iBAAE,OAAO;EAClB,eAAe,iBAAE,OAAO,EAAE,SAAS;EACnC,SAAS,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY;EACtC,MAAM,iBAAE,KAAK,iBAAiB;EAC9B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,YAAY,iBAAE,OAAO,EAAE,YAAY,EAAE,SAAS;EAC9C,kBAAkB,iBAAE,QAAQ,EAAE,SAAS;EACvC,WAAW,iBAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS;EACnD,SAAS,mBAAmB,SAAS;EACrC,OAAO,iBAAiB,SAAS;EACjC,MAAM,gBAAgB,SAAS;EAC/B,OAAO,iBAAiB,SAAS;EACjC,WAAW,qBAAqB,SAAS;EACzC,aAAa,iBAAE,MAAM,sBAAsB,EAAE,SAAS;EACtD,WAAW,0BAA0B,SAAS;EAC9C,UAAU,eAAe,SAAS;AACpC,CAAC;AAQM,IAAM,sBAAsB,iBAAE,OAAO;EAC1C,QAAQ;EACR,SAAS;EACT,QAAQ,kBAAkB,SAAS;EACnC,QAAQ,iBAAE,MAAM,gBAAgB;EAChC,YAAY,iBAAiB,SAAS;EACtC,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAKM,IAAM,iBAAiB,iBAAE,OAAO;EACrC,MAAM,iBAAE,OAAO;EACf,OAAO,iBAAE,QAAQ,EAAE,SAAS;EAC5B,QAAQ,iBAAE,QAAQ,EAAE,SAAS;EAC7B,IAAI,iBAAE,OAAO,EAAE,SAAS;EACxB,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;AAClC,CAAC;AAOM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,QAAQ,MAAM;EACtB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAMM,IAAM,qBAAqB,iBAAE,OAAO;EACzC,MAAM,iBAAE,QAAQ,OAAO;EACvB,YAAY,iBAAE,OAAO;EACrB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAGM,IAAM,oBAAoB,iBAAE,OAAO;EACxC,MAAM,iBAAE,QAAQ,MAAM;EACtB,YAAY,iBAAE,OAAO;EACrB,MAAM,iBAAE,OAAO;AACjB,CAAC;AAGM,IAAM,gBAAgB,iBAAE,mBAAmB,QAAQ;EACxD;EACA;EACA;AACF,CAAC;AAED,IAAM,4BAA4B,iBAAE,MAAM,CAAC,eAAe,iBAAE,OAAO,iBAAE,QAAQ,CAAC,CAAC,CAAC;AASzE,IAAM,gBAAgB,iBAAE,OAAO;EACpC,MAAM,iBAAE,KAAK,CAAC,aAAa,QAAQ,UAAU,MAAM,CAAC;EACpD,SAAS,iBACN,MAAM,CAAC,iBAAE,OAAO,GAAG,iBAAE,MAAM,yBAAyB,GAAG,iBAAE,OAAO,iBAAE,QAAQ,CAAC,CAAC,CAAC,EAC7E,SAAS;EACZ,WAAW,iBAAE,MAAM,cAAc,EAAE,SAAS;EAC5C,MAAM,iBAAE,OAAO,EAAE,SAAS;EAC1B,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,UAAU,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS;AAC3C,CAAC;AASM,IAAM,cAAc,mBAAmB,OAAO;EACnD,UAAU,iBAAE,MAAM,aAAa;EAC/B,QAAQ,iBAAE,MAAM,gBAAgB;EAChC,YAAY,iBAAiB,SAAS;EACtC,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,WAAW,iBAAE,OAAO,EAAE,SAAS;EAC/B,SAAS,iBAAE,OAAO,EAAE,SAAS;EAC7B,UAAU,eAAe,SAAS;AACpC,CAAC;AAQM,IAAM,0BAA0B,iBAAE,OAAO;EAC9C,UAAU,iBAAE,OAAO;EACnB,gBAAgB,iBAAE,MAAM,aAAa;EACrC,QAAQ,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACvC,UAAU,iBAAE,MAAM,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;;EAEtD,YAAY,iBAAE,OAAO,EAAE,SAAS;EAChC,YAAY,iBAAE,MAAM,iBAAE,OAAO,CAAC;EAC9B,OAAO,iBAAE,MAAM,aAAa;EAC5B,UAAU,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;EACpD,OAAO,YAAY,SAAS,EAAE,SAAS;EACvC,cAAc,mBAAmB,SAAS,EAAE,SAAS;EACrD,YAAY,iBAAiB,SAAS,EAAE,SAAS;EACjD,SAAS,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACxC,YAAY,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC3C,WAAW,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC1C,SAAS,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EACxC,aAAa,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC5C,eAAe,iBAAE,OAAO,EAAE,SAAS,EAAE,SAAS;EAC9C,QAAQ,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,SAAS;AACpD,CAAC;AAEM,IAAM,0BAA0B,iBAAE,OAAO;EAC9C,IAAI,iBAAE,OAAO,EAAE,SAAS;EACxB,MAAM,iBAAE,OAAO;EACf,MAAM,iBAAE,QAAQ;EAChB,OAAO,iBAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,SAAS;EACzC,QAAQ,iBAAE,OAAO;EACjB,UAAU,iBAAE,OAAO,EAAE,SAAS;AAChC,CAAC;AAKM,IAAM,2BAA2B,iBAAE,OAAO;EAC/C,MAAM,iBAAE,QAAQ;EAChB,OAAO,iBAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC;EAC9B,QAAQ,iBAAE,OAAO;EACjB,QAAQ,iBAAE,MAAM,uBAAuB,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;;EAE9D,SAAS,iBAAE,OAAO,iBAAE,QAAQ,CAAC,EAAE,SAAS;AAC1C,CAAC;AAoCM,IAAM,4BAA4B;AAMlC,IAAM,wBAAwB;AAE9B,IAAM,yBAAyB;ADlWtC,SAAS,YAAoB;AAC3B,SAAO,aAAa,GAAG,MAAM;AAC/B;AAKA,SAAS,WAAW,OAAuB;AACzC,MAAI,OAAO,MAAM,KAAK,KAAK,CAAC,OAAO,SAAS,KAAK,GAAG;AAClD,WAAO;EACT;AACA,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,KAAK,CAAC;AACvC;AAKA,SAAS,YAAY,OAAwB;AAC3C,MAAI,iBAAiB,OAAO;AAC1B,WAAO,MAAM;EACf;AACA,SAAO,OAAO,KAAK;AACrB;AAMA,eAAsB,gBAAgB,SAA6C;AACjF,MAAI;AAEF,UAAM,QAAQ,UAAU;AAGxB,UAAM,WAAW,KAAK,MAAM,KAAK;AAGjC,UAAM,aAAa,gBAAgB,QAAQ;AAG3C,UAAM,QAAQ,wBAAwB,MAAM,UAAU;AAGtD,QAAI,MAAM,eAAe,MAAM,WAAW,QAAQ,MAAM,WAAW,SAAY;AAC7E,UAAI;AACJ,YAAM,WAAW,MAAM;AACvB,aAAO,eAAe,OAAO,UAAU;QACrC,MAAM;AACJ,cAAI,iBAAiB,QAAW;AAC9B,2BAAe,KAAK,MAAM,aAAa,UAAU,MAAM,CAAC;UAC1D;AACA,iBAAO;QACT;QACA,cAAc;QACd,YAAY;MACd,CAAC;IACH;AAGA,UAAM,YAAY,MAAM,QAAQ,KAAK;AAGrC,UAAM,SAAS,yBAAyB,MAAM;MAC5C,GAAG;MACH,OAAO,WAAW,UAAU,KAAK;MACjC,QAAQ,UAAU,QAAQ,IAAI,CAAC,WAAW;QACxC,GAAG;QACH,GAAI,MAAM,UAAU,SAAY,EAAE,OAAO,WAAW,MAAM,KAAK,EAAE,IAAI,CAAC;MACxE,EAAE;IACJ,CAAC;AAGD,YAAQ,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;EAC7C,SAAS,OAAO;AAEd,UAAM,eAAe,YAAY,KAAK;AACtC,UAAM,cAAkC;MACtC,MAAM;MACN,OAAO;MACP,QAAQ,sBAAsB,YAAY;MAC1C,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN,QAAQ;QACV;MACF;IACF;AACA,YAAQ,IAAI,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC;AAChD,YAAQ,KAAK,CAAC;EAChB;AACF;AAMO,IAAM,gBAAgB;AE/B7B,SAAS,kBAAkB,OAA8C;AACvE,QAAM,gBAAgB,MAAM,iBAAiB,QAAQ,IAAI;AACzD,SAAO,eAAe,KAAK,IAAI,gBAAgB;AACjD;AAEA,SAAS,SAAS,OAAe,YAAY,KAAc;AACzD,MAAI,MAAM,UAAU,WAAW;AAC7B,WAAO;EACT;AACA,SAAO,GAAG,MAAM,MAAM,GAAG,SAAS,CAAC;;AACrC;AAEA,SAAS,uBACP,eACA,cACA,OACA,UAA4C,CAAC,GACrC;AACR,MAAI,CAAC,aAAa,KAAK,GAAG;AACxB,UAAM,IAAI,MAAM,GAAG,KAAK,qBAAqB;EAC/C;AAEA,MAAI,SAAS,WAAW,YAAY,GAAG;AACrC,UAAM,IAAI,MAAM,GAAG,KAAK,uCAAuC,YAAY,EAAE;EAC/E;AAEA,QAAM,OAAO,SAAS,QAAQ,aAAa;AAC3C,QAAM,eAAe,SAAS,QAAQ,MAAM,YAAY;AACxD,QAAM,iBAAiB,SAAS,SAAS,MAAM,YAAY;AAC3D,MAAI,mBAAmB,MAAM,QAAQ,cAAc,MAAM;AACvD,WAAO;EACT;AAEA,MACE,mBAAmB,MACnB,eAAe,WAAW,IAAI,KAC9B,SAAS,WAAW,cAAc,GAClC;AACA,UAAM,IAAI,MAAM,GAAG,KAAK,oCAAoC,YAAY,EAAE;EAC5E;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,UAAqE;AAC/F,MAAI,aAAa,QAAW;AAC1B,WAAO,CAAC;EACV;AACA,SAAO,OAAO,aAAa,WAAW,CAAC,QAAQ,IAAI,CAAC,GAAG,QAAQ;AACjE;AAEA,SAAS,aACP,SACA,YAA+B,mBAAmB,QAAQ,QAAQ,GAClE;AACA,MAAI,QAAQ,WAAW,QAAQ,QAAQ,SAAS,GAAG;AACjD,WAAO,CAAC,GAAG,QAAQ,OAAO;EAC5B;AAEA,QAAM,UAAU,CAAC,GAAI,QAAQ,iBAAiB,CAAC,QAAQ,UAAU,KAAK,CAAE;AACxE,UAAQ,KAAK,GAAG,SAAS;AACzB,SAAO;AACT;AAEA,SAAS,wBAAwB,MAAuB;AACtD,QAAM,UAAU,KAAK,KAAK;AAC1B,MAAI,CAAC,SAAS;AACZ,WAAO;EACT;AAEA,MAAI;AACF,WAAO,KAAK,MAAM,OAAO;EAC3B,QAAQ;AACN,UAAM,QAAQ,QAAQ,QAAQ,GAAG;AACjC,UAAM,MAAM,QAAQ,YAAY,GAAG;AACnC,QAAI,SAAS,KAAK,MAAM,OAAO;AAC7B,aAAO,KAAK,MAAM,QAAQ,MAAM,OAAO,MAAM,CAAC,CAAC;IACjD;AACA,UAAM,IAAI,MAAM,8CAA8C;EAChE;AACF;AAEA,SAAS,mBAAmB,OAA2C;AACrE,SACE,OAAO,UAAU,YACjB,UAAU,QACV,MAAM,QAAS,MAA2B,WAAW;AAEzD;AAEA,SAAS,cAAc,MAAwB,WAA0C;AACvF,QAAM,QACJ,UAAU,YACV,CAAC,GAAI,UAAU,kBAAkB,CAAC,GAAI,UAAU,KAAK,EAAE,OAAO,OAAO,EAAE,KAAK,GAAG;AACjF,SAAO,SAAS,KAAK,QAAQ;AAC/B;AAEO,SAAS,iCACd,QACA,UAAiE,CAAC,GAC9C;AACpB,QAAM,UAAU,OAAO,eAAe,CAAC,GAAG;IAAQ,CAAC,UAChD,KAAK,oBAAoB,CAAC,GAAG,IAAI,CAAC,SAAS;AAC1C,YAAMA,QAAO,KAAK,WAAW;AAC7B,YAAM,WACJ,KAAK,mBAAmB,KAAK,gBAAgB,SAAS,IAClD,SAAS,KAAK,gBAAgB,KAAK,MAAM,CAAC,IAC1C;AACN,aAAO;QACL,MAAM,cAAc,MAAM,IAAI;QAC9B,MAAAA;QACA,QAAQA,QAAO,wBAAwB;QACvC,GAAI,aAAa,SAAY,EAAE,SAAS,IAAI,CAAC;MAC/C;IACF,CAAC;EACH;AAEA,MAAI,OAAO,WAAW,GAAG;AACvB,UAAMA,QAAO,QAAQ,oBAAoB;AACzC,WAAO,yBAAyB,MAAM;MACpC,MAAAA;MACA,OAAOA,QAAO,IAAI;MAClB,QAAQA,QAAO,kDAAkD;MACjE,QAAQ;QACN;UACE,MAAM;UACN,MAAAA;UACA,QAAQA,QAAO,gCAAgC;QACjD;MACF;MACA,SAAS;QACP,gBAAgB,OAAO,WAAW;QAClC,iBAAiB,OAAO,iBAAiB;QACzC,kBAAkB,OAAO,kBAAkB;QAC3C,kBAAkB,OAAO,kBAAkB;QAC3C,mBAAmB,OAAO,mBAAmB;QAC7C,gBAAgB,OAAO,gBAAgB;MACzC;IACF,CAAC;EACH;AAEA,QAAM,cAAc,OAAO,OAAO,CAAC,SAAS,KAAK,IAAI,EAAE;AACvD,QAAM,OAAO,gBAAgB,OAAO;AACpC,SAAO,yBAAyB,MAAM;IACpC;IACA,OAAO,cAAc,OAAO;IAC5B,QAAQ,GAAG,WAAW,IAAI,OAAO,MAAM;IACvC;IACA,SAAS;MACP,gBAAgB,OAAO,WAAW;MAClC,iBAAiB,OAAO,iBAAiB,OAAO;MAChD,kBAAkB,OAAO,kBAAkB;MAC3C,kBAAkB,OAAO,kBAAkB,OAAO,SAAS;MAC3D,mBAAmB,OAAO,mBAAmB;MAC7C,gBAAgB,OAAO,gBAAgB;IACzC;EACF,CAAC;AACH;AAGO,IAAM,iCAAiC;AAE9C,SAAS,WACP,SACA,SAKwB;AACxB,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,QAAQ,OAAO,IAAI,MAAM,mCAAmC,CAAC;EACtE;AAEA,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,QAAQ,MAAM,QAAQ,CAAC,GAAG,QAAQ,MAAM,CAAC,GAAG;MAChD,KAAK,QAAQ;MACb,KAAK,EAAE,GAAG,QAAQ,KAAK,GAAG,QAAQ,IAAI;MACtC,OAAO,CAAC,UAAU,QAAQ,MAAM;IAClC,CAAC;AAED,QAAI,SAAS;AACb,QAAI,SAAS;AACb,QAAI,WAAW;AACf,UAAM,UACJ,QAAQ,cAAc,SAClB,WAAW,MAAM;AACf,iBAAW;AACX,YAAM,KAAK,SAAS;IACtB,GAAG,QAAQ,SAAS,IACpB;AAEN,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,YAAY,MAAM;AAC/B,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;IACZ,CAAC;AACD,UAAM,OAAO,GAAG,QAAQ,CAAC,UAAU;AACjC,gBAAU;IACZ,CAAC;AACD,UAAM,GAAG,SAAS,MAAM;AACxB,UAAM,GAAG,SAAS,CAAC,aAAa;AAC9B,UAAI,SAAS;AACX,qBAAa,OAAO;MACtB;AACA,UAAI,UAAU;AACZ,eAAO,IAAI,MAAM,kCAAkC,QAAQ,SAAS,KAAK,CAAC;AAC1E;MACF;AACA,cAAQ,EAAE,UAAU,QAAQ,OAAO,CAAC;IACtC,CAAC;EACH,CAAC;AACH;AAEA,SAAS,sBAAsB,cAAsB,UAA0B;AAC7E,MAAI,CAAC,SAAS,KAAK,GAAG;AACpB,UAAM,IAAI,MAAM,qCAAqC;EACvD;AAEA,SAAO,SAAS,WAAW,QAAQ,IAC/B,SAAS,QAAQ,QAAQ,IACzB,SAAS,QAAQ,cAAc,QAAQ;AAC7C;AAEA,eAAe,2BACb,WACA,SACA,KAC+E;AAC/E,MAAI,UAAU,WAAW,GAAG;AAC1B,WAAO,EAAE,UAAU;EACrB;AAEA,QAAM,UAAU,MAAM,QAAQ,SAAS,KAAK,KAAK,iBAAiB,CAAC;AACnE,QAAM,eAAe,SAAS,QAAQ,QAAQ,gBAAgB,QAAQ,IAAI,CAAC;AAC3E,QAAM,cAAc,MAAM,QAAQ;IAChC,UAAU,IAAI,OAAO,UAAU,UAAU;AACvC,YAAM,aAAa,sBAAsB,cAAc,QAAQ;AAC/D,YAAM,kBAAkB,SAAS,KAAK,SAAS,GAAG,KAAK,IAAI,SAAS,SAAS,QAAQ,CAAC,EAAE;AACxF,YAAM,SAAS,YAAY,eAAe;AAC1C,aAAO,SAAS,SAAS,KAAK,eAAe;IAC/C,CAAC;EACH;AACA,SAAO,EAAE,WAAW,aAAa,QAAQ;AAC3C;AAEA,eAAe,iBACb,eACA,YAC2B;AAC3B,QAAM,YACJ,eAAe,SAAY,MAAM,SAAS,YAAY,MAAM,IAAI,cAAc;AAChF,QAAM,SAAS,wBAAwB,SAAS;AAChD,MAAI,CAAC,mBAAmB,MAAM,GAAG;AAC/B,UAAM,IAAI,MAAM,mDAAmD;EACrE;AACA,SAAO;AACT;AAEA,eAAsB,yBACpB,SACA,OAC6B;AAC7B,QAAM,gBAAgB,kBAAkB,KAAK;AAC7C,MAAI,CAAC,eAAe;AAClB,WAAO;MACL,MAAM;MACN,OAAO;MACP,QAAQ;MACR,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN,QACE;QACJ;MACF;IACF;EACF;AAEA,QAAM,WAAqB,CAAC;AAC5B,MAAI;AACF,UAAM,MAAM,QAAQ,MAChB,uBAAuB,eAAe,QAAQ,KAAK,OAAO,EAAE,WAAW,KAAK,CAAC,IAC7E;AACJ,UAAM,YAAY,mBAAmB,QAAQ,QAAQ;AACrD,UAAM,oBACJ,QAAQ,6BAA6B,OACjC,MAAM,2BAA2B,WAAW,SAAS,GAAG,IACxD,EAAE,UAAU;AAClB,QAAI,kBAAkB,SAAS;AAC7B,eAAS,KAAK,kBAAkB,OAAO;IACzC;AAEA,UAAM,UAAU,aAAa,SAAS,kBAAkB,SAAS;AACjE,UAAM,qBAAqB,QAAQ,sBAAsB,QAAQ,YAAY;AAC7E,QAAI,aAAa,QAAQ,aACrB,uBAAuB,KAAK,QAAQ,YAAY,YAAY,IAC5D;AAEJ,QAAI,oBAAoB;AACtB,YAAM,UAAU,MAAM,QAAQ,SAAS,KAAK,OAAO,GAAG,gBAAgB,CAAC;AACvE,eAAS,KAAK,OAAO;AACrB,mBAAa,SAAS,KAAK,SAAS,cAAc;AAClD,cAAQ,KAAK,mBAAmB,gBAAgB,UAAU,EAAE;IAC9D;AAEA,UAAM,SAAS,MAAM,WAAW,SAAS;MACvC;MACA,WAAW,QAAQ;MACnB,KAAK;QACH,uBAAuB;QACvB,GAAI,eAAe,SAAY,EAAE,yBAAyB,WAAW,IAAI,CAAC;QAC1E,GAAG,QAAQ;MACb;IACF,CAAC;AAED,UAAM,SAAS,MAAM,iBAAiB,QAAQ,UAAU;AACxD,WAAO,iCAAiC,QAAQ,OAAO;EACzD,SAAS,OAAO;AACd,UAAM,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACpE,WAAO;MACL,MAAM;MACN,OAAO;MACP,QAAQ;MACR,QAAQ;QACN;UACE,MAAM;UACN,MAAM;UACN;QACF;MACF;IACF;EACF,UAAA;AACE,eAAW,WAAW,SAAS,QAAQ,GAAG;AACxC,YAAM,GAAG,SAAS,EAAE,WAAW,MAAM,OAAO,KAAK,CAAC,EAAE,MAAM,MAAM;MAAC,CAAC;IACpE;EACF;AACF;AAEO,SAAS,4BAA4B,SAA6C;AACvF,kBAAgB,CAAC,UAAU,yBAAyB,SAAS,KAAK,CAAC;AACrE;","names":["pass"]}
|
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
EXECUTION_TRACE_SCHEMA_VERSION,
|
|
7
7
|
buildPromptInputs,
|
|
8
8
|
external_exports,
|
|
9
|
-
|
|
9
|
+
extractLastAssistantContent2,
|
|
10
10
|
getAgentvDataDir,
|
|
11
11
|
groupTranscriptJsonLines,
|
|
12
12
|
loadTests,
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
prepareSharedWorkspaceSetup,
|
|
17
17
|
readTranscriptJsonl,
|
|
18
18
|
releaseSharedWorkspaceSetup
|
|
19
|
-
} from "./chunk-
|
|
19
|
+
} from "./chunk-6262OKXM.js";
|
|
20
20
|
|
|
21
21
|
// ../../packages/core/dist/index.js
|
|
22
22
|
import { readFileSync } from "node:fs";
|
|
@@ -43,12 +43,15 @@ import { promisify } from "node:util";
|
|
|
43
43
|
import * as childProcess from "node:child_process";
|
|
44
44
|
import { existsSync as existsSync2 } from "node:fs";
|
|
45
45
|
import { spawnSync } from "node:child_process";
|
|
46
|
+
import { spawnSync as spawnSync2 } from "node:child_process";
|
|
47
|
+
import { readFile as readFile2 } from "node:fs/promises";
|
|
48
|
+
import path4 from "node:path";
|
|
46
49
|
import { readdir as readdir2, stat as stat2 } from "node:fs/promises";
|
|
47
50
|
import { homedir } from "node:os";
|
|
48
|
-
import
|
|
51
|
+
import path5 from "node:path";
|
|
49
52
|
import { readdir as readdir3, stat as stat3 } from "node:fs/promises";
|
|
50
53
|
import { homedir as homedir2 } from "node:os";
|
|
51
|
-
import
|
|
54
|
+
import path6 from "node:path";
|
|
52
55
|
var TRACE_SESSION_SCHEMA_VERSION = "agentv.dashboard.trace_session.v1";
|
|
53
56
|
function isRecord(value) {
|
|
54
57
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -1489,6 +1492,9 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1489
1492
|
const type = entry.type;
|
|
1490
1493
|
switch (type) {
|
|
1491
1494
|
case "skill-trigger":
|
|
1495
|
+
throw new Error(staleSkillTriggerMessage(entry));
|
|
1496
|
+
case "skill-used":
|
|
1497
|
+
case "not-skill-used":
|
|
1492
1498
|
return null;
|
|
1493
1499
|
case "llm-rubric":
|
|
1494
1500
|
return typeof entry.value === "string" ? entry.value : null;
|
|
@@ -1515,10 +1521,9 @@ function assertionToNaturalLanguage(entry) {
|
|
|
1515
1521
|
case "ends-with":
|
|
1516
1522
|
return `Output ends with '${entry.value}'`;
|
|
1517
1523
|
case "llm-grader":
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
return typeof entry.prompt === "string" ? entry.prompt : null;
|
|
1524
|
+
throw new Error(
|
|
1525
|
+
"Authored assertion type 'llm-grader' has been removed. Use 'llm-rubric' for free-form rubric checks or 'agent-rubric' for agentic rubric checks."
|
|
1526
|
+
);
|
|
1522
1527
|
case "tool-trajectory": {
|
|
1523
1528
|
const expectedArr = Array.isArray(entry.expected) ? entry.expected : [];
|
|
1524
1529
|
const tools = expectedArr.map((e) => e.tool).filter(Boolean).join(", ");
|
|
@@ -1562,16 +1567,42 @@ function assertionToNaturalLanguageList(entry) {
|
|
|
1562
1567
|
}).filter((value) => typeof value === "string");
|
|
1563
1568
|
}
|
|
1564
1569
|
}
|
|
1565
|
-
if (entry.type === "llm-grader") {
|
|
1566
|
-
if (Array.isArray(entry.rubrics) && entry.rubrics.length > 0) {
|
|
1567
|
-
return entry.rubrics.map((r) => r.outcome ?? r.criteria ?? r.id).filter((s) => typeof s === "string");
|
|
1568
|
-
}
|
|
1569
|
-
}
|
|
1570
1570
|
const nl = assertionToNaturalLanguage(entry);
|
|
1571
1571
|
return nl !== null ? [nl] : [];
|
|
1572
1572
|
}
|
|
1573
|
-
function
|
|
1574
|
-
|
|
1573
|
+
function staleSkillTriggerMessage(entry) {
|
|
1574
|
+
const skill = typeof entry.skill === "string" ? entry.skill.trim() : "";
|
|
1575
|
+
const shouldTrigger = entry.should_trigger !== false;
|
|
1576
|
+
if (!skill) {
|
|
1577
|
+
return "Authored assertion type 'skill-trigger' has been removed. Use 'skill-used' with value: <skill> for expected skill use, or 'not-skill-used' with value: <skill> when the skill must not be used.";
|
|
1578
|
+
}
|
|
1579
|
+
const replacementType = shouldTrigger ? "skill-used" : "not-skill-used";
|
|
1580
|
+
return `Authored assertion type 'skill-trigger' has been removed. Replace skill: ${skill} with type: ${replacementType}, value: ${skill}.`;
|
|
1581
|
+
}
|
|
1582
|
+
function skillNameFromValue(value) {
|
|
1583
|
+
if (typeof value === "string" && value.trim()) {
|
|
1584
|
+
return value.trim();
|
|
1585
|
+
}
|
|
1586
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
1587
|
+
const name = value.name;
|
|
1588
|
+
return typeof name === "string" && name.trim() ? name.trim() : void 0;
|
|
1589
|
+
}
|
|
1590
|
+
return void 0;
|
|
1591
|
+
}
|
|
1592
|
+
function extractSkillUseAssertions(assertions) {
|
|
1593
|
+
return assertions.flatMap((entry) => {
|
|
1594
|
+
if (entry.type === "skill-trigger") {
|
|
1595
|
+
throw new Error(staleSkillTriggerMessage(entry));
|
|
1596
|
+
}
|
|
1597
|
+
if (entry.type !== "skill-used" && entry.type !== "not-skill-used") {
|
|
1598
|
+
return [];
|
|
1599
|
+
}
|
|
1600
|
+
const skill = skillNameFromValue(entry.value);
|
|
1601
|
+
if (!skill) {
|
|
1602
|
+
return [];
|
|
1603
|
+
}
|
|
1604
|
+
return [{ skill, shouldTrigger: entry.type === "skill-used" }];
|
|
1605
|
+
});
|
|
1575
1606
|
}
|
|
1576
1607
|
function extractInput(rawCase) {
|
|
1577
1608
|
const files = Array.isArray(rawCase.input_files) ? rawCase.input_files.filter((f) => typeof f === "string") : [];
|
|
@@ -1619,7 +1650,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1619
1650
|
throw new Error(`Invalid EVAL.yaml: missing 'tests' array in '${source}'`);
|
|
1620
1651
|
}
|
|
1621
1652
|
const suiteAssertions = rawSuite.assert ?? [];
|
|
1622
|
-
const suiteNlAssertions = suiteAssertions.
|
|
1653
|
+
const suiteNlAssertions = suiteAssertions.flatMap(assertionToNaturalLanguageList);
|
|
1623
1654
|
function getSkillFile(skillName) {
|
|
1624
1655
|
const existing = files.get(skillName);
|
|
1625
1656
|
if (existing) return existing;
|
|
@@ -1641,12 +1672,12 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1641
1672
|
nlAssertions.push(rawCase.criteria.trim());
|
|
1642
1673
|
}
|
|
1643
1674
|
for (const entry of caseAssertions) {
|
|
1644
|
-
if (entry.type !== "skill-
|
|
1675
|
+
if (entry.type !== "skill-used" && entry.type !== "not-skill-used") {
|
|
1645
1676
|
nlAssertions.push(...assertionToNaturalLanguageList(entry));
|
|
1646
1677
|
}
|
|
1647
1678
|
}
|
|
1648
1679
|
nlAssertions.push(...suiteNlAssertions);
|
|
1649
|
-
const triggerJudges =
|
|
1680
|
+
const triggerJudges = extractSkillUseAssertions(caseAssertions);
|
|
1650
1681
|
const { prompt, files: inputFiles } = extractInput(rawCase);
|
|
1651
1682
|
const expectedOutput = extractExpectedOutput(rawCase.expected_output);
|
|
1652
1683
|
const numericId = idx + 1;
|
|
@@ -1662,10 +1693,8 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
1662
1693
|
noSkillFile2.evals.push({ ...baseCase });
|
|
1663
1694
|
} else {
|
|
1664
1695
|
for (const tj of triggerJudges) {
|
|
1665
|
-
const
|
|
1666
|
-
|
|
1667
|
-
const skillFile = getSkillFile(skillName);
|
|
1668
|
-
skillFile.evals.push({ ...baseCase, should_trigger: shouldTrigger });
|
|
1696
|
+
const skillFile = getSkillFile(tj.skill);
|
|
1697
|
+
skillFile.evals.push({ ...baseCase, should_trigger: tj.shouldTrigger });
|
|
1669
1698
|
}
|
|
1670
1699
|
}
|
|
1671
1700
|
}
|
|
@@ -1971,7 +2000,7 @@ You must return a valid JSON object matching this schema:
|
|
|
1971
2000
|
question: prompt,
|
|
1972
2001
|
systemPrompt: system
|
|
1973
2002
|
});
|
|
1974
|
-
const text =
|
|
2003
|
+
const text = extractLastAssistantContent2(response.output);
|
|
1975
2004
|
const cleaned = text.replace(/```json\n?|```/g, "").trim();
|
|
1976
2005
|
result = rubricGenerationSchema.parse(JSON.parse(cleaned));
|
|
1977
2006
|
break;
|
|
@@ -5374,6 +5403,127 @@ function runBeforeSessionHook(command) {
|
|
|
5374
5403
|
console.log(`before_session hook injected ${injected} environment variable(s).`);
|
|
5375
5404
|
}
|
|
5376
5405
|
}
|
|
5406
|
+
var ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
5407
|
+
function parseShellExportsEnv(content) {
|
|
5408
|
+
const result = {};
|
|
5409
|
+
for (const line of content.split("\n")) {
|
|
5410
|
+
const trimmed = line.trim();
|
|
5411
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
5412
|
+
const match = trimmed.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/);
|
|
5413
|
+
if (!match) continue;
|
|
5414
|
+
const key = match[1];
|
|
5415
|
+
let value = match[2];
|
|
5416
|
+
if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
|
|
5417
|
+
value = value.slice(1, -1);
|
|
5418
|
+
}
|
|
5419
|
+
result[key] = value;
|
|
5420
|
+
}
|
|
5421
|
+
return result;
|
|
5422
|
+
}
|
|
5423
|
+
function parseJsonEnv(content) {
|
|
5424
|
+
let parsed;
|
|
5425
|
+
try {
|
|
5426
|
+
parsed = JSON.parse(content);
|
|
5427
|
+
} catch {
|
|
5428
|
+
throw new Error("invalid JSON");
|
|
5429
|
+
}
|
|
5430
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
5431
|
+
throw new Error("expected a flat JSON object of string values");
|
|
5432
|
+
}
|
|
5433
|
+
const result = {};
|
|
5434
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
5435
|
+
if (!ENV_NAME_PATTERN.test(key)) {
|
|
5436
|
+
throw new Error(`invalid environment variable name "${key}"`);
|
|
5437
|
+
}
|
|
5438
|
+
if (typeof value !== "string") {
|
|
5439
|
+
throw new Error(`value for "${key}" must be a string`);
|
|
5440
|
+
}
|
|
5441
|
+
result[key] = value;
|
|
5442
|
+
}
|
|
5443
|
+
return result;
|
|
5444
|
+
}
|
|
5445
|
+
function injectEnv(vars) {
|
|
5446
|
+
let injected = 0;
|
|
5447
|
+
for (const [key, value] of Object.entries(vars)) {
|
|
5448
|
+
if (!ENV_NAME_PATTERN.test(key)) continue;
|
|
5449
|
+
if (process.env[key] === void 0) {
|
|
5450
|
+
process.env[key] = value;
|
|
5451
|
+
injected++;
|
|
5452
|
+
}
|
|
5453
|
+
}
|
|
5454
|
+
return injected;
|
|
5455
|
+
}
|
|
5456
|
+
async function loadEnvPathFiles(envPaths, baseDir) {
|
|
5457
|
+
const loaded = [];
|
|
5458
|
+
const missing = [];
|
|
5459
|
+
let injectedCount = 0;
|
|
5460
|
+
for (const envPath of envPaths) {
|
|
5461
|
+
const resolvedPath = path4.isAbsolute(envPath) ? envPath : path4.join(baseDir, envPath);
|
|
5462
|
+
let content;
|
|
5463
|
+
try {
|
|
5464
|
+
content = await readFile2(resolvedPath, "utf8");
|
|
5465
|
+
} catch (error) {
|
|
5466
|
+
if (error.code === "ENOENT") {
|
|
5467
|
+
missing.push(resolvedPath);
|
|
5468
|
+
logWarning(`env_path file not found: ${resolvedPath}`);
|
|
5469
|
+
continue;
|
|
5470
|
+
}
|
|
5471
|
+
throw new Error(`Could not read env_path file ${resolvedPath}: ${error.message}`);
|
|
5472
|
+
}
|
|
5473
|
+
injectedCount += injectEnv(parseShellExportsEnv(content));
|
|
5474
|
+
loaded.push(resolvedPath);
|
|
5475
|
+
}
|
|
5476
|
+
if (injectedCount > 0) {
|
|
5477
|
+
console.log(
|
|
5478
|
+
`env_path injected ${injectedCount} environment variable(s) from ${loaded.length} file(s).`
|
|
5479
|
+
);
|
|
5480
|
+
}
|
|
5481
|
+
return { loaded, missing, injectedCount };
|
|
5482
|
+
}
|
|
5483
|
+
function parseEnvFromOutput(stdout, format) {
|
|
5484
|
+
return format === "json" ? parseJsonEnv(stdout) : parseShellExportsEnv(stdout);
|
|
5485
|
+
}
|
|
5486
|
+
async function runEnvFromEntries(entries, options) {
|
|
5487
|
+
let injectedCount = 0;
|
|
5488
|
+
for (const entry of entries) {
|
|
5489
|
+
const [command, ...args] = entry.command;
|
|
5490
|
+
const commandLabel = entry.command.join(" ");
|
|
5491
|
+
console.log(`Running env_from command: ${commandLabel}`);
|
|
5492
|
+
const result = spawnSync2(command, args, {
|
|
5493
|
+
cwd: options.cwd,
|
|
5494
|
+
encoding: "utf8",
|
|
5495
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
5496
|
+
});
|
|
5497
|
+
if (result.stderr) {
|
|
5498
|
+
process.stderr.write(result.stderr);
|
|
5499
|
+
}
|
|
5500
|
+
if (result.error) {
|
|
5501
|
+
throw new Error(`env_from command failed to start: ${commandLabel}: ${result.error.message}`);
|
|
5502
|
+
}
|
|
5503
|
+
if (result.status !== 0) {
|
|
5504
|
+
throw new Error(
|
|
5505
|
+
`env_from command exited with code ${result.status ?? "unknown"}: ${commandLabel}`
|
|
5506
|
+
);
|
|
5507
|
+
}
|
|
5508
|
+
const format = entry.format ?? "shell_exports";
|
|
5509
|
+
let vars;
|
|
5510
|
+
try {
|
|
5511
|
+
vars = parseEnvFromOutput(result.stdout ?? "", format);
|
|
5512
|
+
} catch (error) {
|
|
5513
|
+
throw new Error(
|
|
5514
|
+
`env_from command produced invalid ${format} output: ${commandLabel}: ${error.message}`
|
|
5515
|
+
);
|
|
5516
|
+
}
|
|
5517
|
+
injectedCount += injectEnv(vars);
|
|
5518
|
+
}
|
|
5519
|
+
if (injectedCount > 0) {
|
|
5520
|
+
console.log(`env_from injected ${injectedCount} environment variable(s).`);
|
|
5521
|
+
}
|
|
5522
|
+
return { injectedCount };
|
|
5523
|
+
}
|
|
5524
|
+
function logWarning(message) {
|
|
5525
|
+
console.warn(`Warning: ${message}`);
|
|
5526
|
+
}
|
|
5377
5527
|
var SKIPPED_TYPES = /* @__PURE__ */ new Set(["progress", "system", "file-history-snapshot"]);
|
|
5378
5528
|
function parseClaudeSession(jsonl) {
|
|
5379
5529
|
const messages = [];
|
|
@@ -5727,7 +5877,7 @@ function extractResponseItemContent(content) {
|
|
|
5727
5877
|
}
|
|
5728
5878
|
return parts.length > 0 ? parts.join("") : void 0;
|
|
5729
5879
|
}
|
|
5730
|
-
var DEFAULT_SESSIONS_DIR = () =>
|
|
5880
|
+
var DEFAULT_SESSIONS_DIR = () => path5.join(homedir(), ".codex", "sessions");
|
|
5731
5881
|
async function discoverCodexSessions(opts) {
|
|
5732
5882
|
const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
|
|
5733
5883
|
const limit = opts?.latest ? 1 : opts?.limit ?? 10;
|
|
@@ -5739,7 +5889,7 @@ async function discoverCodexSessions(opts) {
|
|
|
5739
5889
|
return [];
|
|
5740
5890
|
}
|
|
5741
5891
|
for (const year of yearDirs) {
|
|
5742
|
-
const yearPath =
|
|
5892
|
+
const yearPath = path5.join(sessionsDir, year);
|
|
5743
5893
|
let monthDirs;
|
|
5744
5894
|
try {
|
|
5745
5895
|
monthDirs = await readdir2(yearPath);
|
|
@@ -5747,7 +5897,7 @@ async function discoverCodexSessions(opts) {
|
|
|
5747
5897
|
continue;
|
|
5748
5898
|
}
|
|
5749
5899
|
for (const month of monthDirs) {
|
|
5750
|
-
const monthPath =
|
|
5900
|
+
const monthPath = path5.join(yearPath, month);
|
|
5751
5901
|
let dayDirs;
|
|
5752
5902
|
try {
|
|
5753
5903
|
dayDirs = await readdir2(monthPath);
|
|
@@ -5759,7 +5909,7 @@ async function discoverCodexSessions(opts) {
|
|
|
5759
5909
|
const dirDate = `${year}-${month}-${day}`;
|
|
5760
5910
|
if (dirDate !== opts.date) continue;
|
|
5761
5911
|
}
|
|
5762
|
-
const dayPath =
|
|
5912
|
+
const dayPath = path5.join(monthPath, day);
|
|
5763
5913
|
let files;
|
|
5764
5914
|
try {
|
|
5765
5915
|
files = await readdir2(dayPath);
|
|
@@ -5768,7 +5918,7 @@ async function discoverCodexSessions(opts) {
|
|
|
5768
5918
|
}
|
|
5769
5919
|
for (const file of files) {
|
|
5770
5920
|
if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
|
|
5771
|
-
const filePath =
|
|
5921
|
+
const filePath = path5.join(dayPath, file);
|
|
5772
5922
|
const nameWithoutExt = file.replace(/\.jsonl$/, "");
|
|
5773
5923
|
const parts = nameWithoutExt.split("-");
|
|
5774
5924
|
const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
|
|
@@ -5787,7 +5937,7 @@ async function discoverCodexSessions(opts) {
|
|
|
5787
5937
|
sessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
5788
5938
|
return sessions.slice(0, limit);
|
|
5789
5939
|
}
|
|
5790
|
-
var DEFAULT_PROJECTS_DIR = () =>
|
|
5940
|
+
var DEFAULT_PROJECTS_DIR = () => path6.join(homedir2(), ".claude", "projects");
|
|
5791
5941
|
function encodeProjectPath(projectPath) {
|
|
5792
5942
|
return projectPath.replace(/\//g, "-");
|
|
5793
5943
|
}
|
|
@@ -5806,7 +5956,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
5806
5956
|
}
|
|
5807
5957
|
const sessions = [];
|
|
5808
5958
|
for (const projectDir of projectDirs) {
|
|
5809
|
-
const dirPath =
|
|
5959
|
+
const dirPath = path6.join(projectsDir, projectDir);
|
|
5810
5960
|
let entries;
|
|
5811
5961
|
try {
|
|
5812
5962
|
entries = await readdir3(dirPath);
|
|
@@ -5817,7 +5967,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
5817
5967
|
if (!entry.endsWith(".jsonl")) continue;
|
|
5818
5968
|
const sessionId = entry.replace(/\.jsonl$/, "");
|
|
5819
5969
|
if (opts?.sessionId && sessionId !== opts.sessionId) continue;
|
|
5820
|
-
const filePath =
|
|
5970
|
+
const filePath = path6.join(dirPath, entry);
|
|
5821
5971
|
let updatedAt;
|
|
5822
5972
|
try {
|
|
5823
5973
|
const fileStat = await stat3(filePath);
|
|
@@ -6081,6 +6231,10 @@ export {
|
|
|
6081
6231
|
RunBudgetTracker,
|
|
6082
6232
|
parseEnvOutput,
|
|
6083
6233
|
runBeforeSessionHook,
|
|
6234
|
+
parseShellExportsEnv,
|
|
6235
|
+
parseJsonEnv,
|
|
6236
|
+
loadEnvPathFiles,
|
|
6237
|
+
runEnvFromEntries,
|
|
6084
6238
|
parseClaudeSession,
|
|
6085
6239
|
parseCodexSession,
|
|
6086
6240
|
discoverCodexSessions,
|
|
@@ -6089,4 +6243,4 @@ export {
|
|
|
6089
6243
|
parseCopilotEvents,
|
|
6090
6244
|
createAgentKernel
|
|
6091
6245
|
};
|
|
6092
|
-
//# sourceMappingURL=chunk-
|
|
6246
|
+
//# sourceMappingURL=chunk-JGSRUJZQ.js.map
|