agentv 0.2.8 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -585,7 +585,7 @@ var require_utc = __commonJS({
585
585
  import { Command } from "commander";
586
586
  import { readFileSync as readFileSync2 } from "node:fs";
587
587
 
588
- // ../../packages/core/dist/chunk-XXNQA4EW.js
588
+ // ../../packages/core/dist/chunk-NL7K4CAK.js
589
589
  import { constants } from "node:fs";
590
590
  import { access } from "node:fs/promises";
591
591
  import path from "node:path";
@@ -692,6 +692,8 @@ var KNOWN_PROVIDERS = [
692
692
  "azure",
693
693
  "anthropic",
694
694
  "gemini",
695
+ "codex",
696
+ "cli",
695
697
  "mock",
696
698
  "vscode",
697
699
  "vscode-insiders"
@@ -703,6 +705,8 @@ var PROVIDER_ALIASES = [
703
705
  // alias for "gemini"
704
706
  "google-gemini",
705
707
  // alias for "gemini"
708
+ "codex-cli",
709
+ // alias for "codex"
706
710
  "openai",
707
711
  // legacy/future support
708
712
  "bedrock",
@@ -716,7 +720,7 @@ var TARGETS_SCHEMA_V2 = "agentv-targets-v2";
716
720
  import micromatch from "micromatch";
717
721
  import { constants as constants3 } from "node:fs";
718
722
  import { access as access3, readFile as readFile2 } from "node:fs/promises";
719
- import path7 from "node:path";
723
+ import path8 from "node:path";
720
724
  import { fileURLToPath } from "node:url";
721
725
  import { parse as parse3 } from "yaml";
722
726
 
@@ -5031,6 +5035,11 @@ var _c = pr();
5031
5035
  var ya = new Error("Agent description must be at least 20 characters (explain in detail what the agent does)");
5032
5036
  var ba = new Error("Agent definition is the prompt you give to the LLM for the agent. It must be detailed and at least 100 characters");
5033
5037
 
5038
+ // ../../packages/core/dist/index.js
5039
+ import { exec as execWithCallback } from "node:child_process";
5040
+ import path22 from "node:path";
5041
+ import { promisify as promisify2 } from "node:util";
5042
+
5034
5043
  // ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/external.js
5035
5044
  var external_exports = {};
5036
5045
  __export(external_exports, {
@@ -5509,8 +5518,8 @@ function getErrorMap() {
5509
5518
 
5510
5519
  // ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/helpers/parseUtil.js
5511
5520
  var makeIssue = (params) => {
5512
- const { data, path: path17, errorMaps, issueData } = params;
5513
- const fullPath = [...path17, ...issueData.path || []];
5521
+ const { data, path: path18, errorMaps, issueData } = params;
5522
+ const fullPath = [...path18, ...issueData.path || []];
5514
5523
  const fullIssue = {
5515
5524
  ...issueData,
5516
5525
  path: fullPath
@@ -5626,11 +5635,11 @@ var errorUtil;
5626
5635
 
5627
5636
  // ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/types.js
5628
5637
  var ParseInputLazyPath = class {
5629
- constructor(parent, value, path17, key2) {
5638
+ constructor(parent, value, path18, key2) {
5630
5639
  this._cachedPath = [];
5631
5640
  this.parent = parent;
5632
5641
  this.data = value;
5633
- this._path = path17;
5642
+ this._path = path18;
5634
5643
  this._key = key2;
5635
5644
  }
5636
5645
  get path() {
@@ -9074,27 +9083,26 @@ var NEVER = INVALID;
9074
9083
 
9075
9084
  // ../../packages/core/dist/index.js
9076
9085
  import { readFile as readFile22 } from "node:fs/promises";
9077
- import path22 from "node:path";
9086
+ import path32 from "node:path";
9078
9087
 
9079
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/vscode/agentDispatch.js
9088
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/agentDispatch.js
9080
9089
  import { exec, spawn } from "child_process";
9081
9090
  import { copyFile, mkdir as mkdir2, readdir as readdir2, readFile, stat as stat2, writeFile } from "fs/promises";
9082
- import path5 from "path";
9091
+ import path6 from "path";
9083
9092
  import { promisify } from "util";
9084
9093
 
9085
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/vscode/constants.js
9094
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/constants.js
9086
9095
  import os from "os";
9087
9096
  import path2 from "path";
9088
9097
  var DEFAULT_LOCK_NAME = "subagent.lock";
9098
+ var DEFAULT_ALIVE_FILENAME = ".alive";
9089
9099
  function getDefaultSubagentRoot(vscodeCmd = "code") {
9090
9100
  const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
9091
9101
  return path2.join(os.homedir(), ".subagent", folder);
9092
9102
  }
9093
9103
  var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
9094
- var DEFAULT_WAKEUP_FILENAME = "wakeup.chatmode.md";
9095
- var DEFAULT_ALIVE_FILENAME = ".alive";
9096
9104
 
9097
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/utils/fs.js
9105
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/fs.js
9098
9106
  import { constants as constants2 } from "fs";
9099
9107
  import { access as access2, mkdir, readdir, rm, stat } from "fs/promises";
9100
9108
  import path3 from "path";
@@ -9127,15 +9135,26 @@ async function removeIfExists(target) {
9127
9135
  }
9128
9136
  }
9129
9137
 
9130
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/utils/time.js
9138
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/path.js
9139
+ import path4 from "path";
9140
+ function pathToFileUri(filePath) {
9141
+ const absolutePath = path4.isAbsolute(filePath) ? filePath : path4.resolve(filePath);
9142
+ const normalizedPath = absolutePath.replace(/\\/g, "/");
9143
+ if (/^[a-zA-Z]:\//.test(normalizedPath)) {
9144
+ return `file:///${normalizedPath}`;
9145
+ }
9146
+ return `file://${normalizedPath}`;
9147
+ }
9148
+
9149
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/time.js
9131
9150
  function sleep(ms2) {
9132
9151
  return new Promise((resolve) => {
9133
9152
  setTimeout(resolve, ms2);
9134
9153
  });
9135
9154
  }
9136
9155
 
9137
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/utils/workspace.js
9138
- import path4 from "path";
9156
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/workspace.js
9157
+ import path5 from "path";
9139
9158
 
9140
9159
  // ../../node_modules/.pnpm/json5@2.2.3/node_modules/json5/dist/index.mjs
9141
9160
  var Space_Separator = /[\u1680\u2000-\u200A\u202F\u205F\u3000]/;
@@ -10222,7 +10241,7 @@ var JSON5 = {
10222
10241
  var lib = JSON5;
10223
10242
  var dist_default = lib;
10224
10243
 
10225
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/utils/workspace.js
10244
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/workspace.js
10226
10245
  function transformWorkspacePaths(workspaceContent, templateDir) {
10227
10246
  let workspace;
10228
10247
  try {
@@ -10238,10 +10257,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
10238
10257
  }
10239
10258
  const transformedFolders = workspace.folders.map((folder) => {
10240
10259
  const folderPath = folder.path;
10241
- if (path4.isAbsolute(folderPath)) {
10260
+ if (path5.isAbsolute(folderPath)) {
10242
10261
  return folder;
10243
10262
  }
10244
- const absolutePath = path4.resolve(templateDir, folderPath);
10263
+ const absolutePath = path5.resolve(templateDir, folderPath);
10245
10264
  return {
10246
10265
  ...folder,
10247
10266
  path: absolutePath
@@ -10266,19 +10285,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
10266
10285
  if (locationMap && typeof locationMap === "object") {
10267
10286
  const transformedMap = {};
10268
10287
  for (const [locationPath, value] of Object.entries(locationMap)) {
10269
- const isAbsolute = path4.isAbsolute(locationPath);
10288
+ const isAbsolute = path5.isAbsolute(locationPath);
10270
10289
  if (isAbsolute) {
10271
10290
  transformedMap[locationPath] = value;
10272
10291
  } else {
10273
10292
  const firstGlobIndex = locationPath.search(/[*]/);
10274
10293
  if (firstGlobIndex === -1) {
10275
- const resolvedPath = path4.resolve(templateDir, locationPath).replace(/\\/g, "/");
10294
+ const resolvedPath = path5.resolve(templateDir, locationPath).replace(/\\/g, "/");
10276
10295
  transformedMap[resolvedPath] = value;
10277
10296
  } else {
10278
10297
  const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
10279
10298
  const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
10280
10299
  const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
10281
- const resolvedPath = (path4.resolve(templateDir, basePath) + patternPath).replace(/\\/g, "/");
10300
+ const resolvedPath = (path5.resolve(templateDir, basePath) + patternPath).replace(/\\/g, "/");
10282
10301
  transformedMap[resolvedPath] = value;
10283
10302
  }
10284
10303
  }
@@ -10295,28 +10314,39 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
10295
10314
  return JSON.stringify(transformedWorkspace, null, 2);
10296
10315
  }
10297
10316
 
10298
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/vscode/agentDispatch.js
10317
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/agentDispatch.js
10299
10318
  var execAsync = promisify(exec);
10319
+ function generateTimestamp() {
10320
+ return (/* @__PURE__ */ new Date()).toISOString().replace(/[-:TZ.]/g, "").slice(0, 14);
10321
+ }
10300
10322
  var DEFAULT_WORKSPACE_TEMPLATE = {
10301
10323
  folders: [
10302
10324
  {
10303
10325
  path: "."
10304
10326
  }
10305
- ],
10306
- settings: {
10307
- "chat.modeFilesLocations": {
10308
- "**/*.chatmode.md": true
10309
- }
10310
- }
10327
+ ]
10311
10328
  };
10312
10329
  var DEFAULT_WAKEUP_CONTENT = `---
10313
10330
  description: 'Wake-up Signal'
10314
- tools: ['edit', 'runNotebooks', 'search', 'new', 'runCommands', 'runTasks', 'usages', 'vscodeAPI', 'problems', 'changes', 'testFailure', 'openSimpleBrowser', 'fetch', 'githubRepo']
10315
- model: GPT-4.1 (copilot)
10331
+ model: Grok Code Fast 1 (copilot)
10316
10332
  ---`;
10317
10333
  function getSubagentRoot(vscodeCmd = "code") {
10318
10334
  return getDefaultSubagentRoot(vscodeCmd);
10319
10335
  }
10336
+ async function resolvePromptFile(promptFile) {
10337
+ if (!promptFile) {
10338
+ return void 0;
10339
+ }
10340
+ const resolvedPrompt = path6.resolve(promptFile);
10341
+ if (!await pathExists(resolvedPrompt)) {
10342
+ throw new Error(`Prompt file not found: ${resolvedPrompt}`);
10343
+ }
10344
+ const promptStats = await stat2(resolvedPrompt);
10345
+ if (!promptStats.isFile()) {
10346
+ throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
10347
+ }
10348
+ return resolvedPrompt;
10349
+ }
10320
10350
  async function findUnlockedSubagent(subagentRoot) {
10321
10351
  if (!await pathExists(subagentRoot)) {
10322
10352
  return null;
@@ -10327,7 +10357,7 @@ async function findUnlockedSubagent(subagentRoot) {
10327
10357
  number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
10328
10358
  })).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
10329
10359
  for (const subagent of subagents) {
10330
- const lockFile = path5.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
10360
+ const lockFile = path6.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
10331
10361
  if (!await pathExists(lockFile)) {
10332
10362
  return subagent.absolutePath;
10333
10363
  }
@@ -10348,9 +10378,12 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
10348
10378
  spawn(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
10349
10379
  return true;
10350
10380
  }
10351
- const aliveFile = path5.join(subagentDir, DEFAULT_ALIVE_FILENAME);
10381
+ const aliveFile = path6.join(subagentDir, DEFAULT_ALIVE_FILENAME);
10352
10382
  await removeIfExists(aliveFile);
10353
- const wakeupDst = path5.join(subagentDir, DEFAULT_WAKEUP_FILENAME);
10383
+ const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
10384
+ await mkdir2(githubAgentsDir, { recursive: true });
10385
+ const wakeupDst = path6.join(githubAgentsDir, "wakeup.md");
10386
+ const subagentDst = path6.join(githubAgentsDir, "subagent.md");
10354
10387
  await writeFile(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
10355
10388
  spawn(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
10356
10389
  await sleep(100);
@@ -10370,7 +10403,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
10370
10403
  async function copyAgentConfig(subagentDir, workspaceTemplate) {
10371
10404
  let workspaceContent;
10372
10405
  if (workspaceTemplate) {
10373
- const workspaceSrc = path5.resolve(workspaceTemplate);
10406
+ const workspaceSrc = path6.resolve(workspaceTemplate);
10374
10407
  if (!await pathExists(workspaceSrc)) {
10375
10408
  throw new Error(`workspace template not found: ${workspaceSrc}`);
10376
10409
  }
@@ -10383,33 +10416,37 @@ async function copyAgentConfig(subagentDir, workspaceTemplate) {
10383
10416
  } else {
10384
10417
  workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
10385
10418
  }
10386
- const workspaceName = `${path5.basename(subagentDir)}.code-workspace`;
10387
- const workspaceDst = path5.join(subagentDir, workspaceName);
10388
- const templateDir = workspaceTemplate ? path5.dirname(path5.resolve(workspaceTemplate)) : subagentDir;
10419
+ const workspaceName = `${path6.basename(subagentDir)}.code-workspace`;
10420
+ const workspaceDst = path6.join(subagentDir, workspaceName);
10421
+ const templateDir = workspaceTemplate ? path6.dirname(path6.resolve(workspaceTemplate)) : subagentDir;
10389
10422
  const workspaceJson = JSON.stringify(workspaceContent, null, 2);
10390
10423
  const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
10391
10424
  await writeFile(workspaceDst, transformedContent, "utf8");
10392
- const messagesDir = path5.join(subagentDir, "messages");
10425
+ const messagesDir = path6.join(subagentDir, "messages");
10393
10426
  await mkdir2(messagesDir, { recursive: true });
10394
10427
  return { workspace: workspaceDst, messagesDir };
10395
10428
  }
10396
10429
  async function createSubagentLock(subagentDir) {
10397
- const messagesDir = path5.join(subagentDir, "messages");
10430
+ const messagesDir = path6.join(subagentDir, "messages");
10398
10431
  if (await pathExists(messagesDir)) {
10399
10432
  const files = await readdir2(messagesDir);
10400
10433
  await Promise.all(files.map(async (file) => {
10401
- const target = path5.join(messagesDir, file);
10434
+ const target = path6.join(messagesDir, file);
10402
10435
  await removeIfExists(target);
10403
10436
  }));
10404
10437
  }
10405
- const chatmodeFiles = await readdir2(subagentDir);
10406
- await Promise.all(chatmodeFiles.filter((file) => file.endsWith(".chatmode.md")).map((file) => removeIfExists(path5.join(subagentDir, file))));
10407
- const lockFile = path5.join(subagentDir, DEFAULT_LOCK_NAME);
10438
+ const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
10439
+ if (await pathExists(githubAgentsDir)) {
10440
+ const agentFiles = await readdir2(githubAgentsDir);
10441
+ const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
10442
+ await Promise.all(agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path6.join(githubAgentsDir, file))));
10443
+ }
10444
+ const lockFile = path6.join(subagentDir, DEFAULT_LOCK_NAME);
10408
10445
  await writeFile(lockFile, "", { encoding: "utf8" });
10409
10446
  return lockFile;
10410
10447
  }
10411
10448
  async function removeSubagentLock(subagentDir) {
10412
- const lockFile = path5.join(subagentDir, DEFAULT_LOCK_NAME);
10449
+ const lockFile = path6.join(subagentDir, DEFAULT_LOCK_NAME);
10413
10450
  await removeIfExists(lockFile);
10414
10451
  }
10415
10452
  async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, silent = false) {
@@ -10449,6 +10486,54 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
10449
10486
  }
10450
10487
  return false;
10451
10488
  }
10489
+ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
10490
+ if (!silent) {
10491
+ const fileList = responseFilesFinal.map((file) => path6.basename(file)).join(", ");
10492
+ console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
10493
+ }
10494
+ try {
10495
+ const pending = new Set(responseFilesFinal);
10496
+ while (pending.size > 0) {
10497
+ for (const file of [...pending]) {
10498
+ if (await pathExists(file)) {
10499
+ pending.delete(file);
10500
+ }
10501
+ }
10502
+ if (pending.size > 0) {
10503
+ await sleep(pollInterval);
10504
+ }
10505
+ }
10506
+ } catch (error) {
10507
+ if (error.code === "ENOENT") {
10508
+ return false;
10509
+ }
10510
+ throw error;
10511
+ }
10512
+ for (const file of responseFilesFinal) {
10513
+ let attempts = 0;
10514
+ const maxAttempts = 10;
10515
+ while (attempts < maxAttempts) {
10516
+ try {
10517
+ const content = await readFile(file, { encoding: "utf8" });
10518
+ if (!silent) {
10519
+ process.stdout.write(`${content}
10520
+ `);
10521
+ }
10522
+ break;
10523
+ } catch (error) {
10524
+ attempts += 1;
10525
+ if (error.code !== "EBUSY" || attempts >= maxAttempts) {
10526
+ if (!silent) {
10527
+ console.error(`error: failed to read agent response: ${error.message}`);
10528
+ }
10529
+ return false;
10530
+ }
10531
+ await sleep(pollInterval);
10532
+ }
10533
+ }
10534
+ }
10535
+ return true;
10536
+ }
10452
10537
  async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
10453
10538
  if (dryRun) {
10454
10539
  return 0;
@@ -10466,17 +10551,19 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
10466
10551
  return 1;
10467
10552
  }
10468
10553
  if (promptFile) {
10469
- const chatmodeFile = path5.join(subagentDir, `${chatId}.chatmode.md`);
10554
+ const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
10555
+ await mkdir2(githubAgentsDir, { recursive: true });
10556
+ const agentFile = path6.join(githubAgentsDir, `${chatId}.md`);
10470
10557
  try {
10471
- await copyFile(promptFile, chatmodeFile);
10558
+ await copyFile(promptFile, agentFile);
10472
10559
  } catch (error) {
10473
- console.error(`error: Failed to copy prompt file to chatmode: ${error.message}`);
10560
+ console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
10474
10561
  return 1;
10475
10562
  }
10476
10563
  }
10477
10564
  return 0;
10478
10565
  }
10479
- function createRequestPrompt(userQuery, responseFileTmp, responseFileFinal, subagentName, vscodeCmd) {
10566
+ function createRequestPrompt(userQuery, responseFileTmp, responseFileFinal) {
10480
10567
  return `[[ ## task ## ]]
10481
10568
 
10482
10569
  ${userQuery}
@@ -10488,18 +10575,46 @@ ${userQuery}
10488
10575
  2. When completely finished, run these PowerShell commands to signal completion:
10489
10576
  \`\`\`
10490
10577
  Move-Item -LiteralPath '${responseFileTmp}' -Destination '${responseFileFinal}'
10491
- subagent ${vscodeCmd} unlock --subagent ${subagentName}
10578
+ if (Test-Path subagent.lock) { del subagent.lock }
10492
10579
  \`\`\`
10493
10580
 
10494
10581
  Do not proceed to step 2 until your response is completely written to the temporary file.`;
10495
10582
  }
10583
+ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal) {
10584
+ return `[[ ## task ## ]]
10585
+
10586
+ ${userQuery}
10587
+
10588
+ [[ ## system_instructions ## ]]
10589
+
10590
+ Write your complete response to: ${responseFileTmp}
10591
+ When completely finished and the response is stable, rename it to: ${responseFileFinal}
10592
+ Do not unlock the workspace from this request; batch orchestration will handle unlocking after all responses are ready.`;
10593
+ }
10594
+ function createBatchOrchestratorPrompt(requestFiles, responseFiles) {
10595
+ const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path6.basename(file)}`).join("\n");
10596
+ const responseList = responseFiles.map((file) => `"${path6.basename(file)}"`).join(", ");
10597
+ return `MANDATORY: Run #runSubagent tool in your Available Actions for each request file to process them in isolated contexts.
10598
+ DO NOT read the request files yourself - only pass the file paths to each subagent:
10599
+
10600
+ ${requestLines}
10601
+
10602
+ After ALL queries complete, verify all responses exist and unlock:
10603
+
10604
+ \`\`\`powershell
10605
+ $responses = @(${responseList})
10606
+ $missing = $responses | Where-Object { -not (Test-Path "messages/$_") }
10607
+ if ($missing.Count -eq 0) { del subagent.lock }
10608
+ \`\`\`
10609
+ `;
10610
+ }
10496
10611
  async function resolveAttachments(extraAttachments) {
10497
10612
  if (!extraAttachments) {
10498
10613
  return [];
10499
10614
  }
10500
10615
  const resolved = [];
10501
10616
  for (const attachment of extraAttachments) {
10502
- const resolvedPath = path5.resolve(attachment);
10617
+ const resolvedPath = path6.resolve(attachment);
10503
10618
  if (!await pathExists(resolvedPath)) {
10504
10619
  throw new Error(`Attachment not found: ${resolvedPath}`);
10505
10620
  }
@@ -10509,18 +10624,40 @@ async function resolveAttachments(extraAttachments) {
10509
10624
  }
10510
10625
  async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
10511
10626
  try {
10512
- const workspacePath = path5.join(subagentDir, `${path5.basename(subagentDir)}.code-workspace`);
10513
- const messagesDir = path5.join(subagentDir, "messages");
10627
+ const workspacePath = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
10628
+ const messagesDir = path6.join(subagentDir, "messages");
10514
10629
  await mkdir2(messagesDir, { recursive: true });
10515
- const reqFile = path5.join(messagesDir, `${timestamp}_req.md`);
10630
+ const reqFile = path6.join(messagesDir, `${timestamp}_req.md`);
10516
10631
  await writeFile(reqFile, requestInstructions, { encoding: "utf8" });
10517
10632
  const chatArgs = ["-r", "chat", "-m", chatId];
10518
10633
  for (const attachment of attachmentPaths) {
10519
10634
  chatArgs.push("-a", attachment);
10520
10635
  }
10521
10636
  chatArgs.push("-a", reqFile);
10522
- chatArgs.push(`Follow instructions in ${path5.basename(reqFile)}`);
10523
- const workspaceReady = await ensureWorkspaceFocused(workspacePath, path5.basename(subagentDir), subagentDir, vscodeCmd);
10637
+ chatArgs.push(`Follow instructions in ${path6.basename(reqFile)}`);
10638
+ const workspaceReady = await ensureWorkspaceFocused(workspacePath, path6.basename(subagentDir), subagentDir, vscodeCmd);
10639
+ if (!workspaceReady) {
10640
+ console.error("warning: Workspace may not be fully ready");
10641
+ }
10642
+ await sleep(500);
10643
+ spawn(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
10644
+ return true;
10645
+ } catch (error) {
10646
+ console.error(`warning: Failed to launch VS Code: ${error.message}`);
10647
+ return false;
10648
+ }
10649
+ }
10650
+ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
10651
+ try {
10652
+ const workspacePath = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
10653
+ const messagesDir = path6.join(subagentDir, "messages");
10654
+ await mkdir2(messagesDir, { recursive: true });
10655
+ const chatArgs = ["-r", "chat", "-m", chatId];
10656
+ for (const attachment of attachmentPaths) {
10657
+ chatArgs.push("-a", attachment);
10658
+ }
10659
+ chatArgs.push(chatInstruction);
10660
+ const workspaceReady = await ensureWorkspaceFocused(workspacePath, path6.basename(subagentDir), subagentDir, vscodeCmd);
10524
10661
  if (!workspaceReady) {
10525
10662
  console.error("warning: Workspace may not be fully ready");
10526
10663
  }
@@ -10536,21 +10673,13 @@ async function dispatchAgentSession(options) {
10536
10673
  const { userQuery, promptFile, extraAttachments, workspaceTemplate, dryRun = false, wait = true, vscodeCmd = "code", subagentRoot, silent = false } = options;
10537
10674
  try {
10538
10675
  let resolvedPrompt;
10539
- if (promptFile) {
10540
- resolvedPrompt = path5.resolve(promptFile);
10541
- if (!await pathExists(resolvedPrompt)) {
10542
- return {
10543
- exitCode: 1,
10544
- error: `Prompt file not found: ${resolvedPrompt}`
10545
- };
10546
- }
10547
- const promptStats = await stat2(resolvedPrompt);
10548
- if (!promptStats.isFile()) {
10549
- return {
10550
- exitCode: 1,
10551
- error: `Prompt file must be a file, not a directory: ${resolvedPrompt}`
10552
- };
10553
- }
10676
+ try {
10677
+ resolvedPrompt = await resolvePromptFile(promptFile);
10678
+ } catch (error) {
10679
+ return {
10680
+ exitCode: 1,
10681
+ error: error.message
10682
+ };
10554
10683
  }
10555
10684
  const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
10556
10685
  const subagentDir = await findUnlockedSubagent(subagentRootPath);
@@ -10560,7 +10689,7 @@ async function dispatchAgentSession(options) {
10560
10689
  error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
10561
10690
  };
10562
10691
  }
10563
- const subagentName = path5.basename(subagentDir);
10692
+ const subagentName = path6.basename(subagentDir);
10564
10693
  const chatId = Math.random().toString(16).slice(2, 10);
10565
10694
  const preparationResult = await prepareSubagentDirectory(subagentDir, resolvedPrompt, chatId, workspaceTemplate, dryRun);
10566
10695
  if (preparationResult !== 0) {
@@ -10580,11 +10709,11 @@ async function dispatchAgentSession(options) {
10580
10709
  error: attachmentError.message
10581
10710
  };
10582
10711
  }
10583
- const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[-:TZ.]/g, "").slice(0, 14);
10584
- const messagesDir = path5.join(subagentDir, "messages");
10585
- const responseFileTmp = path5.join(messagesDir, `${timestamp}_res.tmp.md`);
10586
- const responseFileFinal = path5.join(messagesDir, `${timestamp}_res.md`);
10587
- const requestInstructions = createRequestPrompt(userQuery, responseFileTmp, responseFileFinal, subagentName, vscodeCmd);
10712
+ const timestamp = generateTimestamp();
10713
+ const messagesDir = path6.join(subagentDir, "messages");
10714
+ const responseFileTmp = path6.join(messagesDir, `${timestamp}_res.tmp.md`);
10715
+ const responseFileFinal = path6.join(messagesDir, `${timestamp}_res.md`);
10716
+ const requestInstructions = createRequestPrompt(userQuery, responseFileTmp, responseFileFinal);
10588
10717
  if (dryRun) {
10589
10718
  return {
10590
10719
  exitCode: 0,
@@ -10635,10 +10764,141 @@ async function dispatchAgentSession(options) {
10635
10764
  };
10636
10765
  }
10637
10766
  }
10767
+ async function dispatchBatchAgent(options) {
10768
+ const { userQueries, promptFile, extraAttachments, workspaceTemplate, dryRun = false, wait = false, vscodeCmd = "code", subagentRoot, silent = false } = options;
10769
+ if (!userQueries || userQueries.length === 0) {
10770
+ return {
10771
+ exitCode: 1,
10772
+ requestFiles: [],
10773
+ queryCount: 0,
10774
+ error: "At least one query is required for batch dispatch"
10775
+ };
10776
+ }
10777
+ const queryCount = userQueries.length;
10778
+ let requestFiles = [];
10779
+ let responseFilesFinal = [];
10780
+ let subagentName;
10781
+ try {
10782
+ let resolvedPrompt;
10783
+ try {
10784
+ resolvedPrompt = await resolvePromptFile(promptFile);
10785
+ } catch (error) {
10786
+ return {
10787
+ exitCode: 1,
10788
+ requestFiles,
10789
+ queryCount,
10790
+ error: error.message
10791
+ };
10792
+ }
10793
+ const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
10794
+ const subagentDir = await findUnlockedSubagent(subagentRootPath);
10795
+ if (!subagentDir) {
10796
+ return {
10797
+ exitCode: 1,
10798
+ requestFiles,
10799
+ queryCount,
10800
+ error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
10801
+ };
10802
+ }
10803
+ subagentName = path6.basename(subagentDir);
10804
+ const chatId = Math.random().toString(16).slice(2, 10);
10805
+ const preparationResult = await prepareSubagentDirectory(subagentDir, resolvedPrompt, chatId, workspaceTemplate, dryRun);
10806
+ if (preparationResult !== 0) {
10807
+ return {
10808
+ exitCode: preparationResult,
10809
+ subagentName,
10810
+ requestFiles,
10811
+ queryCount,
10812
+ error: "Failed to prepare subagent workspace"
10813
+ };
10814
+ }
10815
+ let attachments;
10816
+ try {
10817
+ attachments = await resolveAttachments(extraAttachments);
10818
+ } catch (attachmentError) {
10819
+ return {
10820
+ exitCode: 1,
10821
+ subagentName,
10822
+ requestFiles,
10823
+ queryCount,
10824
+ error: attachmentError.message
10825
+ };
10826
+ }
10827
+ const timestamp = generateTimestamp();
10828
+ const messagesDir = path6.join(subagentDir, "messages");
10829
+ requestFiles = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_req.md`));
10830
+ const responseTmpFiles = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_res.tmp.md`));
10831
+ responseFilesFinal = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_res.md`));
10832
+ const orchestratorFile = path6.join(messagesDir, `${timestamp}_orchestrator.md`);
10833
+ if (!dryRun) {
10834
+ await Promise.all(userQueries.map((query, index) => writeFile(requestFiles[index], createBatchRequestPrompt(query, responseTmpFiles[index], responseFilesFinal[index]), { encoding: "utf8" })));
10835
+ const orchestratorContent = createBatchOrchestratorPrompt(requestFiles, responseFilesFinal);
10836
+ await writeFile(orchestratorFile, orchestratorContent, { encoding: "utf8" });
10837
+ }
10838
+ const chatAttachments = [orchestratorFile, ...attachments];
10839
+ const orchestratorUri = pathToFileUri(orchestratorFile);
10840
+ const chatInstruction = `Follow instructions in [${timestamp}_orchestrator.md](${orchestratorUri}). Use #runSubagent tool.`;
10841
+ if (dryRun) {
10842
+ return {
10843
+ exitCode: 0,
10844
+ subagentName,
10845
+ requestFiles,
10846
+ responseFiles: wait ? responseFilesFinal : void 0,
10847
+ queryCount
10848
+ };
10849
+ }
10850
+ const launchSuccess = await launchVsCodeWithBatchChat(subagentDir, chatId, chatAttachments, chatInstruction, vscodeCmd);
10851
+ if (!launchSuccess) {
10852
+ return {
10853
+ exitCode: 1,
10854
+ subagentName,
10855
+ requestFiles,
10856
+ queryCount,
10857
+ error: "Failed to launch VS Code for batch dispatch"
10858
+ };
10859
+ }
10860
+ if (!wait) {
10861
+ return {
10862
+ exitCode: 0,
10863
+ subagentName,
10864
+ requestFiles,
10865
+ queryCount
10866
+ };
10867
+ }
10868
+ const responsesCompleted = await waitForBatchResponses(responseFilesFinal, 1e3, silent);
10869
+ if (!responsesCompleted) {
10870
+ return {
10871
+ exitCode: 1,
10872
+ subagentName,
10873
+ requestFiles,
10874
+ responseFiles: responseFilesFinal,
10875
+ queryCount,
10876
+ error: "Timed out waiting for batch responses"
10877
+ };
10878
+ }
10879
+ await removeSubagentLock(subagentDir);
10880
+ return {
10881
+ exitCode: 0,
10882
+ subagentName,
10883
+ requestFiles,
10884
+ responseFiles: responseFilesFinal,
10885
+ queryCount
10886
+ };
10887
+ } catch (error) {
10888
+ return {
10889
+ exitCode: 1,
10890
+ subagentName,
10891
+ requestFiles,
10892
+ responseFiles: responseFilesFinal.length > 0 ? responseFilesFinal : void 0,
10893
+ queryCount,
10894
+ error: error.message
10895
+ };
10896
+ }
10897
+ }
10638
10898
 
10639
- // ../../node_modules/.pnpm/subagent@0.4.2/node_modules/subagent/dist/vscode/provision.js
10899
+ // ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/provision.js
10640
10900
  import { writeFile as writeFile2 } from "fs/promises";
10641
- import path6 from "path";
10901
+ import path7 from "path";
10642
10902
  var DEFAULT_WORKSPACE_TEMPLATE2 = {
10643
10903
  folders: [
10644
10904
  {
@@ -10647,7 +10907,7 @@ var DEFAULT_WORKSPACE_TEMPLATE2 = {
10647
10907
  ],
10648
10908
  settings: {
10649
10909
  "chat.modeFilesLocations": {
10650
- "**/*.chatmode.md": true
10910
+ ".github/agents/**/*.md": true
10651
10911
  }
10652
10912
  }
10653
10913
  };
@@ -10661,7 +10921,7 @@ async function provisionSubagents(options) {
10661
10921
  if (!Number.isInteger(subagents) || subagents < 1) {
10662
10922
  throw new Error("subagents must be a positive integer");
10663
10923
  }
10664
- const targetPath = path6.resolve(targetRoot);
10924
+ const targetPath = path7.resolve(targetRoot);
10665
10925
  if (!dryRun) {
10666
10926
  await ensureDir(targetPath);
10667
10927
  }
@@ -10680,7 +10940,7 @@ async function provisionSubagents(options) {
10680
10940
  continue;
10681
10941
  }
10682
10942
  highestNumber = Math.max(highestNumber, parsed);
10683
- const lockFile = path6.join(entry.absolutePath, lockName);
10943
+ const lockFile = path7.join(entry.absolutePath, lockName);
10684
10944
  const locked = await pathExists(lockFile);
10685
10945
  if (locked) {
10686
10946
  lockedSubagents.add(entry.absolutePath);
@@ -10697,9 +10957,11 @@ async function provisionSubagents(options) {
10697
10957
  break;
10698
10958
  }
10699
10959
  const subagentDir = subagent.absolutePath;
10700
- const lockFile = path6.join(subagentDir, lockName);
10701
- const workspaceDst = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
10702
- const wakeupDst = path6.join(subagentDir, DEFAULT_WAKEUP_FILENAME);
10960
+ const githubAgentsDir = path7.join(subagentDir, ".github", "agents");
10961
+ const lockFile = path7.join(subagentDir, lockName);
10962
+ const workspaceDst = path7.join(subagentDir, `${path7.basename(subagentDir)}.code-workspace`);
10963
+ const wakeupDst = path7.join(githubAgentsDir, "wakeup.md");
10964
+ const subagentDst = path7.join(githubAgentsDir, "subagent.md");
10703
10965
  const isLocked = await pathExists(lockFile);
10704
10966
  if (isLocked && !force) {
10705
10967
  continue;
@@ -10707,6 +10969,7 @@ async function provisionSubagents(options) {
10707
10969
  if (isLocked && force) {
10708
10970
  if (!dryRun) {
10709
10971
  await removeIfExists(lockFile);
10972
+ await ensureDir(githubAgentsDir);
10710
10973
  await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
10711
10974
  await writeFile2(wakeupDst, wakeupContent, "utf8");
10712
10975
  }
@@ -10717,6 +10980,7 @@ async function provisionSubagents(options) {
10717
10980
  }
10718
10981
  if (!isLocked && force) {
10719
10982
  if (!dryRun) {
10983
+ await ensureDir(githubAgentsDir);
10720
10984
  await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
10721
10985
  await writeFile2(wakeupDst, wakeupContent, "utf8");
10722
10986
  }
@@ -10725,6 +10989,7 @@ async function provisionSubagents(options) {
10725
10989
  continue;
10726
10990
  }
10727
10991
  if (!dryRun && !await pathExists(workspaceDst)) {
10992
+ await ensureDir(githubAgentsDir);
10728
10993
  await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
10729
10994
  await writeFile2(wakeupDst, wakeupContent, "utf8");
10730
10995
  }
@@ -10734,11 +10999,14 @@ async function provisionSubagents(options) {
10734
10999
  let nextIndex = highestNumber;
10735
11000
  while (subagentsProvisioned < subagents) {
10736
11001
  nextIndex += 1;
10737
- const subagentDir = path6.join(targetPath, `subagent-${nextIndex}`);
10738
- const workspaceDst = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
10739
- const wakeupDst = path6.join(subagentDir, DEFAULT_WAKEUP_FILENAME);
11002
+ const subagentDir = path7.join(targetPath, `subagent-${nextIndex}`);
11003
+ const githubAgentsDir = path7.join(subagentDir, ".github", "agents");
11004
+ const workspaceDst = path7.join(subagentDir, `${path7.basename(subagentDir)}.code-workspace`);
11005
+ const wakeupDst = path7.join(githubAgentsDir, "wakeup.md");
11006
+ const subagentDst = path7.join(githubAgentsDir, "subagent.md");
10740
11007
  if (!dryRun) {
10741
11008
  await ensureDir(subagentDir);
11009
+ await ensureDir(githubAgentsDir);
10742
11010
  await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
10743
11011
  await writeFile2(wakeupDst, wakeupContent, "utf8");
10744
11012
  }
@@ -10753,14 +11021,21 @@ async function provisionSubagents(options) {
10753
11021
  }
10754
11022
 
10755
11023
  // ../../packages/core/dist/index.js
11024
+ import { exec as execCallback, spawn as spawn2 } from "node:child_process";
10756
11025
  import { constants as constants22 } from "node:fs";
10757
- import { access as access22, readFile as readFile3 } from "node:fs/promises";
10758
- import path32 from "node:path";
11026
+ import { access as access22, copyFile as copyFile2, mkdtemp, mkdir as mkdir3, rm as rm2, writeFile as writeFile3 } from "node:fs/promises";
11027
+ import { tmpdir } from "node:os";
11028
+ import path52 from "node:path";
11029
+ import { promisify as promisify22 } from "node:util";
11030
+ import path42 from "node:path";
11031
+ import { constants as constants32 } from "node:fs";
11032
+ import { access as access32, readFile as readFile3 } from "node:fs/promises";
11033
+ import path62 from "node:path";
10759
11034
  import { parse as parse22 } from "yaml";
10760
11035
  import { randomUUID } from "node:crypto";
10761
11036
  import { createHash, randomUUID as randomUUID2 } from "node:crypto";
10762
- import { mkdir as mkdir3, writeFile as writeFile22 } from "node:fs/promises";
10763
- import path42 from "node:path";
11037
+ import { mkdir as mkdir22, readFile as readFile4, writeFile as writeFile22 } from "node:fs/promises";
11038
+ import path72 from "node:path";
10764
11039
  var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
10765
11040
  var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
10766
11041
  function isTestMessageRole(value) {
@@ -10800,10 +11075,10 @@ function isTestMessage(value) {
10800
11075
  }
10801
11076
  return candidate.content.every(isJsonObject);
10802
11077
  }
10803
- var GRADER_KIND_VALUES = ["heuristic", "llm_judge"];
10804
- var GRADER_KIND_SET = new Set(GRADER_KIND_VALUES);
10805
- function isGraderKind(value) {
10806
- return typeof value === "string" && GRADER_KIND_SET.has(value);
11078
+ var EVALUATOR_KIND_VALUES = ["code", "llm_judge"];
11079
+ var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
11080
+ function isEvaluatorKind(value) {
11081
+ return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
10807
11082
  }
10808
11083
  var CODE_BLOCK_PATTERN = /```[\s\S]*?```/g;
10809
11084
  var ANSI_YELLOW = "\x1B[33m";
@@ -10813,7 +11088,7 @@ var SCHEMA_CONFIG_V2 = "agentv-config-v2";
10813
11088
  async function loadConfig(evalFilePath, repoRoot) {
10814
11089
  const directories = buildDirectoryChain(evalFilePath, repoRoot);
10815
11090
  for (const directory of directories) {
10816
- const configPath = path7.join(directory, ".agentv", "config.yaml");
11091
+ const configPath = path8.join(directory, ".agentv", "config.yaml");
10817
11092
  if (!await fileExists2(configPath)) {
10818
11093
  continue;
10819
11094
  }
@@ -10876,7 +11151,7 @@ function extractCodeBlocks(segments) {
10876
11151
  }
10877
11152
  async function loadEvalCases(evalFilePath, repoRoot, options) {
10878
11153
  const verbose = options?.verbose ?? false;
10879
- const absoluteTestPath = path7.resolve(evalFilePath);
11154
+ const absoluteTestPath = path8.resolve(evalFilePath);
10880
11155
  if (!await fileExists2(absoluteTestPath)) {
10881
11156
  throw new Error(`Test file not found: ${evalFilePath}`);
10882
11157
  }
@@ -10900,7 +11175,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
10900
11175
  if (!Array.isArray(rawTestcases)) {
10901
11176
  throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
10902
11177
  }
10903
- const globalGrader = coerceGrader(suite.grader) ?? "llm_judge";
11178
+ const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
10904
11179
  const results = [];
10905
11180
  for (const rawEvalcase of rawTestcases) {
10906
11181
  if (!isJsonObject(rawEvalcase)) {
@@ -10987,9 +11262,9 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
10987
11262
  }
10988
11263
  try {
10989
11264
  const fileContent = (await readFile2(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
10990
- const relativeToRepo = path7.relative(repoRootPath, resolvedPath);
11265
+ const relativeToRepo = path8.relative(repoRootPath, resolvedPath);
10991
11266
  if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
10992
- guidelinePaths.push(path7.resolve(resolvedPath));
11267
+ guidelinePaths.push(path8.resolve(resolvedPath));
10993
11268
  if (verbose) {
10994
11269
  console.log(` [Guideline] Found: ${displayPath}`);
10995
11270
  console.log(` Resolved to: ${resolvedPath}`);
@@ -10999,7 +11274,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
10999
11274
  type: "file",
11000
11275
  path: displayPath,
11001
11276
  text: fileContent,
11002
- resolvedPath: path7.resolve(resolvedPath)
11277
+ resolvedPath: path8.resolve(resolvedPath)
11003
11278
  });
11004
11279
  if (verbose) {
11005
11280
  console.log(` [File] Found: ${displayPath}`);
@@ -11023,7 +11298,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
11023
11298
  const assistantContent = assistantMessages[0]?.content;
11024
11299
  const expectedAssistantRaw = await resolveAssistantContent(assistantContent, searchRoots, verbose);
11025
11300
  const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
11026
- const testCaseGrader = coerceGrader(evalcase.grader) ?? globalGrader;
11301
+ const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
11302
+ const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
11027
11303
  const userFilePaths = [];
11028
11304
  for (const segment of userSegments) {
11029
11305
  if (segment.type === "file" && typeof segment.resolvedPath === "string") {
@@ -11031,7 +11307,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
11031
11307
  }
11032
11308
  }
11033
11309
  const allFilePaths = [
11034
- ...guidelinePaths.map((guidelinePath) => path7.resolve(guidelinePath)),
11310
+ ...guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
11035
11311
  ...userFilePaths
11036
11312
  ];
11037
11313
  const testCase = {
@@ -11041,12 +11317,13 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
11041
11317
  user_segments: userSegments,
11042
11318
  system_message: systemMessageContent,
11043
11319
  expected_assistant_raw: expectedAssistantRaw,
11044
- guideline_paths: guidelinePaths.map((guidelinePath) => path7.resolve(guidelinePath)),
11320
+ guideline_paths: guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
11045
11321
  guideline_patterns: guidelinePatterns,
11046
11322
  file_paths: allFilePaths,
11047
11323
  code_snippets: codeSnippets,
11048
11324
  outcome,
11049
- grader: testCaseGrader
11325
+ evaluator: testCaseEvaluatorKind,
11326
+ evaluators
11050
11327
  };
11051
11328
  if (verbose) {
11052
11329
  console.log(`
@@ -11067,14 +11344,14 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
11067
11344
  async function buildPromptInputs(testCase) {
11068
11345
  const guidelineContents = [];
11069
11346
  for (const rawPath of testCase.guideline_paths) {
11070
- const absolutePath = path7.resolve(rawPath);
11347
+ const absolutePath = path8.resolve(rawPath);
11071
11348
  if (!await fileExists2(absolutePath)) {
11072
11349
  logWarning(`Could not read guideline file ${absolutePath}: file does not exist`);
11073
11350
  continue;
11074
11351
  }
11075
11352
  try {
11076
11353
  const content = (await readFile2(absolutePath, "utf8")).replace(/\r\n/g, "\n");
11077
- guidelineContents.push(`=== ${path7.basename(absolutePath)} ===
11354
+ guidelineContents.push(`=== ${path8.basename(absolutePath)} ===
11078
11355
  ${content}`);
11079
11356
  } catch (error) {
11080
11357
  logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
@@ -11127,7 +11404,7 @@ function resolveToAbsolutePath(candidate) {
11127
11404
  if (candidate.startsWith("file://")) {
11128
11405
  return fileURLToPath(new URL(candidate));
11129
11406
  }
11130
- return path7.resolve(candidate);
11407
+ return path8.resolve(candidate);
11131
11408
  }
11132
11409
  throw new TypeError("Unsupported repoRoot value. Expected string or URL.");
11133
11410
  }
@@ -11207,14 +11484,88 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
11207
11484
  }
11208
11485
  return parts.join(" ");
11209
11486
  }
11210
- function coerceGrader(candidate) {
11487
+ async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
11488
+ const execution = rawEvalCase.execution;
11489
+ const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
11490
+ if (candidateEvaluators === void 0) {
11491
+ return void 0;
11492
+ }
11493
+ if (!Array.isArray(candidateEvaluators)) {
11494
+ logWarning(`Skipping evaluators for '${evalId}': expected array`);
11495
+ return void 0;
11496
+ }
11497
+ const evaluators = [];
11498
+ for (const rawEvaluator of candidateEvaluators) {
11499
+ if (!isJsonObject(rawEvaluator)) {
11500
+ logWarning(`Skipping invalid evaluator entry for '${evalId}' (expected object)`);
11501
+ continue;
11502
+ }
11503
+ const name = asString(rawEvaluator.name);
11504
+ const typeValue = rawEvaluator.type;
11505
+ if (!name || !isEvaluatorKind(typeValue)) {
11506
+ logWarning(`Skipping evaluator with invalid name/type in '${evalId}'`);
11507
+ continue;
11508
+ }
11509
+ if (typeValue === "code") {
11510
+ const script = asString(rawEvaluator.script);
11511
+ if (!script) {
11512
+ logWarning(`Skipping code evaluator '${name}' in '${evalId}': missing script`);
11513
+ continue;
11514
+ }
11515
+ const cwd = asString(rawEvaluator.cwd);
11516
+ let resolvedCwd;
11517
+ if (cwd) {
11518
+ const resolved = await resolveFileReference(cwd, searchRoots);
11519
+ if (resolved.resolvedPath) {
11520
+ resolvedCwd = path8.resolve(resolved.resolvedPath);
11521
+ } else {
11522
+ logWarning(
11523
+ `Code evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
11524
+ resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => ` Tried: ${attempt}`) : void 0
11525
+ );
11526
+ }
11527
+ }
11528
+ evaluators.push({
11529
+ name,
11530
+ type: "code",
11531
+ script,
11532
+ cwd,
11533
+ resolvedCwd
11534
+ });
11535
+ continue;
11536
+ }
11537
+ const prompt = asString(rawEvaluator.prompt);
11538
+ let promptPath;
11539
+ if (prompt) {
11540
+ const resolved = await resolveFileReference(prompt, searchRoots);
11541
+ if (resolved.resolvedPath) {
11542
+ promptPath = path8.resolve(resolved.resolvedPath);
11543
+ } else {
11544
+ logWarning(
11545
+ `Inline prompt used for evaluator '${name}' in '${evalId}' (file not found: ${resolved.displayPath})`,
11546
+ resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => ` Tried: ${attempt}`) : void 0
11547
+ );
11548
+ }
11549
+ }
11550
+ const model = asString(rawEvaluator.model);
11551
+ evaluators.push({
11552
+ name,
11553
+ type: "llm_judge",
11554
+ prompt,
11555
+ promptPath,
11556
+ model
11557
+ });
11558
+ }
11559
+ return evaluators.length > 0 ? evaluators : void 0;
11560
+ }
11561
+ function coerceEvaluator(candidate, contextId) {
11211
11562
  if (typeof candidate !== "string") {
11212
11563
  return void 0;
11213
11564
  }
11214
- if (isGraderKind(candidate)) {
11565
+ if (isEvaluatorKind(candidate)) {
11215
11566
  return candidate;
11216
11567
  }
11217
- logWarning(`Unknown grader '${candidate}', falling back to default`);
11568
+ logWarning(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
11218
11569
  return void 0;
11219
11570
  }
11220
11571
  function logWarning(message, details) {
@@ -11406,49 +11757,253 @@ var GeminiProvider = class {
11406
11757
  return mapResponse(ensureChatResponse(response));
11407
11758
  }
11408
11759
  };
11409
- var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
11410
- var MockProvider = class {
11760
+ var execAsync2 = promisify2(execWithCallback);
11761
+ var DEFAULT_MAX_BUFFER = 10 * 1024 * 1024;
11762
+ async function defaultCommandRunner(command, options) {
11763
+ const execOptions = {
11764
+ cwd: options.cwd,
11765
+ env: options.env,
11766
+ timeout: options.timeoutMs,
11767
+ signal: options.signal,
11768
+ maxBuffer: DEFAULT_MAX_BUFFER,
11769
+ shell: process.platform === "win32" ? "powershell.exe" : void 0
11770
+ };
11771
+ try {
11772
+ const { stdout, stderr } = await execAsync2(command, execOptions);
11773
+ return {
11774
+ stdout,
11775
+ stderr,
11776
+ exitCode: 0,
11777
+ failed: false,
11778
+ timedOut: false,
11779
+ signal: null
11780
+ };
11781
+ } catch (error) {
11782
+ const execError = error;
11783
+ return {
11784
+ stdout: execError.stdout ?? "",
11785
+ stderr: execError.stderr ?? "",
11786
+ exitCode: typeof execError.code === "number" ? execError.code : null,
11787
+ failed: true,
11788
+ timedOut: execError.timedOut === true || execError.killed === true,
11789
+ signal: execError.signal ?? null
11790
+ };
11791
+ }
11792
+ }
11793
+ var CliProvider = class {
11411
11794
  id;
11412
- kind = "mock";
11795
+ kind = "cli";
11413
11796
  targetName;
11414
- cannedResponse;
11415
- delayMs;
11416
- delayMinMs;
11417
- delayMaxMs;
11418
- constructor(targetName, config) {
11419
- this.id = `mock:${targetName}`;
11797
+ supportsBatch = false;
11798
+ config;
11799
+ runCommand;
11800
+ healthcheckPromise;
11801
+ constructor(targetName, config, runner = defaultCommandRunner) {
11420
11802
  this.targetName = targetName;
11421
- this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
11422
- this.delayMs = config.delayMs ?? 0;
11423
- this.delayMinMs = config.delayMinMs ?? 0;
11424
- this.delayMaxMs = config.delayMaxMs ?? 0;
11803
+ this.id = `cli:${targetName}`;
11804
+ this.config = config;
11805
+ this.runCommand = runner;
11425
11806
  }
11426
11807
  async invoke(request) {
11427
- const delay = this.calculateDelay();
11428
- if (delay > 0) {
11429
- await new Promise((resolve) => setTimeout(resolve, delay));
11808
+ if (request.signal?.aborted) {
11809
+ throw new Error("CLI provider request was aborted before execution");
11810
+ }
11811
+ await this.ensureHealthy(request.signal);
11812
+ const templateValues = buildTemplateValues(request, this.config);
11813
+ const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
11814
+ const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
11815
+ const result = await this.runCommand(renderedCommand, {
11816
+ cwd: this.config.cwd,
11817
+ env,
11818
+ timeoutMs: this.config.timeoutMs,
11819
+ signal: request.signal
11820
+ });
11821
+ if (result.failed || (result.exitCode ?? 0) !== 0) {
11822
+ if (request.signal?.aborted) {
11823
+ throw new Error("CLI provider request was aborted");
11824
+ }
11825
+ if (result.timedOut) {
11826
+ throw new Error(
11827
+ `CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
11828
+ );
11829
+ }
11830
+ const codeText = result.exitCode !== null ? result.exitCode : "unknown";
11831
+ const detail = result.stderr.trim() || result.stdout.trim();
11832
+ const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
11833
+ throw new Error(message);
11430
11834
  }
11431
11835
  return {
11432
- text: this.cannedResponse,
11836
+ text: result.stdout,
11433
11837
  raw: {
11434
- prompt: request.prompt,
11435
- guidelines: request.guidelines
11838
+ command: renderedCommand,
11839
+ stderr: result.stderr,
11840
+ exitCode: result.exitCode ?? 0,
11841
+ cwd: this.config.cwd
11436
11842
  }
11437
11843
  };
11438
11844
  }
11439
- calculateDelay() {
11440
- if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
11441
- const min = Math.max(0, this.delayMinMs);
11442
- const max = Math.max(min, this.delayMaxMs);
11443
- return Math.floor(Math.random() * (max - min + 1)) + min;
11845
+ async ensureHealthy(signal) {
11846
+ if (!this.config.healthcheck) {
11847
+ return;
11444
11848
  }
11445
- return this.delayMs;
11849
+ if (!this.healthcheckPromise) {
11850
+ this.healthcheckPromise = this.runHealthcheck(this.config.healthcheck, signal);
11851
+ }
11852
+ return this.healthcheckPromise;
11446
11853
  }
11447
- };
11448
- var BASE_TARGET_SCHEMA = external_exports.object({
11449
- name: external_exports.string().min(1, "target name is required"),
11450
- provider: external_exports.string().min(1, "provider is required"),
11451
- settings: external_exports.record(external_exports.unknown()).optional(),
11854
+ async runHealthcheck(healthcheck, signal) {
11855
+ if (!healthcheck) {
11856
+ return;
11857
+ }
11858
+ const timeoutMs = healthcheck.timeoutMs ?? this.config.timeoutMs;
11859
+ if (healthcheck.type === "http") {
11860
+ const controller = new AbortController();
11861
+ const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
11862
+ signal?.addEventListener("abort", () => controller.abort(), { once: true });
11863
+ try {
11864
+ const response = await fetch(healthcheck.url, { method: "GET", signal: controller.signal });
11865
+ if (!response.ok) {
11866
+ throw new Error(`HTTP ${response.status} ${response.statusText}`);
11867
+ }
11868
+ } catch (error) {
11869
+ const reason = error instanceof Error ? error.message : String(error);
11870
+ throw new Error(`CLI healthcheck failed for '${this.targetName}': ${reason}`);
11871
+ } finally {
11872
+ if (timer !== void 0) {
11873
+ clearTimeout(timer);
11874
+ }
11875
+ }
11876
+ return;
11877
+ }
11878
+ const renderedCommand = renderTemplate(
11879
+ healthcheck.commandTemplate,
11880
+ buildTemplateValues(
11881
+ {
11882
+ prompt: "",
11883
+ guidelines: "",
11884
+ inputFiles: [],
11885
+ evalCaseId: "",
11886
+ attempt: 0
11887
+ },
11888
+ this.config
11889
+ )
11890
+ );
11891
+ const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
11892
+ const result = await this.runCommand(renderedCommand, {
11893
+ cwd: healthcheck.cwd ?? this.config.cwd,
11894
+ env,
11895
+ timeoutMs,
11896
+ signal
11897
+ });
11898
+ if (result.failed || (result.exitCode ?? 0) !== 0) {
11899
+ const codeText = result.exitCode !== null ? result.exitCode : "unknown";
11900
+ const detail = result.stderr.trim() || result.stdout.trim();
11901
+ const message = detail ? `${detail} (exit code ${codeText})` : `CLI healthcheck command exited with code ${codeText}`;
11902
+ throw new Error(`CLI healthcheck failed for '${this.targetName}': ${message}`);
11903
+ }
11904
+ }
11905
+ };
11906
+ function buildTemplateValues(request, config) {
11907
+ const inputFiles = normalizeInputFiles(request.inputFiles);
11908
+ return {
11909
+ PROMPT: shellEscape(request.prompt ?? ""),
11910
+ GUIDELINES: shellEscape(request.guidelines ?? ""),
11911
+ EVAL_ID: shellEscape(request.evalCaseId ?? ""),
11912
+ ATTEMPT: shellEscape(String(request.attempt ?? 0)),
11913
+ FILES: formatFileList(inputFiles, config.filesFormat)
11914
+ };
11915
+ }
11916
+ function normalizeInputFiles(inputFiles) {
11917
+ if (!inputFiles || inputFiles.length === 0) {
11918
+ return void 0;
11919
+ }
11920
+ const unique = /* @__PURE__ */ new Map();
11921
+ for (const inputFile of inputFiles) {
11922
+ const absolutePath = path22.resolve(inputFile);
11923
+ if (!unique.has(absolutePath)) {
11924
+ unique.set(absolutePath, absolutePath);
11925
+ }
11926
+ }
11927
+ return Array.from(unique.values());
11928
+ }
11929
+ function formatFileList(files, template) {
11930
+ if (!files || files.length === 0) {
11931
+ return "";
11932
+ }
11933
+ const formatter = template ?? "{path}";
11934
+ return files.map((filePath) => {
11935
+ const escapedPath = shellEscape(filePath);
11936
+ const escapedName = shellEscape(path22.basename(filePath));
11937
+ return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
11938
+ }).join(" ");
11939
+ }
11940
+ function renderTemplate(template, values) {
11941
+ return template.replace(/\{([A-Z_]+)\}/g, (match, key2) => {
11942
+ const replacement = values[key2];
11943
+ return replacement !== void 0 ? replacement : match;
11944
+ });
11945
+ }
11946
+ function shellEscape(value) {
11947
+ if (value.length === 0) {
11948
+ return "''";
11949
+ }
11950
+ if (process.platform === "win32") {
11951
+ const escaped = value.replace(/"/g, '\\"');
11952
+ return `"${escaped}"`;
11953
+ }
11954
+ return `'${value.replace(/'/g, `'"'"'`)}'`;
11955
+ }
11956
+ function formatTimeoutSuffix(timeoutMs) {
11957
+ if (!timeoutMs || timeoutMs <= 0) {
11958
+ return "";
11959
+ }
11960
+ const seconds = Math.ceil(timeoutMs / 1e3);
11961
+ return ` after ${seconds}s`;
11962
+ }
11963
+ var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
11964
+ var MockProvider = class {
11965
+ id;
11966
+ kind = "mock";
11967
+ targetName;
11968
+ cannedResponse;
11969
+ delayMs;
11970
+ delayMinMs;
11971
+ delayMaxMs;
11972
+ constructor(targetName, config) {
11973
+ this.id = `mock:${targetName}`;
11974
+ this.targetName = targetName;
11975
+ this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
11976
+ this.delayMs = config.delayMs ?? 0;
11977
+ this.delayMinMs = config.delayMinMs ?? 0;
11978
+ this.delayMaxMs = config.delayMaxMs ?? 0;
11979
+ }
11980
+ async invoke(request) {
11981
+ const delay = this.calculateDelay();
11982
+ if (delay > 0) {
11983
+ await new Promise((resolve) => setTimeout(resolve, delay));
11984
+ }
11985
+ return {
11986
+ text: this.cannedResponse,
11987
+ raw: {
11988
+ prompt: request.prompt,
11989
+ guidelines: request.guidelines
11990
+ }
11991
+ };
11992
+ }
11993
+ calculateDelay() {
11994
+ if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
11995
+ const min = Math.max(0, this.delayMinMs);
11996
+ const max = Math.max(min, this.delayMaxMs);
11997
+ return Math.floor(Math.random() * (max - min + 1)) + min;
11998
+ }
11999
+ return this.delayMs;
12000
+ }
12001
+ };
12002
+ var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
12003
+ var BASE_TARGET_SCHEMA = external_exports.object({
12004
+ name: external_exports.string().min(1, "target name is required"),
12005
+ provider: external_exports.string().min(1, "provider is required"),
12006
+ settings: external_exports.record(external_exports.unknown()).optional(),
11452
12007
  judge_target: external_exports.string().optional(),
11453
12008
  workers: external_exports.number().int().min(1).optional()
11454
12009
  });
@@ -11467,6 +12022,9 @@ function normalizeAzureApiVersion(value) {
11467
12022
  function resolveTargetDefinition(definition, env = process.env) {
11468
12023
  const parsed = BASE_TARGET_SCHEMA.parse(definition);
11469
12024
  const provider = parsed.provider.toLowerCase();
12025
+ const providerBatching = resolveOptionalBoolean(
12026
+ parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
12027
+ );
11470
12028
  switch (provider) {
11471
12029
  case "azure":
11472
12030
  case "azure-openai":
@@ -11475,6 +12033,7 @@ function resolveTargetDefinition(definition, env = process.env) {
11475
12033
  name: parsed.name,
11476
12034
  judgeTarget: parsed.judge_target,
11477
12035
  workers: parsed.workers,
12036
+ providerBatching,
11478
12037
  config: resolveAzureConfig(parsed, env)
11479
12038
  };
11480
12039
  case "anthropic":
@@ -11483,6 +12042,7 @@ function resolveTargetDefinition(definition, env = process.env) {
11483
12042
  name: parsed.name,
11484
12043
  judgeTarget: parsed.judge_target,
11485
12044
  workers: parsed.workers,
12045
+ providerBatching,
11486
12046
  config: resolveAnthropicConfig(parsed, env)
11487
12047
  };
11488
12048
  case "gemini":
@@ -11493,14 +12053,26 @@ function resolveTargetDefinition(definition, env = process.env) {
11493
12053
  name: parsed.name,
11494
12054
  judgeTarget: parsed.judge_target,
11495
12055
  workers: parsed.workers,
12056
+ providerBatching,
11496
12057
  config: resolveGeminiConfig(parsed, env)
11497
12058
  };
12059
+ case "codex":
12060
+ case "codex-cli":
12061
+ return {
12062
+ kind: "codex",
12063
+ name: parsed.name,
12064
+ judgeTarget: parsed.judge_target,
12065
+ workers: parsed.workers,
12066
+ providerBatching,
12067
+ config: resolveCodexConfig(parsed, env)
12068
+ };
11498
12069
  case "mock":
11499
12070
  return {
11500
12071
  kind: "mock",
11501
12072
  name: parsed.name,
11502
12073
  judgeTarget: parsed.judge_target,
11503
12074
  workers: parsed.workers,
12075
+ providerBatching,
11504
12076
  config: resolveMockConfig(parsed)
11505
12077
  };
11506
12078
  case "vscode":
@@ -11510,8 +12082,18 @@ function resolveTargetDefinition(definition, env = process.env) {
11510
12082
  name: parsed.name,
11511
12083
  judgeTarget: parsed.judge_target,
11512
12084
  workers: parsed.workers,
12085
+ providerBatching,
11513
12086
  config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
11514
12087
  };
12088
+ case "cli":
12089
+ return {
12090
+ kind: "cli",
12091
+ name: parsed.name,
12092
+ judgeTarget: parsed.judge_target,
12093
+ workers: parsed.workers,
12094
+ providerBatching,
12095
+ config: resolveCliConfig(parsed, env)
12096
+ };
11515
12097
  default:
11516
12098
  throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
11517
12099
  }
@@ -11579,6 +12161,29 @@ function resolveGeminiConfig(target, env) {
11579
12161
  maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
11580
12162
  };
11581
12163
  }
12164
+ function resolveCodexConfig(target, env) {
12165
+ const settings = target.settings ?? {};
12166
+ const executableSource = settings.executable ?? settings.command ?? settings.binary;
12167
+ const argsSource = settings.args ?? settings.arguments;
12168
+ const cwdSource = settings.cwd;
12169
+ const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
12170
+ const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
12171
+ allowLiteral: true,
12172
+ optionalEnv: true
12173
+ }) ?? "codex";
12174
+ const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
12175
+ const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
12176
+ allowLiteral: true,
12177
+ optionalEnv: true
12178
+ });
12179
+ const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
12180
+ return {
12181
+ executable,
12182
+ args,
12183
+ cwd,
12184
+ timeoutMs
12185
+ };
12186
+ }
11582
12187
  function resolveMockConfig(target) {
11583
12188
  const settings = target.settings ?? {};
11584
12189
  const response = typeof settings.response === "string" ? settings.response : void 0;
@@ -11608,6 +12213,125 @@ function resolveVSCodeConfig(target, env, insiders) {
11608
12213
  workspaceTemplate
11609
12214
  };
11610
12215
  }
12216
+ function resolveCliConfig(target, env) {
12217
+ const settings = target.settings ?? {};
12218
+ const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
12219
+ const filesFormat = resolveOptionalLiteralString(
12220
+ settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
12221
+ );
12222
+ const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
12223
+ allowLiteral: true,
12224
+ optionalEnv: true
12225
+ });
12226
+ const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
12227
+ const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
12228
+ const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
12229
+ const commandTemplate = resolveString(
12230
+ commandTemplateSource,
12231
+ env,
12232
+ `${target.name} CLI command template`,
12233
+ true
12234
+ );
12235
+ assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
12236
+ return {
12237
+ commandTemplate,
12238
+ filesFormat,
12239
+ cwd,
12240
+ env: envOverrides,
12241
+ timeoutMs,
12242
+ healthcheck
12243
+ };
12244
+ }
12245
+ function resolveEnvOverrides(source2, env, targetName) {
12246
+ if (source2 === void 0 || source2 === null) {
12247
+ return void 0;
12248
+ }
12249
+ if (typeof source2 !== "object" || Array.isArray(source2)) {
12250
+ throw new Error(`${targetName} env overrides must be an object map of strings`);
12251
+ }
12252
+ const entries = Object.entries(source2);
12253
+ const resolved = {};
12254
+ for (const [key2, value] of entries) {
12255
+ if (typeof value !== "string") {
12256
+ throw new Error(`${targetName} env override '${key2}' must be a string`);
12257
+ }
12258
+ const resolvedValue = resolveString(value, env, `${targetName} env override '${key2}'`);
12259
+ resolved[key2] = resolvedValue;
12260
+ }
12261
+ return Object.keys(resolved).length > 0 ? resolved : void 0;
12262
+ }
12263
+ function resolveTimeoutMs(source2, description) {
12264
+ const seconds = resolveOptionalNumber(source2, `${description} (seconds)`);
12265
+ if (seconds === void 0) {
12266
+ return void 0;
12267
+ }
12268
+ if (seconds <= 0) {
12269
+ throw new Error(`${description} must be greater than zero seconds`);
12270
+ }
12271
+ return Math.floor(seconds * 1e3);
12272
+ }
12273
+ function resolveCliHealthcheck(source2, env, targetName) {
12274
+ if (source2 === void 0 || source2 === null) {
12275
+ return void 0;
12276
+ }
12277
+ if (typeof source2 !== "object" || Array.isArray(source2)) {
12278
+ throw new Error(`${targetName} healthcheck must be an object`);
12279
+ }
12280
+ const candidate = source2;
12281
+ const type = candidate.type;
12282
+ const timeoutMs = resolveTimeoutMs(
12283
+ candidate.timeout_seconds ?? candidate.timeoutSeconds,
12284
+ `${targetName} healthcheck timeout`
12285
+ );
12286
+ if (type === "http") {
12287
+ const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
12288
+ return {
12289
+ type: "http",
12290
+ url,
12291
+ timeoutMs
12292
+ };
12293
+ }
12294
+ if (type === "command") {
12295
+ const commandTemplate = resolveString(
12296
+ candidate.command_template ?? candidate.commandTemplate,
12297
+ env,
12298
+ `${targetName} healthcheck command template`,
12299
+ true
12300
+ );
12301
+ assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
12302
+ const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
12303
+ allowLiteral: true,
12304
+ optionalEnv: true
12305
+ });
12306
+ return {
12307
+ type: "command",
12308
+ commandTemplate,
12309
+ timeoutMs,
12310
+ cwd
12311
+ };
12312
+ }
12313
+ throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
12314
+ }
12315
+ function assertSupportedCliPlaceholders(template, description) {
12316
+ const placeholders = extractCliPlaceholders(template);
12317
+ for (const placeholder of placeholders) {
12318
+ if (!CLI_PLACEHOLDERS.has(placeholder)) {
12319
+ throw new Error(
12320
+ `${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
12321
+ );
12322
+ }
12323
+ }
12324
+ }
12325
+ function extractCliPlaceholders(template) {
12326
+ const matches = template.matchAll(/\{([A-Z_]+)\}/g);
12327
+ const results = [];
12328
+ for (const match of matches) {
12329
+ if (match[1]) {
12330
+ results.push(match[1]);
12331
+ }
12332
+ }
12333
+ return results;
12334
+ }
11611
12335
  function resolveString(source2, env, description, allowLiteral = false) {
11612
12336
  const value = resolveOptionalString(source2, env, description, {
11613
12337
  allowLiteral,
@@ -11638,11 +12362,14 @@ function resolveOptionalString(source2, env, description, options) {
11638
12362
  }
11639
12363
  const allowLiteral = options?.allowLiteral ?? false;
11640
12364
  const optionalEnv = options?.optionalEnv ?? false;
11641
- if (!allowLiteral && isLikelyEnvReference(trimmed)) {
12365
+ const looksLikeEnv = isLikelyEnvReference(trimmed);
12366
+ if (looksLikeEnv) {
11642
12367
  if (optionalEnv) {
11643
12368
  return void 0;
11644
12369
  }
11645
- throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
12370
+ if (!allowLiteral) {
12371
+ throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
12372
+ }
11646
12373
  }
11647
12374
  return trimmed;
11648
12375
  }
@@ -11692,10 +12419,43 @@ function resolveOptionalBoolean(source2) {
11692
12419
  function isLikelyEnvReference(value) {
11693
12420
  return /^[A-Z0-9_]+$/.test(value);
11694
12421
  }
12422
+ function resolveOptionalStringArray(source2, env, description) {
12423
+ if (source2 === void 0 || source2 === null) {
12424
+ return void 0;
12425
+ }
12426
+ if (!Array.isArray(source2)) {
12427
+ throw new Error(`${description} must be an array of strings`);
12428
+ }
12429
+ if (source2.length === 0) {
12430
+ return void 0;
12431
+ }
12432
+ const resolved = [];
12433
+ for (let i6 = 0; i6 < source2.length; i6++) {
12434
+ const item = source2[i6];
12435
+ if (typeof item !== "string") {
12436
+ throw new Error(`${description}[${i6}] must be a string`);
12437
+ }
12438
+ const trimmed = item.trim();
12439
+ if (trimmed.length === 0) {
12440
+ throw new Error(`${description}[${i6}] cannot be empty`);
12441
+ }
12442
+ const envValue = env[trimmed];
12443
+ if (envValue !== void 0) {
12444
+ if (envValue.trim().length === 0) {
12445
+ throw new Error(`Environment variable '${trimmed}' for ${description}[${i6}] is empty`);
12446
+ }
12447
+ resolved.push(envValue);
12448
+ } else {
12449
+ resolved.push(trimmed);
12450
+ }
12451
+ }
12452
+ return resolved.length > 0 ? resolved : void 0;
12453
+ }
11695
12454
  var VSCodeProvider = class {
11696
12455
  id;
11697
12456
  kind;
11698
12457
  targetName;
12458
+ supportsBatch = true;
11699
12459
  config;
11700
12460
  constructor(targetName, config, kind) {
11701
12461
  this.id = `${kind}:${targetName}`;
@@ -11707,12 +12467,11 @@ var VSCodeProvider = class {
11707
12467
  if (request.signal?.aborted) {
11708
12468
  throw new Error("VS Code provider request was aborted before dispatch");
11709
12469
  }
11710
- const attachments = normalizeAttachments(request.attachments);
11711
- const promptContent = buildPromptDocument(request, attachments, request.guideline_patterns);
12470
+ const inputFiles = normalizeAttachments(request.inputFiles);
12471
+ const promptContent = buildPromptDocument(request, inputFiles, request.guideline_patterns);
11712
12472
  const session = await dispatchAgentSession({
11713
12473
  userQuery: promptContent,
11714
- // Use full prompt content instead of just request.prompt
11715
- extraAttachments: attachments,
12474
+ extraAttachments: inputFiles,
11716
12475
  wait: this.config.waitForResponse,
11717
12476
  dryRun: this.config.dryRun,
11718
12477
  vscodeCmd: this.config.command,
@@ -11729,7 +12488,7 @@ var VSCodeProvider = class {
11729
12488
  text: "",
11730
12489
  raw: {
11731
12490
  session,
11732
- attachments
12491
+ inputFiles
11733
12492
  }
11734
12493
  };
11735
12494
  }
@@ -11738,42 +12497,106 @@ var VSCodeProvider = class {
11738
12497
  text: responseText,
11739
12498
  raw: {
11740
12499
  session,
11741
- attachments
12500
+ inputFiles
11742
12501
  }
11743
12502
  };
11744
12503
  }
12504
+ async invokeBatch(requests) {
12505
+ if (requests.length === 0) {
12506
+ return [];
12507
+ }
12508
+ const normalizedRequests = requests.map((req) => ({
12509
+ request: req,
12510
+ inputFiles: normalizeAttachments(req.inputFiles)
12511
+ }));
12512
+ const combinedInputFiles = mergeAttachments(
12513
+ normalizedRequests.map(({ inputFiles }) => inputFiles)
12514
+ );
12515
+ const userQueries = normalizedRequests.map(
12516
+ ({ request, inputFiles }) => buildPromptDocument(request, inputFiles, request.guideline_patterns)
12517
+ );
12518
+ const session = await dispatchBatchAgent({
12519
+ userQueries,
12520
+ extraAttachments: combinedInputFiles,
12521
+ wait: this.config.waitForResponse,
12522
+ dryRun: this.config.dryRun,
12523
+ vscodeCmd: this.config.command,
12524
+ subagentRoot: this.config.subagentRoot,
12525
+ workspaceTemplate: this.config.workspaceTemplate,
12526
+ silent: true
12527
+ });
12528
+ if (session.exitCode !== 0 || !session.responseFiles) {
12529
+ const failure = session.error ?? "VS Code subagent did not produce batch responses";
12530
+ throw new Error(failure);
12531
+ }
12532
+ if (this.config.dryRun) {
12533
+ return normalizedRequests.map(({ inputFiles }) => ({
12534
+ text: "",
12535
+ raw: {
12536
+ session,
12537
+ inputFiles,
12538
+ allInputFiles: combinedInputFiles
12539
+ }
12540
+ }));
12541
+ }
12542
+ if (session.responseFiles.length !== requests.length) {
12543
+ throw new Error(
12544
+ `VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
12545
+ );
12546
+ }
12547
+ const responses = [];
12548
+ for (const [index, responseFile] of session.responseFiles.entries()) {
12549
+ const responseText = await readFile22(responseFile, "utf8");
12550
+ responses.push({
12551
+ text: responseText,
12552
+ raw: {
12553
+ session,
12554
+ inputFiles: normalizedRequests[index]?.inputFiles,
12555
+ allInputFiles: combinedInputFiles,
12556
+ responseFile
12557
+ }
12558
+ });
12559
+ }
12560
+ return responses;
12561
+ }
11745
12562
  };
11746
12563
  function buildPromptDocument(request, attachments, guidelinePatterns) {
11747
12564
  const parts = [];
11748
12565
  const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
11749
- if (guidelineFiles.length > 0) {
11750
- parts.push("\n", buildMandatoryPrereadBlock(guidelineFiles));
12566
+ const attachmentFiles = collectAttachmentFiles(attachments);
12567
+ const nonGuidelineAttachments = attachmentFiles.filter(
12568
+ (file) => !guidelineFiles.includes(file)
12569
+ );
12570
+ const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
12571
+ if (prereadBlock.length > 0) {
12572
+ parts.push("\n", prereadBlock);
11751
12573
  }
11752
12574
  parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
11753
12575
  return parts.join("\n").trim();
11754
12576
  }
11755
- function buildMandatoryPrereadBlock(guidelineFiles) {
11756
- if (guidelineFiles.length === 0) {
12577
+ function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
12578
+ if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
11757
12579
  return "";
11758
12580
  }
11759
- const fileList = [];
11760
- let counter = 0;
11761
- for (const absolutePath of guidelineFiles) {
11762
- counter += 1;
11763
- const fileName = path22.basename(absolutePath);
11764
- const fileUri = pathToFileUri(absolutePath);
11765
- fileList.push(`* [${fileName}](${fileUri})`);
11766
- }
11767
- const filesText = fileList.join("\n");
11768
- const instruction = [
11769
- `Read all guideline files:
11770
- ${filesText}.
11771
- `,
11772
- `If any file is missing, fail with ERROR: missing-file <filename> and stop.
11773
- `,
11774
- `Then apply system_instructions on the user query below.`
11775
- ].join("");
11776
- return `${instruction}`;
12581
+ const buildList = (files) => files.map((absolutePath) => {
12582
+ const fileName = path32.basename(absolutePath);
12583
+ const fileUri = pathToFileUri2(absolutePath);
12584
+ return `* [${fileName}](${fileUri})`;
12585
+ });
12586
+ const sections = [];
12587
+ if (guidelineFiles.length > 0) {
12588
+ sections.push(`Read all guideline files:
12589
+ ${buildList(guidelineFiles).join("\n")}.`);
12590
+ }
12591
+ if (attachmentFiles.length > 0) {
12592
+ sections.push(`Read all attachment files:
12593
+ ${buildList(attachmentFiles).join("\n")}.`);
12594
+ }
12595
+ sections.push(
12596
+ "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
12597
+ "Then apply system_instructions on the user query below."
12598
+ );
12599
+ return sections.join("\n");
11777
12600
  }
11778
12601
  function collectGuidelineFiles(attachments, guidelinePatterns) {
11779
12602
  if (!attachments || attachments.length === 0) {
@@ -11781,8 +12604,8 @@ function collectGuidelineFiles(attachments, guidelinePatterns) {
11781
12604
  }
11782
12605
  const unique = /* @__PURE__ */ new Map();
11783
12606
  for (const attachment of attachments) {
11784
- const absolutePath = path22.resolve(attachment);
11785
- const normalized = absolutePath.split(path22.sep).join("/");
12607
+ const absolutePath = path32.resolve(attachment);
12608
+ const normalized = absolutePath.split(path32.sep).join("/");
11786
12609
  if (isGuidelineFile(normalized, guidelinePatterns)) {
11787
12610
  if (!unique.has(absolutePath)) {
11788
12611
  unique.set(absolutePath, absolutePath);
@@ -11791,8 +12614,21 @@ function collectGuidelineFiles(attachments, guidelinePatterns) {
11791
12614
  }
11792
12615
  return Array.from(unique.values());
11793
12616
  }
11794
- function pathToFileUri(filePath) {
11795
- const absolutePath = path22.isAbsolute(filePath) ? filePath : path22.resolve(filePath);
12617
+ function collectAttachmentFiles(attachments) {
12618
+ if (!attachments || attachments.length === 0) {
12619
+ return [];
12620
+ }
12621
+ const unique = /* @__PURE__ */ new Map();
12622
+ for (const attachment of attachments) {
12623
+ const absolutePath = path32.resolve(attachment);
12624
+ if (!unique.has(absolutePath)) {
12625
+ unique.set(absolutePath, absolutePath);
12626
+ }
12627
+ }
12628
+ return Array.from(unique.values());
12629
+ }
12630
+ function pathToFileUri2(filePath) {
12631
+ const absolutePath = path32.isAbsolute(filePath) ? filePath : path32.resolve(filePath);
11796
12632
  const normalizedPath = absolutePath.replace(/\\/g, "/");
11797
12633
  if (/^[a-zA-Z]:\//.test(normalizedPath)) {
11798
12634
  return `file:///${normalizedPath}`;
@@ -11805,10 +12641,20 @@ function normalizeAttachments(attachments) {
11805
12641
  }
11806
12642
  const deduped = /* @__PURE__ */ new Set();
11807
12643
  for (const attachment of attachments) {
11808
- deduped.add(path22.resolve(attachment));
12644
+ deduped.add(path32.resolve(attachment));
11809
12645
  }
11810
12646
  return Array.from(deduped);
11811
12647
  }
12648
+ function mergeAttachments(all) {
12649
+ const deduped = /* @__PURE__ */ new Set();
12650
+ for (const list of all) {
12651
+ if (!list) continue;
12652
+ for (const inputFile of list) {
12653
+ deduped.add(path32.resolve(inputFile));
12654
+ }
12655
+ }
12656
+ return deduped.size > 0 ? Array.from(deduped) : void 0;
12657
+ }
11812
12658
  async function ensureVSCodeSubagents(options) {
11813
12659
  const { kind, count, verbose = false } = options;
11814
12660
  const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
@@ -11847,6 +12693,569 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
11847
12693
  };
11848
12694
  }
11849
12695
  }
12696
+ function buildPromptDocument2(request, inputFiles, options) {
12697
+ const parts = [];
12698
+ const guidelineFiles = collectGuidelineFiles2(
12699
+ inputFiles,
12700
+ options?.guidelinePatterns ?? request.guideline_patterns,
12701
+ options?.guidelineOverrides
12702
+ );
12703
+ const inputFilesList = collectInputFiles(inputFiles);
12704
+ const nonGuidelineInputFiles = inputFilesList.filter(
12705
+ (file) => !guidelineFiles.includes(file)
12706
+ );
12707
+ const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineInputFiles);
12708
+ if (prereadBlock.length > 0) {
12709
+ parts.push("\n", prereadBlock);
12710
+ }
12711
+ parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
12712
+ return parts.join("\n").trim();
12713
+ }
12714
+ function normalizeInputFiles2(inputFiles) {
12715
+ if (!inputFiles || inputFiles.length === 0) {
12716
+ return void 0;
12717
+ }
12718
+ const deduped = /* @__PURE__ */ new Map();
12719
+ for (const inputFile of inputFiles) {
12720
+ const absolutePath = path42.resolve(inputFile);
12721
+ if (!deduped.has(absolutePath)) {
12722
+ deduped.set(absolutePath, absolutePath);
12723
+ }
12724
+ }
12725
+ return Array.from(deduped.values());
12726
+ }
12727
+ function collectGuidelineFiles2(inputFiles, guidelinePatterns, overrides) {
12728
+ if (!inputFiles || inputFiles.length === 0) {
12729
+ return [];
12730
+ }
12731
+ const unique = /* @__PURE__ */ new Map();
12732
+ for (const inputFile of inputFiles) {
12733
+ const absolutePath = path42.resolve(inputFile);
12734
+ if (overrides?.has(absolutePath)) {
12735
+ if (!unique.has(absolutePath)) {
12736
+ unique.set(absolutePath, absolutePath);
12737
+ }
12738
+ continue;
12739
+ }
12740
+ const normalized = absolutePath.split(path42.sep).join("/");
12741
+ if (isGuidelineFile(normalized, guidelinePatterns)) {
12742
+ if (!unique.has(absolutePath)) {
12743
+ unique.set(absolutePath, absolutePath);
12744
+ }
12745
+ }
12746
+ }
12747
+ return Array.from(unique.values());
12748
+ }
12749
+ function collectInputFiles(inputFiles) {
12750
+ if (!inputFiles || inputFiles.length === 0) {
12751
+ return [];
12752
+ }
12753
+ const unique = /* @__PURE__ */ new Map();
12754
+ for (const inputFile of inputFiles) {
12755
+ const absolutePath = path42.resolve(inputFile);
12756
+ if (!unique.has(absolutePath)) {
12757
+ unique.set(absolutePath, absolutePath);
12758
+ }
12759
+ }
12760
+ return Array.from(unique.values());
12761
+ }
12762
+ function buildMandatoryPrereadBlock2(guidelineFiles, inputFiles) {
12763
+ if (guidelineFiles.length === 0 && inputFiles.length === 0) {
12764
+ return "";
12765
+ }
12766
+ const buildList = (files) => files.map((absolutePath) => {
12767
+ const fileName = path42.basename(absolutePath);
12768
+ const fileUri = pathToFileUri22(absolutePath);
12769
+ return `* [${fileName}](${fileUri})`;
12770
+ });
12771
+ const sections = [];
12772
+ if (guidelineFiles.length > 0) {
12773
+ sections.push(`Read all guideline files:
12774
+ ${buildList(guidelineFiles).join("\n")}.`);
12775
+ }
12776
+ if (inputFiles.length > 0) {
12777
+ sections.push(`Read all input files:
12778
+ ${buildList(inputFiles).join("\n")}.`);
12779
+ }
12780
+ sections.push(
12781
+ "If any file is missing, fail with ERROR: missing-file <filename> and stop.",
12782
+ "Then apply system_instructions on the user query below."
12783
+ );
12784
+ return sections.join("\n");
12785
+ }
12786
+ function pathToFileUri22(filePath) {
12787
+ const absolutePath = path42.isAbsolute(filePath) ? filePath : path42.resolve(filePath);
12788
+ const normalizedPath = absolutePath.replace(/\\/g, "/");
12789
+ if (/^[a-zA-Z]:\//.test(normalizedPath)) {
12790
+ return `file:///${normalizedPath}`;
12791
+ }
12792
+ return `file://${normalizedPath}`;
12793
+ }
12794
+ var execAsync22 = promisify22(execCallback);
12795
+ var WORKSPACE_PREFIX = "agentv-codex-";
12796
+ var PROMPT_FILENAME = "prompt.md";
12797
+ var FILES_DIR = "files";
12798
+ var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
12799
+ var CodexProvider = class {
12800
+ id;
12801
+ kind = "codex";
12802
+ targetName;
12803
+ supportsBatch = false;
12804
+ config;
12805
+ runCodex;
12806
+ environmentCheck;
12807
+ resolvedExecutable;
12808
+ constructor(targetName, config, runner = defaultCodexRunner) {
12809
+ this.id = `codex:${targetName}`;
12810
+ this.targetName = targetName;
12811
+ this.config = config;
12812
+ this.runCodex = runner;
12813
+ }
12814
+ async invoke(request) {
12815
+ if (request.signal?.aborted) {
12816
+ throw new Error("Codex provider request was aborted before execution");
12817
+ }
12818
+ await this.ensureEnvironmentReady();
12819
+ const inputFiles = normalizeInputFiles2(request.inputFiles);
12820
+ const originalGuidelines = new Set(
12821
+ collectGuidelineFiles2(inputFiles, request.guideline_patterns).map((file) => path52.resolve(file))
12822
+ );
12823
+ const workspaceRoot = await this.createWorkspace();
12824
+ try {
12825
+ const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
12826
+ inputFiles,
12827
+ workspaceRoot,
12828
+ originalGuidelines
12829
+ );
12830
+ const promptContent = buildPromptDocument2(request, mirroredInputFiles, {
12831
+ guidelinePatterns: request.guideline_patterns,
12832
+ guidelineOverrides: guidelineMirrors
12833
+ });
12834
+ const promptFile = path52.join(workspaceRoot, PROMPT_FILENAME);
12835
+ await writeFile3(promptFile, promptContent, "utf8");
12836
+ const args = this.buildCodexArgs();
12837
+ const cwd = this.resolveCwd(workspaceRoot);
12838
+ const result = await this.executeCodex(args, cwd, promptContent, request.signal);
12839
+ if (result.timedOut) {
12840
+ throw new Error(
12841
+ `Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
12842
+ );
12843
+ }
12844
+ if (result.exitCode !== 0) {
12845
+ const detail = pickDetail(result.stderr, result.stdout);
12846
+ const prefix = `Codex CLI exited with code ${result.exitCode}`;
12847
+ throw new Error(detail ? `${prefix}: ${detail}` : prefix);
12848
+ }
12849
+ const parsed = parseCodexJson(result.stdout);
12850
+ const assistantText = extractAssistantText(parsed);
12851
+ return {
12852
+ text: assistantText,
12853
+ raw: {
12854
+ response: parsed,
12855
+ stdout: result.stdout,
12856
+ stderr: result.stderr,
12857
+ exitCode: result.exitCode,
12858
+ args,
12859
+ executable: this.resolvedExecutable ?? this.config.executable,
12860
+ promptFile,
12861
+ workspace: workspaceRoot,
12862
+ inputFiles: mirroredInputFiles
12863
+ }
12864
+ };
12865
+ } finally {
12866
+ await this.cleanupWorkspace(workspaceRoot);
12867
+ }
12868
+ }
12869
+ async ensureEnvironmentReady() {
12870
+ if (!this.environmentCheck) {
12871
+ this.environmentCheck = this.validateEnvironment();
12872
+ }
12873
+ await this.environmentCheck;
12874
+ }
12875
+ async validateEnvironment() {
12876
+ this.resolvedExecutable = await locateExecutable(this.config.executable);
12877
+ }
12878
+ resolveCwd(workspaceRoot) {
12879
+ if (!this.config.cwd) {
12880
+ return workspaceRoot;
12881
+ }
12882
+ return path52.resolve(this.config.cwd);
12883
+ }
12884
+ buildCodexArgs() {
12885
+ const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
12886
+ if (this.config.args && this.config.args.length > 0) {
12887
+ args.push(...this.config.args);
12888
+ }
12889
+ args.push("-");
12890
+ return args;
12891
+ }
12892
+ async executeCodex(args, cwd, promptContent, signal) {
12893
+ try {
12894
+ return await this.runCodex({
12895
+ executable: this.resolvedExecutable ?? this.config.executable,
12896
+ args,
12897
+ cwd,
12898
+ prompt: promptContent,
12899
+ timeoutMs: this.config.timeoutMs,
12900
+ env: process.env,
12901
+ signal
12902
+ });
12903
+ } catch (error) {
12904
+ const err = error;
12905
+ if (err.code === "ENOENT") {
12906
+ throw new Error(
12907
+ `Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
12908
+ );
12909
+ }
12910
+ throw error;
12911
+ }
12912
+ }
12913
+ async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
12914
+ if (!inputFiles || inputFiles.length === 0) {
12915
+ return {
12916
+ mirroredInputFiles: void 0,
12917
+ guidelineMirrors: /* @__PURE__ */ new Set()
12918
+ };
12919
+ }
12920
+ const filesRoot = path52.join(workspaceRoot, FILES_DIR);
12921
+ await mkdir3(filesRoot, { recursive: true });
12922
+ const mirrored = [];
12923
+ const guidelineMirrors = /* @__PURE__ */ new Set();
12924
+ const nameCounts = /* @__PURE__ */ new Map();
12925
+ for (const inputFile of inputFiles) {
12926
+ const absoluteSource = path52.resolve(inputFile);
12927
+ const baseName = path52.basename(absoluteSource);
12928
+ const count = nameCounts.get(baseName) ?? 0;
12929
+ nameCounts.set(baseName, count + 1);
12930
+ const finalName = count === 0 ? baseName : `${baseName}.${count}`;
12931
+ const destination = path52.join(filesRoot, finalName);
12932
+ await copyFile2(absoluteSource, destination);
12933
+ const resolvedDestination = path52.resolve(destination);
12934
+ mirrored.push(resolvedDestination);
12935
+ if (guidelineOriginals.has(absoluteSource)) {
12936
+ guidelineMirrors.add(resolvedDestination);
12937
+ }
12938
+ }
12939
+ return {
12940
+ mirroredInputFiles: mirrored,
12941
+ guidelineMirrors
12942
+ };
12943
+ }
12944
+ async createWorkspace() {
12945
+ return await mkdtemp(path52.join(tmpdir(), WORKSPACE_PREFIX));
12946
+ }
12947
+ async cleanupWorkspace(workspaceRoot) {
12948
+ try {
12949
+ await rm2(workspaceRoot, { recursive: true, force: true });
12950
+ } catch {
12951
+ }
12952
+ }
12953
+ };
12954
+ async function locateExecutable(candidate) {
12955
+ const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
12956
+ if (includesPathSeparator) {
12957
+ const resolved = path52.isAbsolute(candidate) ? candidate : path52.resolve(candidate);
12958
+ const executablePath = await ensureWindowsExecutableVariant(resolved);
12959
+ await access22(executablePath, constants22.F_OK);
12960
+ return executablePath;
12961
+ }
12962
+ const locator = process.platform === "win32" ? "where" : "which";
12963
+ try {
12964
+ const { stdout } = await execAsync22(`${locator} ${candidate}`);
12965
+ const lines = stdout.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
12966
+ const preferred = selectExecutableCandidate(lines);
12967
+ if (preferred) {
12968
+ const executablePath = await ensureWindowsExecutableVariant(preferred);
12969
+ await access22(executablePath, constants22.F_OK);
12970
+ return executablePath;
12971
+ }
12972
+ } catch {
12973
+ }
12974
+ throw new Error(`Codex executable '${candidate}' was not found on PATH`);
12975
+ }
12976
+ function selectExecutableCandidate(candidates) {
12977
+ if (candidates.length === 0) {
12978
+ return void 0;
12979
+ }
12980
+ if (process.platform !== "win32") {
12981
+ return candidates[0];
12982
+ }
12983
+ const extensions = getWindowsExecutableExtensions();
12984
+ for (const ext of extensions) {
12985
+ const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
12986
+ if (match) {
12987
+ return match;
12988
+ }
12989
+ }
12990
+ return candidates[0];
12991
+ }
12992
+ async function ensureWindowsExecutableVariant(candidate) {
12993
+ if (process.platform !== "win32") {
12994
+ return candidate;
12995
+ }
12996
+ if (hasExecutableExtension(candidate)) {
12997
+ return candidate;
12998
+ }
12999
+ const extensions = getWindowsExecutableExtensions();
13000
+ for (const ext of extensions) {
13001
+ const withExtension = `${candidate}${ext}`;
13002
+ try {
13003
+ await access22(withExtension, constants22.F_OK);
13004
+ return withExtension;
13005
+ } catch {
13006
+ }
13007
+ }
13008
+ return candidate;
13009
+ }
13010
+ function hasExecutableExtension(candidate) {
13011
+ const lower = candidate.toLowerCase();
13012
+ return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
13013
+ }
13014
+ var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
13015
+ function getWindowsExecutableExtensions() {
13016
+ if (process.platform !== "win32") {
13017
+ return [];
13018
+ }
13019
+ const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
13020
+ return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
13021
+ }
13022
+ function parseCodexJson(output) {
13023
+ const trimmed = output.trim();
13024
+ if (trimmed.length === 0) {
13025
+ throw new Error("Codex CLI produced no output in --json mode");
13026
+ }
13027
+ try {
13028
+ return JSON.parse(trimmed);
13029
+ } catch {
13030
+ const lineObjects = parseJsonLines(trimmed);
13031
+ if (lineObjects) {
13032
+ return lineObjects;
13033
+ }
13034
+ const lastBrace = trimmed.lastIndexOf("{");
13035
+ if (lastBrace >= 0) {
13036
+ const candidate = trimmed.slice(lastBrace);
13037
+ try {
13038
+ return JSON.parse(candidate);
13039
+ } catch {
13040
+ }
13041
+ }
13042
+ const preview = trimmed.slice(0, 200);
13043
+ throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
13044
+ }
13045
+ }
13046
+ function extractAssistantText(parsed) {
13047
+ if (Array.isArray(parsed)) {
13048
+ const text = extractFromEventStream(parsed);
13049
+ if (text) {
13050
+ return text;
13051
+ }
13052
+ }
13053
+ if (!parsed || typeof parsed !== "object") {
13054
+ throw new Error("Codex CLI JSON response did not include an assistant message");
13055
+ }
13056
+ const record = parsed;
13057
+ const eventText = extractFromEvent(record);
13058
+ if (eventText) {
13059
+ return eventText;
13060
+ }
13061
+ const messages = Array.isArray(record.messages) ? record.messages : void 0;
13062
+ if (messages) {
13063
+ for (let index = messages.length - 1; index >= 0; index -= 1) {
13064
+ const entry = messages[index];
13065
+ if (!entry || typeof entry !== "object") {
13066
+ continue;
13067
+ }
13068
+ const role = entry.role;
13069
+ if (role !== "assistant") {
13070
+ continue;
13071
+ }
13072
+ const content = entry.content;
13073
+ const flattened = flattenContent(content);
13074
+ if (flattened) {
13075
+ return flattened;
13076
+ }
13077
+ }
13078
+ }
13079
+ const response = record.response;
13080
+ if (response && typeof response === "object") {
13081
+ const content = response.content;
13082
+ const flattened = flattenContent(content);
13083
+ if (flattened) {
13084
+ return flattened;
13085
+ }
13086
+ }
13087
+ const output = record.output;
13088
+ const flattenedOutput = flattenContent(output);
13089
+ if (flattenedOutput) {
13090
+ return flattenedOutput;
13091
+ }
13092
+ throw new Error("Codex CLI JSON response did not include an assistant message");
13093
+ }
13094
+ function extractFromEventStream(events) {
13095
+ for (let index = events.length - 1; index >= 0; index -= 1) {
13096
+ const candidate = events[index];
13097
+ const text = extractFromEvent(candidate);
13098
+ if (text) {
13099
+ return text;
13100
+ }
13101
+ }
13102
+ return void 0;
13103
+ }
13104
+ function extractFromEvent(event) {
13105
+ if (!event || typeof event !== "object") {
13106
+ return void 0;
13107
+ }
13108
+ const record = event;
13109
+ const type = typeof record.type === "string" ? record.type : void 0;
13110
+ if (type === JSONL_TYPE_ITEM_COMPLETED) {
13111
+ const item = record.item;
13112
+ const text = extractFromItem(item);
13113
+ if (text) {
13114
+ return text;
13115
+ }
13116
+ }
13117
+ const output = record.output ?? record.content;
13118
+ const flattened = flattenContent(output);
13119
+ if (flattened) {
13120
+ return flattened;
13121
+ }
13122
+ return void 0;
13123
+ }
13124
+ function extractFromItem(item) {
13125
+ if (!item || typeof item !== "object") {
13126
+ return void 0;
13127
+ }
13128
+ const record = item;
13129
+ const itemType = typeof record.type === "string" ? record.type : void 0;
13130
+ if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
13131
+ const text = flattenContent(record.text ?? record.content ?? record.output);
13132
+ if (text) {
13133
+ return text;
13134
+ }
13135
+ }
13136
+ return void 0;
13137
+ }
13138
+ function flattenContent(value) {
13139
+ if (typeof value === "string") {
13140
+ return value;
13141
+ }
13142
+ if (Array.isArray(value)) {
13143
+ const parts = value.map((segment) => {
13144
+ if (typeof segment === "string") {
13145
+ return segment;
13146
+ }
13147
+ if (segment && typeof segment === "object" && "text" in segment) {
13148
+ const text = segment.text;
13149
+ return typeof text === "string" ? text : void 0;
13150
+ }
13151
+ return void 0;
13152
+ }).filter((part) => typeof part === "string" && part.length > 0);
13153
+ return parts.length > 0 ? parts.join(" \n") : void 0;
13154
+ }
13155
+ if (value && typeof value === "object" && "text" in value) {
13156
+ const text = value.text;
13157
+ return typeof text === "string" ? text : void 0;
13158
+ }
13159
+ return void 0;
13160
+ }
13161
+ function parseJsonLines(output) {
13162
+ const lines = output.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
13163
+ if (lines.length <= 1) {
13164
+ return void 0;
13165
+ }
13166
+ const parsed = [];
13167
+ for (const line2 of lines) {
13168
+ try {
13169
+ parsed.push(JSON.parse(line2));
13170
+ } catch {
13171
+ return void 0;
13172
+ }
13173
+ }
13174
+ return parsed;
13175
+ }
13176
+ function pickDetail(stderr, stdout) {
13177
+ const errorText = stderr.trim();
13178
+ if (errorText.length > 0) {
13179
+ return errorText;
13180
+ }
13181
+ const stdoutText = stdout.trim();
13182
+ return stdoutText.length > 0 ? stdoutText : void 0;
13183
+ }
13184
+ function formatTimeoutSuffix2(timeoutMs) {
13185
+ if (!timeoutMs || timeoutMs <= 0) {
13186
+ return "";
13187
+ }
13188
+ const seconds = Math.ceil(timeoutMs / 1e3);
13189
+ return ` after ${seconds}s`;
13190
+ }
13191
+ async function defaultCodexRunner(options) {
13192
+ return await new Promise((resolve, reject) => {
13193
+ const child = spawn2(options.executable, options.args, {
13194
+ cwd: options.cwd,
13195
+ env: options.env,
13196
+ stdio: ["pipe", "pipe", "pipe"],
13197
+ shell: shouldShellExecute(options.executable)
13198
+ });
13199
+ let stdout = "";
13200
+ let stderr = "";
13201
+ let timedOut = false;
13202
+ const onAbort = () => {
13203
+ child.kill("SIGTERM");
13204
+ };
13205
+ if (options.signal) {
13206
+ if (options.signal.aborted) {
13207
+ onAbort();
13208
+ } else {
13209
+ options.signal.addEventListener("abort", onAbort, { once: true });
13210
+ }
13211
+ }
13212
+ let timeoutHandle;
13213
+ if (options.timeoutMs && options.timeoutMs > 0) {
13214
+ timeoutHandle = setTimeout(() => {
13215
+ timedOut = true;
13216
+ child.kill("SIGTERM");
13217
+ }, options.timeoutMs);
13218
+ timeoutHandle.unref?.();
13219
+ }
13220
+ child.stdout.setEncoding("utf8");
13221
+ child.stdout.on("data", (chunk) => {
13222
+ stdout += chunk;
13223
+ });
13224
+ child.stderr.setEncoding("utf8");
13225
+ child.stderr.on("data", (chunk) => {
13226
+ stderr += chunk;
13227
+ });
13228
+ child.stdin.end(options.prompt);
13229
+ const cleanup = () => {
13230
+ if (timeoutHandle) {
13231
+ clearTimeout(timeoutHandle);
13232
+ }
13233
+ if (options.signal) {
13234
+ options.signal.removeEventListener("abort", onAbort);
13235
+ }
13236
+ };
13237
+ child.on("error", (error) => {
13238
+ cleanup();
13239
+ reject(error);
13240
+ });
13241
+ child.on("close", (code) => {
13242
+ cleanup();
13243
+ resolve({
13244
+ stdout,
13245
+ stderr,
13246
+ exitCode: typeof code === "number" ? code : -1,
13247
+ timedOut
13248
+ });
13249
+ });
13250
+ });
13251
+ }
13252
+ function shouldShellExecute(executable) {
13253
+ if (process.platform !== "win32") {
13254
+ return false;
13255
+ }
13256
+ const lower = executable.toLowerCase();
13257
+ return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
13258
+ }
11850
13259
  function isRecord(value) {
11851
13260
  return typeof value === "object" && value !== null && !Array.isArray(value);
11852
13261
  }
@@ -11901,14 +13310,14 @@ function assertTargetDefinition(value, index, filePath) {
11901
13310
  }
11902
13311
  async function fileExists3(filePath) {
11903
13312
  try {
11904
- await access22(filePath, constants22.F_OK);
13313
+ await access32(filePath, constants32.F_OK);
11905
13314
  return true;
11906
13315
  } catch {
11907
13316
  return false;
11908
13317
  }
11909
13318
  }
11910
13319
  async function readTargetDefinitions(filePath) {
11911
- const absolutePath = path32.resolve(filePath);
13320
+ const absolutePath = path62.resolve(filePath);
11912
13321
  if (!await fileExists3(absolutePath)) {
11913
13322
  throw new Error(`targets.yaml not found at ${absolutePath}`);
11914
13323
  }
@@ -11933,6 +13342,10 @@ function createProvider(target) {
11933
13342
  return new AnthropicProvider(target.name, target.config);
11934
13343
  case "gemini":
11935
13344
  return new GeminiProvider(target.name, target.config);
13345
+ case "cli":
13346
+ return new CliProvider(target.name, target.config);
13347
+ case "codex":
13348
+ return new CodexProvider(target.name, target.config);
11936
13349
  case "mock":
11937
13350
  return new MockProvider(target.name, target.config);
11938
13351
  case "vscode":
@@ -11944,222 +13357,28 @@ function createProvider(target) {
11944
13357
  }
11945
13358
  }
11946
13359
  }
11947
- var KEY_TERM_MATCH_THRESHOLD = 0.5;
11948
- var ACTION_WORDS = /* @__PURE__ */ new Set([
11949
- "use",
11950
- "avoid",
11951
- "prefer",
11952
- "replace",
11953
- "consider",
11954
- "ensure",
11955
- "remove",
11956
- "add"
11957
- ]);
11958
- var STOP_WORDS = /* @__PURE__ */ new Set([
11959
- "the",
11960
- "a",
11961
- "an",
11962
- "and",
11963
- "or",
11964
- "but",
11965
- "in",
11966
- "on",
11967
- "at",
11968
- "to",
11969
- "for",
11970
- "of",
11971
- "with",
11972
- "by",
11973
- "is",
11974
- "are",
11975
- "was",
11976
- "were",
11977
- "be",
11978
- "been",
11979
- "being",
11980
- "have",
11981
- "has",
11982
- "had",
11983
- "do",
11984
- "does",
11985
- "did",
11986
- "will",
11987
- "would",
11988
- "could",
11989
- "should"
11990
- ]);
11991
- var ERROR_PREFIXES = [
11992
- "error:",
11993
- "err:",
11994
- "vs code command failed",
11995
- "exception",
11996
- "traceback",
11997
- "no response file was generated",
11998
- "timed out",
11999
- "cli not found"
12000
- ];
12001
- function extractAspects(expectedResponse) {
12002
- const lines = expectedResponse.split(/\r?\n/).map((line2) => line2.trim());
12003
- const aspects = [];
12004
- for (const line2 of lines) {
12005
- if (line2.length === 0) {
12006
- continue;
12007
- }
12008
- const bulletMatch = /^([-*•]|[0-9]+\.)\s*(.+)$/.exec(line2);
12009
- if (bulletMatch) {
12010
- const normalized = normalizeAspect(bulletMatch[2]);
12011
- if (normalized.length > 0) {
12012
- aspects.push(normalized);
12013
- }
12014
- continue;
12015
- }
12016
- const lowered = line2.toLowerCase();
12017
- if (Array.from(ACTION_WORDS).some((word) => lowered.startsWith(word))) {
12018
- const normalized = normalizeAspect(line2);
12019
- if (normalized.length > 0) {
12020
- aspects.push(normalized);
12021
- }
12022
- }
12023
- }
12024
- return aspects;
12025
- }
12026
- function calculateHits(candidateResponse, expectedAspects) {
12027
- const { normalizedText, words } = normalizeCandidate(candidateResponse);
12028
- const hits = [];
12029
- for (const aspect of expectedAspects) {
12030
- if (matchesAspect(aspect, normalizedText, words)) {
12031
- hits.push(aspect);
12032
- }
12033
- }
12034
- return hits;
12035
- }
12036
- function scoreCandidateResponse(candidateResponse, expectedAspects) {
12037
- if (expectedAspects.length === 0) {
12038
- if (isErrorLike(candidateResponse)) {
12039
- return {
12040
- score: 0,
12041
- hits: [],
12042
- misses: ["Model produced an error instead of an answer."],
12043
- hitCount: 0,
12044
- totalAspects: 0,
12045
- rawAspects: []
12046
- };
12047
- }
12048
- return {
12049
- score: 1,
12050
- hits: [],
12051
- misses: [],
12052
- hitCount: 0,
12053
- totalAspects: 0,
12054
- rawAspects: []
12055
- };
12056
- }
12057
- const hits = calculateHits(candidateResponse, expectedAspects);
12058
- const misses = expectedAspects.filter((aspect) => !hits.includes(aspect));
12059
- const score = expectedAspects.length > 0 ? hits.length / expectedAspects.length : 0;
12060
- return {
12061
- score,
12062
- hits,
12063
- misses,
12064
- hitCount: hits.length,
12065
- totalAspects: expectedAspects.length,
12066
- rawAspects: expectedAspects
12067
- };
12068
- }
12069
- function isErrorLike(text) {
12070
- if (!text) {
12071
- return false;
12072
- }
12073
- const lowered = text.trim().toLowerCase();
12074
- return ERROR_PREFIXES.some((prefix) => lowered.startsWith(prefix));
12075
- }
12076
- function normalizeAspect(aspect) {
12077
- const sanitized = aspect.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim();
12078
- return sanitized;
12079
- }
12080
- function normalizeCandidate(candidate) {
12081
- const lowered = candidate.toLowerCase();
12082
- const normalizedText = lowered.replace(/[^\w\s]/g, " ");
12083
- const words = new Set(normalizedText.split(/\s+/).filter((word) => word.length > 0));
12084
- return { normalizedText, words };
12085
- }
12086
- function matchesAspect(aspect, candidateNormalized, candidateWords) {
12087
- const keyTerms = extractKeyTerms(aspect);
12088
- if (keyTerms.length === 0) {
12089
- return false;
12090
- }
12091
- const matches = keyTerms.filter((term) => candidateWords.has(term)).length;
12092
- const ratio = matches / keyTerms.length;
12093
- if (ratio >= KEY_TERM_MATCH_THRESHOLD) {
12094
- return true;
12095
- }
12096
- const aspectWords = aspect.split(" ");
12097
- if (aspectWords.length >= 2) {
12098
- for (let index = 0; index < aspectWords.length - 1; index += 1) {
12099
- const phrase = `${aspectWords[index]} ${aspectWords[index + 1]}`;
12100
- if (candidateNormalized.includes(phrase)) {
12101
- return true;
12102
- }
12103
- }
12104
- }
12105
- return false;
12106
- }
12107
- function extractKeyTerms(aspect, maxTerms = 5) {
12108
- const terms = [];
12109
- const words = aspect.split(" ");
12110
- for (const word of words) {
12111
- if (word.length <= 2) {
12112
- continue;
12113
- }
12114
- if (STOP_WORDS.has(word)) {
12115
- continue;
12116
- }
12117
- terms.push(word);
12118
- if (terms.length >= maxTerms) {
12119
- break;
12120
- }
12121
- }
12122
- return terms;
12123
- }
12124
- var HeuristicGrader = class {
12125
- kind = "heuristic";
12126
- grade(context2) {
12127
- const expectedAspects = extractAspects(context2.evalCase.expected_assistant_raw);
12128
- const result = scoreCandidateResponse(context2.candidate, expectedAspects);
12129
- const misses = [...result.misses];
12130
- if (expectedAspects.length === 0 && isErrorLike(context2.candidate)) {
12131
- const firstLine = context2.candidate.split(/\r?\n/)[0]?.trim();
12132
- if (firstLine && !misses.includes(firstLine)) {
12133
- misses.unshift(firstLine);
12134
- }
12135
- }
12136
- return {
12137
- score: result.score,
12138
- hits: result.hits,
12139
- misses,
12140
- expectedAspectCount: result.totalAspects,
12141
- rawAspects: result.rawAspects
12142
- };
12143
- }
12144
- };
12145
- var QualityGrader = class {
13360
+ var LlmJudgeEvaluator = class {
12146
13361
  kind = "llm_judge";
12147
13362
  resolveJudgeProvider;
12148
13363
  maxOutputTokens;
12149
13364
  temperature;
13365
+ customPrompt;
12150
13366
  constructor(options) {
12151
13367
  this.resolveJudgeProvider = options.resolveJudgeProvider;
12152
13368
  this.maxOutputTokens = options.maxOutputTokens;
12153
13369
  this.temperature = options.temperature;
13370
+ this.customPrompt = options.customPrompt;
12154
13371
  }
12155
- async grade(context2) {
13372
+ async evaluate(context2) {
12156
13373
  const judgeProvider = await this.resolveJudgeProvider(context2);
12157
13374
  if (!judgeProvider) {
12158
13375
  throw new Error("No judge provider available for LLM grading");
12159
13376
  }
12160
13377
  const prompt = buildQualityPrompt(context2.evalCase, context2.candidate);
13378
+ const systemPrompt = context2.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
12161
13379
  const metadata = {
12162
- systemPrompt: QUALITY_SYSTEM_PROMPT
13380
+ ...systemPrompt !== void 0 ? { systemPrompt } : {},
13381
+ ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
12163
13382
  };
12164
13383
  const response = await judgeProvider.invoke({
12165
13384
  prompt,
@@ -12174,12 +13393,13 @@ var QualityGrader = class {
12174
13393
  const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
12175
13394
  const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
12176
13395
  const reasoning = parsed.reasoning ?? response.reasoning;
12177
- const graderRawRequest = {
13396
+ const evaluatorRawRequest = {
12178
13397
  id: randomUUID(),
12179
13398
  provider: judgeProvider.id,
12180
13399
  prompt,
12181
- systemPrompt: QUALITY_SYSTEM_PROMPT,
12182
- target: context2.target.name
13400
+ target: context2.target.name,
13401
+ ...systemPrompt !== void 0 ? { systemPrompt } : {},
13402
+ ...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
12183
13403
  };
12184
13404
  return {
12185
13405
  score,
@@ -12187,7 +13407,7 @@ var QualityGrader = class {
12187
13407
  misses,
12188
13408
  expectedAspectCount: hits.length + misses.length || 1,
12189
13409
  reasoning,
12190
- graderRawRequest
13410
+ evaluatorRawRequest
12191
13411
  };
12192
13412
  }
12193
13413
  };
@@ -12302,8 +13522,114 @@ function extractJsonBlob(text) {
12302
13522
  const match = text.match(/\{[\s\S]*\}/);
12303
13523
  return match?.[0];
12304
13524
  }
12305
- function isNonEmptyString(value) {
12306
- return typeof value === "string" && value.trim().length > 0;
13525
+ function isNonEmptyString(value) {
13526
+ return typeof value === "string" && value.trim().length > 0;
13527
+ }
13528
+ var CodeEvaluator = class {
13529
+ kind = "code";
13530
+ script;
13531
+ cwd;
13532
+ agentTimeoutMs;
13533
+ constructor(options) {
13534
+ this.script = options.script;
13535
+ this.cwd = options.cwd;
13536
+ this.agentTimeoutMs = options.agentTimeoutMs;
13537
+ }
13538
+ async evaluate(context2) {
13539
+ const inputPayload = JSON.stringify(
13540
+ {
13541
+ task: context2.evalCase.task,
13542
+ outcome: context2.evalCase.outcome,
13543
+ expected: context2.evalCase.expected_assistant_raw,
13544
+ output: context2.candidate,
13545
+ system_message: context2.promptInputs.systemMessage ?? "",
13546
+ guideline_paths: context2.evalCase.guideline_paths,
13547
+ attachments: context2.evalCase.file_paths,
13548
+ user_segments: context2.evalCase.user_segments
13549
+ },
13550
+ null,
13551
+ 2
13552
+ );
13553
+ try {
13554
+ const stdout = await executeScript(this.script, inputPayload, this.agentTimeoutMs, this.cwd);
13555
+ const parsed = parseJsonSafe(stdout);
13556
+ const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
13557
+ const hits = Array.isArray(parsed?.hits) ? parsed.hits.filter(isNonEmptyString) : [];
13558
+ const misses = Array.isArray(parsed?.misses) ? parsed.misses.filter(isNonEmptyString) : [];
13559
+ const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
13560
+ return {
13561
+ score,
13562
+ hits,
13563
+ misses,
13564
+ expectedAspectCount: hits.length + misses.length || 1,
13565
+ reasoning,
13566
+ evaluatorRawRequest: {
13567
+ script: this.script,
13568
+ ...this.cwd ? { cwd: this.cwd } : {}
13569
+ }
13570
+ };
13571
+ } catch (error) {
13572
+ const message = error instanceof Error ? error.message : String(error);
13573
+ return {
13574
+ score: 0,
13575
+ hits: [],
13576
+ misses: [`Code evaluator failed: ${message}`],
13577
+ expectedAspectCount: 1,
13578
+ reasoning: message,
13579
+ evaluatorRawRequest: {
13580
+ script: this.script,
13581
+ ...this.cwd ? { cwd: this.cwd } : {},
13582
+ error: message
13583
+ }
13584
+ };
13585
+ }
13586
+ }
13587
+ };
13588
+ async function executeScript(scriptPath, input, agentTimeoutMs, cwd) {
13589
+ const { spawn: spawn22 } = await import("node:child_process");
13590
+ return await new Promise((resolve, reject) => {
13591
+ const child = spawn22(scriptPath, {
13592
+ shell: true,
13593
+ cwd
13594
+ });
13595
+ let stdout = "";
13596
+ let stderr = "";
13597
+ const timeout = agentTimeoutMs ? setTimeout(() => {
13598
+ child.kill();
13599
+ reject(new Error(`Code evaluator timed out after ${agentTimeoutMs}ms`));
13600
+ }, agentTimeoutMs) : void 0;
13601
+ child.stdout?.on("data", (data) => {
13602
+ stdout += data.toString();
13603
+ });
13604
+ child.stderr?.on("data", (data) => {
13605
+ stderr += data.toString();
13606
+ });
13607
+ child.on("error", (error) => {
13608
+ if (timeout !== void 0) {
13609
+ clearTimeout(timeout);
13610
+ }
13611
+ reject(error);
13612
+ });
13613
+ child.on("exit", (code) => {
13614
+ if (timeout !== void 0) {
13615
+ clearTimeout(timeout);
13616
+ }
13617
+ if (code && code !== 0 && stderr.length > 0) {
13618
+ reject(new Error(`Code evaluator exited with code ${code}: ${stderr.trim()}`));
13619
+ return;
13620
+ }
13621
+ resolve(stdout.trim());
13622
+ });
13623
+ child.stdin?.write(input);
13624
+ child.stdin?.end();
13625
+ });
13626
+ }
13627
+ function parseJsonSafe(payload) {
13628
+ try {
13629
+ return JSON.parse(payload);
13630
+ } catch {
13631
+ return void 0;
13632
+ }
12307
13633
  }
12308
13634
  var Node = class {
12309
13635
  value;
@@ -12445,7 +13771,7 @@ async function runEvaluation(options) {
12445
13771
  targets,
12446
13772
  env,
12447
13773
  providerFactory,
12448
- graders,
13774
+ evaluators,
12449
13775
  maxRetries,
12450
13776
  agentTimeoutMs,
12451
13777
  promptDumpDir,
@@ -12504,8 +13830,14 @@ async function runEvaluation(options) {
12504
13830
  }
12505
13831
  return getOrCreateProvider(resolvedJudge);
12506
13832
  };
12507
- const graderRegistry = buildGraderRegistry(graders, resolveJudgeProvider);
13833
+ const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
12508
13834
  const primaryProvider = getOrCreateProvider(target);
13835
+ const providerSupportsBatch = target.providerBatching === true && primaryProvider.supportsBatch === true && typeof primaryProvider.invokeBatch === "function";
13836
+ if (target.providerBatching && !providerSupportsBatch && verbose) {
13837
+ console.warn(
13838
+ `Provider batching requested for target '${target.name}', but provider does not advertise batch support. Using per-case dispatch.`
13839
+ );
13840
+ }
12509
13841
  if (onProgress && filteredEvalCases.length > 0) {
12510
13842
  for (let i6 = 0; i6 < filteredEvalCases.length; i6++) {
12511
13843
  await onProgress({
@@ -12515,6 +13847,28 @@ async function runEvaluation(options) {
12515
13847
  });
12516
13848
  }
12517
13849
  }
13850
+ if (providerSupportsBatch) {
13851
+ try {
13852
+ return await runBatchEvaluation({
13853
+ evalCases: filteredEvalCases,
13854
+ provider: primaryProvider,
13855
+ target,
13856
+ evaluatorRegistry,
13857
+ promptDumpDir,
13858
+ nowFn: now ?? (() => /* @__PURE__ */ new Date()),
13859
+ onProgress,
13860
+ onResult,
13861
+ verbose,
13862
+ resolveJudgeProvider,
13863
+ agentTimeoutMs
13864
+ });
13865
+ } catch (error) {
13866
+ if (verbose) {
13867
+ const message = error instanceof Error ? error.message : String(error);
13868
+ console.warn(`Provider batch execution failed, falling back to per-case dispatch: ${message}`);
13869
+ }
13870
+ }
13871
+ }
12518
13872
  const workers = options.maxConcurrency ?? target.workers ?? 1;
12519
13873
  const limit = pLimit(workers);
12520
13874
  let nextWorkerId = 1;
@@ -12537,7 +13891,7 @@ async function runEvaluation(options) {
12537
13891
  evalCase,
12538
13892
  provider: primaryProvider,
12539
13893
  target,
12540
- graders: graderRegistry,
13894
+ evaluators: evaluatorRegistry,
12541
13895
  maxRetries,
12542
13896
  agentTimeoutMs,
12543
13897
  promptDumpDir,
@@ -12598,12 +13952,118 @@ async function runEvaluation(options) {
12598
13952
  }
12599
13953
  return results;
12600
13954
  }
13955
+ async function runBatchEvaluation(options) {
13956
+ const {
13957
+ evalCases,
13958
+ provider,
13959
+ target,
13960
+ evaluatorRegistry,
13961
+ promptDumpDir,
13962
+ nowFn,
13963
+ onProgress,
13964
+ onResult,
13965
+ resolveJudgeProvider,
13966
+ agentTimeoutMs
13967
+ } = options;
13968
+ const promptInputsList = [];
13969
+ for (const evalCase of evalCases) {
13970
+ const promptInputs = await buildPromptInputs(evalCase);
13971
+ if (promptDumpDir) {
13972
+ await dumpPrompt(promptDumpDir, evalCase, promptInputs);
13973
+ }
13974
+ promptInputsList.push(promptInputs);
13975
+ }
13976
+ const batchRequests = evalCases.map((evalCase, index) => {
13977
+ const promptInputs = promptInputsList[index];
13978
+ return {
13979
+ prompt: promptInputs.request,
13980
+ guidelines: promptInputs.guidelines,
13981
+ guideline_patterns: evalCase.guideline_patterns,
13982
+ inputFiles: evalCase.file_paths,
13983
+ evalCaseId: evalCase.id,
13984
+ metadata: {
13985
+ systemPrompt: promptInputs.systemMessage ?? ""
13986
+ }
13987
+ };
13988
+ });
13989
+ const batchResponse = await provider.invokeBatch?.(batchRequests);
13990
+ if (!Array.isArray(batchResponse)) {
13991
+ throw new Error("Provider batching failed: invokeBatch did not return an array");
13992
+ }
13993
+ if (batchResponse.length !== evalCases.length) {
13994
+ throw new Error(
13995
+ `Provider batching failed: expected ${evalCases.length} responses, received ${batchResponse.length}`
13996
+ );
13997
+ }
13998
+ if (onProgress) {
13999
+ const startedAt = Date.now();
14000
+ for (let i6 = 0; i6 < evalCases.length; i6++) {
14001
+ await onProgress({
14002
+ workerId: 1,
14003
+ evalId: evalCases[i6].id,
14004
+ status: "running",
14005
+ startedAt
14006
+ });
14007
+ }
14008
+ }
14009
+ const results = [];
14010
+ for (let i6 = 0; i6 < evalCases.length; i6++) {
14011
+ const evalCase = evalCases[i6];
14012
+ const promptInputs = promptInputsList[i6];
14013
+ const providerResponse = batchResponse[i6];
14014
+ let result;
14015
+ try {
14016
+ result = await evaluateCandidate({
14017
+ evalCase,
14018
+ candidate: providerResponse.text ?? "",
14019
+ target,
14020
+ provider,
14021
+ evaluators: evaluatorRegistry,
14022
+ promptInputs,
14023
+ nowFn,
14024
+ attempt: 0,
14025
+ judgeProvider: await resolveJudgeProvider(target),
14026
+ agentTimeoutMs
14027
+ });
14028
+ } catch (error) {
14029
+ const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
14030
+ results.push(errorResult);
14031
+ if (onResult) {
14032
+ await onResult(errorResult);
14033
+ }
14034
+ if (onProgress) {
14035
+ await onProgress({
14036
+ workerId: 1,
14037
+ evalId: evalCase.id,
14038
+ status: "failed",
14039
+ completedAt: Date.now(),
14040
+ error: error instanceof Error ? error.message : String(error)
14041
+ });
14042
+ }
14043
+ continue;
14044
+ }
14045
+ results.push(result);
14046
+ if (onResult) {
14047
+ await onResult(result);
14048
+ }
14049
+ if (onProgress) {
14050
+ await onProgress({
14051
+ workerId: 1,
14052
+ evalId: evalCase.id,
14053
+ status: "completed",
14054
+ startedAt: 0,
14055
+ completedAt: Date.now()
14056
+ });
14057
+ }
14058
+ }
14059
+ return results;
14060
+ }
12601
14061
  async function runEvalCase(options) {
12602
14062
  const {
12603
14063
  evalCase,
12604
14064
  provider,
12605
14065
  target,
12606
- graders,
14066
+ evaluators,
12607
14067
  now,
12608
14068
  maxRetries,
12609
14069
  agentTimeoutMs,
@@ -12658,27 +14118,49 @@ async function runEvalCase(options) {
12658
14118
  if (cacheKey && cache && !cachedResponse) {
12659
14119
  await cache.set(cacheKey, providerResponse);
12660
14120
  }
12661
- const graderKind = evalCase.grader ?? "heuristic";
12662
- const activeGrader = graders[graderKind] ?? graders.heuristic;
12663
- if (!activeGrader) {
12664
- throw new Error(`No grader registered for kind '${graderKind}'`);
12665
- }
12666
- let grade;
12667
14121
  try {
12668
- const gradeTimestamp = nowFn();
12669
- grade = await activeGrader.grade({
14122
+ return await evaluateCandidate({
12670
14123
  evalCase,
12671
14124
  candidate: providerResponse.text ?? "",
12672
14125
  target,
12673
14126
  provider,
12674
- attempt,
14127
+ evaluators,
12675
14128
  promptInputs,
12676
- now: gradeTimestamp,
12677
- judgeProvider
14129
+ nowFn,
14130
+ attempt,
14131
+ judgeProvider,
14132
+ agentTimeoutMs
12678
14133
  });
12679
14134
  } catch (error) {
12680
14135
  return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
12681
14136
  }
14137
+ }
14138
+ async function evaluateCandidate(options) {
14139
+ const {
14140
+ evalCase,
14141
+ candidate,
14142
+ target,
14143
+ provider,
14144
+ evaluators,
14145
+ promptInputs,
14146
+ nowFn,
14147
+ attempt,
14148
+ judgeProvider,
14149
+ agentTimeoutMs
14150
+ } = options;
14151
+ const gradeTimestamp = nowFn();
14152
+ const { score, evaluatorResults } = await runEvaluatorsForCase({
14153
+ evalCase,
14154
+ candidate,
14155
+ target,
14156
+ provider,
14157
+ evaluators,
14158
+ attempt,
14159
+ promptInputs,
14160
+ now: gradeTimestamp,
14161
+ judgeProvider,
14162
+ agentTimeoutMs
14163
+ });
12682
14164
  const completedAt = nowFn();
12683
14165
  const rawRequest = {
12684
14166
  request: promptInputs.request,
@@ -12689,18 +14171,191 @@ async function runEvalCase(options) {
12689
14171
  return {
12690
14172
  eval_id: evalCase.id,
12691
14173
  conversation_id: evalCase.conversation_id,
12692
- score: grade.score,
12693
- hits: grade.hits,
12694
- misses: grade.misses,
12695
- model_answer: providerResponse.text ?? "",
12696
- expected_aspect_count: grade.expectedAspectCount,
14174
+ score: score.score,
14175
+ hits: score.hits,
14176
+ misses: score.misses,
14177
+ model_answer: candidate,
14178
+ expected_aspect_count: score.expectedAspectCount,
12697
14179
  target: target.name,
12698
14180
  timestamp: completedAt.toISOString(),
12699
- reasoning: grade.reasoning,
12700
- raw_aspects: grade.rawAspects,
14181
+ reasoning: score.reasoning,
14182
+ raw_aspects: score.rawAspects,
12701
14183
  raw_request: rawRequest,
12702
- grader_raw_request: grade.graderRawRequest
14184
+ evaluator_raw_request: evaluatorResults ? void 0 : score.evaluatorRawRequest,
14185
+ evaluator_results: evaluatorResults
14186
+ };
14187
+ }
14188
+ async function runEvaluatorsForCase(options) {
14189
+ const { evalCase, candidate, target, provider, evaluators, attempt, promptInputs, now, judgeProvider, agentTimeoutMs } = options;
14190
+ if (evalCase.evaluators && evalCase.evaluators.length > 0) {
14191
+ return runEvaluatorList({
14192
+ evalCase,
14193
+ evaluators: evalCase.evaluators,
14194
+ candidate,
14195
+ target,
14196
+ provider,
14197
+ evaluatorRegistry: evaluators,
14198
+ attempt,
14199
+ promptInputs,
14200
+ now,
14201
+ judgeProvider,
14202
+ agentTimeoutMs
14203
+ });
14204
+ }
14205
+ const evaluatorKind = evalCase.evaluator ?? "llm_judge";
14206
+ const activeEvaluator = evaluators[evaluatorKind] ?? evaluators.llm_judge;
14207
+ if (!activeEvaluator) {
14208
+ throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
14209
+ }
14210
+ const score = await activeEvaluator.evaluate({
14211
+ evalCase,
14212
+ candidate,
14213
+ target,
14214
+ provider,
14215
+ attempt,
14216
+ promptInputs,
14217
+ now,
14218
+ judgeProvider
14219
+ });
14220
+ return { score };
14221
+ }
14222
+ async function runEvaluatorList(options) {
14223
+ const {
14224
+ evalCase,
14225
+ evaluators,
14226
+ candidate,
14227
+ target,
14228
+ provider,
14229
+ evaluatorRegistry,
14230
+ attempt,
14231
+ promptInputs,
14232
+ now,
14233
+ judgeProvider,
14234
+ agentTimeoutMs
14235
+ } = options;
14236
+ const scored = [];
14237
+ const evaluatorResults = [];
14238
+ for (const evaluator of evaluators ?? []) {
14239
+ try {
14240
+ if (evaluator.type === "llm_judge") {
14241
+ const score2 = await runLlmJudgeEvaluator({
14242
+ config: evaluator,
14243
+ evalCase,
14244
+ candidate,
14245
+ target,
14246
+ provider,
14247
+ evaluatorRegistry,
14248
+ attempt,
14249
+ promptInputs,
14250
+ now,
14251
+ judgeProvider
14252
+ });
14253
+ scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
14254
+ evaluatorResults.push({
14255
+ name: evaluator.name,
14256
+ type: evaluator.type,
14257
+ score: score2.score,
14258
+ hits: score2.hits,
14259
+ misses: score2.misses,
14260
+ reasoning: score2.reasoning,
14261
+ evaluator_raw_request: score2.evaluatorRawRequest
14262
+ });
14263
+ continue;
14264
+ }
14265
+ if (evaluator.type === "code") {
14266
+ const codeEvaluator = new CodeEvaluator({
14267
+ script: evaluator.script,
14268
+ cwd: evaluator.resolvedCwd ?? evaluator.cwd,
14269
+ agentTimeoutMs
14270
+ });
14271
+ const score2 = await codeEvaluator.evaluate({
14272
+ evalCase,
14273
+ candidate,
14274
+ target,
14275
+ provider,
14276
+ attempt,
14277
+ promptInputs,
14278
+ now
14279
+ });
14280
+ scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
14281
+ evaluatorResults.push({
14282
+ name: evaluator.name,
14283
+ type: evaluator.type,
14284
+ score: score2.score,
14285
+ hits: score2.hits,
14286
+ misses: score2.misses,
14287
+ reasoning: score2.reasoning,
14288
+ evaluator_raw_request: score2.evaluatorRawRequest
14289
+ });
14290
+ continue;
14291
+ }
14292
+ } catch (error) {
14293
+ const message = error instanceof Error ? error.message : String(error);
14294
+ const fallbackScore = {
14295
+ score: 0,
14296
+ hits: [],
14297
+ misses: [`Evaluator '${evaluator.name}' failed: ${message}`],
14298
+ expectedAspectCount: 1,
14299
+ reasoning: message
14300
+ };
14301
+ scored.push({ score: fallbackScore, name: evaluator.name ?? "unknown", type: evaluator.type ?? "unknown" });
14302
+ evaluatorResults.push({
14303
+ name: evaluator.name ?? "unknown",
14304
+ type: evaluator.type ?? "unknown",
14305
+ score: 0,
14306
+ hits: [],
14307
+ misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
14308
+ reasoning: message
14309
+ });
14310
+ }
14311
+ }
14312
+ const aggregateScore = scored.length > 0 ? scored.reduce((total, entry) => total + entry.score.score, 0) / scored.length : 0;
14313
+ const hits = scored.flatMap((entry) => entry.score.hits);
14314
+ const misses = scored.flatMap((entry) => entry.score.misses);
14315
+ const expectedAspectCount = scored.reduce((total, entry) => total + (entry.score.expectedAspectCount ?? 0), 0);
14316
+ const rawAspects = scored.flatMap((entry) => entry.score.rawAspects ?? []);
14317
+ const reasoningParts = scored.map((entry) => entry.score.reasoning ? `${entry.name}: ${entry.score.reasoning}` : void 0).filter(isNonEmptyString2);
14318
+ const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
14319
+ const score = {
14320
+ score: aggregateScore,
14321
+ hits,
14322
+ misses,
14323
+ expectedAspectCount,
14324
+ reasoning,
14325
+ rawAspects: rawAspects.length > 0 ? rawAspects : void 0
12703
14326
  };
14327
+ return { score, evaluatorResults };
14328
+ }
14329
+ async function runLlmJudgeEvaluator(options) {
14330
+ const { config, evalCase, candidate, target, provider, evaluatorRegistry, attempt, promptInputs, now, judgeProvider } = options;
14331
+ const customPrompt = await resolveCustomPrompt(config);
14332
+ return evaluatorRegistry.llm_judge.evaluate({
14333
+ evalCase,
14334
+ candidate,
14335
+ target,
14336
+ provider,
14337
+ attempt,
14338
+ promptInputs,
14339
+ now,
14340
+ judgeProvider,
14341
+ systemPrompt: customPrompt,
14342
+ evaluator: config,
14343
+ judgeModel: config.model
14344
+ });
14345
+ }
14346
+ async function resolveCustomPrompt(config) {
14347
+ if (config.promptPath) {
14348
+ try {
14349
+ return await readFile4(config.promptPath, "utf8");
14350
+ } catch (error) {
14351
+ const message = error instanceof Error ? error.message : String(error);
14352
+ console.warn(`Could not read custom prompt at ${config.promptPath}: ${message}`);
14353
+ }
14354
+ }
14355
+ return config.prompt;
14356
+ }
14357
+ function isNonEmptyString2(value) {
14358
+ return typeof value === "string" && value.trim().length > 0;
12704
14359
  }
12705
14360
  function filterEvalCases(evalCases, evalId) {
12706
14361
  if (!evalId) {
@@ -12708,9 +14363,8 @@ function filterEvalCases(evalCases, evalId) {
12708
14363
  }
12709
14364
  return evalCases.filter((evalCase) => evalCase.id === evalId);
12710
14365
  }
12711
- function buildGraderRegistry(overrides, resolveJudgeProvider) {
12712
- const heuristic = overrides?.heuristic ?? new HeuristicGrader();
12713
- const llmJudge = overrides?.llm_judge ?? new QualityGrader({
14366
+ function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
14367
+ const llmJudge = overrides?.llm_judge ?? new LlmJudgeEvaluator({
12714
14368
  resolveJudgeProvider: async (context2) => {
12715
14369
  if (context2.judgeProvider) {
12716
14370
  return context2.judgeProvider;
@@ -12720,15 +14374,14 @@ function buildGraderRegistry(overrides, resolveJudgeProvider) {
12720
14374
  });
12721
14375
  return {
12722
14376
  ...overrides,
12723
- heuristic,
12724
14377
  llm_judge: llmJudge
12725
14378
  };
12726
14379
  }
12727
14380
  async function dumpPrompt(directory, evalCase, promptInputs) {
12728
14381
  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
12729
14382
  const filename = `${timestamp}_${sanitizeFilename(evalCase.id)}.json`;
12730
- const filePath = path42.resolve(directory, filename);
12731
- await mkdir3(path42.dirname(filePath), { recursive: true });
14383
+ const filePath = path72.resolve(directory, filename);
14384
+ await mkdir22(path72.dirname(filePath), { recursive: true });
12732
14385
  const payload = {
12733
14386
  eval_id: evalCase.id,
12734
14387
  request: promptInputs.request,
@@ -12745,7 +14398,7 @@ function sanitizeFilename(value) {
12745
14398
  return sanitized.length > 0 ? sanitized : randomUUID2();
12746
14399
  }
12747
14400
  async function invokeProvider(provider, options) {
12748
- const { evalCase, target, promptInputs, attempt, agentTimeoutMs, signal } = options;
14401
+ const { evalCase, promptInputs, attempt, agentTimeoutMs, signal } = options;
12749
14402
  const controller = new AbortController();
12750
14403
  const timeout = agentTimeoutMs ? setTimeout(() => controller.abort(), agentTimeoutMs) : void 0;
12751
14404
  if (signal) {
@@ -12756,7 +14409,7 @@ async function invokeProvider(provider, options) {
12756
14409
  prompt: promptInputs.request,
12757
14410
  guidelines: promptInputs.guidelines,
12758
14411
  guideline_patterns: evalCase.guideline_patterns,
12759
- attachments: evalCase.file_paths,
14412
+ inputFiles: evalCase.file_paths,
12760
14413
  evalCaseId: evalCase.id,
12761
14414
  attempt,
12762
14415
  metadata: {
@@ -12825,19 +14478,19 @@ function createAgentKernel() {
12825
14478
  // src/commands/eval/run-eval.ts
12826
14479
  import { constants as constants6 } from "node:fs";
12827
14480
  import { access as access6, mkdir as mkdir6 } from "node:fs/promises";
12828
- import path12 from "node:path";
14481
+ import path13 from "node:path";
12829
14482
  import { pathToFileURL } from "node:url";
12830
14483
 
12831
14484
  // src/commands/eval/env.ts
12832
14485
  import { config as loadDotenv } from "dotenv";
12833
14486
  import { constants as constants4 } from "node:fs";
12834
14487
  import { access as access4 } from "node:fs/promises";
12835
- import path8 from "node:path";
14488
+ import path9 from "node:path";
12836
14489
  function uniqueDirs(directories) {
12837
14490
  const seen = /* @__PURE__ */ new Set();
12838
14491
  const result = [];
12839
14492
  for (const dir of directories) {
12840
- const absolute = path8.resolve(dir);
14493
+ const absolute = path9.resolve(dir);
12841
14494
  if (seen.has(absolute)) {
12842
14495
  continue;
12843
14496
  }
@@ -12856,14 +14509,14 @@ async function fileExists4(filePath) {
12856
14509
  }
12857
14510
  function collectAncestorDirectories(start, boundary) {
12858
14511
  const directories = [];
12859
- const boundaryDir = path8.resolve(boundary);
12860
- let current = path8.resolve(start);
14512
+ const boundaryDir = path9.resolve(boundary);
14513
+ let current = path9.resolve(start);
12861
14514
  while (current !== void 0) {
12862
14515
  directories.push(current);
12863
14516
  if (current === boundaryDir) {
12864
14517
  break;
12865
14518
  }
12866
- const parent = path8.dirname(current);
14519
+ const parent = path9.dirname(current);
12867
14520
  if (parent === current) {
12868
14521
  break;
12869
14522
  }
@@ -12873,7 +14526,7 @@ function collectAncestorDirectories(start, boundary) {
12873
14526
  }
12874
14527
  async function loadEnvFromHierarchy(options) {
12875
14528
  const { testFilePath, repoRoot, verbose } = options;
12876
- const testDir = path8.dirname(path8.resolve(testFilePath));
14529
+ const testDir = path9.dirname(path9.resolve(testFilePath));
12877
14530
  const cwd = process.cwd();
12878
14531
  const searchDirs = uniqueDirs([
12879
14532
  ...collectAncestorDirectories(testDir, repoRoot),
@@ -12881,7 +14534,7 @@ async function loadEnvFromHierarchy(options) {
12881
14534
  cwd
12882
14535
  ]);
12883
14536
  for (const dir of searchDirs) {
12884
- const candidate = path8.join(dir, ".env");
14537
+ const candidate = path9.join(dir, ".env");
12885
14538
  if (await fileExists4(candidate)) {
12886
14539
  loadDotenv({ path: candidate, override: false });
12887
14540
  if (verbose) {
@@ -13105,7 +14758,7 @@ var Mutex = class {
13105
14758
  // src/commands/eval/jsonl-writer.ts
13106
14759
  import { createWriteStream } from "node:fs";
13107
14760
  import { mkdir as mkdir4 } from "node:fs/promises";
13108
- import path9 from "node:path";
14761
+ import path10 from "node:path";
13109
14762
  import { finished } from "node:stream/promises";
13110
14763
  var JsonlWriter = class _JsonlWriter {
13111
14764
  stream;
@@ -13115,7 +14768,7 @@ var JsonlWriter = class _JsonlWriter {
13115
14768
  this.stream = stream;
13116
14769
  }
13117
14770
  static async open(filePath) {
13118
- await mkdir4(path9.dirname(filePath), { recursive: true });
14771
+ await mkdir4(path10.dirname(filePath), { recursive: true });
13119
14772
  const stream = createWriteStream(filePath, { flags: "w", encoding: "utf8" });
13120
14773
  return new _JsonlWriter(stream);
13121
14774
  }
@@ -13147,7 +14800,7 @@ var JsonlWriter = class _JsonlWriter {
13147
14800
  // src/commands/eval/yaml-writer.ts
13148
14801
  import { createWriteStream as createWriteStream2 } from "node:fs";
13149
14802
  import { mkdir as mkdir5 } from "node:fs/promises";
13150
- import path10 from "node:path";
14803
+ import path11 from "node:path";
13151
14804
  import { finished as finished2 } from "node:stream/promises";
13152
14805
  import { stringify as stringifyYaml } from "yaml";
13153
14806
  var YamlWriter = class _YamlWriter {
@@ -13159,7 +14812,7 @@ var YamlWriter = class _YamlWriter {
13159
14812
  this.stream = stream;
13160
14813
  }
13161
14814
  static async open(filePath) {
13162
- await mkdir5(path10.dirname(filePath), { recursive: true });
14815
+ await mkdir5(path11.dirname(filePath), { recursive: true });
13163
14816
  const stream = createWriteStream2(filePath, { flags: "w", encoding: "utf8" });
13164
14817
  return new _YamlWriter(stream);
13165
14818
  }
@@ -13467,14 +15120,14 @@ function formatEvaluationSummary(summary) {
13467
15120
 
13468
15121
  // src/commands/eval/targets.ts
13469
15122
  import { constants as constants5 } from "node:fs";
13470
- import { access as access5, readFile as readFile4 } from "node:fs/promises";
13471
- import path11 from "node:path";
15123
+ import { access as access5, readFile as readFile5 } from "node:fs/promises";
15124
+ import path12 from "node:path";
13472
15125
  import { parse as parse4 } from "yaml";
13473
15126
  var TARGET_FILE_CANDIDATES = [
13474
15127
  "targets.yaml",
13475
15128
  "targets.yml",
13476
- path11.join(".agentv", "targets.yaml"),
13477
- path11.join(".agentv", "targets.yml")
15129
+ path12.join(".agentv", "targets.yaml"),
15130
+ path12.join(".agentv", "targets.yml")
13478
15131
  ];
13479
15132
  async function fileExists5(filePath) {
13480
15133
  try {
@@ -13486,7 +15139,7 @@ async function fileExists5(filePath) {
13486
15139
  }
13487
15140
  async function readTestSuiteTarget(testFilePath) {
13488
15141
  try {
13489
- const raw = await readFile4(path11.resolve(testFilePath), "utf8");
15142
+ const raw = await readFile5(path12.resolve(testFilePath), "utf8");
13490
15143
  const parsed = parse4(raw);
13491
15144
  if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
13492
15145
  const targetValue = parsed.target;
@@ -13501,12 +15154,12 @@ async function readTestSuiteTarget(testFilePath) {
13501
15154
  async function discoverTargetsFile(options) {
13502
15155
  const { explicitPath, testFilePath, repoRoot, cwd } = options;
13503
15156
  if (explicitPath) {
13504
- const resolvedExplicit = path11.resolve(explicitPath);
15157
+ const resolvedExplicit = path12.resolve(explicitPath);
13505
15158
  if (await fileExists5(resolvedExplicit)) {
13506
15159
  return resolvedExplicit;
13507
15160
  }
13508
15161
  for (const candidate of TARGET_FILE_CANDIDATES) {
13509
- const nested = path11.join(resolvedExplicit, candidate);
15162
+ const nested = path12.join(resolvedExplicit, candidate);
13510
15163
  if (await fileExists5(nested)) {
13511
15164
  return nested;
13512
15165
  }
@@ -13514,13 +15167,13 @@ async function discoverTargetsFile(options) {
13514
15167
  throw new Error(`targets.yaml not found at provided path: ${resolvedExplicit}`);
13515
15168
  }
13516
15169
  const directories = [...buildDirectoryChain(testFilePath, repoRoot)];
13517
- const resolvedCwd = path11.resolve(cwd);
15170
+ const resolvedCwd = path12.resolve(cwd);
13518
15171
  if (!directories.includes(resolvedCwd)) {
13519
15172
  directories.push(resolvedCwd);
13520
15173
  }
13521
15174
  for (const directory of directories) {
13522
15175
  for (const candidate of TARGET_FILE_CANDIDATES) {
13523
- const fullPath = path11.join(directory, candidate);
15176
+ const fullPath = path12.join(directory, candidate);
13524
15177
  if (await fileExists5(fullPath)) {
13525
15178
  return fullPath;
13526
15179
  }
@@ -13645,15 +15298,15 @@ async function ensureFileExists(filePath, description) {
13645
15298
  }
13646
15299
  }
13647
15300
  async function findRepoRoot(start) {
13648
- const fallback = path12.resolve(start);
15301
+ const fallback = path13.resolve(start);
13649
15302
  let current = fallback;
13650
15303
  while (current !== void 0) {
13651
- const candidate = path12.join(current, ".git");
15304
+ const candidate = path13.join(current, ".git");
13652
15305
  try {
13653
15306
  await access6(candidate, constants6.F_OK);
13654
15307
  return current;
13655
15308
  } catch {
13656
- const parent = path12.dirname(current);
15309
+ const parent = path13.dirname(current);
13657
15310
  if (parent === current) {
13658
15311
  break;
13659
15312
  }
@@ -13663,21 +15316,21 @@ async function findRepoRoot(start) {
13663
15316
  return fallback;
13664
15317
  }
13665
15318
  function buildDefaultOutputPath(testFilePath, cwd, format) {
13666
- const testFileName = path12.basename(testFilePath);
15319
+ const testFileName = path13.basename(testFilePath);
13667
15320
  const withoutExtension = testFileName.replace(/\.test\.ya?ml$/i, "").replace(/\.ya?ml$/i, "");
13668
15321
  const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
13669
15322
  const baseName = withoutExtension.length > 0 ? withoutExtension : "results";
13670
15323
  const extension = getDefaultExtension(format);
13671
- return path12.join(cwd, ".agentv", "results", `${baseName}_${timestamp}${extension}`);
15324
+ return path13.join(cwd, ".agentv", "results", `${baseName}_${timestamp}${extension}`);
13672
15325
  }
13673
15326
  function resolvePromptDirectory(option, cwd) {
13674
15327
  if (option === void 0) {
13675
15328
  return void 0;
13676
15329
  }
13677
15330
  if (typeof option === "string" && option.trim().length > 0) {
13678
- return path12.resolve(cwd, option);
15331
+ return path13.resolve(cwd, option);
13679
15332
  }
13680
- return path12.join(cwd, ".agentv", "prompts");
15333
+ return path13.join(cwd, ".agentv", "prompts");
13681
15334
  }
13682
15335
  function createEvaluationCache() {
13683
15336
  const store = /* @__PURE__ */ new Map();
@@ -13693,7 +15346,7 @@ function createEvaluationCache() {
13693
15346
  async function runEvalCommand(input) {
13694
15347
  const options = normalizeOptions(input.rawOptions);
13695
15348
  const cwd = process.cwd();
13696
- const testFilePath = path12.resolve(input.testFile);
15349
+ const testFilePath = path13.resolve(input.testFile);
13697
15350
  await ensureFileExists(testFilePath, "Test file");
13698
15351
  const repoRoot = await findRepoRoot(cwd);
13699
15352
  if (options.verbose) {
@@ -13719,7 +15372,7 @@ async function runEvalCommand(input) {
13719
15372
  const providerLabel = options.dryRun ? `${targetSelection.resolvedTarget.kind} (dry-run)` : targetSelection.resolvedTarget.kind;
13720
15373
  const targetMessage = options.verbose ? `Using target (${targetSelection.targetSource}): ${targetSelection.targetName} [provider=${providerLabel}] via ${targetSelection.targetsFilePath}` : `Using target: ${targetSelection.targetName} [provider=${providerLabel}]`;
13721
15374
  console.log(targetMessage);
13722
- const outputPath = options.outPath ? path12.resolve(options.outPath) : buildDefaultOutputPath(testFilePath, cwd, options.format);
15375
+ const outputPath = options.outPath ? path13.resolve(options.outPath) : buildDefaultOutputPath(testFilePath, cwd, options.format);
13723
15376
  console.log(`Output path: ${outputPath}`);
13724
15377
  const promptDumpDir = resolvePromptDirectory(options.dumpPrompts, cwd);
13725
15378
  if (promptDumpDir) {
@@ -13811,7 +15464,7 @@ async function resolveEvaluationRunner() {
13811
15464
  if (!overridePath) {
13812
15465
  return runEvaluation;
13813
15466
  }
13814
- const resolved = path12.isAbsolute(overridePath) ? overridePath : path12.resolve(process.cwd(), overridePath);
15467
+ const resolved = path13.isAbsolute(overridePath) ? overridePath : path13.resolve(process.cwd(), overridePath);
13815
15468
  const moduleUrl = pathToFileURL(resolved).href;
13816
15469
  const mod = await import(moduleUrl);
13817
15470
  const candidate = mod.runEvaluation;
@@ -13872,31 +15525,31 @@ function registerEvalCommand(program) {
13872
15525
 
13873
15526
  // src/commands/init/index.ts
13874
15527
  import { existsSync, mkdirSync, writeFileSync } from "node:fs";
13875
- import path14 from "node:path";
15528
+ import path15 from "node:path";
13876
15529
 
13877
15530
  // src/templates/index.ts
13878
15531
  import { readFileSync } from "node:fs";
13879
- import path13 from "node:path";
15532
+ import path14 from "node:path";
13880
15533
  import { fileURLToPath as fileURLToPath2 } from "node:url";
13881
15534
  var TemplateManager = class {
13882
15535
  static getTemplates() {
13883
- const currentDir = path13.dirname(fileURLToPath2(import.meta.url));
15536
+ const currentDir = path14.dirname(fileURLToPath2(import.meta.url));
13884
15537
  let templatesDir;
13885
- if (currentDir.includes(path13.sep + "dist")) {
13886
- templatesDir = path13.join(currentDir, "templates");
15538
+ if (currentDir.includes(path14.sep + "dist")) {
15539
+ templatesDir = path14.join(currentDir, "templates");
13887
15540
  } else {
13888
15541
  templatesDir = currentDir;
13889
15542
  }
13890
15543
  const evalBuildPrompt = readFileSync(
13891
- path13.join(templatesDir, "eval-build.prompt.md"),
15544
+ path14.join(templatesDir, "eval-build.prompt.md"),
13892
15545
  "utf-8"
13893
15546
  );
13894
15547
  const evalSchema = readFileSync(
13895
- path13.join(templatesDir, "eval-schema.json"),
15548
+ path14.join(templatesDir, "eval-schema.json"),
13896
15549
  "utf-8"
13897
15550
  );
13898
15551
  const configSchema = readFileSync(
13899
- path13.join(templatesDir, "config-schema.json"),
15552
+ path14.join(templatesDir, "config-schema.json"),
13900
15553
  "utf-8"
13901
15554
  );
13902
15555
  return [
@@ -13918,28 +15571,37 @@ var TemplateManager = class {
13918
15571
 
13919
15572
  // src/commands/init/index.ts
13920
15573
  async function initCommand(options = {}) {
13921
- const targetPath = path14.resolve(options.targetPath ?? ".");
13922
- const githubDir = path14.join(targetPath, ".github");
15574
+ const targetPath = path15.resolve(options.targetPath ?? ".");
15575
+ const githubDir = path15.join(targetPath, ".github");
13923
15576
  if (!existsSync(githubDir)) {
13924
15577
  mkdirSync(githubDir, { recursive: true });
13925
15578
  }
13926
15579
  const templates = TemplateManager.getTemplates();
13927
15580
  for (const template of templates) {
13928
- const targetFilePath = path14.join(githubDir, template.path);
13929
- const targetDirPath = path14.dirname(targetFilePath);
15581
+ const targetFilePath = path15.join(githubDir, template.path);
15582
+ const targetDirPath = path15.dirname(targetFilePath);
13930
15583
  if (!existsSync(targetDirPath)) {
13931
15584
  mkdirSync(targetDirPath, { recursive: true });
13932
15585
  }
13933
15586
  writeFileSync(targetFilePath, template.content, "utf-8");
13934
- console.log(`Created ${path14.relative(targetPath, targetFilePath)}`);
15587
+ console.log(`Created ${path15.relative(targetPath, targetFilePath)}`);
13935
15588
  }
13936
15589
  console.log("\nAgentV initialized successfully!");
13937
15590
  console.log(`
13938
- Files installed to ${path14.relative(targetPath, githubDir)}:`);
15591
+ Files installed to ${path15.relative(targetPath, githubDir)}:`);
13939
15592
  templates.forEach((t) => console.log(` - ${t.path}`));
13940
15593
  console.log("\nYou can now create eval files using the schema and prompt templates.");
13941
15594
  }
13942
15595
 
15596
+ // src/commands/status.ts
15597
+ function registerStatusCommand(program) {
15598
+ program.command("status").description("Show the latest AgentV kernel status").action(() => {
15599
+ const kernel = createAgentKernel();
15600
+ console.log(`Kernel status: ${kernel.status}`);
15601
+ });
15602
+ return program;
15603
+ }
15604
+
13943
15605
  // src/commands/validate/format-output.ts
13944
15606
  var ANSI_RED = "\x1B[31m";
13945
15607
  var ANSI_YELLOW2 = "\x1B[33m";
@@ -14012,10 +15674,10 @@ function isTTY() {
14012
15674
  }
14013
15675
 
14014
15676
  // ../../packages/core/dist/evaluation/validation/index.js
14015
- import { readFile as readFile5 } from "node:fs/promises";
15677
+ import { readFile as readFile6 } from "node:fs/promises";
14016
15678
  import { parse as parse5 } from "yaml";
14017
15679
  import { readFile as readFile23 } from "node:fs/promises";
14018
- import path15 from "node:path";
15680
+ import path16 from "node:path";
14019
15681
  import { parse as parse23 } from "yaml";
14020
15682
  import { readFile as readFile32 } from "node:fs/promises";
14021
15683
  import path23 from "node:path";
@@ -14030,7 +15692,7 @@ var SCHEMA_TARGETS_V2 = "agentv-targets-v2";
14030
15692
  var SCHEMA_CONFIG_V22 = "agentv-config-v2";
14031
15693
  async function detectFileType(filePath) {
14032
15694
  try {
14033
- const content = await readFile5(filePath, "utf8");
15695
+ const content = await readFile6(filePath, "utf8");
14034
15696
  const parsed = parse5(content);
14035
15697
  if (typeof parsed !== "object" || parsed === null) {
14036
15698
  return "unknown";
@@ -14060,7 +15722,7 @@ function isObject(value) {
14060
15722
  }
14061
15723
  async function validateEvalFile(filePath) {
14062
15724
  const errors = [];
14063
- const absolutePath = path15.resolve(filePath);
15725
+ const absolutePath = path16.resolve(filePath);
14064
15726
  let parsed;
14065
15727
  try {
14066
15728
  const content = await readFile23(absolutePath, "utf8");
@@ -14249,6 +15911,7 @@ function validateMessages(messages, location, filePath, errors) {
14249
15911
  function isObject2(value) {
14250
15912
  return typeof value === "object" && value !== null && !Array.isArray(value);
14251
15913
  }
15914
+ var CLI_PLACEHOLDERS2 = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
14252
15915
  async function validateTargetsFile(filePath) {
14253
15916
  const errors = [];
14254
15917
  const absolutePath = path23.resolve(filePath);
@@ -14269,6 +15932,182 @@ async function validateTargetsFile(filePath) {
14269
15932
  errors
14270
15933
  };
14271
15934
  }
15935
+ function validateCliSettings(settings, absolutePath2, location, errors2) {
15936
+ if (!isObject2(settings)) {
15937
+ errors2.push({
15938
+ severity: "error",
15939
+ filePath: absolutePath2,
15940
+ location,
15941
+ message: "CLI provider requires a 'settings' object"
15942
+ });
15943
+ return;
15944
+ }
15945
+ const commandTemplate = settings["command_template"] ?? settings["commandTemplate"];
15946
+ if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
15947
+ errors2.push({
15948
+ severity: "error",
15949
+ filePath: absolutePath2,
15950
+ location: `${location}.commandTemplate`,
15951
+ message: "CLI provider requires 'commandTemplate' as a non-empty string"
15952
+ });
15953
+ } else {
15954
+ recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
15955
+ }
15956
+ const attachmentsFormat = settings["attachments_format"] ?? settings["attachmentsFormat"];
15957
+ if (attachmentsFormat !== void 0 && typeof attachmentsFormat !== "string") {
15958
+ errors2.push({
15959
+ severity: "error",
15960
+ filePath: absolutePath2,
15961
+ location: `${location}.attachmentsFormat`,
15962
+ message: "'attachmentsFormat' must be a string when provided"
15963
+ });
15964
+ }
15965
+ const filesFormat = settings["files_format"] ?? settings["filesFormat"];
15966
+ if (filesFormat !== void 0 && typeof filesFormat !== "string") {
15967
+ errors2.push({
15968
+ severity: "error",
15969
+ filePath: absolutePath2,
15970
+ location: `${location}.filesFormat`,
15971
+ message: "'filesFormat' must be a string when provided"
15972
+ });
15973
+ }
15974
+ const cwd = settings["cwd"];
15975
+ if (cwd !== void 0 && typeof cwd !== "string") {
15976
+ errors2.push({
15977
+ severity: "error",
15978
+ filePath: absolutePath2,
15979
+ location: `${location}.cwd`,
15980
+ message: "'cwd' must be a string when provided"
15981
+ });
15982
+ }
15983
+ const timeoutSeconds = settings["timeout_seconds"] ?? settings["timeoutSeconds"];
15984
+ if (timeoutSeconds !== void 0) {
15985
+ const numericTimeout = Number(timeoutSeconds);
15986
+ if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
15987
+ errors2.push({
15988
+ severity: "error",
15989
+ filePath: absolutePath2,
15990
+ location: `${location}.timeoutSeconds`,
15991
+ message: "'timeoutSeconds' must be a positive number when provided"
15992
+ });
15993
+ }
15994
+ }
15995
+ const envOverrides = settings["env"];
15996
+ if (envOverrides !== void 0) {
15997
+ if (!isObject2(envOverrides)) {
15998
+ errors2.push({
15999
+ severity: "error",
16000
+ filePath: absolutePath2,
16001
+ location: `${location}.env`,
16002
+ message: "'env' must be an object with string values"
16003
+ });
16004
+ } else {
16005
+ for (const [key2, value] of Object.entries(envOverrides)) {
16006
+ if (typeof value !== "string" || value.trim().length === 0) {
16007
+ errors2.push({
16008
+ severity: "error",
16009
+ filePath: absolutePath2,
16010
+ location: `${location}.env.${key2}`,
16011
+ message: `Environment override '${key2}' must be a non-empty string`
16012
+ });
16013
+ }
16014
+ }
16015
+ }
16016
+ }
16017
+ const healthcheck = settings["healthcheck"];
16018
+ if (healthcheck !== void 0) {
16019
+ validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
16020
+ }
16021
+ }
16022
+ function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
16023
+ if (!isObject2(healthcheck)) {
16024
+ errors2.push({
16025
+ severity: "error",
16026
+ filePath: absolutePath2,
16027
+ location,
16028
+ message: "'healthcheck' must be an object when provided"
16029
+ });
16030
+ return;
16031
+ }
16032
+ const type = healthcheck["type"];
16033
+ if (type !== "http" && type !== "command") {
16034
+ errors2.push({
16035
+ severity: "error",
16036
+ filePath: absolutePath2,
16037
+ location: `${location}.type`,
16038
+ message: "healthcheck.type must be either 'http' or 'command'"
16039
+ });
16040
+ return;
16041
+ }
16042
+ const timeoutSeconds = healthcheck["timeout_seconds"] ?? healthcheck["timeoutSeconds"];
16043
+ if (timeoutSeconds !== void 0) {
16044
+ const numericTimeout = Number(timeoutSeconds);
16045
+ if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
16046
+ errors2.push({
16047
+ severity: "error",
16048
+ filePath: absolutePath2,
16049
+ location: `${location}.timeoutSeconds`,
16050
+ message: "healthcheck.timeoutSeconds must be a positive number when provided"
16051
+ });
16052
+ }
16053
+ }
16054
+ if (type === "http") {
16055
+ const url = healthcheck["url"];
16056
+ if (typeof url !== "string" || url.trim().length === 0) {
16057
+ errors2.push({
16058
+ severity: "error",
16059
+ filePath: absolutePath2,
16060
+ location: `${location}.url`,
16061
+ message: "healthcheck.url must be a non-empty string for http checks"
16062
+ });
16063
+ }
16064
+ return;
16065
+ }
16066
+ const commandTemplate = healthcheck["command_template"] ?? healthcheck["commandTemplate"];
16067
+ if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
16068
+ errors2.push({
16069
+ severity: "error",
16070
+ filePath: absolutePath2,
16071
+ location: `${location}.commandTemplate`,
16072
+ message: "healthcheck.commandTemplate must be a non-empty string for command checks"
16073
+ });
16074
+ } else {
16075
+ recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
16076
+ }
16077
+ const cwd = healthcheck["cwd"];
16078
+ if (cwd !== void 0 && typeof cwd !== "string") {
16079
+ errors2.push({
16080
+ severity: "error",
16081
+ filePath: absolutePath2,
16082
+ location: `${location}.cwd`,
16083
+ message: "healthcheck.cwd must be a string when provided"
16084
+ });
16085
+ }
16086
+ }
16087
+ function recordUnknownPlaceholders(template, absolutePath2, location, errors2) {
16088
+ const placeholders = extractPlaceholders(template);
16089
+ for (const placeholder of placeholders) {
16090
+ if (!CLI_PLACEHOLDERS2.has(placeholder)) {
16091
+ errors2.push({
16092
+ severity: "error",
16093
+ filePath: absolutePath2,
16094
+ location,
16095
+ message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS2).join(", ")}`
16096
+ });
16097
+ }
16098
+ }
16099
+ }
16100
+ function extractPlaceholders(template) {
16101
+ const matches = template.matchAll(/\{([A-Z_]+)\}/g);
16102
+ const result = [];
16103
+ for (const match of matches) {
16104
+ const placeholder = match[1];
16105
+ if (placeholder) {
16106
+ result.push(placeholder);
16107
+ }
16108
+ }
16109
+ return result;
16110
+ }
14272
16111
  if (!isObject2(parsed)) {
14273
16112
  errors.push({
14274
16113
  severity: "error",
@@ -14330,6 +16169,7 @@ async function validateTargetsFile(filePath) {
14330
16169
  });
14331
16170
  }
14332
16171
  const provider = target["provider"];
16172
+ const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
14333
16173
  if (typeof provider !== "string" || provider.trim().length === 0) {
14334
16174
  errors.push({
14335
16175
  severity: "error",
@@ -14346,7 +16186,7 @@ async function validateTargetsFile(filePath) {
14346
16186
  });
14347
16187
  }
14348
16188
  const settings = target["settings"];
14349
- if (settings !== void 0 && !isObject2(settings)) {
16189
+ if (providerValue !== "cli" && settings !== void 0 && !isObject2(settings)) {
14350
16190
  errors.push({
14351
16191
  severity: "error",
14352
16192
  filePath: absolutePath,
@@ -14354,6 +16194,9 @@ async function validateTargetsFile(filePath) {
14354
16194
  message: "Invalid 'settings' field (must be an object)"
14355
16195
  });
14356
16196
  }
16197
+ if (providerValue === "cli") {
16198
+ validateCliSettings(settings, absolutePath, `${location}.settings`, errors);
16199
+ }
14357
16200
  const judgeTarget = target["judge_target"];
14358
16201
  if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {
14359
16202
  errors.push({
@@ -14558,7 +16401,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
14558
16401
  // src/commands/validate/validate-files.ts
14559
16402
  import { constants as constants7 } from "node:fs";
14560
16403
  import { access as access7, readdir as readdir3, stat as stat3 } from "node:fs/promises";
14561
- import path16 from "node:path";
16404
+ import path17 from "node:path";
14562
16405
  async function validateFiles(paths) {
14563
16406
  const filePaths = await expandPaths(paths);
14564
16407
  const results = [];
@@ -14576,7 +16419,7 @@ async function validateFiles(paths) {
14576
16419
  };
14577
16420
  }
14578
16421
  async function validateSingleFile(filePath) {
14579
- const absolutePath = path16.resolve(filePath);
16422
+ const absolutePath = path17.resolve(filePath);
14580
16423
  const fileType = await detectFileType(absolutePath);
14581
16424
  if (fileType === "unknown") {
14582
16425
  return {
@@ -14615,7 +16458,7 @@ async function validateSingleFile(filePath) {
14615
16458
  async function expandPaths(paths) {
14616
16459
  const expanded = [];
14617
16460
  for (const inputPath of paths) {
14618
- const absolutePath = path16.resolve(inputPath);
16461
+ const absolutePath = path17.resolve(inputPath);
14619
16462
  try {
14620
16463
  await access7(absolutePath, constants7.F_OK);
14621
16464
  } catch {
@@ -14639,7 +16482,7 @@ async function findYamlFiles(dirPath) {
14639
16482
  try {
14640
16483
  const entries = await readdir3(dirPath, { withFileTypes: true });
14641
16484
  for (const entry of entries) {
14642
- const fullPath = path16.join(dirPath, entry.name);
16485
+ const fullPath = path17.join(dirPath, entry.name);
14643
16486
  if (entry.isDirectory()) {
14644
16487
  if (entry.name === "node_modules" || entry.name.startsWith(".")) {
14645
16488
  continue;
@@ -14656,7 +16499,7 @@ async function findYamlFiles(dirPath) {
14656
16499
  return results;
14657
16500
  }
14658
16501
  function isYamlFile(filePath) {
14659
- const ext = path16.extname(filePath).toLowerCase();
16502
+ const ext = path17.extname(filePath).toLowerCase();
14660
16503
  return ext === ".yaml" || ext === ".yml";
14661
16504
  }
14662
16505
 
@@ -14685,15 +16528,6 @@ function registerValidateCommand(program) {
14685
16528
  return program;
14686
16529
  }
14687
16530
 
14688
- // src/commands/status.ts
14689
- function registerStatusCommand(program) {
14690
- program.command("status").description("Show the latest AgentV kernel status").action(() => {
14691
- const kernel = createAgentKernel();
14692
- console.log(`Kernel status: ${kernel.status}`);
14693
- });
14694
- return program;
14695
- }
14696
-
14697
16531
  // src/index.ts
14698
16532
  var packageJson = JSON.parse(readFileSync2(new URL("../package.json", import.meta.url), "utf8"));
14699
16533
  function createProgram() {
@@ -14722,4 +16556,4 @@ export {
14722
16556
  createProgram,
14723
16557
  runCli
14724
16558
  };
14725
- //# sourceMappingURL=chunk-RLBRJX7V.js.map
16559
+ //# sourceMappingURL=chunk-THVRLL37.js.map