agentv 0.2.8 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -63
- package/dist/{chunk-RLBRJX7V.js → chunk-THVRLL37.js} +2316 -482
- package/dist/chunk-THVRLL37.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +2 -2
- package/dist/chunk-RLBRJX7V.js.map +0 -1
|
@@ -585,7 +585,7 @@ var require_utc = __commonJS({
|
|
|
585
585
|
import { Command } from "commander";
|
|
586
586
|
import { readFileSync as readFileSync2 } from "node:fs";
|
|
587
587
|
|
|
588
|
-
// ../../packages/core/dist/chunk-
|
|
588
|
+
// ../../packages/core/dist/chunk-NL7K4CAK.js
|
|
589
589
|
import { constants } from "node:fs";
|
|
590
590
|
import { access } from "node:fs/promises";
|
|
591
591
|
import path from "node:path";
|
|
@@ -692,6 +692,8 @@ var KNOWN_PROVIDERS = [
|
|
|
692
692
|
"azure",
|
|
693
693
|
"anthropic",
|
|
694
694
|
"gemini",
|
|
695
|
+
"codex",
|
|
696
|
+
"cli",
|
|
695
697
|
"mock",
|
|
696
698
|
"vscode",
|
|
697
699
|
"vscode-insiders"
|
|
@@ -703,6 +705,8 @@ var PROVIDER_ALIASES = [
|
|
|
703
705
|
// alias for "gemini"
|
|
704
706
|
"google-gemini",
|
|
705
707
|
// alias for "gemini"
|
|
708
|
+
"codex-cli",
|
|
709
|
+
// alias for "codex"
|
|
706
710
|
"openai",
|
|
707
711
|
// legacy/future support
|
|
708
712
|
"bedrock",
|
|
@@ -716,7 +720,7 @@ var TARGETS_SCHEMA_V2 = "agentv-targets-v2";
|
|
|
716
720
|
import micromatch from "micromatch";
|
|
717
721
|
import { constants as constants3 } from "node:fs";
|
|
718
722
|
import { access as access3, readFile as readFile2 } from "node:fs/promises";
|
|
719
|
-
import
|
|
723
|
+
import path8 from "node:path";
|
|
720
724
|
import { fileURLToPath } from "node:url";
|
|
721
725
|
import { parse as parse3 } from "yaml";
|
|
722
726
|
|
|
@@ -5031,6 +5035,11 @@ var _c = pr();
|
|
|
5031
5035
|
var ya = new Error("Agent description must be at least 20 characters (explain in detail what the agent does)");
|
|
5032
5036
|
var ba = new Error("Agent definition is the prompt you give to the LLM for the agent. It must be detailed and at least 100 characters");
|
|
5033
5037
|
|
|
5038
|
+
// ../../packages/core/dist/index.js
|
|
5039
|
+
import { exec as execWithCallback } from "node:child_process";
|
|
5040
|
+
import path22 from "node:path";
|
|
5041
|
+
import { promisify as promisify2 } from "node:util";
|
|
5042
|
+
|
|
5034
5043
|
// ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/external.js
|
|
5035
5044
|
var external_exports = {};
|
|
5036
5045
|
__export(external_exports, {
|
|
@@ -5509,8 +5518,8 @@ function getErrorMap() {
|
|
|
5509
5518
|
|
|
5510
5519
|
// ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/helpers/parseUtil.js
|
|
5511
5520
|
var makeIssue = (params) => {
|
|
5512
|
-
const { data, path:
|
|
5513
|
-
const fullPath = [...
|
|
5521
|
+
const { data, path: path18, errorMaps, issueData } = params;
|
|
5522
|
+
const fullPath = [...path18, ...issueData.path || []];
|
|
5514
5523
|
const fullIssue = {
|
|
5515
5524
|
...issueData,
|
|
5516
5525
|
path: fullPath
|
|
@@ -5626,11 +5635,11 @@ var errorUtil;
|
|
|
5626
5635
|
|
|
5627
5636
|
// ../../node_modules/.pnpm/zod@3.25.76/node_modules/zod/v3/types.js
|
|
5628
5637
|
var ParseInputLazyPath = class {
|
|
5629
|
-
constructor(parent, value,
|
|
5638
|
+
constructor(parent, value, path18, key2) {
|
|
5630
5639
|
this._cachedPath = [];
|
|
5631
5640
|
this.parent = parent;
|
|
5632
5641
|
this.data = value;
|
|
5633
|
-
this._path =
|
|
5642
|
+
this._path = path18;
|
|
5634
5643
|
this._key = key2;
|
|
5635
5644
|
}
|
|
5636
5645
|
get path() {
|
|
@@ -9074,27 +9083,26 @@ var NEVER = INVALID;
|
|
|
9074
9083
|
|
|
9075
9084
|
// ../../packages/core/dist/index.js
|
|
9076
9085
|
import { readFile as readFile22 } from "node:fs/promises";
|
|
9077
|
-
import
|
|
9086
|
+
import path32 from "node:path";
|
|
9078
9087
|
|
|
9079
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
9088
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/agentDispatch.js
|
|
9080
9089
|
import { exec, spawn } from "child_process";
|
|
9081
9090
|
import { copyFile, mkdir as mkdir2, readdir as readdir2, readFile, stat as stat2, writeFile } from "fs/promises";
|
|
9082
|
-
import
|
|
9091
|
+
import path6 from "path";
|
|
9083
9092
|
import { promisify } from "util";
|
|
9084
9093
|
|
|
9085
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
9094
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/constants.js
|
|
9086
9095
|
import os from "os";
|
|
9087
9096
|
import path2 from "path";
|
|
9088
9097
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
9098
|
+
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
9089
9099
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
9090
9100
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
9091
9101
|
return path2.join(os.homedir(), ".subagent", folder);
|
|
9092
9102
|
}
|
|
9093
9103
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
9094
|
-
var DEFAULT_WAKEUP_FILENAME = "wakeup.chatmode.md";
|
|
9095
|
-
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
9096
9104
|
|
|
9097
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
9105
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/fs.js
|
|
9098
9106
|
import { constants as constants2 } from "fs";
|
|
9099
9107
|
import { access as access2, mkdir, readdir, rm, stat } from "fs/promises";
|
|
9100
9108
|
import path3 from "path";
|
|
@@ -9127,15 +9135,26 @@ async function removeIfExists(target) {
|
|
|
9127
9135
|
}
|
|
9128
9136
|
}
|
|
9129
9137
|
|
|
9130
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
9138
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/path.js
|
|
9139
|
+
import path4 from "path";
|
|
9140
|
+
function pathToFileUri(filePath) {
|
|
9141
|
+
const absolutePath = path4.isAbsolute(filePath) ? filePath : path4.resolve(filePath);
|
|
9142
|
+
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
9143
|
+
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
9144
|
+
return `file:///${normalizedPath}`;
|
|
9145
|
+
}
|
|
9146
|
+
return `file://${normalizedPath}`;
|
|
9147
|
+
}
|
|
9148
|
+
|
|
9149
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/time.js
|
|
9131
9150
|
function sleep(ms2) {
|
|
9132
9151
|
return new Promise((resolve) => {
|
|
9133
9152
|
setTimeout(resolve, ms2);
|
|
9134
9153
|
});
|
|
9135
9154
|
}
|
|
9136
9155
|
|
|
9137
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
9138
|
-
import
|
|
9156
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/workspace.js
|
|
9157
|
+
import path5 from "path";
|
|
9139
9158
|
|
|
9140
9159
|
// ../../node_modules/.pnpm/json5@2.2.3/node_modules/json5/dist/index.mjs
|
|
9141
9160
|
var Space_Separator = /[\u1680\u2000-\u200A\u202F\u205F\u3000]/;
|
|
@@ -10222,7 +10241,7 @@ var JSON5 = {
|
|
|
10222
10241
|
var lib = JSON5;
|
|
10223
10242
|
var dist_default = lib;
|
|
10224
10243
|
|
|
10225
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
10244
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/utils/workspace.js
|
|
10226
10245
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
10227
10246
|
let workspace;
|
|
10228
10247
|
try {
|
|
@@ -10238,10 +10257,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
10238
10257
|
}
|
|
10239
10258
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
10240
10259
|
const folderPath = folder.path;
|
|
10241
|
-
if (
|
|
10260
|
+
if (path5.isAbsolute(folderPath)) {
|
|
10242
10261
|
return folder;
|
|
10243
10262
|
}
|
|
10244
|
-
const absolutePath =
|
|
10263
|
+
const absolutePath = path5.resolve(templateDir, folderPath);
|
|
10245
10264
|
return {
|
|
10246
10265
|
...folder,
|
|
10247
10266
|
path: absolutePath
|
|
@@ -10266,19 +10285,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
10266
10285
|
if (locationMap && typeof locationMap === "object") {
|
|
10267
10286
|
const transformedMap = {};
|
|
10268
10287
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
10269
|
-
const isAbsolute =
|
|
10288
|
+
const isAbsolute = path5.isAbsolute(locationPath);
|
|
10270
10289
|
if (isAbsolute) {
|
|
10271
10290
|
transformedMap[locationPath] = value;
|
|
10272
10291
|
} else {
|
|
10273
10292
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
10274
10293
|
if (firstGlobIndex === -1) {
|
|
10275
|
-
const resolvedPath =
|
|
10294
|
+
const resolvedPath = path5.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
10276
10295
|
transformedMap[resolvedPath] = value;
|
|
10277
10296
|
} else {
|
|
10278
10297
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
10279
10298
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
10280
10299
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
10281
|
-
const resolvedPath = (
|
|
10300
|
+
const resolvedPath = (path5.resolve(templateDir, basePath) + patternPath).replace(/\\/g, "/");
|
|
10282
10301
|
transformedMap[resolvedPath] = value;
|
|
10283
10302
|
}
|
|
10284
10303
|
}
|
|
@@ -10295,28 +10314,39 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
10295
10314
|
return JSON.stringify(transformedWorkspace, null, 2);
|
|
10296
10315
|
}
|
|
10297
10316
|
|
|
10298
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
10317
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/agentDispatch.js
|
|
10299
10318
|
var execAsync = promisify(exec);
|
|
10319
|
+
function generateTimestamp() {
|
|
10320
|
+
return (/* @__PURE__ */ new Date()).toISOString().replace(/[-:TZ.]/g, "").slice(0, 14);
|
|
10321
|
+
}
|
|
10300
10322
|
var DEFAULT_WORKSPACE_TEMPLATE = {
|
|
10301
10323
|
folders: [
|
|
10302
10324
|
{
|
|
10303
10325
|
path: "."
|
|
10304
10326
|
}
|
|
10305
|
-
]
|
|
10306
|
-
settings: {
|
|
10307
|
-
"chat.modeFilesLocations": {
|
|
10308
|
-
"**/*.chatmode.md": true
|
|
10309
|
-
}
|
|
10310
|
-
}
|
|
10327
|
+
]
|
|
10311
10328
|
};
|
|
10312
10329
|
var DEFAULT_WAKEUP_CONTENT = `---
|
|
10313
10330
|
description: 'Wake-up Signal'
|
|
10314
|
-
|
|
10315
|
-
model: GPT-4.1 (copilot)
|
|
10331
|
+
model: Grok Code Fast 1 (copilot)
|
|
10316
10332
|
---`;
|
|
10317
10333
|
function getSubagentRoot(vscodeCmd = "code") {
|
|
10318
10334
|
return getDefaultSubagentRoot(vscodeCmd);
|
|
10319
10335
|
}
|
|
10336
|
+
async function resolvePromptFile(promptFile) {
|
|
10337
|
+
if (!promptFile) {
|
|
10338
|
+
return void 0;
|
|
10339
|
+
}
|
|
10340
|
+
const resolvedPrompt = path6.resolve(promptFile);
|
|
10341
|
+
if (!await pathExists(resolvedPrompt)) {
|
|
10342
|
+
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
10343
|
+
}
|
|
10344
|
+
const promptStats = await stat2(resolvedPrompt);
|
|
10345
|
+
if (!promptStats.isFile()) {
|
|
10346
|
+
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
10347
|
+
}
|
|
10348
|
+
return resolvedPrompt;
|
|
10349
|
+
}
|
|
10320
10350
|
async function findUnlockedSubagent(subagentRoot) {
|
|
10321
10351
|
if (!await pathExists(subagentRoot)) {
|
|
10322
10352
|
return null;
|
|
@@ -10327,7 +10357,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
10327
10357
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
10328
10358
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
10329
10359
|
for (const subagent of subagents) {
|
|
10330
|
-
const lockFile =
|
|
10360
|
+
const lockFile = path6.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
10331
10361
|
if (!await pathExists(lockFile)) {
|
|
10332
10362
|
return subagent.absolutePath;
|
|
10333
10363
|
}
|
|
@@ -10348,9 +10378,12 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
10348
10378
|
spawn(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
10349
10379
|
return true;
|
|
10350
10380
|
}
|
|
10351
|
-
const aliveFile =
|
|
10381
|
+
const aliveFile = path6.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
10352
10382
|
await removeIfExists(aliveFile);
|
|
10353
|
-
const
|
|
10383
|
+
const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
|
|
10384
|
+
await mkdir2(githubAgentsDir, { recursive: true });
|
|
10385
|
+
const wakeupDst = path6.join(githubAgentsDir, "wakeup.md");
|
|
10386
|
+
const subagentDst = path6.join(githubAgentsDir, "subagent.md");
|
|
10354
10387
|
await writeFile(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
10355
10388
|
spawn(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
10356
10389
|
await sleep(100);
|
|
@@ -10370,7 +10403,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
10370
10403
|
async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
10371
10404
|
let workspaceContent;
|
|
10372
10405
|
if (workspaceTemplate) {
|
|
10373
|
-
const workspaceSrc =
|
|
10406
|
+
const workspaceSrc = path6.resolve(workspaceTemplate);
|
|
10374
10407
|
if (!await pathExists(workspaceSrc)) {
|
|
10375
10408
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
10376
10409
|
}
|
|
@@ -10383,33 +10416,37 @@ async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
|
10383
10416
|
} else {
|
|
10384
10417
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
10385
10418
|
}
|
|
10386
|
-
const workspaceName = `${
|
|
10387
|
-
const workspaceDst =
|
|
10388
|
-
const templateDir = workspaceTemplate ?
|
|
10419
|
+
const workspaceName = `${path6.basename(subagentDir)}.code-workspace`;
|
|
10420
|
+
const workspaceDst = path6.join(subagentDir, workspaceName);
|
|
10421
|
+
const templateDir = workspaceTemplate ? path6.dirname(path6.resolve(workspaceTemplate)) : subagentDir;
|
|
10389
10422
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
10390
10423
|
const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
10391
10424
|
await writeFile(workspaceDst, transformedContent, "utf8");
|
|
10392
|
-
const messagesDir =
|
|
10425
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10393
10426
|
await mkdir2(messagesDir, { recursive: true });
|
|
10394
10427
|
return { workspace: workspaceDst, messagesDir };
|
|
10395
10428
|
}
|
|
10396
10429
|
async function createSubagentLock(subagentDir) {
|
|
10397
|
-
const messagesDir =
|
|
10430
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10398
10431
|
if (await pathExists(messagesDir)) {
|
|
10399
10432
|
const files = await readdir2(messagesDir);
|
|
10400
10433
|
await Promise.all(files.map(async (file) => {
|
|
10401
|
-
const target =
|
|
10434
|
+
const target = path6.join(messagesDir, file);
|
|
10402
10435
|
await removeIfExists(target);
|
|
10403
10436
|
}));
|
|
10404
10437
|
}
|
|
10405
|
-
const
|
|
10406
|
-
|
|
10407
|
-
|
|
10438
|
+
const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
|
|
10439
|
+
if (await pathExists(githubAgentsDir)) {
|
|
10440
|
+
const agentFiles = await readdir2(githubAgentsDir);
|
|
10441
|
+
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
10442
|
+
await Promise.all(agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path6.join(githubAgentsDir, file))));
|
|
10443
|
+
}
|
|
10444
|
+
const lockFile = path6.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
10408
10445
|
await writeFile(lockFile, "", { encoding: "utf8" });
|
|
10409
10446
|
return lockFile;
|
|
10410
10447
|
}
|
|
10411
10448
|
async function removeSubagentLock(subagentDir) {
|
|
10412
|
-
const lockFile =
|
|
10449
|
+
const lockFile = path6.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
10413
10450
|
await removeIfExists(lockFile);
|
|
10414
10451
|
}
|
|
10415
10452
|
async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, silent = false) {
|
|
@@ -10449,6 +10486,54 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
10449
10486
|
}
|
|
10450
10487
|
return false;
|
|
10451
10488
|
}
|
|
10489
|
+
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
|
|
10490
|
+
if (!silent) {
|
|
10491
|
+
const fileList = responseFilesFinal.map((file) => path6.basename(file)).join(", ");
|
|
10492
|
+
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
10493
|
+
}
|
|
10494
|
+
try {
|
|
10495
|
+
const pending = new Set(responseFilesFinal);
|
|
10496
|
+
while (pending.size > 0) {
|
|
10497
|
+
for (const file of [...pending]) {
|
|
10498
|
+
if (await pathExists(file)) {
|
|
10499
|
+
pending.delete(file);
|
|
10500
|
+
}
|
|
10501
|
+
}
|
|
10502
|
+
if (pending.size > 0) {
|
|
10503
|
+
await sleep(pollInterval);
|
|
10504
|
+
}
|
|
10505
|
+
}
|
|
10506
|
+
} catch (error) {
|
|
10507
|
+
if (error.code === "ENOENT") {
|
|
10508
|
+
return false;
|
|
10509
|
+
}
|
|
10510
|
+
throw error;
|
|
10511
|
+
}
|
|
10512
|
+
for (const file of responseFilesFinal) {
|
|
10513
|
+
let attempts = 0;
|
|
10514
|
+
const maxAttempts = 10;
|
|
10515
|
+
while (attempts < maxAttempts) {
|
|
10516
|
+
try {
|
|
10517
|
+
const content = await readFile(file, { encoding: "utf8" });
|
|
10518
|
+
if (!silent) {
|
|
10519
|
+
process.stdout.write(`${content}
|
|
10520
|
+
`);
|
|
10521
|
+
}
|
|
10522
|
+
break;
|
|
10523
|
+
} catch (error) {
|
|
10524
|
+
attempts += 1;
|
|
10525
|
+
if (error.code !== "EBUSY" || attempts >= maxAttempts) {
|
|
10526
|
+
if (!silent) {
|
|
10527
|
+
console.error(`error: failed to read agent response: ${error.message}`);
|
|
10528
|
+
}
|
|
10529
|
+
return false;
|
|
10530
|
+
}
|
|
10531
|
+
await sleep(pollInterval);
|
|
10532
|
+
}
|
|
10533
|
+
}
|
|
10534
|
+
}
|
|
10535
|
+
return true;
|
|
10536
|
+
}
|
|
10452
10537
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
|
|
10453
10538
|
if (dryRun) {
|
|
10454
10539
|
return 0;
|
|
@@ -10466,17 +10551,19 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
10466
10551
|
return 1;
|
|
10467
10552
|
}
|
|
10468
10553
|
if (promptFile) {
|
|
10469
|
-
const
|
|
10554
|
+
const githubAgentsDir = path6.join(subagentDir, ".github", "agents");
|
|
10555
|
+
await mkdir2(githubAgentsDir, { recursive: true });
|
|
10556
|
+
const agentFile = path6.join(githubAgentsDir, `${chatId}.md`);
|
|
10470
10557
|
try {
|
|
10471
|
-
await copyFile(promptFile,
|
|
10558
|
+
await copyFile(promptFile, agentFile);
|
|
10472
10559
|
} catch (error) {
|
|
10473
|
-
console.error(`error: Failed to copy prompt file to
|
|
10560
|
+
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
10474
10561
|
return 1;
|
|
10475
10562
|
}
|
|
10476
10563
|
}
|
|
10477
10564
|
return 0;
|
|
10478
10565
|
}
|
|
10479
|
-
function createRequestPrompt(userQuery, responseFileTmp, responseFileFinal
|
|
10566
|
+
function createRequestPrompt(userQuery, responseFileTmp, responseFileFinal) {
|
|
10480
10567
|
return `[[ ## task ## ]]
|
|
10481
10568
|
|
|
10482
10569
|
${userQuery}
|
|
@@ -10488,18 +10575,46 @@ ${userQuery}
|
|
|
10488
10575
|
2. When completely finished, run these PowerShell commands to signal completion:
|
|
10489
10576
|
\`\`\`
|
|
10490
10577
|
Move-Item -LiteralPath '${responseFileTmp}' -Destination '${responseFileFinal}'
|
|
10491
|
-
subagent
|
|
10578
|
+
if (Test-Path subagent.lock) { del subagent.lock }
|
|
10492
10579
|
\`\`\`
|
|
10493
10580
|
|
|
10494
10581
|
Do not proceed to step 2 until your response is completely written to the temporary file.`;
|
|
10495
10582
|
}
|
|
10583
|
+
function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal) {
|
|
10584
|
+
return `[[ ## task ## ]]
|
|
10585
|
+
|
|
10586
|
+
${userQuery}
|
|
10587
|
+
|
|
10588
|
+
[[ ## system_instructions ## ]]
|
|
10589
|
+
|
|
10590
|
+
Write your complete response to: ${responseFileTmp}
|
|
10591
|
+
When completely finished and the response is stable, rename it to: ${responseFileFinal}
|
|
10592
|
+
Do not unlock the workspace from this request; batch orchestration will handle unlocking after all responses are ready.`;
|
|
10593
|
+
}
|
|
10594
|
+
function createBatchOrchestratorPrompt(requestFiles, responseFiles) {
|
|
10595
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path6.basename(file)}`).join("\n");
|
|
10596
|
+
const responseList = responseFiles.map((file) => `"${path6.basename(file)}"`).join(", ");
|
|
10597
|
+
return `MANDATORY: Run #runSubagent tool in your Available Actions for each request file to process them in isolated contexts.
|
|
10598
|
+
DO NOT read the request files yourself - only pass the file paths to each subagent:
|
|
10599
|
+
|
|
10600
|
+
${requestLines}
|
|
10601
|
+
|
|
10602
|
+
After ALL queries complete, verify all responses exist and unlock:
|
|
10603
|
+
|
|
10604
|
+
\`\`\`powershell
|
|
10605
|
+
$responses = @(${responseList})
|
|
10606
|
+
$missing = $responses | Where-Object { -not (Test-Path "messages/$_") }
|
|
10607
|
+
if ($missing.Count -eq 0) { del subagent.lock }
|
|
10608
|
+
\`\`\`
|
|
10609
|
+
`;
|
|
10610
|
+
}
|
|
10496
10611
|
async function resolveAttachments(extraAttachments) {
|
|
10497
10612
|
if (!extraAttachments) {
|
|
10498
10613
|
return [];
|
|
10499
10614
|
}
|
|
10500
10615
|
const resolved = [];
|
|
10501
10616
|
for (const attachment of extraAttachments) {
|
|
10502
|
-
const resolvedPath =
|
|
10617
|
+
const resolvedPath = path6.resolve(attachment);
|
|
10503
10618
|
if (!await pathExists(resolvedPath)) {
|
|
10504
10619
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
10505
10620
|
}
|
|
@@ -10509,18 +10624,40 @@ async function resolveAttachments(extraAttachments) {
|
|
|
10509
10624
|
}
|
|
10510
10625
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
10511
10626
|
try {
|
|
10512
|
-
const workspacePath =
|
|
10513
|
-
const messagesDir =
|
|
10627
|
+
const workspacePath = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
|
|
10628
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10514
10629
|
await mkdir2(messagesDir, { recursive: true });
|
|
10515
|
-
const reqFile =
|
|
10630
|
+
const reqFile = path6.join(messagesDir, `${timestamp}_req.md`);
|
|
10516
10631
|
await writeFile(reqFile, requestInstructions, { encoding: "utf8" });
|
|
10517
10632
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
10518
10633
|
for (const attachment of attachmentPaths) {
|
|
10519
10634
|
chatArgs.push("-a", attachment);
|
|
10520
10635
|
}
|
|
10521
10636
|
chatArgs.push("-a", reqFile);
|
|
10522
|
-
chatArgs.push(`Follow instructions in ${
|
|
10523
|
-
const workspaceReady = await ensureWorkspaceFocused(workspacePath,
|
|
10637
|
+
chatArgs.push(`Follow instructions in ${path6.basename(reqFile)}`);
|
|
10638
|
+
const workspaceReady = await ensureWorkspaceFocused(workspacePath, path6.basename(subagentDir), subagentDir, vscodeCmd);
|
|
10639
|
+
if (!workspaceReady) {
|
|
10640
|
+
console.error("warning: Workspace may not be fully ready");
|
|
10641
|
+
}
|
|
10642
|
+
await sleep(500);
|
|
10643
|
+
spawn(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
10644
|
+
return true;
|
|
10645
|
+
} catch (error) {
|
|
10646
|
+
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
10647
|
+
return false;
|
|
10648
|
+
}
|
|
10649
|
+
}
|
|
10650
|
+
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
10651
|
+
try {
|
|
10652
|
+
const workspacePath = path6.join(subagentDir, `${path6.basename(subagentDir)}.code-workspace`);
|
|
10653
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10654
|
+
await mkdir2(messagesDir, { recursive: true });
|
|
10655
|
+
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
10656
|
+
for (const attachment of attachmentPaths) {
|
|
10657
|
+
chatArgs.push("-a", attachment);
|
|
10658
|
+
}
|
|
10659
|
+
chatArgs.push(chatInstruction);
|
|
10660
|
+
const workspaceReady = await ensureWorkspaceFocused(workspacePath, path6.basename(subagentDir), subagentDir, vscodeCmd);
|
|
10524
10661
|
if (!workspaceReady) {
|
|
10525
10662
|
console.error("warning: Workspace may not be fully ready");
|
|
10526
10663
|
}
|
|
@@ -10536,21 +10673,13 @@ async function dispatchAgentSession(options) {
|
|
|
10536
10673
|
const { userQuery, promptFile, extraAttachments, workspaceTemplate, dryRun = false, wait = true, vscodeCmd = "code", subagentRoot, silent = false } = options;
|
|
10537
10674
|
try {
|
|
10538
10675
|
let resolvedPrompt;
|
|
10539
|
-
|
|
10540
|
-
resolvedPrompt =
|
|
10541
|
-
|
|
10542
|
-
|
|
10543
|
-
|
|
10544
|
-
|
|
10545
|
-
|
|
10546
|
-
}
|
|
10547
|
-
const promptStats = await stat2(resolvedPrompt);
|
|
10548
|
-
if (!promptStats.isFile()) {
|
|
10549
|
-
return {
|
|
10550
|
-
exitCode: 1,
|
|
10551
|
-
error: `Prompt file must be a file, not a directory: ${resolvedPrompt}`
|
|
10552
|
-
};
|
|
10553
|
-
}
|
|
10676
|
+
try {
|
|
10677
|
+
resolvedPrompt = await resolvePromptFile(promptFile);
|
|
10678
|
+
} catch (error) {
|
|
10679
|
+
return {
|
|
10680
|
+
exitCode: 1,
|
|
10681
|
+
error: error.message
|
|
10682
|
+
};
|
|
10554
10683
|
}
|
|
10555
10684
|
const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
|
|
10556
10685
|
const subagentDir = await findUnlockedSubagent(subagentRootPath);
|
|
@@ -10560,7 +10689,7 @@ async function dispatchAgentSession(options) {
|
|
|
10560
10689
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
10561
10690
|
};
|
|
10562
10691
|
}
|
|
10563
|
-
const subagentName =
|
|
10692
|
+
const subagentName = path6.basename(subagentDir);
|
|
10564
10693
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
10565
10694
|
const preparationResult = await prepareSubagentDirectory(subagentDir, resolvedPrompt, chatId, workspaceTemplate, dryRun);
|
|
10566
10695
|
if (preparationResult !== 0) {
|
|
@@ -10580,11 +10709,11 @@ async function dispatchAgentSession(options) {
|
|
|
10580
10709
|
error: attachmentError.message
|
|
10581
10710
|
};
|
|
10582
10711
|
}
|
|
10583
|
-
const timestamp = (
|
|
10584
|
-
const messagesDir =
|
|
10585
|
-
const responseFileTmp =
|
|
10586
|
-
const responseFileFinal =
|
|
10587
|
-
const requestInstructions = createRequestPrompt(userQuery, responseFileTmp, responseFileFinal
|
|
10712
|
+
const timestamp = generateTimestamp();
|
|
10713
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10714
|
+
const responseFileTmp = path6.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
10715
|
+
const responseFileFinal = path6.join(messagesDir, `${timestamp}_res.md`);
|
|
10716
|
+
const requestInstructions = createRequestPrompt(userQuery, responseFileTmp, responseFileFinal);
|
|
10588
10717
|
if (dryRun) {
|
|
10589
10718
|
return {
|
|
10590
10719
|
exitCode: 0,
|
|
@@ -10635,10 +10764,141 @@ async function dispatchAgentSession(options) {
|
|
|
10635
10764
|
};
|
|
10636
10765
|
}
|
|
10637
10766
|
}
|
|
10767
|
+
async function dispatchBatchAgent(options) {
|
|
10768
|
+
const { userQueries, promptFile, extraAttachments, workspaceTemplate, dryRun = false, wait = false, vscodeCmd = "code", subagentRoot, silent = false } = options;
|
|
10769
|
+
if (!userQueries || userQueries.length === 0) {
|
|
10770
|
+
return {
|
|
10771
|
+
exitCode: 1,
|
|
10772
|
+
requestFiles: [],
|
|
10773
|
+
queryCount: 0,
|
|
10774
|
+
error: "At least one query is required for batch dispatch"
|
|
10775
|
+
};
|
|
10776
|
+
}
|
|
10777
|
+
const queryCount = userQueries.length;
|
|
10778
|
+
let requestFiles = [];
|
|
10779
|
+
let responseFilesFinal = [];
|
|
10780
|
+
let subagentName;
|
|
10781
|
+
try {
|
|
10782
|
+
let resolvedPrompt;
|
|
10783
|
+
try {
|
|
10784
|
+
resolvedPrompt = await resolvePromptFile(promptFile);
|
|
10785
|
+
} catch (error) {
|
|
10786
|
+
return {
|
|
10787
|
+
exitCode: 1,
|
|
10788
|
+
requestFiles,
|
|
10789
|
+
queryCount,
|
|
10790
|
+
error: error.message
|
|
10791
|
+
};
|
|
10792
|
+
}
|
|
10793
|
+
const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
|
|
10794
|
+
const subagentDir = await findUnlockedSubagent(subagentRootPath);
|
|
10795
|
+
if (!subagentDir) {
|
|
10796
|
+
return {
|
|
10797
|
+
exitCode: 1,
|
|
10798
|
+
requestFiles,
|
|
10799
|
+
queryCount,
|
|
10800
|
+
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
10801
|
+
};
|
|
10802
|
+
}
|
|
10803
|
+
subagentName = path6.basename(subagentDir);
|
|
10804
|
+
const chatId = Math.random().toString(16).slice(2, 10);
|
|
10805
|
+
const preparationResult = await prepareSubagentDirectory(subagentDir, resolvedPrompt, chatId, workspaceTemplate, dryRun);
|
|
10806
|
+
if (preparationResult !== 0) {
|
|
10807
|
+
return {
|
|
10808
|
+
exitCode: preparationResult,
|
|
10809
|
+
subagentName,
|
|
10810
|
+
requestFiles,
|
|
10811
|
+
queryCount,
|
|
10812
|
+
error: "Failed to prepare subagent workspace"
|
|
10813
|
+
};
|
|
10814
|
+
}
|
|
10815
|
+
let attachments;
|
|
10816
|
+
try {
|
|
10817
|
+
attachments = await resolveAttachments(extraAttachments);
|
|
10818
|
+
} catch (attachmentError) {
|
|
10819
|
+
return {
|
|
10820
|
+
exitCode: 1,
|
|
10821
|
+
subagentName,
|
|
10822
|
+
requestFiles,
|
|
10823
|
+
queryCount,
|
|
10824
|
+
error: attachmentError.message
|
|
10825
|
+
};
|
|
10826
|
+
}
|
|
10827
|
+
const timestamp = generateTimestamp();
|
|
10828
|
+
const messagesDir = path6.join(subagentDir, "messages");
|
|
10829
|
+
requestFiles = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_req.md`));
|
|
10830
|
+
const responseTmpFiles = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_res.tmp.md`));
|
|
10831
|
+
responseFilesFinal = userQueries.map((_, index) => path6.join(messagesDir, `${timestamp}_${index}_res.md`));
|
|
10832
|
+
const orchestratorFile = path6.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
10833
|
+
if (!dryRun) {
|
|
10834
|
+
await Promise.all(userQueries.map((query, index) => writeFile(requestFiles[index], createBatchRequestPrompt(query, responseTmpFiles[index], responseFilesFinal[index]), { encoding: "utf8" })));
|
|
10835
|
+
const orchestratorContent = createBatchOrchestratorPrompt(requestFiles, responseFilesFinal);
|
|
10836
|
+
await writeFile(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
10837
|
+
}
|
|
10838
|
+
const chatAttachments = [orchestratorFile, ...attachments];
|
|
10839
|
+
const orchestratorUri = pathToFileUri(orchestratorFile);
|
|
10840
|
+
const chatInstruction = `Follow instructions in [${timestamp}_orchestrator.md](${orchestratorUri}). Use #runSubagent tool.`;
|
|
10841
|
+
if (dryRun) {
|
|
10842
|
+
return {
|
|
10843
|
+
exitCode: 0,
|
|
10844
|
+
subagentName,
|
|
10845
|
+
requestFiles,
|
|
10846
|
+
responseFiles: wait ? responseFilesFinal : void 0,
|
|
10847
|
+
queryCount
|
|
10848
|
+
};
|
|
10849
|
+
}
|
|
10850
|
+
const launchSuccess = await launchVsCodeWithBatchChat(subagentDir, chatId, chatAttachments, chatInstruction, vscodeCmd);
|
|
10851
|
+
if (!launchSuccess) {
|
|
10852
|
+
return {
|
|
10853
|
+
exitCode: 1,
|
|
10854
|
+
subagentName,
|
|
10855
|
+
requestFiles,
|
|
10856
|
+
queryCount,
|
|
10857
|
+
error: "Failed to launch VS Code for batch dispatch"
|
|
10858
|
+
};
|
|
10859
|
+
}
|
|
10860
|
+
if (!wait) {
|
|
10861
|
+
return {
|
|
10862
|
+
exitCode: 0,
|
|
10863
|
+
subagentName,
|
|
10864
|
+
requestFiles,
|
|
10865
|
+
queryCount
|
|
10866
|
+
};
|
|
10867
|
+
}
|
|
10868
|
+
const responsesCompleted = await waitForBatchResponses(responseFilesFinal, 1e3, silent);
|
|
10869
|
+
if (!responsesCompleted) {
|
|
10870
|
+
return {
|
|
10871
|
+
exitCode: 1,
|
|
10872
|
+
subagentName,
|
|
10873
|
+
requestFiles,
|
|
10874
|
+
responseFiles: responseFilesFinal,
|
|
10875
|
+
queryCount,
|
|
10876
|
+
error: "Timed out waiting for batch responses"
|
|
10877
|
+
};
|
|
10878
|
+
}
|
|
10879
|
+
await removeSubagentLock(subagentDir);
|
|
10880
|
+
return {
|
|
10881
|
+
exitCode: 0,
|
|
10882
|
+
subagentName,
|
|
10883
|
+
requestFiles,
|
|
10884
|
+
responseFiles: responseFilesFinal,
|
|
10885
|
+
queryCount
|
|
10886
|
+
};
|
|
10887
|
+
} catch (error) {
|
|
10888
|
+
return {
|
|
10889
|
+
exitCode: 1,
|
|
10890
|
+
subagentName,
|
|
10891
|
+
requestFiles,
|
|
10892
|
+
responseFiles: responseFilesFinal.length > 0 ? responseFilesFinal : void 0,
|
|
10893
|
+
queryCount,
|
|
10894
|
+
error: error.message
|
|
10895
|
+
};
|
|
10896
|
+
}
|
|
10897
|
+
}
|
|
10638
10898
|
|
|
10639
|
-
// ../../node_modules/.pnpm/subagent@0.4.
|
|
10899
|
+
// ../../node_modules/.pnpm/subagent@0.4.6/node_modules/subagent/dist/vscode/provision.js
|
|
10640
10900
|
import { writeFile as writeFile2 } from "fs/promises";
|
|
10641
|
-
import
|
|
10901
|
+
import path7 from "path";
|
|
10642
10902
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
10643
10903
|
folders: [
|
|
10644
10904
|
{
|
|
@@ -10647,7 +10907,7 @@ var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
|
10647
10907
|
],
|
|
10648
10908
|
settings: {
|
|
10649
10909
|
"chat.modeFilesLocations": {
|
|
10650
|
-
"
|
|
10910
|
+
".github/agents/**/*.md": true
|
|
10651
10911
|
}
|
|
10652
10912
|
}
|
|
10653
10913
|
};
|
|
@@ -10661,7 +10921,7 @@ async function provisionSubagents(options) {
|
|
|
10661
10921
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
10662
10922
|
throw new Error("subagents must be a positive integer");
|
|
10663
10923
|
}
|
|
10664
|
-
const targetPath =
|
|
10924
|
+
const targetPath = path7.resolve(targetRoot);
|
|
10665
10925
|
if (!dryRun) {
|
|
10666
10926
|
await ensureDir(targetPath);
|
|
10667
10927
|
}
|
|
@@ -10680,7 +10940,7 @@ async function provisionSubagents(options) {
|
|
|
10680
10940
|
continue;
|
|
10681
10941
|
}
|
|
10682
10942
|
highestNumber = Math.max(highestNumber, parsed);
|
|
10683
|
-
const lockFile =
|
|
10943
|
+
const lockFile = path7.join(entry.absolutePath, lockName);
|
|
10684
10944
|
const locked = await pathExists(lockFile);
|
|
10685
10945
|
if (locked) {
|
|
10686
10946
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -10697,9 +10957,11 @@ async function provisionSubagents(options) {
|
|
|
10697
10957
|
break;
|
|
10698
10958
|
}
|
|
10699
10959
|
const subagentDir = subagent.absolutePath;
|
|
10700
|
-
const
|
|
10701
|
-
const
|
|
10702
|
-
const
|
|
10960
|
+
const githubAgentsDir = path7.join(subagentDir, ".github", "agents");
|
|
10961
|
+
const lockFile = path7.join(subagentDir, lockName);
|
|
10962
|
+
const workspaceDst = path7.join(subagentDir, `${path7.basename(subagentDir)}.code-workspace`);
|
|
10963
|
+
const wakeupDst = path7.join(githubAgentsDir, "wakeup.md");
|
|
10964
|
+
const subagentDst = path7.join(githubAgentsDir, "subagent.md");
|
|
10703
10965
|
const isLocked = await pathExists(lockFile);
|
|
10704
10966
|
if (isLocked && !force) {
|
|
10705
10967
|
continue;
|
|
@@ -10707,6 +10969,7 @@ async function provisionSubagents(options) {
|
|
|
10707
10969
|
if (isLocked && force) {
|
|
10708
10970
|
if (!dryRun) {
|
|
10709
10971
|
await removeIfExists(lockFile);
|
|
10972
|
+
await ensureDir(githubAgentsDir);
|
|
10710
10973
|
await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
10711
10974
|
await writeFile2(wakeupDst, wakeupContent, "utf8");
|
|
10712
10975
|
}
|
|
@@ -10717,6 +10980,7 @@ async function provisionSubagents(options) {
|
|
|
10717
10980
|
}
|
|
10718
10981
|
if (!isLocked && force) {
|
|
10719
10982
|
if (!dryRun) {
|
|
10983
|
+
await ensureDir(githubAgentsDir);
|
|
10720
10984
|
await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
10721
10985
|
await writeFile2(wakeupDst, wakeupContent, "utf8");
|
|
10722
10986
|
}
|
|
@@ -10725,6 +10989,7 @@ async function provisionSubagents(options) {
|
|
|
10725
10989
|
continue;
|
|
10726
10990
|
}
|
|
10727
10991
|
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
10992
|
+
await ensureDir(githubAgentsDir);
|
|
10728
10993
|
await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
10729
10994
|
await writeFile2(wakeupDst, wakeupContent, "utf8");
|
|
10730
10995
|
}
|
|
@@ -10734,11 +10999,14 @@ async function provisionSubagents(options) {
|
|
|
10734
10999
|
let nextIndex = highestNumber;
|
|
10735
11000
|
while (subagentsProvisioned < subagents) {
|
|
10736
11001
|
nextIndex += 1;
|
|
10737
|
-
const subagentDir =
|
|
10738
|
-
const
|
|
10739
|
-
const
|
|
11002
|
+
const subagentDir = path7.join(targetPath, `subagent-${nextIndex}`);
|
|
11003
|
+
const githubAgentsDir = path7.join(subagentDir, ".github", "agents");
|
|
11004
|
+
const workspaceDst = path7.join(subagentDir, `${path7.basename(subagentDir)}.code-workspace`);
|
|
11005
|
+
const wakeupDst = path7.join(githubAgentsDir, "wakeup.md");
|
|
11006
|
+
const subagentDst = path7.join(githubAgentsDir, "subagent.md");
|
|
10740
11007
|
if (!dryRun) {
|
|
10741
11008
|
await ensureDir(subagentDir);
|
|
11009
|
+
await ensureDir(githubAgentsDir);
|
|
10742
11010
|
await writeFile2(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
10743
11011
|
await writeFile2(wakeupDst, wakeupContent, "utf8");
|
|
10744
11012
|
}
|
|
@@ -10753,14 +11021,21 @@ async function provisionSubagents(options) {
|
|
|
10753
11021
|
}
|
|
10754
11022
|
|
|
10755
11023
|
// ../../packages/core/dist/index.js
|
|
11024
|
+
import { exec as execCallback, spawn as spawn2 } from "node:child_process";
|
|
10756
11025
|
import { constants as constants22 } from "node:fs";
|
|
10757
|
-
import { access as access22,
|
|
10758
|
-
import
|
|
11026
|
+
import { access as access22, copyFile as copyFile2, mkdtemp, mkdir as mkdir3, rm as rm2, writeFile as writeFile3 } from "node:fs/promises";
|
|
11027
|
+
import { tmpdir } from "node:os";
|
|
11028
|
+
import path52 from "node:path";
|
|
11029
|
+
import { promisify as promisify22 } from "node:util";
|
|
11030
|
+
import path42 from "node:path";
|
|
11031
|
+
import { constants as constants32 } from "node:fs";
|
|
11032
|
+
import { access as access32, readFile as readFile3 } from "node:fs/promises";
|
|
11033
|
+
import path62 from "node:path";
|
|
10759
11034
|
import { parse as parse22 } from "yaml";
|
|
10760
11035
|
import { randomUUID } from "node:crypto";
|
|
10761
11036
|
import { createHash, randomUUID as randomUUID2 } from "node:crypto";
|
|
10762
|
-
import { mkdir as
|
|
10763
|
-
import
|
|
11037
|
+
import { mkdir as mkdir22, readFile as readFile4, writeFile as writeFile22 } from "node:fs/promises";
|
|
11038
|
+
import path72 from "node:path";
|
|
10764
11039
|
var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
|
|
10765
11040
|
var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
|
|
10766
11041
|
function isTestMessageRole(value) {
|
|
@@ -10800,10 +11075,10 @@ function isTestMessage(value) {
|
|
|
10800
11075
|
}
|
|
10801
11076
|
return candidate.content.every(isJsonObject);
|
|
10802
11077
|
}
|
|
10803
|
-
var
|
|
10804
|
-
var
|
|
10805
|
-
function
|
|
10806
|
-
return typeof value === "string" &&
|
|
11078
|
+
var EVALUATOR_KIND_VALUES = ["code", "llm_judge"];
|
|
11079
|
+
var EVALUATOR_KIND_SET = new Set(EVALUATOR_KIND_VALUES);
|
|
11080
|
+
function isEvaluatorKind(value) {
|
|
11081
|
+
return typeof value === "string" && EVALUATOR_KIND_SET.has(value);
|
|
10807
11082
|
}
|
|
10808
11083
|
var CODE_BLOCK_PATTERN = /```[\s\S]*?```/g;
|
|
10809
11084
|
var ANSI_YELLOW = "\x1B[33m";
|
|
@@ -10813,7 +11088,7 @@ var SCHEMA_CONFIG_V2 = "agentv-config-v2";
|
|
|
10813
11088
|
async function loadConfig(evalFilePath, repoRoot) {
|
|
10814
11089
|
const directories = buildDirectoryChain(evalFilePath, repoRoot);
|
|
10815
11090
|
for (const directory of directories) {
|
|
10816
|
-
const configPath =
|
|
11091
|
+
const configPath = path8.join(directory, ".agentv", "config.yaml");
|
|
10817
11092
|
if (!await fileExists2(configPath)) {
|
|
10818
11093
|
continue;
|
|
10819
11094
|
}
|
|
@@ -10876,7 +11151,7 @@ function extractCodeBlocks(segments) {
|
|
|
10876
11151
|
}
|
|
10877
11152
|
async function loadEvalCases(evalFilePath, repoRoot, options) {
|
|
10878
11153
|
const verbose = options?.verbose ?? false;
|
|
10879
|
-
const absoluteTestPath =
|
|
11154
|
+
const absoluteTestPath = path8.resolve(evalFilePath);
|
|
10880
11155
|
if (!await fileExists2(absoluteTestPath)) {
|
|
10881
11156
|
throw new Error(`Test file not found: ${evalFilePath}`);
|
|
10882
11157
|
}
|
|
@@ -10900,7 +11175,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
10900
11175
|
if (!Array.isArray(rawTestcases)) {
|
|
10901
11176
|
throw new Error(`Invalid test file format: ${evalFilePath} - missing 'evalcases' field`);
|
|
10902
11177
|
}
|
|
10903
|
-
const
|
|
11178
|
+
const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm_judge";
|
|
10904
11179
|
const results = [];
|
|
10905
11180
|
for (const rawEvalcase of rawTestcases) {
|
|
10906
11181
|
if (!isJsonObject(rawEvalcase)) {
|
|
@@ -10987,9 +11262,9 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
10987
11262
|
}
|
|
10988
11263
|
try {
|
|
10989
11264
|
const fileContent = (await readFile2(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
10990
|
-
const relativeToRepo =
|
|
11265
|
+
const relativeToRepo = path8.relative(repoRootPath, resolvedPath);
|
|
10991
11266
|
if (isGuidelineFile(relativeToRepo, guidelinePatterns)) {
|
|
10992
|
-
guidelinePaths.push(
|
|
11267
|
+
guidelinePaths.push(path8.resolve(resolvedPath));
|
|
10993
11268
|
if (verbose) {
|
|
10994
11269
|
console.log(` [Guideline] Found: ${displayPath}`);
|
|
10995
11270
|
console.log(` Resolved to: ${resolvedPath}`);
|
|
@@ -10999,7 +11274,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
10999
11274
|
type: "file",
|
|
11000
11275
|
path: displayPath,
|
|
11001
11276
|
text: fileContent,
|
|
11002
|
-
resolvedPath:
|
|
11277
|
+
resolvedPath: path8.resolve(resolvedPath)
|
|
11003
11278
|
});
|
|
11004
11279
|
if (verbose) {
|
|
11005
11280
|
console.log(` [File] Found: ${displayPath}`);
|
|
@@ -11023,7 +11298,8 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
11023
11298
|
const assistantContent = assistantMessages[0]?.content;
|
|
11024
11299
|
const expectedAssistantRaw = await resolveAssistantContent(assistantContent, searchRoots, verbose);
|
|
11025
11300
|
const userTextPrompt = userTextParts.map((part) => part.trim()).filter((part) => part.length > 0).join(" ");
|
|
11026
|
-
const
|
|
11301
|
+
const testCaseEvaluatorKind = coerceEvaluator(evalcase.evaluator, id) ?? globalEvaluator;
|
|
11302
|
+
const evaluators = await parseEvaluators(evalcase, searchRoots, id ?? "unknown");
|
|
11027
11303
|
const userFilePaths = [];
|
|
11028
11304
|
for (const segment of userSegments) {
|
|
11029
11305
|
if (segment.type === "file" && typeof segment.resolvedPath === "string") {
|
|
@@ -11031,7 +11307,7 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
11031
11307
|
}
|
|
11032
11308
|
}
|
|
11033
11309
|
const allFilePaths = [
|
|
11034
|
-
...guidelinePaths.map((guidelinePath) =>
|
|
11310
|
+
...guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
11035
11311
|
...userFilePaths
|
|
11036
11312
|
];
|
|
11037
11313
|
const testCase = {
|
|
@@ -11041,12 +11317,13 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
11041
11317
|
user_segments: userSegments,
|
|
11042
11318
|
system_message: systemMessageContent,
|
|
11043
11319
|
expected_assistant_raw: expectedAssistantRaw,
|
|
11044
|
-
guideline_paths: guidelinePaths.map((guidelinePath) =>
|
|
11320
|
+
guideline_paths: guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
11045
11321
|
guideline_patterns: guidelinePatterns,
|
|
11046
11322
|
file_paths: allFilePaths,
|
|
11047
11323
|
code_snippets: codeSnippets,
|
|
11048
11324
|
outcome,
|
|
11049
|
-
|
|
11325
|
+
evaluator: testCaseEvaluatorKind,
|
|
11326
|
+
evaluators
|
|
11050
11327
|
};
|
|
11051
11328
|
if (verbose) {
|
|
11052
11329
|
console.log(`
|
|
@@ -11067,14 +11344,14 @@ Please add '$schema: ${SCHEMA_EVAL_V2}' at the top of the file.`;
|
|
|
11067
11344
|
async function buildPromptInputs(testCase) {
|
|
11068
11345
|
const guidelineContents = [];
|
|
11069
11346
|
for (const rawPath of testCase.guideline_paths) {
|
|
11070
|
-
const absolutePath =
|
|
11347
|
+
const absolutePath = path8.resolve(rawPath);
|
|
11071
11348
|
if (!await fileExists2(absolutePath)) {
|
|
11072
11349
|
logWarning(`Could not read guideline file ${absolutePath}: file does not exist`);
|
|
11073
11350
|
continue;
|
|
11074
11351
|
}
|
|
11075
11352
|
try {
|
|
11076
11353
|
const content = (await readFile2(absolutePath, "utf8")).replace(/\r\n/g, "\n");
|
|
11077
|
-
guidelineContents.push(`=== ${
|
|
11354
|
+
guidelineContents.push(`=== ${path8.basename(absolutePath)} ===
|
|
11078
11355
|
${content}`);
|
|
11079
11356
|
} catch (error) {
|
|
11080
11357
|
logWarning(`Could not read guideline file ${absolutePath}: ${error.message}`);
|
|
@@ -11127,7 +11404,7 @@ function resolveToAbsolutePath(candidate) {
|
|
|
11127
11404
|
if (candidate.startsWith("file://")) {
|
|
11128
11405
|
return fileURLToPath(new URL(candidate));
|
|
11129
11406
|
}
|
|
11130
|
-
return
|
|
11407
|
+
return path8.resolve(candidate);
|
|
11131
11408
|
}
|
|
11132
11409
|
throw new TypeError("Unsupported repoRoot value. Expected string or URL.");
|
|
11133
11410
|
}
|
|
@@ -11207,14 +11484,88 @@ async function resolveAssistantContent(content, searchRoots, verbose) {
|
|
|
11207
11484
|
}
|
|
11208
11485
|
return parts.join(" ");
|
|
11209
11486
|
}
|
|
11210
|
-
function
|
|
11487
|
+
async function parseEvaluators(rawEvalCase, searchRoots, evalId) {
|
|
11488
|
+
const execution = rawEvalCase.execution;
|
|
11489
|
+
const candidateEvaluators = isJsonObject(execution) ? execution.evaluators ?? rawEvalCase.evaluators : rawEvalCase.evaluators;
|
|
11490
|
+
if (candidateEvaluators === void 0) {
|
|
11491
|
+
return void 0;
|
|
11492
|
+
}
|
|
11493
|
+
if (!Array.isArray(candidateEvaluators)) {
|
|
11494
|
+
logWarning(`Skipping evaluators for '${evalId}': expected array`);
|
|
11495
|
+
return void 0;
|
|
11496
|
+
}
|
|
11497
|
+
const evaluators = [];
|
|
11498
|
+
for (const rawEvaluator of candidateEvaluators) {
|
|
11499
|
+
if (!isJsonObject(rawEvaluator)) {
|
|
11500
|
+
logWarning(`Skipping invalid evaluator entry for '${evalId}' (expected object)`);
|
|
11501
|
+
continue;
|
|
11502
|
+
}
|
|
11503
|
+
const name = asString(rawEvaluator.name);
|
|
11504
|
+
const typeValue = rawEvaluator.type;
|
|
11505
|
+
if (!name || !isEvaluatorKind(typeValue)) {
|
|
11506
|
+
logWarning(`Skipping evaluator with invalid name/type in '${evalId}'`);
|
|
11507
|
+
continue;
|
|
11508
|
+
}
|
|
11509
|
+
if (typeValue === "code") {
|
|
11510
|
+
const script = asString(rawEvaluator.script);
|
|
11511
|
+
if (!script) {
|
|
11512
|
+
logWarning(`Skipping code evaluator '${name}' in '${evalId}': missing script`);
|
|
11513
|
+
continue;
|
|
11514
|
+
}
|
|
11515
|
+
const cwd = asString(rawEvaluator.cwd);
|
|
11516
|
+
let resolvedCwd;
|
|
11517
|
+
if (cwd) {
|
|
11518
|
+
const resolved = await resolveFileReference(cwd, searchRoots);
|
|
11519
|
+
if (resolved.resolvedPath) {
|
|
11520
|
+
resolvedCwd = path8.resolve(resolved.resolvedPath);
|
|
11521
|
+
} else {
|
|
11522
|
+
logWarning(
|
|
11523
|
+
`Code evaluator '${name}' in '${evalId}': cwd not found (${resolved.displayPath})`,
|
|
11524
|
+
resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => ` Tried: ${attempt}`) : void 0
|
|
11525
|
+
);
|
|
11526
|
+
}
|
|
11527
|
+
}
|
|
11528
|
+
evaluators.push({
|
|
11529
|
+
name,
|
|
11530
|
+
type: "code",
|
|
11531
|
+
script,
|
|
11532
|
+
cwd,
|
|
11533
|
+
resolvedCwd
|
|
11534
|
+
});
|
|
11535
|
+
continue;
|
|
11536
|
+
}
|
|
11537
|
+
const prompt = asString(rawEvaluator.prompt);
|
|
11538
|
+
let promptPath;
|
|
11539
|
+
if (prompt) {
|
|
11540
|
+
const resolved = await resolveFileReference(prompt, searchRoots);
|
|
11541
|
+
if (resolved.resolvedPath) {
|
|
11542
|
+
promptPath = path8.resolve(resolved.resolvedPath);
|
|
11543
|
+
} else {
|
|
11544
|
+
logWarning(
|
|
11545
|
+
`Inline prompt used for evaluator '${name}' in '${evalId}' (file not found: ${resolved.displayPath})`,
|
|
11546
|
+
resolved.attempted.length > 0 ? resolved.attempted.map((attempt) => ` Tried: ${attempt}`) : void 0
|
|
11547
|
+
);
|
|
11548
|
+
}
|
|
11549
|
+
}
|
|
11550
|
+
const model = asString(rawEvaluator.model);
|
|
11551
|
+
evaluators.push({
|
|
11552
|
+
name,
|
|
11553
|
+
type: "llm_judge",
|
|
11554
|
+
prompt,
|
|
11555
|
+
promptPath,
|
|
11556
|
+
model
|
|
11557
|
+
});
|
|
11558
|
+
}
|
|
11559
|
+
return evaluators.length > 0 ? evaluators : void 0;
|
|
11560
|
+
}
|
|
11561
|
+
function coerceEvaluator(candidate, contextId) {
|
|
11211
11562
|
if (typeof candidate !== "string") {
|
|
11212
11563
|
return void 0;
|
|
11213
11564
|
}
|
|
11214
|
-
if (
|
|
11565
|
+
if (isEvaluatorKind(candidate)) {
|
|
11215
11566
|
return candidate;
|
|
11216
11567
|
}
|
|
11217
|
-
logWarning(`Unknown
|
|
11568
|
+
logWarning(`Unknown evaluator '${candidate}' in ${contextId}, falling back to default`);
|
|
11218
11569
|
return void 0;
|
|
11219
11570
|
}
|
|
11220
11571
|
function logWarning(message, details) {
|
|
@@ -11406,49 +11757,253 @@ var GeminiProvider = class {
|
|
|
11406
11757
|
return mapResponse(ensureChatResponse(response));
|
|
11407
11758
|
}
|
|
11408
11759
|
};
|
|
11409
|
-
var
|
|
11410
|
-
var
|
|
11760
|
+
var execAsync2 = promisify2(execWithCallback);
|
|
11761
|
+
var DEFAULT_MAX_BUFFER = 10 * 1024 * 1024;
|
|
11762
|
+
async function defaultCommandRunner(command, options) {
|
|
11763
|
+
const execOptions = {
|
|
11764
|
+
cwd: options.cwd,
|
|
11765
|
+
env: options.env,
|
|
11766
|
+
timeout: options.timeoutMs,
|
|
11767
|
+
signal: options.signal,
|
|
11768
|
+
maxBuffer: DEFAULT_MAX_BUFFER,
|
|
11769
|
+
shell: process.platform === "win32" ? "powershell.exe" : void 0
|
|
11770
|
+
};
|
|
11771
|
+
try {
|
|
11772
|
+
const { stdout, stderr } = await execAsync2(command, execOptions);
|
|
11773
|
+
return {
|
|
11774
|
+
stdout,
|
|
11775
|
+
stderr,
|
|
11776
|
+
exitCode: 0,
|
|
11777
|
+
failed: false,
|
|
11778
|
+
timedOut: false,
|
|
11779
|
+
signal: null
|
|
11780
|
+
};
|
|
11781
|
+
} catch (error) {
|
|
11782
|
+
const execError = error;
|
|
11783
|
+
return {
|
|
11784
|
+
stdout: execError.stdout ?? "",
|
|
11785
|
+
stderr: execError.stderr ?? "",
|
|
11786
|
+
exitCode: typeof execError.code === "number" ? execError.code : null,
|
|
11787
|
+
failed: true,
|
|
11788
|
+
timedOut: execError.timedOut === true || execError.killed === true,
|
|
11789
|
+
signal: execError.signal ?? null
|
|
11790
|
+
};
|
|
11791
|
+
}
|
|
11792
|
+
}
|
|
11793
|
+
var CliProvider = class {
|
|
11411
11794
|
id;
|
|
11412
|
-
kind = "
|
|
11795
|
+
kind = "cli";
|
|
11413
11796
|
targetName;
|
|
11414
|
-
|
|
11415
|
-
|
|
11416
|
-
|
|
11417
|
-
|
|
11418
|
-
constructor(targetName, config) {
|
|
11419
|
-
this.id = `mock:${targetName}`;
|
|
11797
|
+
supportsBatch = false;
|
|
11798
|
+
config;
|
|
11799
|
+
runCommand;
|
|
11800
|
+
healthcheckPromise;
|
|
11801
|
+
constructor(targetName, config, runner = defaultCommandRunner) {
|
|
11420
11802
|
this.targetName = targetName;
|
|
11421
|
-
this.
|
|
11422
|
-
this.
|
|
11423
|
-
this.
|
|
11424
|
-
this.delayMaxMs = config.delayMaxMs ?? 0;
|
|
11803
|
+
this.id = `cli:${targetName}`;
|
|
11804
|
+
this.config = config;
|
|
11805
|
+
this.runCommand = runner;
|
|
11425
11806
|
}
|
|
11426
11807
|
async invoke(request) {
|
|
11427
|
-
|
|
11428
|
-
|
|
11429
|
-
|
|
11808
|
+
if (request.signal?.aborted) {
|
|
11809
|
+
throw new Error("CLI provider request was aborted before execution");
|
|
11810
|
+
}
|
|
11811
|
+
await this.ensureHealthy(request.signal);
|
|
11812
|
+
const templateValues = buildTemplateValues(request, this.config);
|
|
11813
|
+
const renderedCommand = renderTemplate(this.config.commandTemplate, templateValues);
|
|
11814
|
+
const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
|
|
11815
|
+
const result = await this.runCommand(renderedCommand, {
|
|
11816
|
+
cwd: this.config.cwd,
|
|
11817
|
+
env,
|
|
11818
|
+
timeoutMs: this.config.timeoutMs,
|
|
11819
|
+
signal: request.signal
|
|
11820
|
+
});
|
|
11821
|
+
if (result.failed || (result.exitCode ?? 0) !== 0) {
|
|
11822
|
+
if (request.signal?.aborted) {
|
|
11823
|
+
throw new Error("CLI provider request was aborted");
|
|
11824
|
+
}
|
|
11825
|
+
if (result.timedOut) {
|
|
11826
|
+
throw new Error(
|
|
11827
|
+
`CLI provider timed out${formatTimeoutSuffix(this.config.timeoutMs ?? void 0)}`
|
|
11828
|
+
);
|
|
11829
|
+
}
|
|
11830
|
+
const codeText = result.exitCode !== null ? result.exitCode : "unknown";
|
|
11831
|
+
const detail = result.stderr.trim() || result.stdout.trim();
|
|
11832
|
+
const message = detail ? `${detail} (exit code ${codeText})` : `CLI exited with code ${codeText}`;
|
|
11833
|
+
throw new Error(message);
|
|
11430
11834
|
}
|
|
11431
11835
|
return {
|
|
11432
|
-
text:
|
|
11836
|
+
text: result.stdout,
|
|
11433
11837
|
raw: {
|
|
11434
|
-
|
|
11435
|
-
|
|
11838
|
+
command: renderedCommand,
|
|
11839
|
+
stderr: result.stderr,
|
|
11840
|
+
exitCode: result.exitCode ?? 0,
|
|
11841
|
+
cwd: this.config.cwd
|
|
11436
11842
|
}
|
|
11437
11843
|
};
|
|
11438
11844
|
}
|
|
11439
|
-
|
|
11440
|
-
if (this.
|
|
11441
|
-
|
|
11442
|
-
const max = Math.max(min, this.delayMaxMs);
|
|
11443
|
-
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
11845
|
+
async ensureHealthy(signal) {
|
|
11846
|
+
if (!this.config.healthcheck) {
|
|
11847
|
+
return;
|
|
11444
11848
|
}
|
|
11445
|
-
|
|
11849
|
+
if (!this.healthcheckPromise) {
|
|
11850
|
+
this.healthcheckPromise = this.runHealthcheck(this.config.healthcheck, signal);
|
|
11851
|
+
}
|
|
11852
|
+
return this.healthcheckPromise;
|
|
11446
11853
|
}
|
|
11447
|
-
|
|
11448
|
-
|
|
11449
|
-
|
|
11450
|
-
|
|
11451
|
-
|
|
11854
|
+
async runHealthcheck(healthcheck, signal) {
|
|
11855
|
+
if (!healthcheck) {
|
|
11856
|
+
return;
|
|
11857
|
+
}
|
|
11858
|
+
const timeoutMs = healthcheck.timeoutMs ?? this.config.timeoutMs;
|
|
11859
|
+
if (healthcheck.type === "http") {
|
|
11860
|
+
const controller = new AbortController();
|
|
11861
|
+
const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : void 0;
|
|
11862
|
+
signal?.addEventListener("abort", () => controller.abort(), { once: true });
|
|
11863
|
+
try {
|
|
11864
|
+
const response = await fetch(healthcheck.url, { method: "GET", signal: controller.signal });
|
|
11865
|
+
if (!response.ok) {
|
|
11866
|
+
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
|
11867
|
+
}
|
|
11868
|
+
} catch (error) {
|
|
11869
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
11870
|
+
throw new Error(`CLI healthcheck failed for '${this.targetName}': ${reason}`);
|
|
11871
|
+
} finally {
|
|
11872
|
+
if (timer !== void 0) {
|
|
11873
|
+
clearTimeout(timer);
|
|
11874
|
+
}
|
|
11875
|
+
}
|
|
11876
|
+
return;
|
|
11877
|
+
}
|
|
11878
|
+
const renderedCommand = renderTemplate(
|
|
11879
|
+
healthcheck.commandTemplate,
|
|
11880
|
+
buildTemplateValues(
|
|
11881
|
+
{
|
|
11882
|
+
prompt: "",
|
|
11883
|
+
guidelines: "",
|
|
11884
|
+
inputFiles: [],
|
|
11885
|
+
evalCaseId: "",
|
|
11886
|
+
attempt: 0
|
|
11887
|
+
},
|
|
11888
|
+
this.config
|
|
11889
|
+
)
|
|
11890
|
+
);
|
|
11891
|
+
const env = this.config.env ? { ...process.env, ...this.config.env } : process.env;
|
|
11892
|
+
const result = await this.runCommand(renderedCommand, {
|
|
11893
|
+
cwd: healthcheck.cwd ?? this.config.cwd,
|
|
11894
|
+
env,
|
|
11895
|
+
timeoutMs,
|
|
11896
|
+
signal
|
|
11897
|
+
});
|
|
11898
|
+
if (result.failed || (result.exitCode ?? 0) !== 0) {
|
|
11899
|
+
const codeText = result.exitCode !== null ? result.exitCode : "unknown";
|
|
11900
|
+
const detail = result.stderr.trim() || result.stdout.trim();
|
|
11901
|
+
const message = detail ? `${detail} (exit code ${codeText})` : `CLI healthcheck command exited with code ${codeText}`;
|
|
11902
|
+
throw new Error(`CLI healthcheck failed for '${this.targetName}': ${message}`);
|
|
11903
|
+
}
|
|
11904
|
+
}
|
|
11905
|
+
};
|
|
11906
|
+
function buildTemplateValues(request, config) {
|
|
11907
|
+
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
11908
|
+
return {
|
|
11909
|
+
PROMPT: shellEscape(request.prompt ?? ""),
|
|
11910
|
+
GUIDELINES: shellEscape(request.guidelines ?? ""),
|
|
11911
|
+
EVAL_ID: shellEscape(request.evalCaseId ?? ""),
|
|
11912
|
+
ATTEMPT: shellEscape(String(request.attempt ?? 0)),
|
|
11913
|
+
FILES: formatFileList(inputFiles, config.filesFormat)
|
|
11914
|
+
};
|
|
11915
|
+
}
|
|
11916
|
+
function normalizeInputFiles(inputFiles) {
|
|
11917
|
+
if (!inputFiles || inputFiles.length === 0) {
|
|
11918
|
+
return void 0;
|
|
11919
|
+
}
|
|
11920
|
+
const unique = /* @__PURE__ */ new Map();
|
|
11921
|
+
for (const inputFile of inputFiles) {
|
|
11922
|
+
const absolutePath = path22.resolve(inputFile);
|
|
11923
|
+
if (!unique.has(absolutePath)) {
|
|
11924
|
+
unique.set(absolutePath, absolutePath);
|
|
11925
|
+
}
|
|
11926
|
+
}
|
|
11927
|
+
return Array.from(unique.values());
|
|
11928
|
+
}
|
|
11929
|
+
function formatFileList(files, template) {
|
|
11930
|
+
if (!files || files.length === 0) {
|
|
11931
|
+
return "";
|
|
11932
|
+
}
|
|
11933
|
+
const formatter = template ?? "{path}";
|
|
11934
|
+
return files.map((filePath) => {
|
|
11935
|
+
const escapedPath = shellEscape(filePath);
|
|
11936
|
+
const escapedName = shellEscape(path22.basename(filePath));
|
|
11937
|
+
return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
|
|
11938
|
+
}).join(" ");
|
|
11939
|
+
}
|
|
11940
|
+
function renderTemplate(template, values) {
|
|
11941
|
+
return template.replace(/\{([A-Z_]+)\}/g, (match, key2) => {
|
|
11942
|
+
const replacement = values[key2];
|
|
11943
|
+
return replacement !== void 0 ? replacement : match;
|
|
11944
|
+
});
|
|
11945
|
+
}
|
|
11946
|
+
function shellEscape(value) {
|
|
11947
|
+
if (value.length === 0) {
|
|
11948
|
+
return "''";
|
|
11949
|
+
}
|
|
11950
|
+
if (process.platform === "win32") {
|
|
11951
|
+
const escaped = value.replace(/"/g, '\\"');
|
|
11952
|
+
return `"${escaped}"`;
|
|
11953
|
+
}
|
|
11954
|
+
return `'${value.replace(/'/g, `'"'"'`)}'`;
|
|
11955
|
+
}
|
|
11956
|
+
function formatTimeoutSuffix(timeoutMs) {
|
|
11957
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
11958
|
+
return "";
|
|
11959
|
+
}
|
|
11960
|
+
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
11961
|
+
return ` after ${seconds}s`;
|
|
11962
|
+
}
|
|
11963
|
+
var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
|
|
11964
|
+
var MockProvider = class {
|
|
11965
|
+
id;
|
|
11966
|
+
kind = "mock";
|
|
11967
|
+
targetName;
|
|
11968
|
+
cannedResponse;
|
|
11969
|
+
delayMs;
|
|
11970
|
+
delayMinMs;
|
|
11971
|
+
delayMaxMs;
|
|
11972
|
+
constructor(targetName, config) {
|
|
11973
|
+
this.id = `mock:${targetName}`;
|
|
11974
|
+
this.targetName = targetName;
|
|
11975
|
+
this.cannedResponse = config.response ?? DEFAULT_MOCK_RESPONSE;
|
|
11976
|
+
this.delayMs = config.delayMs ?? 0;
|
|
11977
|
+
this.delayMinMs = config.delayMinMs ?? 0;
|
|
11978
|
+
this.delayMaxMs = config.delayMaxMs ?? 0;
|
|
11979
|
+
}
|
|
11980
|
+
async invoke(request) {
|
|
11981
|
+
const delay = this.calculateDelay();
|
|
11982
|
+
if (delay > 0) {
|
|
11983
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
11984
|
+
}
|
|
11985
|
+
return {
|
|
11986
|
+
text: this.cannedResponse,
|
|
11987
|
+
raw: {
|
|
11988
|
+
prompt: request.prompt,
|
|
11989
|
+
guidelines: request.guidelines
|
|
11990
|
+
}
|
|
11991
|
+
};
|
|
11992
|
+
}
|
|
11993
|
+
calculateDelay() {
|
|
11994
|
+
if (this.delayMinMs > 0 || this.delayMaxMs > 0) {
|
|
11995
|
+
const min = Math.max(0, this.delayMinMs);
|
|
11996
|
+
const max = Math.max(min, this.delayMaxMs);
|
|
11997
|
+
return Math.floor(Math.random() * (max - min + 1)) + min;
|
|
11998
|
+
}
|
|
11999
|
+
return this.delayMs;
|
|
12000
|
+
}
|
|
12001
|
+
};
|
|
12002
|
+
var CLI_PLACEHOLDERS = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
|
|
12003
|
+
var BASE_TARGET_SCHEMA = external_exports.object({
|
|
12004
|
+
name: external_exports.string().min(1, "target name is required"),
|
|
12005
|
+
provider: external_exports.string().min(1, "provider is required"),
|
|
12006
|
+
settings: external_exports.record(external_exports.unknown()).optional(),
|
|
11452
12007
|
judge_target: external_exports.string().optional(),
|
|
11453
12008
|
workers: external_exports.number().int().min(1).optional()
|
|
11454
12009
|
});
|
|
@@ -11467,6 +12022,9 @@ function normalizeAzureApiVersion(value) {
|
|
|
11467
12022
|
function resolveTargetDefinition(definition, env = process.env) {
|
|
11468
12023
|
const parsed = BASE_TARGET_SCHEMA.parse(definition);
|
|
11469
12024
|
const provider = parsed.provider.toLowerCase();
|
|
12025
|
+
const providerBatching = resolveOptionalBoolean(
|
|
12026
|
+
parsed.settings?.provider_batching ?? parsed.settings?.providerBatching
|
|
12027
|
+
);
|
|
11470
12028
|
switch (provider) {
|
|
11471
12029
|
case "azure":
|
|
11472
12030
|
case "azure-openai":
|
|
@@ -11475,6 +12033,7 @@ function resolveTargetDefinition(definition, env = process.env) {
|
|
|
11475
12033
|
name: parsed.name,
|
|
11476
12034
|
judgeTarget: parsed.judge_target,
|
|
11477
12035
|
workers: parsed.workers,
|
|
12036
|
+
providerBatching,
|
|
11478
12037
|
config: resolveAzureConfig(parsed, env)
|
|
11479
12038
|
};
|
|
11480
12039
|
case "anthropic":
|
|
@@ -11483,6 +12042,7 @@ function resolveTargetDefinition(definition, env = process.env) {
|
|
|
11483
12042
|
name: parsed.name,
|
|
11484
12043
|
judgeTarget: parsed.judge_target,
|
|
11485
12044
|
workers: parsed.workers,
|
|
12045
|
+
providerBatching,
|
|
11486
12046
|
config: resolveAnthropicConfig(parsed, env)
|
|
11487
12047
|
};
|
|
11488
12048
|
case "gemini":
|
|
@@ -11493,14 +12053,26 @@ function resolveTargetDefinition(definition, env = process.env) {
|
|
|
11493
12053
|
name: parsed.name,
|
|
11494
12054
|
judgeTarget: parsed.judge_target,
|
|
11495
12055
|
workers: parsed.workers,
|
|
12056
|
+
providerBatching,
|
|
11496
12057
|
config: resolveGeminiConfig(parsed, env)
|
|
11497
12058
|
};
|
|
12059
|
+
case "codex":
|
|
12060
|
+
case "codex-cli":
|
|
12061
|
+
return {
|
|
12062
|
+
kind: "codex",
|
|
12063
|
+
name: parsed.name,
|
|
12064
|
+
judgeTarget: parsed.judge_target,
|
|
12065
|
+
workers: parsed.workers,
|
|
12066
|
+
providerBatching,
|
|
12067
|
+
config: resolveCodexConfig(parsed, env)
|
|
12068
|
+
};
|
|
11498
12069
|
case "mock":
|
|
11499
12070
|
return {
|
|
11500
12071
|
kind: "mock",
|
|
11501
12072
|
name: parsed.name,
|
|
11502
12073
|
judgeTarget: parsed.judge_target,
|
|
11503
12074
|
workers: parsed.workers,
|
|
12075
|
+
providerBatching,
|
|
11504
12076
|
config: resolveMockConfig(parsed)
|
|
11505
12077
|
};
|
|
11506
12078
|
case "vscode":
|
|
@@ -11510,8 +12082,18 @@ function resolveTargetDefinition(definition, env = process.env) {
|
|
|
11510
12082
|
name: parsed.name,
|
|
11511
12083
|
judgeTarget: parsed.judge_target,
|
|
11512
12084
|
workers: parsed.workers,
|
|
12085
|
+
providerBatching,
|
|
11513
12086
|
config: resolveVSCodeConfig(parsed, env, provider === "vscode-insiders")
|
|
11514
12087
|
};
|
|
12088
|
+
case "cli":
|
|
12089
|
+
return {
|
|
12090
|
+
kind: "cli",
|
|
12091
|
+
name: parsed.name,
|
|
12092
|
+
judgeTarget: parsed.judge_target,
|
|
12093
|
+
workers: parsed.workers,
|
|
12094
|
+
providerBatching,
|
|
12095
|
+
config: resolveCliConfig(parsed, env)
|
|
12096
|
+
};
|
|
11515
12097
|
default:
|
|
11516
12098
|
throw new Error(`Unsupported provider '${parsed.provider}' in target '${parsed.name}'`);
|
|
11517
12099
|
}
|
|
@@ -11579,6 +12161,29 @@ function resolveGeminiConfig(target, env) {
|
|
|
11579
12161
|
maxOutputTokens: resolveOptionalNumber(maxTokensSource, `${target.name} max output tokens`)
|
|
11580
12162
|
};
|
|
11581
12163
|
}
|
|
12164
|
+
function resolveCodexConfig(target, env) {
|
|
12165
|
+
const settings = target.settings ?? {};
|
|
12166
|
+
const executableSource = settings.executable ?? settings.command ?? settings.binary;
|
|
12167
|
+
const argsSource = settings.args ?? settings.arguments;
|
|
12168
|
+
const cwdSource = settings.cwd;
|
|
12169
|
+
const timeoutSource = settings.timeout_seconds ?? settings.timeoutSeconds;
|
|
12170
|
+
const executable = resolveOptionalString(executableSource, env, `${target.name} codex executable`, {
|
|
12171
|
+
allowLiteral: true,
|
|
12172
|
+
optionalEnv: true
|
|
12173
|
+
}) ?? "codex";
|
|
12174
|
+
const args = resolveOptionalStringArray(argsSource, env, `${target.name} codex args`);
|
|
12175
|
+
const cwd = resolveOptionalString(cwdSource, env, `${target.name} codex cwd`, {
|
|
12176
|
+
allowLiteral: true,
|
|
12177
|
+
optionalEnv: true
|
|
12178
|
+
});
|
|
12179
|
+
const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} codex timeout`);
|
|
12180
|
+
return {
|
|
12181
|
+
executable,
|
|
12182
|
+
args,
|
|
12183
|
+
cwd,
|
|
12184
|
+
timeoutMs
|
|
12185
|
+
};
|
|
12186
|
+
}
|
|
11582
12187
|
function resolveMockConfig(target) {
|
|
11583
12188
|
const settings = target.settings ?? {};
|
|
11584
12189
|
const response = typeof settings.response === "string" ? settings.response : void 0;
|
|
@@ -11608,6 +12213,125 @@ function resolveVSCodeConfig(target, env, insiders) {
|
|
|
11608
12213
|
workspaceTemplate
|
|
11609
12214
|
};
|
|
11610
12215
|
}
|
|
12216
|
+
function resolveCliConfig(target, env) {
|
|
12217
|
+
const settings = target.settings ?? {};
|
|
12218
|
+
const commandTemplateSource = settings.command_template ?? settings.commandTemplate;
|
|
12219
|
+
const filesFormat = resolveOptionalLiteralString(
|
|
12220
|
+
settings.files_format ?? settings.filesFormat ?? settings.attachments_format ?? settings.attachmentsFormat
|
|
12221
|
+
);
|
|
12222
|
+
const cwd = resolveOptionalString(settings.cwd, env, `${target.name} working directory`, {
|
|
12223
|
+
allowLiteral: true,
|
|
12224
|
+
optionalEnv: true
|
|
12225
|
+
});
|
|
12226
|
+
const envOverrides = resolveEnvOverrides(settings.env, env, target.name);
|
|
12227
|
+
const timeoutMs = resolveTimeoutMs(settings.timeout_seconds ?? settings.timeoutSeconds, `${target.name} timeout`);
|
|
12228
|
+
const healthcheck = resolveCliHealthcheck(settings.healthcheck, env, target.name);
|
|
12229
|
+
const commandTemplate = resolveString(
|
|
12230
|
+
commandTemplateSource,
|
|
12231
|
+
env,
|
|
12232
|
+
`${target.name} CLI command template`,
|
|
12233
|
+
true
|
|
12234
|
+
);
|
|
12235
|
+
assertSupportedCliPlaceholders(commandTemplate, `${target.name} CLI command template`);
|
|
12236
|
+
return {
|
|
12237
|
+
commandTemplate,
|
|
12238
|
+
filesFormat,
|
|
12239
|
+
cwd,
|
|
12240
|
+
env: envOverrides,
|
|
12241
|
+
timeoutMs,
|
|
12242
|
+
healthcheck
|
|
12243
|
+
};
|
|
12244
|
+
}
|
|
12245
|
+
function resolveEnvOverrides(source2, env, targetName) {
|
|
12246
|
+
if (source2 === void 0 || source2 === null) {
|
|
12247
|
+
return void 0;
|
|
12248
|
+
}
|
|
12249
|
+
if (typeof source2 !== "object" || Array.isArray(source2)) {
|
|
12250
|
+
throw new Error(`${targetName} env overrides must be an object map of strings`);
|
|
12251
|
+
}
|
|
12252
|
+
const entries = Object.entries(source2);
|
|
12253
|
+
const resolved = {};
|
|
12254
|
+
for (const [key2, value] of entries) {
|
|
12255
|
+
if (typeof value !== "string") {
|
|
12256
|
+
throw new Error(`${targetName} env override '${key2}' must be a string`);
|
|
12257
|
+
}
|
|
12258
|
+
const resolvedValue = resolveString(value, env, `${targetName} env override '${key2}'`);
|
|
12259
|
+
resolved[key2] = resolvedValue;
|
|
12260
|
+
}
|
|
12261
|
+
return Object.keys(resolved).length > 0 ? resolved : void 0;
|
|
12262
|
+
}
|
|
12263
|
+
function resolveTimeoutMs(source2, description) {
|
|
12264
|
+
const seconds = resolveOptionalNumber(source2, `${description} (seconds)`);
|
|
12265
|
+
if (seconds === void 0) {
|
|
12266
|
+
return void 0;
|
|
12267
|
+
}
|
|
12268
|
+
if (seconds <= 0) {
|
|
12269
|
+
throw new Error(`${description} must be greater than zero seconds`);
|
|
12270
|
+
}
|
|
12271
|
+
return Math.floor(seconds * 1e3);
|
|
12272
|
+
}
|
|
12273
|
+
function resolveCliHealthcheck(source2, env, targetName) {
|
|
12274
|
+
if (source2 === void 0 || source2 === null) {
|
|
12275
|
+
return void 0;
|
|
12276
|
+
}
|
|
12277
|
+
if (typeof source2 !== "object" || Array.isArray(source2)) {
|
|
12278
|
+
throw new Error(`${targetName} healthcheck must be an object`);
|
|
12279
|
+
}
|
|
12280
|
+
const candidate = source2;
|
|
12281
|
+
const type = candidate.type;
|
|
12282
|
+
const timeoutMs = resolveTimeoutMs(
|
|
12283
|
+
candidate.timeout_seconds ?? candidate.timeoutSeconds,
|
|
12284
|
+
`${targetName} healthcheck timeout`
|
|
12285
|
+
);
|
|
12286
|
+
if (type === "http") {
|
|
12287
|
+
const url = resolveString(candidate.url, env, `${targetName} healthcheck URL`);
|
|
12288
|
+
return {
|
|
12289
|
+
type: "http",
|
|
12290
|
+
url,
|
|
12291
|
+
timeoutMs
|
|
12292
|
+
};
|
|
12293
|
+
}
|
|
12294
|
+
if (type === "command") {
|
|
12295
|
+
const commandTemplate = resolveString(
|
|
12296
|
+
candidate.command_template ?? candidate.commandTemplate,
|
|
12297
|
+
env,
|
|
12298
|
+
`${targetName} healthcheck command template`,
|
|
12299
|
+
true
|
|
12300
|
+
);
|
|
12301
|
+
assertSupportedCliPlaceholders(commandTemplate, `${targetName} healthcheck command template`);
|
|
12302
|
+
const cwd = resolveOptionalString(candidate.cwd, env, `${targetName} healthcheck cwd`, {
|
|
12303
|
+
allowLiteral: true,
|
|
12304
|
+
optionalEnv: true
|
|
12305
|
+
});
|
|
12306
|
+
return {
|
|
12307
|
+
type: "command",
|
|
12308
|
+
commandTemplate,
|
|
12309
|
+
timeoutMs,
|
|
12310
|
+
cwd
|
|
12311
|
+
};
|
|
12312
|
+
}
|
|
12313
|
+
throw new Error(`${targetName} healthcheck type must be 'http' or 'command'`);
|
|
12314
|
+
}
|
|
12315
|
+
function assertSupportedCliPlaceholders(template, description) {
|
|
12316
|
+
const placeholders = extractCliPlaceholders(template);
|
|
12317
|
+
for (const placeholder of placeholders) {
|
|
12318
|
+
if (!CLI_PLACEHOLDERS.has(placeholder)) {
|
|
12319
|
+
throw new Error(
|
|
12320
|
+
`${description} includes unsupported placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS).join(", ")}`
|
|
12321
|
+
);
|
|
12322
|
+
}
|
|
12323
|
+
}
|
|
12324
|
+
}
|
|
12325
|
+
function extractCliPlaceholders(template) {
|
|
12326
|
+
const matches = template.matchAll(/\{([A-Z_]+)\}/g);
|
|
12327
|
+
const results = [];
|
|
12328
|
+
for (const match of matches) {
|
|
12329
|
+
if (match[1]) {
|
|
12330
|
+
results.push(match[1]);
|
|
12331
|
+
}
|
|
12332
|
+
}
|
|
12333
|
+
return results;
|
|
12334
|
+
}
|
|
11611
12335
|
function resolveString(source2, env, description, allowLiteral = false) {
|
|
11612
12336
|
const value = resolveOptionalString(source2, env, description, {
|
|
11613
12337
|
allowLiteral,
|
|
@@ -11638,11 +12362,14 @@ function resolveOptionalString(source2, env, description, options) {
|
|
|
11638
12362
|
}
|
|
11639
12363
|
const allowLiteral = options?.allowLiteral ?? false;
|
|
11640
12364
|
const optionalEnv = options?.optionalEnv ?? false;
|
|
11641
|
-
|
|
12365
|
+
const looksLikeEnv = isLikelyEnvReference(trimmed);
|
|
12366
|
+
if (looksLikeEnv) {
|
|
11642
12367
|
if (optionalEnv) {
|
|
11643
12368
|
return void 0;
|
|
11644
12369
|
}
|
|
11645
|
-
|
|
12370
|
+
if (!allowLiteral) {
|
|
12371
|
+
throw new Error(`Environment variable '${trimmed}' required for ${description} is not set`);
|
|
12372
|
+
}
|
|
11646
12373
|
}
|
|
11647
12374
|
return trimmed;
|
|
11648
12375
|
}
|
|
@@ -11692,10 +12419,43 @@ function resolveOptionalBoolean(source2) {
|
|
|
11692
12419
|
function isLikelyEnvReference(value) {
|
|
11693
12420
|
return /^[A-Z0-9_]+$/.test(value);
|
|
11694
12421
|
}
|
|
12422
|
+
function resolveOptionalStringArray(source2, env, description) {
|
|
12423
|
+
if (source2 === void 0 || source2 === null) {
|
|
12424
|
+
return void 0;
|
|
12425
|
+
}
|
|
12426
|
+
if (!Array.isArray(source2)) {
|
|
12427
|
+
throw new Error(`${description} must be an array of strings`);
|
|
12428
|
+
}
|
|
12429
|
+
if (source2.length === 0) {
|
|
12430
|
+
return void 0;
|
|
12431
|
+
}
|
|
12432
|
+
const resolved = [];
|
|
12433
|
+
for (let i6 = 0; i6 < source2.length; i6++) {
|
|
12434
|
+
const item = source2[i6];
|
|
12435
|
+
if (typeof item !== "string") {
|
|
12436
|
+
throw new Error(`${description}[${i6}] must be a string`);
|
|
12437
|
+
}
|
|
12438
|
+
const trimmed = item.trim();
|
|
12439
|
+
if (trimmed.length === 0) {
|
|
12440
|
+
throw new Error(`${description}[${i6}] cannot be empty`);
|
|
12441
|
+
}
|
|
12442
|
+
const envValue = env[trimmed];
|
|
12443
|
+
if (envValue !== void 0) {
|
|
12444
|
+
if (envValue.trim().length === 0) {
|
|
12445
|
+
throw new Error(`Environment variable '${trimmed}' for ${description}[${i6}] is empty`);
|
|
12446
|
+
}
|
|
12447
|
+
resolved.push(envValue);
|
|
12448
|
+
} else {
|
|
12449
|
+
resolved.push(trimmed);
|
|
12450
|
+
}
|
|
12451
|
+
}
|
|
12452
|
+
return resolved.length > 0 ? resolved : void 0;
|
|
12453
|
+
}
|
|
11695
12454
|
var VSCodeProvider = class {
|
|
11696
12455
|
id;
|
|
11697
12456
|
kind;
|
|
11698
12457
|
targetName;
|
|
12458
|
+
supportsBatch = true;
|
|
11699
12459
|
config;
|
|
11700
12460
|
constructor(targetName, config, kind) {
|
|
11701
12461
|
this.id = `${kind}:${targetName}`;
|
|
@@ -11707,12 +12467,11 @@ var VSCodeProvider = class {
|
|
|
11707
12467
|
if (request.signal?.aborted) {
|
|
11708
12468
|
throw new Error("VS Code provider request was aborted before dispatch");
|
|
11709
12469
|
}
|
|
11710
|
-
const
|
|
11711
|
-
const promptContent = buildPromptDocument(request,
|
|
12470
|
+
const inputFiles = normalizeAttachments(request.inputFiles);
|
|
12471
|
+
const promptContent = buildPromptDocument(request, inputFiles, request.guideline_patterns);
|
|
11712
12472
|
const session = await dispatchAgentSession({
|
|
11713
12473
|
userQuery: promptContent,
|
|
11714
|
-
|
|
11715
|
-
extraAttachments: attachments,
|
|
12474
|
+
extraAttachments: inputFiles,
|
|
11716
12475
|
wait: this.config.waitForResponse,
|
|
11717
12476
|
dryRun: this.config.dryRun,
|
|
11718
12477
|
vscodeCmd: this.config.command,
|
|
@@ -11729,7 +12488,7 @@ var VSCodeProvider = class {
|
|
|
11729
12488
|
text: "",
|
|
11730
12489
|
raw: {
|
|
11731
12490
|
session,
|
|
11732
|
-
|
|
12491
|
+
inputFiles
|
|
11733
12492
|
}
|
|
11734
12493
|
};
|
|
11735
12494
|
}
|
|
@@ -11738,42 +12497,106 @@ var VSCodeProvider = class {
|
|
|
11738
12497
|
text: responseText,
|
|
11739
12498
|
raw: {
|
|
11740
12499
|
session,
|
|
11741
|
-
|
|
12500
|
+
inputFiles
|
|
11742
12501
|
}
|
|
11743
12502
|
};
|
|
11744
12503
|
}
|
|
12504
|
+
async invokeBatch(requests) {
|
|
12505
|
+
if (requests.length === 0) {
|
|
12506
|
+
return [];
|
|
12507
|
+
}
|
|
12508
|
+
const normalizedRequests = requests.map((req) => ({
|
|
12509
|
+
request: req,
|
|
12510
|
+
inputFiles: normalizeAttachments(req.inputFiles)
|
|
12511
|
+
}));
|
|
12512
|
+
const combinedInputFiles = mergeAttachments(
|
|
12513
|
+
normalizedRequests.map(({ inputFiles }) => inputFiles)
|
|
12514
|
+
);
|
|
12515
|
+
const userQueries = normalizedRequests.map(
|
|
12516
|
+
({ request, inputFiles }) => buildPromptDocument(request, inputFiles, request.guideline_patterns)
|
|
12517
|
+
);
|
|
12518
|
+
const session = await dispatchBatchAgent({
|
|
12519
|
+
userQueries,
|
|
12520
|
+
extraAttachments: combinedInputFiles,
|
|
12521
|
+
wait: this.config.waitForResponse,
|
|
12522
|
+
dryRun: this.config.dryRun,
|
|
12523
|
+
vscodeCmd: this.config.command,
|
|
12524
|
+
subagentRoot: this.config.subagentRoot,
|
|
12525
|
+
workspaceTemplate: this.config.workspaceTemplate,
|
|
12526
|
+
silent: true
|
|
12527
|
+
});
|
|
12528
|
+
if (session.exitCode !== 0 || !session.responseFiles) {
|
|
12529
|
+
const failure = session.error ?? "VS Code subagent did not produce batch responses";
|
|
12530
|
+
throw new Error(failure);
|
|
12531
|
+
}
|
|
12532
|
+
if (this.config.dryRun) {
|
|
12533
|
+
return normalizedRequests.map(({ inputFiles }) => ({
|
|
12534
|
+
text: "",
|
|
12535
|
+
raw: {
|
|
12536
|
+
session,
|
|
12537
|
+
inputFiles,
|
|
12538
|
+
allInputFiles: combinedInputFiles
|
|
12539
|
+
}
|
|
12540
|
+
}));
|
|
12541
|
+
}
|
|
12542
|
+
if (session.responseFiles.length !== requests.length) {
|
|
12543
|
+
throw new Error(
|
|
12544
|
+
`VS Code batch returned ${session.responseFiles.length} responses for ${requests.length} requests`
|
|
12545
|
+
);
|
|
12546
|
+
}
|
|
12547
|
+
const responses = [];
|
|
12548
|
+
for (const [index, responseFile] of session.responseFiles.entries()) {
|
|
12549
|
+
const responseText = await readFile22(responseFile, "utf8");
|
|
12550
|
+
responses.push({
|
|
12551
|
+
text: responseText,
|
|
12552
|
+
raw: {
|
|
12553
|
+
session,
|
|
12554
|
+
inputFiles: normalizedRequests[index]?.inputFiles,
|
|
12555
|
+
allInputFiles: combinedInputFiles,
|
|
12556
|
+
responseFile
|
|
12557
|
+
}
|
|
12558
|
+
});
|
|
12559
|
+
}
|
|
12560
|
+
return responses;
|
|
12561
|
+
}
|
|
11745
12562
|
};
|
|
11746
12563
|
function buildPromptDocument(request, attachments, guidelinePatterns) {
|
|
11747
12564
|
const parts = [];
|
|
11748
12565
|
const guidelineFiles = collectGuidelineFiles(attachments, guidelinePatterns);
|
|
11749
|
-
|
|
11750
|
-
|
|
12566
|
+
const attachmentFiles = collectAttachmentFiles(attachments);
|
|
12567
|
+
const nonGuidelineAttachments = attachmentFiles.filter(
|
|
12568
|
+
(file) => !guidelineFiles.includes(file)
|
|
12569
|
+
);
|
|
12570
|
+
const prereadBlock = buildMandatoryPrereadBlock(guidelineFiles, nonGuidelineAttachments);
|
|
12571
|
+
if (prereadBlock.length > 0) {
|
|
12572
|
+
parts.push("\n", prereadBlock);
|
|
11751
12573
|
}
|
|
11752
12574
|
parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
|
|
11753
12575
|
return parts.join("\n").trim();
|
|
11754
12576
|
}
|
|
11755
|
-
function buildMandatoryPrereadBlock(guidelineFiles) {
|
|
11756
|
-
if (guidelineFiles.length === 0) {
|
|
12577
|
+
function buildMandatoryPrereadBlock(guidelineFiles, attachmentFiles) {
|
|
12578
|
+
if (guidelineFiles.length === 0 && attachmentFiles.length === 0) {
|
|
11757
12579
|
return "";
|
|
11758
12580
|
}
|
|
11759
|
-
const
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
12581
|
+
const buildList = (files) => files.map((absolutePath) => {
|
|
12582
|
+
const fileName = path32.basename(absolutePath);
|
|
12583
|
+
const fileUri = pathToFileUri2(absolutePath);
|
|
12584
|
+
return `* [${fileName}](${fileUri})`;
|
|
12585
|
+
});
|
|
12586
|
+
const sections = [];
|
|
12587
|
+
if (guidelineFiles.length > 0) {
|
|
12588
|
+
sections.push(`Read all guideline files:
|
|
12589
|
+
${buildList(guidelineFiles).join("\n")}.`);
|
|
12590
|
+
}
|
|
12591
|
+
if (attachmentFiles.length > 0) {
|
|
12592
|
+
sections.push(`Read all attachment files:
|
|
12593
|
+
${buildList(attachmentFiles).join("\n")}.`);
|
|
12594
|
+
}
|
|
12595
|
+
sections.push(
|
|
12596
|
+
"If any file is missing, fail with ERROR: missing-file <filename> and stop.",
|
|
12597
|
+
"Then apply system_instructions on the user query below."
|
|
12598
|
+
);
|
|
12599
|
+
return sections.join("\n");
|
|
11777
12600
|
}
|
|
11778
12601
|
function collectGuidelineFiles(attachments, guidelinePatterns) {
|
|
11779
12602
|
if (!attachments || attachments.length === 0) {
|
|
@@ -11781,8 +12604,8 @@ function collectGuidelineFiles(attachments, guidelinePatterns) {
|
|
|
11781
12604
|
}
|
|
11782
12605
|
const unique = /* @__PURE__ */ new Map();
|
|
11783
12606
|
for (const attachment of attachments) {
|
|
11784
|
-
const absolutePath =
|
|
11785
|
-
const normalized = absolutePath.split(
|
|
12607
|
+
const absolutePath = path32.resolve(attachment);
|
|
12608
|
+
const normalized = absolutePath.split(path32.sep).join("/");
|
|
11786
12609
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
11787
12610
|
if (!unique.has(absolutePath)) {
|
|
11788
12611
|
unique.set(absolutePath, absolutePath);
|
|
@@ -11791,8 +12614,21 @@ function collectGuidelineFiles(attachments, guidelinePatterns) {
|
|
|
11791
12614
|
}
|
|
11792
12615
|
return Array.from(unique.values());
|
|
11793
12616
|
}
|
|
11794
|
-
function
|
|
11795
|
-
|
|
12617
|
+
function collectAttachmentFiles(attachments) {
|
|
12618
|
+
if (!attachments || attachments.length === 0) {
|
|
12619
|
+
return [];
|
|
12620
|
+
}
|
|
12621
|
+
const unique = /* @__PURE__ */ new Map();
|
|
12622
|
+
for (const attachment of attachments) {
|
|
12623
|
+
const absolutePath = path32.resolve(attachment);
|
|
12624
|
+
if (!unique.has(absolutePath)) {
|
|
12625
|
+
unique.set(absolutePath, absolutePath);
|
|
12626
|
+
}
|
|
12627
|
+
}
|
|
12628
|
+
return Array.from(unique.values());
|
|
12629
|
+
}
|
|
12630
|
+
function pathToFileUri2(filePath) {
|
|
12631
|
+
const absolutePath = path32.isAbsolute(filePath) ? filePath : path32.resolve(filePath);
|
|
11796
12632
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
11797
12633
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
11798
12634
|
return `file:///${normalizedPath}`;
|
|
@@ -11805,10 +12641,20 @@ function normalizeAttachments(attachments) {
|
|
|
11805
12641
|
}
|
|
11806
12642
|
const deduped = /* @__PURE__ */ new Set();
|
|
11807
12643
|
for (const attachment of attachments) {
|
|
11808
|
-
deduped.add(
|
|
12644
|
+
deduped.add(path32.resolve(attachment));
|
|
11809
12645
|
}
|
|
11810
12646
|
return Array.from(deduped);
|
|
11811
12647
|
}
|
|
12648
|
+
function mergeAttachments(all) {
|
|
12649
|
+
const deduped = /* @__PURE__ */ new Set();
|
|
12650
|
+
for (const list of all) {
|
|
12651
|
+
if (!list) continue;
|
|
12652
|
+
for (const inputFile of list) {
|
|
12653
|
+
deduped.add(path32.resolve(inputFile));
|
|
12654
|
+
}
|
|
12655
|
+
}
|
|
12656
|
+
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
12657
|
+
}
|
|
11812
12658
|
async function ensureVSCodeSubagents(options) {
|
|
11813
12659
|
const { kind, count, verbose = false } = options;
|
|
11814
12660
|
const vscodeCmd = kind === "vscode-insiders" ? "code-insiders" : "code";
|
|
@@ -11847,6 +12693,569 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
11847
12693
|
};
|
|
11848
12694
|
}
|
|
11849
12695
|
}
|
|
12696
|
+
function buildPromptDocument2(request, inputFiles, options) {
|
|
12697
|
+
const parts = [];
|
|
12698
|
+
const guidelineFiles = collectGuidelineFiles2(
|
|
12699
|
+
inputFiles,
|
|
12700
|
+
options?.guidelinePatterns ?? request.guideline_patterns,
|
|
12701
|
+
options?.guidelineOverrides
|
|
12702
|
+
);
|
|
12703
|
+
const inputFilesList = collectInputFiles(inputFiles);
|
|
12704
|
+
const nonGuidelineInputFiles = inputFilesList.filter(
|
|
12705
|
+
(file) => !guidelineFiles.includes(file)
|
|
12706
|
+
);
|
|
12707
|
+
const prereadBlock = buildMandatoryPrereadBlock2(guidelineFiles, nonGuidelineInputFiles);
|
|
12708
|
+
if (prereadBlock.length > 0) {
|
|
12709
|
+
parts.push("\n", prereadBlock);
|
|
12710
|
+
}
|
|
12711
|
+
parts.push("\n[[ ## user_query ## ]]\n", request.prompt.trim());
|
|
12712
|
+
return parts.join("\n").trim();
|
|
12713
|
+
}
|
|
12714
|
+
function normalizeInputFiles2(inputFiles) {
|
|
12715
|
+
if (!inputFiles || inputFiles.length === 0) {
|
|
12716
|
+
return void 0;
|
|
12717
|
+
}
|
|
12718
|
+
const deduped = /* @__PURE__ */ new Map();
|
|
12719
|
+
for (const inputFile of inputFiles) {
|
|
12720
|
+
const absolutePath = path42.resolve(inputFile);
|
|
12721
|
+
if (!deduped.has(absolutePath)) {
|
|
12722
|
+
deduped.set(absolutePath, absolutePath);
|
|
12723
|
+
}
|
|
12724
|
+
}
|
|
12725
|
+
return Array.from(deduped.values());
|
|
12726
|
+
}
|
|
12727
|
+
function collectGuidelineFiles2(inputFiles, guidelinePatterns, overrides) {
|
|
12728
|
+
if (!inputFiles || inputFiles.length === 0) {
|
|
12729
|
+
return [];
|
|
12730
|
+
}
|
|
12731
|
+
const unique = /* @__PURE__ */ new Map();
|
|
12732
|
+
for (const inputFile of inputFiles) {
|
|
12733
|
+
const absolutePath = path42.resolve(inputFile);
|
|
12734
|
+
if (overrides?.has(absolutePath)) {
|
|
12735
|
+
if (!unique.has(absolutePath)) {
|
|
12736
|
+
unique.set(absolutePath, absolutePath);
|
|
12737
|
+
}
|
|
12738
|
+
continue;
|
|
12739
|
+
}
|
|
12740
|
+
const normalized = absolutePath.split(path42.sep).join("/");
|
|
12741
|
+
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
12742
|
+
if (!unique.has(absolutePath)) {
|
|
12743
|
+
unique.set(absolutePath, absolutePath);
|
|
12744
|
+
}
|
|
12745
|
+
}
|
|
12746
|
+
}
|
|
12747
|
+
return Array.from(unique.values());
|
|
12748
|
+
}
|
|
12749
|
+
function collectInputFiles(inputFiles) {
|
|
12750
|
+
if (!inputFiles || inputFiles.length === 0) {
|
|
12751
|
+
return [];
|
|
12752
|
+
}
|
|
12753
|
+
const unique = /* @__PURE__ */ new Map();
|
|
12754
|
+
for (const inputFile of inputFiles) {
|
|
12755
|
+
const absolutePath = path42.resolve(inputFile);
|
|
12756
|
+
if (!unique.has(absolutePath)) {
|
|
12757
|
+
unique.set(absolutePath, absolutePath);
|
|
12758
|
+
}
|
|
12759
|
+
}
|
|
12760
|
+
return Array.from(unique.values());
|
|
12761
|
+
}
|
|
12762
|
+
function buildMandatoryPrereadBlock2(guidelineFiles, inputFiles) {
|
|
12763
|
+
if (guidelineFiles.length === 0 && inputFiles.length === 0) {
|
|
12764
|
+
return "";
|
|
12765
|
+
}
|
|
12766
|
+
const buildList = (files) => files.map((absolutePath) => {
|
|
12767
|
+
const fileName = path42.basename(absolutePath);
|
|
12768
|
+
const fileUri = pathToFileUri22(absolutePath);
|
|
12769
|
+
return `* [${fileName}](${fileUri})`;
|
|
12770
|
+
});
|
|
12771
|
+
const sections = [];
|
|
12772
|
+
if (guidelineFiles.length > 0) {
|
|
12773
|
+
sections.push(`Read all guideline files:
|
|
12774
|
+
${buildList(guidelineFiles).join("\n")}.`);
|
|
12775
|
+
}
|
|
12776
|
+
if (inputFiles.length > 0) {
|
|
12777
|
+
sections.push(`Read all input files:
|
|
12778
|
+
${buildList(inputFiles).join("\n")}.`);
|
|
12779
|
+
}
|
|
12780
|
+
sections.push(
|
|
12781
|
+
"If any file is missing, fail with ERROR: missing-file <filename> and stop.",
|
|
12782
|
+
"Then apply system_instructions on the user query below."
|
|
12783
|
+
);
|
|
12784
|
+
return sections.join("\n");
|
|
12785
|
+
}
|
|
12786
|
+
function pathToFileUri22(filePath) {
|
|
12787
|
+
const absolutePath = path42.isAbsolute(filePath) ? filePath : path42.resolve(filePath);
|
|
12788
|
+
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
12789
|
+
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
12790
|
+
return `file:///${normalizedPath}`;
|
|
12791
|
+
}
|
|
12792
|
+
return `file://${normalizedPath}`;
|
|
12793
|
+
}
|
|
12794
|
+
var execAsync22 = promisify22(execCallback);
|
|
12795
|
+
var WORKSPACE_PREFIX = "agentv-codex-";
|
|
12796
|
+
var PROMPT_FILENAME = "prompt.md";
|
|
12797
|
+
var FILES_DIR = "files";
|
|
12798
|
+
var JSONL_TYPE_ITEM_COMPLETED = "item.completed";
|
|
12799
|
+
var CodexProvider = class {
|
|
12800
|
+
id;
|
|
12801
|
+
kind = "codex";
|
|
12802
|
+
targetName;
|
|
12803
|
+
supportsBatch = false;
|
|
12804
|
+
config;
|
|
12805
|
+
runCodex;
|
|
12806
|
+
environmentCheck;
|
|
12807
|
+
resolvedExecutable;
|
|
12808
|
+
constructor(targetName, config, runner = defaultCodexRunner) {
|
|
12809
|
+
this.id = `codex:${targetName}`;
|
|
12810
|
+
this.targetName = targetName;
|
|
12811
|
+
this.config = config;
|
|
12812
|
+
this.runCodex = runner;
|
|
12813
|
+
}
|
|
12814
|
+
async invoke(request) {
|
|
12815
|
+
if (request.signal?.aborted) {
|
|
12816
|
+
throw new Error("Codex provider request was aborted before execution");
|
|
12817
|
+
}
|
|
12818
|
+
await this.ensureEnvironmentReady();
|
|
12819
|
+
const inputFiles = normalizeInputFiles2(request.inputFiles);
|
|
12820
|
+
const originalGuidelines = new Set(
|
|
12821
|
+
collectGuidelineFiles2(inputFiles, request.guideline_patterns).map((file) => path52.resolve(file))
|
|
12822
|
+
);
|
|
12823
|
+
const workspaceRoot = await this.createWorkspace();
|
|
12824
|
+
try {
|
|
12825
|
+
const { mirroredInputFiles, guidelineMirrors } = await this.mirrorInputFiles(
|
|
12826
|
+
inputFiles,
|
|
12827
|
+
workspaceRoot,
|
|
12828
|
+
originalGuidelines
|
|
12829
|
+
);
|
|
12830
|
+
const promptContent = buildPromptDocument2(request, mirroredInputFiles, {
|
|
12831
|
+
guidelinePatterns: request.guideline_patterns,
|
|
12832
|
+
guidelineOverrides: guidelineMirrors
|
|
12833
|
+
});
|
|
12834
|
+
const promptFile = path52.join(workspaceRoot, PROMPT_FILENAME);
|
|
12835
|
+
await writeFile3(promptFile, promptContent, "utf8");
|
|
12836
|
+
const args = this.buildCodexArgs();
|
|
12837
|
+
const cwd = this.resolveCwd(workspaceRoot);
|
|
12838
|
+
const result = await this.executeCodex(args, cwd, promptContent, request.signal);
|
|
12839
|
+
if (result.timedOut) {
|
|
12840
|
+
throw new Error(
|
|
12841
|
+
`Codex CLI timed out${formatTimeoutSuffix2(this.config.timeoutMs ?? void 0)}`
|
|
12842
|
+
);
|
|
12843
|
+
}
|
|
12844
|
+
if (result.exitCode !== 0) {
|
|
12845
|
+
const detail = pickDetail(result.stderr, result.stdout);
|
|
12846
|
+
const prefix = `Codex CLI exited with code ${result.exitCode}`;
|
|
12847
|
+
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
12848
|
+
}
|
|
12849
|
+
const parsed = parseCodexJson(result.stdout);
|
|
12850
|
+
const assistantText = extractAssistantText(parsed);
|
|
12851
|
+
return {
|
|
12852
|
+
text: assistantText,
|
|
12853
|
+
raw: {
|
|
12854
|
+
response: parsed,
|
|
12855
|
+
stdout: result.stdout,
|
|
12856
|
+
stderr: result.stderr,
|
|
12857
|
+
exitCode: result.exitCode,
|
|
12858
|
+
args,
|
|
12859
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
12860
|
+
promptFile,
|
|
12861
|
+
workspace: workspaceRoot,
|
|
12862
|
+
inputFiles: mirroredInputFiles
|
|
12863
|
+
}
|
|
12864
|
+
};
|
|
12865
|
+
} finally {
|
|
12866
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
12867
|
+
}
|
|
12868
|
+
}
|
|
12869
|
+
async ensureEnvironmentReady() {
|
|
12870
|
+
if (!this.environmentCheck) {
|
|
12871
|
+
this.environmentCheck = this.validateEnvironment();
|
|
12872
|
+
}
|
|
12873
|
+
await this.environmentCheck;
|
|
12874
|
+
}
|
|
12875
|
+
async validateEnvironment() {
|
|
12876
|
+
this.resolvedExecutable = await locateExecutable(this.config.executable);
|
|
12877
|
+
}
|
|
12878
|
+
resolveCwd(workspaceRoot) {
|
|
12879
|
+
if (!this.config.cwd) {
|
|
12880
|
+
return workspaceRoot;
|
|
12881
|
+
}
|
|
12882
|
+
return path52.resolve(this.config.cwd);
|
|
12883
|
+
}
|
|
12884
|
+
buildCodexArgs() {
|
|
12885
|
+
const args = ["--ask-for-approval", "never", "exec", "--json", "--color", "never", "--skip-git-repo-check"];
|
|
12886
|
+
if (this.config.args && this.config.args.length > 0) {
|
|
12887
|
+
args.push(...this.config.args);
|
|
12888
|
+
}
|
|
12889
|
+
args.push("-");
|
|
12890
|
+
return args;
|
|
12891
|
+
}
|
|
12892
|
+
async executeCodex(args, cwd, promptContent, signal) {
|
|
12893
|
+
try {
|
|
12894
|
+
return await this.runCodex({
|
|
12895
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
12896
|
+
args,
|
|
12897
|
+
cwd,
|
|
12898
|
+
prompt: promptContent,
|
|
12899
|
+
timeoutMs: this.config.timeoutMs,
|
|
12900
|
+
env: process.env,
|
|
12901
|
+
signal
|
|
12902
|
+
});
|
|
12903
|
+
} catch (error) {
|
|
12904
|
+
const err = error;
|
|
12905
|
+
if (err.code === "ENOENT") {
|
|
12906
|
+
throw new Error(
|
|
12907
|
+
`Codex executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
|
|
12908
|
+
);
|
|
12909
|
+
}
|
|
12910
|
+
throw error;
|
|
12911
|
+
}
|
|
12912
|
+
}
|
|
12913
|
+
async mirrorInputFiles(inputFiles, workspaceRoot, guidelineOriginals) {
|
|
12914
|
+
if (!inputFiles || inputFiles.length === 0) {
|
|
12915
|
+
return {
|
|
12916
|
+
mirroredInputFiles: void 0,
|
|
12917
|
+
guidelineMirrors: /* @__PURE__ */ new Set()
|
|
12918
|
+
};
|
|
12919
|
+
}
|
|
12920
|
+
const filesRoot = path52.join(workspaceRoot, FILES_DIR);
|
|
12921
|
+
await mkdir3(filesRoot, { recursive: true });
|
|
12922
|
+
const mirrored = [];
|
|
12923
|
+
const guidelineMirrors = /* @__PURE__ */ new Set();
|
|
12924
|
+
const nameCounts = /* @__PURE__ */ new Map();
|
|
12925
|
+
for (const inputFile of inputFiles) {
|
|
12926
|
+
const absoluteSource = path52.resolve(inputFile);
|
|
12927
|
+
const baseName = path52.basename(absoluteSource);
|
|
12928
|
+
const count = nameCounts.get(baseName) ?? 0;
|
|
12929
|
+
nameCounts.set(baseName, count + 1);
|
|
12930
|
+
const finalName = count === 0 ? baseName : `${baseName}.${count}`;
|
|
12931
|
+
const destination = path52.join(filesRoot, finalName);
|
|
12932
|
+
await copyFile2(absoluteSource, destination);
|
|
12933
|
+
const resolvedDestination = path52.resolve(destination);
|
|
12934
|
+
mirrored.push(resolvedDestination);
|
|
12935
|
+
if (guidelineOriginals.has(absoluteSource)) {
|
|
12936
|
+
guidelineMirrors.add(resolvedDestination);
|
|
12937
|
+
}
|
|
12938
|
+
}
|
|
12939
|
+
return {
|
|
12940
|
+
mirroredInputFiles: mirrored,
|
|
12941
|
+
guidelineMirrors
|
|
12942
|
+
};
|
|
12943
|
+
}
|
|
12944
|
+
async createWorkspace() {
|
|
12945
|
+
return await mkdtemp(path52.join(tmpdir(), WORKSPACE_PREFIX));
|
|
12946
|
+
}
|
|
12947
|
+
async cleanupWorkspace(workspaceRoot) {
|
|
12948
|
+
try {
|
|
12949
|
+
await rm2(workspaceRoot, { recursive: true, force: true });
|
|
12950
|
+
} catch {
|
|
12951
|
+
}
|
|
12952
|
+
}
|
|
12953
|
+
};
|
|
12954
|
+
async function locateExecutable(candidate) {
|
|
12955
|
+
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
12956
|
+
if (includesPathSeparator) {
|
|
12957
|
+
const resolved = path52.isAbsolute(candidate) ? candidate : path52.resolve(candidate);
|
|
12958
|
+
const executablePath = await ensureWindowsExecutableVariant(resolved);
|
|
12959
|
+
await access22(executablePath, constants22.F_OK);
|
|
12960
|
+
return executablePath;
|
|
12961
|
+
}
|
|
12962
|
+
const locator = process.platform === "win32" ? "where" : "which";
|
|
12963
|
+
try {
|
|
12964
|
+
const { stdout } = await execAsync22(`${locator} ${candidate}`);
|
|
12965
|
+
const lines = stdout.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
|
|
12966
|
+
const preferred = selectExecutableCandidate(lines);
|
|
12967
|
+
if (preferred) {
|
|
12968
|
+
const executablePath = await ensureWindowsExecutableVariant(preferred);
|
|
12969
|
+
await access22(executablePath, constants22.F_OK);
|
|
12970
|
+
return executablePath;
|
|
12971
|
+
}
|
|
12972
|
+
} catch {
|
|
12973
|
+
}
|
|
12974
|
+
throw new Error(`Codex executable '${candidate}' was not found on PATH`);
|
|
12975
|
+
}
|
|
12976
|
+
function selectExecutableCandidate(candidates) {
|
|
12977
|
+
if (candidates.length === 0) {
|
|
12978
|
+
return void 0;
|
|
12979
|
+
}
|
|
12980
|
+
if (process.platform !== "win32") {
|
|
12981
|
+
return candidates[0];
|
|
12982
|
+
}
|
|
12983
|
+
const extensions = getWindowsExecutableExtensions();
|
|
12984
|
+
for (const ext of extensions) {
|
|
12985
|
+
const match = candidates.find((candidate) => candidate.toLowerCase().endsWith(ext));
|
|
12986
|
+
if (match) {
|
|
12987
|
+
return match;
|
|
12988
|
+
}
|
|
12989
|
+
}
|
|
12990
|
+
return candidates[0];
|
|
12991
|
+
}
|
|
12992
|
+
async function ensureWindowsExecutableVariant(candidate) {
|
|
12993
|
+
if (process.platform !== "win32") {
|
|
12994
|
+
return candidate;
|
|
12995
|
+
}
|
|
12996
|
+
if (hasExecutableExtension(candidate)) {
|
|
12997
|
+
return candidate;
|
|
12998
|
+
}
|
|
12999
|
+
const extensions = getWindowsExecutableExtensions();
|
|
13000
|
+
for (const ext of extensions) {
|
|
13001
|
+
const withExtension = `${candidate}${ext}`;
|
|
13002
|
+
try {
|
|
13003
|
+
await access22(withExtension, constants22.F_OK);
|
|
13004
|
+
return withExtension;
|
|
13005
|
+
} catch {
|
|
13006
|
+
}
|
|
13007
|
+
}
|
|
13008
|
+
return candidate;
|
|
13009
|
+
}
|
|
13010
|
+
function hasExecutableExtension(candidate) {
|
|
13011
|
+
const lower = candidate.toLowerCase();
|
|
13012
|
+
return getWindowsExecutableExtensions().some((ext) => lower.endsWith(ext));
|
|
13013
|
+
}
|
|
13014
|
+
var DEFAULT_WINDOWS_EXTENSIONS = [".com", ".exe", ".bat", ".cmd", ".ps1"];
|
|
13015
|
+
function getWindowsExecutableExtensions() {
|
|
13016
|
+
if (process.platform !== "win32") {
|
|
13017
|
+
return [];
|
|
13018
|
+
}
|
|
13019
|
+
const fromEnv = process.env.PATHEXT?.split(";").map((ext) => ext.trim().toLowerCase()).filter((ext) => ext.length > 0);
|
|
13020
|
+
return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_WINDOWS_EXTENSIONS;
|
|
13021
|
+
}
|
|
13022
|
+
function parseCodexJson(output) {
|
|
13023
|
+
const trimmed = output.trim();
|
|
13024
|
+
if (trimmed.length === 0) {
|
|
13025
|
+
throw new Error("Codex CLI produced no output in --json mode");
|
|
13026
|
+
}
|
|
13027
|
+
try {
|
|
13028
|
+
return JSON.parse(trimmed);
|
|
13029
|
+
} catch {
|
|
13030
|
+
const lineObjects = parseJsonLines(trimmed);
|
|
13031
|
+
if (lineObjects) {
|
|
13032
|
+
return lineObjects;
|
|
13033
|
+
}
|
|
13034
|
+
const lastBrace = trimmed.lastIndexOf("{");
|
|
13035
|
+
if (lastBrace >= 0) {
|
|
13036
|
+
const candidate = trimmed.slice(lastBrace);
|
|
13037
|
+
try {
|
|
13038
|
+
return JSON.parse(candidate);
|
|
13039
|
+
} catch {
|
|
13040
|
+
}
|
|
13041
|
+
}
|
|
13042
|
+
const preview = trimmed.slice(0, 200);
|
|
13043
|
+
throw new Error(`Codex CLI emitted invalid JSON: ${preview}${trimmed.length > 200 ? "\u2026" : ""}`);
|
|
13044
|
+
}
|
|
13045
|
+
}
|
|
13046
|
+
function extractAssistantText(parsed) {
|
|
13047
|
+
if (Array.isArray(parsed)) {
|
|
13048
|
+
const text = extractFromEventStream(parsed);
|
|
13049
|
+
if (text) {
|
|
13050
|
+
return text;
|
|
13051
|
+
}
|
|
13052
|
+
}
|
|
13053
|
+
if (!parsed || typeof parsed !== "object") {
|
|
13054
|
+
throw new Error("Codex CLI JSON response did not include an assistant message");
|
|
13055
|
+
}
|
|
13056
|
+
const record = parsed;
|
|
13057
|
+
const eventText = extractFromEvent(record);
|
|
13058
|
+
if (eventText) {
|
|
13059
|
+
return eventText;
|
|
13060
|
+
}
|
|
13061
|
+
const messages = Array.isArray(record.messages) ? record.messages : void 0;
|
|
13062
|
+
if (messages) {
|
|
13063
|
+
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
|
13064
|
+
const entry = messages[index];
|
|
13065
|
+
if (!entry || typeof entry !== "object") {
|
|
13066
|
+
continue;
|
|
13067
|
+
}
|
|
13068
|
+
const role = entry.role;
|
|
13069
|
+
if (role !== "assistant") {
|
|
13070
|
+
continue;
|
|
13071
|
+
}
|
|
13072
|
+
const content = entry.content;
|
|
13073
|
+
const flattened = flattenContent(content);
|
|
13074
|
+
if (flattened) {
|
|
13075
|
+
return flattened;
|
|
13076
|
+
}
|
|
13077
|
+
}
|
|
13078
|
+
}
|
|
13079
|
+
const response = record.response;
|
|
13080
|
+
if (response && typeof response === "object") {
|
|
13081
|
+
const content = response.content;
|
|
13082
|
+
const flattened = flattenContent(content);
|
|
13083
|
+
if (flattened) {
|
|
13084
|
+
return flattened;
|
|
13085
|
+
}
|
|
13086
|
+
}
|
|
13087
|
+
const output = record.output;
|
|
13088
|
+
const flattenedOutput = flattenContent(output);
|
|
13089
|
+
if (flattenedOutput) {
|
|
13090
|
+
return flattenedOutput;
|
|
13091
|
+
}
|
|
13092
|
+
throw new Error("Codex CLI JSON response did not include an assistant message");
|
|
13093
|
+
}
|
|
13094
|
+
function extractFromEventStream(events) {
|
|
13095
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
13096
|
+
const candidate = events[index];
|
|
13097
|
+
const text = extractFromEvent(candidate);
|
|
13098
|
+
if (text) {
|
|
13099
|
+
return text;
|
|
13100
|
+
}
|
|
13101
|
+
}
|
|
13102
|
+
return void 0;
|
|
13103
|
+
}
|
|
13104
|
+
function extractFromEvent(event) {
|
|
13105
|
+
if (!event || typeof event !== "object") {
|
|
13106
|
+
return void 0;
|
|
13107
|
+
}
|
|
13108
|
+
const record = event;
|
|
13109
|
+
const type = typeof record.type === "string" ? record.type : void 0;
|
|
13110
|
+
if (type === JSONL_TYPE_ITEM_COMPLETED) {
|
|
13111
|
+
const item = record.item;
|
|
13112
|
+
const text = extractFromItem(item);
|
|
13113
|
+
if (text) {
|
|
13114
|
+
return text;
|
|
13115
|
+
}
|
|
13116
|
+
}
|
|
13117
|
+
const output = record.output ?? record.content;
|
|
13118
|
+
const flattened = flattenContent(output);
|
|
13119
|
+
if (flattened) {
|
|
13120
|
+
return flattened;
|
|
13121
|
+
}
|
|
13122
|
+
return void 0;
|
|
13123
|
+
}
|
|
13124
|
+
function extractFromItem(item) {
|
|
13125
|
+
if (!item || typeof item !== "object") {
|
|
13126
|
+
return void 0;
|
|
13127
|
+
}
|
|
13128
|
+
const record = item;
|
|
13129
|
+
const itemType = typeof record.type === "string" ? record.type : void 0;
|
|
13130
|
+
if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
|
|
13131
|
+
const text = flattenContent(record.text ?? record.content ?? record.output);
|
|
13132
|
+
if (text) {
|
|
13133
|
+
return text;
|
|
13134
|
+
}
|
|
13135
|
+
}
|
|
13136
|
+
return void 0;
|
|
13137
|
+
}
|
|
13138
|
+
function flattenContent(value) {
|
|
13139
|
+
if (typeof value === "string") {
|
|
13140
|
+
return value;
|
|
13141
|
+
}
|
|
13142
|
+
if (Array.isArray(value)) {
|
|
13143
|
+
const parts = value.map((segment) => {
|
|
13144
|
+
if (typeof segment === "string") {
|
|
13145
|
+
return segment;
|
|
13146
|
+
}
|
|
13147
|
+
if (segment && typeof segment === "object" && "text" in segment) {
|
|
13148
|
+
const text = segment.text;
|
|
13149
|
+
return typeof text === "string" ? text : void 0;
|
|
13150
|
+
}
|
|
13151
|
+
return void 0;
|
|
13152
|
+
}).filter((part) => typeof part === "string" && part.length > 0);
|
|
13153
|
+
return parts.length > 0 ? parts.join(" \n") : void 0;
|
|
13154
|
+
}
|
|
13155
|
+
if (value && typeof value === "object" && "text" in value) {
|
|
13156
|
+
const text = value.text;
|
|
13157
|
+
return typeof text === "string" ? text : void 0;
|
|
13158
|
+
}
|
|
13159
|
+
return void 0;
|
|
13160
|
+
}
|
|
13161
|
+
function parseJsonLines(output) {
|
|
13162
|
+
const lines = output.split(/\r?\n/).map((line2) => line2.trim()).filter((line2) => line2.length > 0);
|
|
13163
|
+
if (lines.length <= 1) {
|
|
13164
|
+
return void 0;
|
|
13165
|
+
}
|
|
13166
|
+
const parsed = [];
|
|
13167
|
+
for (const line2 of lines) {
|
|
13168
|
+
try {
|
|
13169
|
+
parsed.push(JSON.parse(line2));
|
|
13170
|
+
} catch {
|
|
13171
|
+
return void 0;
|
|
13172
|
+
}
|
|
13173
|
+
}
|
|
13174
|
+
return parsed;
|
|
13175
|
+
}
|
|
13176
|
+
function pickDetail(stderr, stdout) {
|
|
13177
|
+
const errorText = stderr.trim();
|
|
13178
|
+
if (errorText.length > 0) {
|
|
13179
|
+
return errorText;
|
|
13180
|
+
}
|
|
13181
|
+
const stdoutText = stdout.trim();
|
|
13182
|
+
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
13183
|
+
}
|
|
13184
|
+
function formatTimeoutSuffix2(timeoutMs) {
|
|
13185
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
13186
|
+
return "";
|
|
13187
|
+
}
|
|
13188
|
+
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
13189
|
+
return ` after ${seconds}s`;
|
|
13190
|
+
}
|
|
13191
|
+
async function defaultCodexRunner(options) {
|
|
13192
|
+
return await new Promise((resolve, reject) => {
|
|
13193
|
+
const child = spawn2(options.executable, options.args, {
|
|
13194
|
+
cwd: options.cwd,
|
|
13195
|
+
env: options.env,
|
|
13196
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
13197
|
+
shell: shouldShellExecute(options.executable)
|
|
13198
|
+
});
|
|
13199
|
+
let stdout = "";
|
|
13200
|
+
let stderr = "";
|
|
13201
|
+
let timedOut = false;
|
|
13202
|
+
const onAbort = () => {
|
|
13203
|
+
child.kill("SIGTERM");
|
|
13204
|
+
};
|
|
13205
|
+
if (options.signal) {
|
|
13206
|
+
if (options.signal.aborted) {
|
|
13207
|
+
onAbort();
|
|
13208
|
+
} else {
|
|
13209
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
13210
|
+
}
|
|
13211
|
+
}
|
|
13212
|
+
let timeoutHandle;
|
|
13213
|
+
if (options.timeoutMs && options.timeoutMs > 0) {
|
|
13214
|
+
timeoutHandle = setTimeout(() => {
|
|
13215
|
+
timedOut = true;
|
|
13216
|
+
child.kill("SIGTERM");
|
|
13217
|
+
}, options.timeoutMs);
|
|
13218
|
+
timeoutHandle.unref?.();
|
|
13219
|
+
}
|
|
13220
|
+
child.stdout.setEncoding("utf8");
|
|
13221
|
+
child.stdout.on("data", (chunk) => {
|
|
13222
|
+
stdout += chunk;
|
|
13223
|
+
});
|
|
13224
|
+
child.stderr.setEncoding("utf8");
|
|
13225
|
+
child.stderr.on("data", (chunk) => {
|
|
13226
|
+
stderr += chunk;
|
|
13227
|
+
});
|
|
13228
|
+
child.stdin.end(options.prompt);
|
|
13229
|
+
const cleanup = () => {
|
|
13230
|
+
if (timeoutHandle) {
|
|
13231
|
+
clearTimeout(timeoutHandle);
|
|
13232
|
+
}
|
|
13233
|
+
if (options.signal) {
|
|
13234
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
13235
|
+
}
|
|
13236
|
+
};
|
|
13237
|
+
child.on("error", (error) => {
|
|
13238
|
+
cleanup();
|
|
13239
|
+
reject(error);
|
|
13240
|
+
});
|
|
13241
|
+
child.on("close", (code) => {
|
|
13242
|
+
cleanup();
|
|
13243
|
+
resolve({
|
|
13244
|
+
stdout,
|
|
13245
|
+
stderr,
|
|
13246
|
+
exitCode: typeof code === "number" ? code : -1,
|
|
13247
|
+
timedOut
|
|
13248
|
+
});
|
|
13249
|
+
});
|
|
13250
|
+
});
|
|
13251
|
+
}
|
|
13252
|
+
function shouldShellExecute(executable) {
|
|
13253
|
+
if (process.platform !== "win32") {
|
|
13254
|
+
return false;
|
|
13255
|
+
}
|
|
13256
|
+
const lower = executable.toLowerCase();
|
|
13257
|
+
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
13258
|
+
}
|
|
11850
13259
|
function isRecord(value) {
|
|
11851
13260
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
11852
13261
|
}
|
|
@@ -11901,14 +13310,14 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
11901
13310
|
}
|
|
11902
13311
|
async function fileExists3(filePath) {
|
|
11903
13312
|
try {
|
|
11904
|
-
await
|
|
13313
|
+
await access32(filePath, constants32.F_OK);
|
|
11905
13314
|
return true;
|
|
11906
13315
|
} catch {
|
|
11907
13316
|
return false;
|
|
11908
13317
|
}
|
|
11909
13318
|
}
|
|
11910
13319
|
async function readTargetDefinitions(filePath) {
|
|
11911
|
-
const absolutePath =
|
|
13320
|
+
const absolutePath = path62.resolve(filePath);
|
|
11912
13321
|
if (!await fileExists3(absolutePath)) {
|
|
11913
13322
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
11914
13323
|
}
|
|
@@ -11933,6 +13342,10 @@ function createProvider(target) {
|
|
|
11933
13342
|
return new AnthropicProvider(target.name, target.config);
|
|
11934
13343
|
case "gemini":
|
|
11935
13344
|
return new GeminiProvider(target.name, target.config);
|
|
13345
|
+
case "cli":
|
|
13346
|
+
return new CliProvider(target.name, target.config);
|
|
13347
|
+
case "codex":
|
|
13348
|
+
return new CodexProvider(target.name, target.config);
|
|
11936
13349
|
case "mock":
|
|
11937
13350
|
return new MockProvider(target.name, target.config);
|
|
11938
13351
|
case "vscode":
|
|
@@ -11944,222 +13357,28 @@ function createProvider(target) {
|
|
|
11944
13357
|
}
|
|
11945
13358
|
}
|
|
11946
13359
|
}
|
|
11947
|
-
var
|
|
11948
|
-
var ACTION_WORDS = /* @__PURE__ */ new Set([
|
|
11949
|
-
"use",
|
|
11950
|
-
"avoid",
|
|
11951
|
-
"prefer",
|
|
11952
|
-
"replace",
|
|
11953
|
-
"consider",
|
|
11954
|
-
"ensure",
|
|
11955
|
-
"remove",
|
|
11956
|
-
"add"
|
|
11957
|
-
]);
|
|
11958
|
-
var STOP_WORDS = /* @__PURE__ */ new Set([
|
|
11959
|
-
"the",
|
|
11960
|
-
"a",
|
|
11961
|
-
"an",
|
|
11962
|
-
"and",
|
|
11963
|
-
"or",
|
|
11964
|
-
"but",
|
|
11965
|
-
"in",
|
|
11966
|
-
"on",
|
|
11967
|
-
"at",
|
|
11968
|
-
"to",
|
|
11969
|
-
"for",
|
|
11970
|
-
"of",
|
|
11971
|
-
"with",
|
|
11972
|
-
"by",
|
|
11973
|
-
"is",
|
|
11974
|
-
"are",
|
|
11975
|
-
"was",
|
|
11976
|
-
"were",
|
|
11977
|
-
"be",
|
|
11978
|
-
"been",
|
|
11979
|
-
"being",
|
|
11980
|
-
"have",
|
|
11981
|
-
"has",
|
|
11982
|
-
"had",
|
|
11983
|
-
"do",
|
|
11984
|
-
"does",
|
|
11985
|
-
"did",
|
|
11986
|
-
"will",
|
|
11987
|
-
"would",
|
|
11988
|
-
"could",
|
|
11989
|
-
"should"
|
|
11990
|
-
]);
|
|
11991
|
-
var ERROR_PREFIXES = [
|
|
11992
|
-
"error:",
|
|
11993
|
-
"err:",
|
|
11994
|
-
"vs code command failed",
|
|
11995
|
-
"exception",
|
|
11996
|
-
"traceback",
|
|
11997
|
-
"no response file was generated",
|
|
11998
|
-
"timed out",
|
|
11999
|
-
"cli not found"
|
|
12000
|
-
];
|
|
12001
|
-
function extractAspects(expectedResponse) {
|
|
12002
|
-
const lines = expectedResponse.split(/\r?\n/).map((line2) => line2.trim());
|
|
12003
|
-
const aspects = [];
|
|
12004
|
-
for (const line2 of lines) {
|
|
12005
|
-
if (line2.length === 0) {
|
|
12006
|
-
continue;
|
|
12007
|
-
}
|
|
12008
|
-
const bulletMatch = /^([-*•]|[0-9]+\.)\s*(.+)$/.exec(line2);
|
|
12009
|
-
if (bulletMatch) {
|
|
12010
|
-
const normalized = normalizeAspect(bulletMatch[2]);
|
|
12011
|
-
if (normalized.length > 0) {
|
|
12012
|
-
aspects.push(normalized);
|
|
12013
|
-
}
|
|
12014
|
-
continue;
|
|
12015
|
-
}
|
|
12016
|
-
const lowered = line2.toLowerCase();
|
|
12017
|
-
if (Array.from(ACTION_WORDS).some((word) => lowered.startsWith(word))) {
|
|
12018
|
-
const normalized = normalizeAspect(line2);
|
|
12019
|
-
if (normalized.length > 0) {
|
|
12020
|
-
aspects.push(normalized);
|
|
12021
|
-
}
|
|
12022
|
-
}
|
|
12023
|
-
}
|
|
12024
|
-
return aspects;
|
|
12025
|
-
}
|
|
12026
|
-
function calculateHits(candidateResponse, expectedAspects) {
|
|
12027
|
-
const { normalizedText, words } = normalizeCandidate(candidateResponse);
|
|
12028
|
-
const hits = [];
|
|
12029
|
-
for (const aspect of expectedAspects) {
|
|
12030
|
-
if (matchesAspect(aspect, normalizedText, words)) {
|
|
12031
|
-
hits.push(aspect);
|
|
12032
|
-
}
|
|
12033
|
-
}
|
|
12034
|
-
return hits;
|
|
12035
|
-
}
|
|
12036
|
-
function scoreCandidateResponse(candidateResponse, expectedAspects) {
|
|
12037
|
-
if (expectedAspects.length === 0) {
|
|
12038
|
-
if (isErrorLike(candidateResponse)) {
|
|
12039
|
-
return {
|
|
12040
|
-
score: 0,
|
|
12041
|
-
hits: [],
|
|
12042
|
-
misses: ["Model produced an error instead of an answer."],
|
|
12043
|
-
hitCount: 0,
|
|
12044
|
-
totalAspects: 0,
|
|
12045
|
-
rawAspects: []
|
|
12046
|
-
};
|
|
12047
|
-
}
|
|
12048
|
-
return {
|
|
12049
|
-
score: 1,
|
|
12050
|
-
hits: [],
|
|
12051
|
-
misses: [],
|
|
12052
|
-
hitCount: 0,
|
|
12053
|
-
totalAspects: 0,
|
|
12054
|
-
rawAspects: []
|
|
12055
|
-
};
|
|
12056
|
-
}
|
|
12057
|
-
const hits = calculateHits(candidateResponse, expectedAspects);
|
|
12058
|
-
const misses = expectedAspects.filter((aspect) => !hits.includes(aspect));
|
|
12059
|
-
const score = expectedAspects.length > 0 ? hits.length / expectedAspects.length : 0;
|
|
12060
|
-
return {
|
|
12061
|
-
score,
|
|
12062
|
-
hits,
|
|
12063
|
-
misses,
|
|
12064
|
-
hitCount: hits.length,
|
|
12065
|
-
totalAspects: expectedAspects.length,
|
|
12066
|
-
rawAspects: expectedAspects
|
|
12067
|
-
};
|
|
12068
|
-
}
|
|
12069
|
-
function isErrorLike(text) {
|
|
12070
|
-
if (!text) {
|
|
12071
|
-
return false;
|
|
12072
|
-
}
|
|
12073
|
-
const lowered = text.trim().toLowerCase();
|
|
12074
|
-
return ERROR_PREFIXES.some((prefix) => lowered.startsWith(prefix));
|
|
12075
|
-
}
|
|
12076
|
-
function normalizeAspect(aspect) {
|
|
12077
|
-
const sanitized = aspect.toLowerCase().replace(/[^\w\s]/g, " ").replace(/\s+/g, " ").trim();
|
|
12078
|
-
return sanitized;
|
|
12079
|
-
}
|
|
12080
|
-
function normalizeCandidate(candidate) {
|
|
12081
|
-
const lowered = candidate.toLowerCase();
|
|
12082
|
-
const normalizedText = lowered.replace(/[^\w\s]/g, " ");
|
|
12083
|
-
const words = new Set(normalizedText.split(/\s+/).filter((word) => word.length > 0));
|
|
12084
|
-
return { normalizedText, words };
|
|
12085
|
-
}
|
|
12086
|
-
function matchesAspect(aspect, candidateNormalized, candidateWords) {
|
|
12087
|
-
const keyTerms = extractKeyTerms(aspect);
|
|
12088
|
-
if (keyTerms.length === 0) {
|
|
12089
|
-
return false;
|
|
12090
|
-
}
|
|
12091
|
-
const matches = keyTerms.filter((term) => candidateWords.has(term)).length;
|
|
12092
|
-
const ratio = matches / keyTerms.length;
|
|
12093
|
-
if (ratio >= KEY_TERM_MATCH_THRESHOLD) {
|
|
12094
|
-
return true;
|
|
12095
|
-
}
|
|
12096
|
-
const aspectWords = aspect.split(" ");
|
|
12097
|
-
if (aspectWords.length >= 2) {
|
|
12098
|
-
for (let index = 0; index < aspectWords.length - 1; index += 1) {
|
|
12099
|
-
const phrase = `${aspectWords[index]} ${aspectWords[index + 1]}`;
|
|
12100
|
-
if (candidateNormalized.includes(phrase)) {
|
|
12101
|
-
return true;
|
|
12102
|
-
}
|
|
12103
|
-
}
|
|
12104
|
-
}
|
|
12105
|
-
return false;
|
|
12106
|
-
}
|
|
12107
|
-
function extractKeyTerms(aspect, maxTerms = 5) {
|
|
12108
|
-
const terms = [];
|
|
12109
|
-
const words = aspect.split(" ");
|
|
12110
|
-
for (const word of words) {
|
|
12111
|
-
if (word.length <= 2) {
|
|
12112
|
-
continue;
|
|
12113
|
-
}
|
|
12114
|
-
if (STOP_WORDS.has(word)) {
|
|
12115
|
-
continue;
|
|
12116
|
-
}
|
|
12117
|
-
terms.push(word);
|
|
12118
|
-
if (terms.length >= maxTerms) {
|
|
12119
|
-
break;
|
|
12120
|
-
}
|
|
12121
|
-
}
|
|
12122
|
-
return terms;
|
|
12123
|
-
}
|
|
12124
|
-
var HeuristicGrader = class {
|
|
12125
|
-
kind = "heuristic";
|
|
12126
|
-
grade(context2) {
|
|
12127
|
-
const expectedAspects = extractAspects(context2.evalCase.expected_assistant_raw);
|
|
12128
|
-
const result = scoreCandidateResponse(context2.candidate, expectedAspects);
|
|
12129
|
-
const misses = [...result.misses];
|
|
12130
|
-
if (expectedAspects.length === 0 && isErrorLike(context2.candidate)) {
|
|
12131
|
-
const firstLine = context2.candidate.split(/\r?\n/)[0]?.trim();
|
|
12132
|
-
if (firstLine && !misses.includes(firstLine)) {
|
|
12133
|
-
misses.unshift(firstLine);
|
|
12134
|
-
}
|
|
12135
|
-
}
|
|
12136
|
-
return {
|
|
12137
|
-
score: result.score,
|
|
12138
|
-
hits: result.hits,
|
|
12139
|
-
misses,
|
|
12140
|
-
expectedAspectCount: result.totalAspects,
|
|
12141
|
-
rawAspects: result.rawAspects
|
|
12142
|
-
};
|
|
12143
|
-
}
|
|
12144
|
-
};
|
|
12145
|
-
var QualityGrader = class {
|
|
13360
|
+
var LlmJudgeEvaluator = class {
|
|
12146
13361
|
kind = "llm_judge";
|
|
12147
13362
|
resolveJudgeProvider;
|
|
12148
13363
|
maxOutputTokens;
|
|
12149
13364
|
temperature;
|
|
13365
|
+
customPrompt;
|
|
12150
13366
|
constructor(options) {
|
|
12151
13367
|
this.resolveJudgeProvider = options.resolveJudgeProvider;
|
|
12152
13368
|
this.maxOutputTokens = options.maxOutputTokens;
|
|
12153
13369
|
this.temperature = options.temperature;
|
|
13370
|
+
this.customPrompt = options.customPrompt;
|
|
12154
13371
|
}
|
|
12155
|
-
async
|
|
13372
|
+
async evaluate(context2) {
|
|
12156
13373
|
const judgeProvider = await this.resolveJudgeProvider(context2);
|
|
12157
13374
|
if (!judgeProvider) {
|
|
12158
13375
|
throw new Error("No judge provider available for LLM grading");
|
|
12159
13376
|
}
|
|
12160
13377
|
const prompt = buildQualityPrompt(context2.evalCase, context2.candidate);
|
|
13378
|
+
const systemPrompt = context2.systemPrompt ?? this.customPrompt ?? QUALITY_SYSTEM_PROMPT;
|
|
12161
13379
|
const metadata = {
|
|
12162
|
-
systemPrompt:
|
|
13380
|
+
...systemPrompt !== void 0 ? { systemPrompt } : {},
|
|
13381
|
+
...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
|
|
12163
13382
|
};
|
|
12164
13383
|
const response = await judgeProvider.invoke({
|
|
12165
13384
|
prompt,
|
|
@@ -12174,12 +13393,13 @@ var QualityGrader = class {
|
|
|
12174
13393
|
const hits = Array.isArray(parsed.hits) ? parsed.hits.filter(isNonEmptyString).slice(0, 4) : [];
|
|
12175
13394
|
const misses = Array.isArray(parsed.misses) ? parsed.misses.filter(isNonEmptyString).slice(0, 4) : [];
|
|
12176
13395
|
const reasoning = parsed.reasoning ?? response.reasoning;
|
|
12177
|
-
const
|
|
13396
|
+
const evaluatorRawRequest = {
|
|
12178
13397
|
id: randomUUID(),
|
|
12179
13398
|
provider: judgeProvider.id,
|
|
12180
13399
|
prompt,
|
|
12181
|
-
|
|
12182
|
-
|
|
13400
|
+
target: context2.target.name,
|
|
13401
|
+
...systemPrompt !== void 0 ? { systemPrompt } : {},
|
|
13402
|
+
...context2.judgeModel !== void 0 ? { model: context2.judgeModel } : {}
|
|
12183
13403
|
};
|
|
12184
13404
|
return {
|
|
12185
13405
|
score,
|
|
@@ -12187,7 +13407,7 @@ var QualityGrader = class {
|
|
|
12187
13407
|
misses,
|
|
12188
13408
|
expectedAspectCount: hits.length + misses.length || 1,
|
|
12189
13409
|
reasoning,
|
|
12190
|
-
|
|
13410
|
+
evaluatorRawRequest
|
|
12191
13411
|
};
|
|
12192
13412
|
}
|
|
12193
13413
|
};
|
|
@@ -12302,8 +13522,114 @@ function extractJsonBlob(text) {
|
|
|
12302
13522
|
const match = text.match(/\{[\s\S]*\}/);
|
|
12303
13523
|
return match?.[0];
|
|
12304
13524
|
}
|
|
12305
|
-
function isNonEmptyString(value) {
|
|
12306
|
-
return typeof value === "string" && value.trim().length > 0;
|
|
13525
|
+
function isNonEmptyString(value) {
|
|
13526
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
13527
|
+
}
|
|
13528
|
+
var CodeEvaluator = class {
|
|
13529
|
+
kind = "code";
|
|
13530
|
+
script;
|
|
13531
|
+
cwd;
|
|
13532
|
+
agentTimeoutMs;
|
|
13533
|
+
constructor(options) {
|
|
13534
|
+
this.script = options.script;
|
|
13535
|
+
this.cwd = options.cwd;
|
|
13536
|
+
this.agentTimeoutMs = options.agentTimeoutMs;
|
|
13537
|
+
}
|
|
13538
|
+
async evaluate(context2) {
|
|
13539
|
+
const inputPayload = JSON.stringify(
|
|
13540
|
+
{
|
|
13541
|
+
task: context2.evalCase.task,
|
|
13542
|
+
outcome: context2.evalCase.outcome,
|
|
13543
|
+
expected: context2.evalCase.expected_assistant_raw,
|
|
13544
|
+
output: context2.candidate,
|
|
13545
|
+
system_message: context2.promptInputs.systemMessage ?? "",
|
|
13546
|
+
guideline_paths: context2.evalCase.guideline_paths,
|
|
13547
|
+
attachments: context2.evalCase.file_paths,
|
|
13548
|
+
user_segments: context2.evalCase.user_segments
|
|
13549
|
+
},
|
|
13550
|
+
null,
|
|
13551
|
+
2
|
|
13552
|
+
);
|
|
13553
|
+
try {
|
|
13554
|
+
const stdout = await executeScript(this.script, inputPayload, this.agentTimeoutMs, this.cwd);
|
|
13555
|
+
const parsed = parseJsonSafe(stdout);
|
|
13556
|
+
const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
|
|
13557
|
+
const hits = Array.isArray(parsed?.hits) ? parsed.hits.filter(isNonEmptyString) : [];
|
|
13558
|
+
const misses = Array.isArray(parsed?.misses) ? parsed.misses.filter(isNonEmptyString) : [];
|
|
13559
|
+
const reasoning = typeof parsed?.reasoning === "string" ? parsed.reasoning : void 0;
|
|
13560
|
+
return {
|
|
13561
|
+
score,
|
|
13562
|
+
hits,
|
|
13563
|
+
misses,
|
|
13564
|
+
expectedAspectCount: hits.length + misses.length || 1,
|
|
13565
|
+
reasoning,
|
|
13566
|
+
evaluatorRawRequest: {
|
|
13567
|
+
script: this.script,
|
|
13568
|
+
...this.cwd ? { cwd: this.cwd } : {}
|
|
13569
|
+
}
|
|
13570
|
+
};
|
|
13571
|
+
} catch (error) {
|
|
13572
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13573
|
+
return {
|
|
13574
|
+
score: 0,
|
|
13575
|
+
hits: [],
|
|
13576
|
+
misses: [`Code evaluator failed: ${message}`],
|
|
13577
|
+
expectedAspectCount: 1,
|
|
13578
|
+
reasoning: message,
|
|
13579
|
+
evaluatorRawRequest: {
|
|
13580
|
+
script: this.script,
|
|
13581
|
+
...this.cwd ? { cwd: this.cwd } : {},
|
|
13582
|
+
error: message
|
|
13583
|
+
}
|
|
13584
|
+
};
|
|
13585
|
+
}
|
|
13586
|
+
}
|
|
13587
|
+
};
|
|
13588
|
+
async function executeScript(scriptPath, input, agentTimeoutMs, cwd) {
|
|
13589
|
+
const { spawn: spawn22 } = await import("node:child_process");
|
|
13590
|
+
return await new Promise((resolve, reject) => {
|
|
13591
|
+
const child = spawn22(scriptPath, {
|
|
13592
|
+
shell: true,
|
|
13593
|
+
cwd
|
|
13594
|
+
});
|
|
13595
|
+
let stdout = "";
|
|
13596
|
+
let stderr = "";
|
|
13597
|
+
const timeout = agentTimeoutMs ? setTimeout(() => {
|
|
13598
|
+
child.kill();
|
|
13599
|
+
reject(new Error(`Code evaluator timed out after ${agentTimeoutMs}ms`));
|
|
13600
|
+
}, agentTimeoutMs) : void 0;
|
|
13601
|
+
child.stdout?.on("data", (data) => {
|
|
13602
|
+
stdout += data.toString();
|
|
13603
|
+
});
|
|
13604
|
+
child.stderr?.on("data", (data) => {
|
|
13605
|
+
stderr += data.toString();
|
|
13606
|
+
});
|
|
13607
|
+
child.on("error", (error) => {
|
|
13608
|
+
if (timeout !== void 0) {
|
|
13609
|
+
clearTimeout(timeout);
|
|
13610
|
+
}
|
|
13611
|
+
reject(error);
|
|
13612
|
+
});
|
|
13613
|
+
child.on("exit", (code) => {
|
|
13614
|
+
if (timeout !== void 0) {
|
|
13615
|
+
clearTimeout(timeout);
|
|
13616
|
+
}
|
|
13617
|
+
if (code && code !== 0 && stderr.length > 0) {
|
|
13618
|
+
reject(new Error(`Code evaluator exited with code ${code}: ${stderr.trim()}`));
|
|
13619
|
+
return;
|
|
13620
|
+
}
|
|
13621
|
+
resolve(stdout.trim());
|
|
13622
|
+
});
|
|
13623
|
+
child.stdin?.write(input);
|
|
13624
|
+
child.stdin?.end();
|
|
13625
|
+
});
|
|
13626
|
+
}
|
|
13627
|
+
function parseJsonSafe(payload) {
|
|
13628
|
+
try {
|
|
13629
|
+
return JSON.parse(payload);
|
|
13630
|
+
} catch {
|
|
13631
|
+
return void 0;
|
|
13632
|
+
}
|
|
12307
13633
|
}
|
|
12308
13634
|
var Node = class {
|
|
12309
13635
|
value;
|
|
@@ -12445,7 +13771,7 @@ async function runEvaluation(options) {
|
|
|
12445
13771
|
targets,
|
|
12446
13772
|
env,
|
|
12447
13773
|
providerFactory,
|
|
12448
|
-
|
|
13774
|
+
evaluators,
|
|
12449
13775
|
maxRetries,
|
|
12450
13776
|
agentTimeoutMs,
|
|
12451
13777
|
promptDumpDir,
|
|
@@ -12504,8 +13830,14 @@ async function runEvaluation(options) {
|
|
|
12504
13830
|
}
|
|
12505
13831
|
return getOrCreateProvider(resolvedJudge);
|
|
12506
13832
|
};
|
|
12507
|
-
const
|
|
13833
|
+
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
|
|
12508
13834
|
const primaryProvider = getOrCreateProvider(target);
|
|
13835
|
+
const providerSupportsBatch = target.providerBatching === true && primaryProvider.supportsBatch === true && typeof primaryProvider.invokeBatch === "function";
|
|
13836
|
+
if (target.providerBatching && !providerSupportsBatch && verbose) {
|
|
13837
|
+
console.warn(
|
|
13838
|
+
`Provider batching requested for target '${target.name}', but provider does not advertise batch support. Using per-case dispatch.`
|
|
13839
|
+
);
|
|
13840
|
+
}
|
|
12509
13841
|
if (onProgress && filteredEvalCases.length > 0) {
|
|
12510
13842
|
for (let i6 = 0; i6 < filteredEvalCases.length; i6++) {
|
|
12511
13843
|
await onProgress({
|
|
@@ -12515,6 +13847,28 @@ async function runEvaluation(options) {
|
|
|
12515
13847
|
});
|
|
12516
13848
|
}
|
|
12517
13849
|
}
|
|
13850
|
+
if (providerSupportsBatch) {
|
|
13851
|
+
try {
|
|
13852
|
+
return await runBatchEvaluation({
|
|
13853
|
+
evalCases: filteredEvalCases,
|
|
13854
|
+
provider: primaryProvider,
|
|
13855
|
+
target,
|
|
13856
|
+
evaluatorRegistry,
|
|
13857
|
+
promptDumpDir,
|
|
13858
|
+
nowFn: now ?? (() => /* @__PURE__ */ new Date()),
|
|
13859
|
+
onProgress,
|
|
13860
|
+
onResult,
|
|
13861
|
+
verbose,
|
|
13862
|
+
resolveJudgeProvider,
|
|
13863
|
+
agentTimeoutMs
|
|
13864
|
+
});
|
|
13865
|
+
} catch (error) {
|
|
13866
|
+
if (verbose) {
|
|
13867
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
13868
|
+
console.warn(`Provider batch execution failed, falling back to per-case dispatch: ${message}`);
|
|
13869
|
+
}
|
|
13870
|
+
}
|
|
13871
|
+
}
|
|
12518
13872
|
const workers = options.maxConcurrency ?? target.workers ?? 1;
|
|
12519
13873
|
const limit = pLimit(workers);
|
|
12520
13874
|
let nextWorkerId = 1;
|
|
@@ -12537,7 +13891,7 @@ async function runEvaluation(options) {
|
|
|
12537
13891
|
evalCase,
|
|
12538
13892
|
provider: primaryProvider,
|
|
12539
13893
|
target,
|
|
12540
|
-
|
|
13894
|
+
evaluators: evaluatorRegistry,
|
|
12541
13895
|
maxRetries,
|
|
12542
13896
|
agentTimeoutMs,
|
|
12543
13897
|
promptDumpDir,
|
|
@@ -12598,12 +13952,118 @@ async function runEvaluation(options) {
|
|
|
12598
13952
|
}
|
|
12599
13953
|
return results;
|
|
12600
13954
|
}
|
|
13955
|
+
async function runBatchEvaluation(options) {
|
|
13956
|
+
const {
|
|
13957
|
+
evalCases,
|
|
13958
|
+
provider,
|
|
13959
|
+
target,
|
|
13960
|
+
evaluatorRegistry,
|
|
13961
|
+
promptDumpDir,
|
|
13962
|
+
nowFn,
|
|
13963
|
+
onProgress,
|
|
13964
|
+
onResult,
|
|
13965
|
+
resolveJudgeProvider,
|
|
13966
|
+
agentTimeoutMs
|
|
13967
|
+
} = options;
|
|
13968
|
+
const promptInputsList = [];
|
|
13969
|
+
for (const evalCase of evalCases) {
|
|
13970
|
+
const promptInputs = await buildPromptInputs(evalCase);
|
|
13971
|
+
if (promptDumpDir) {
|
|
13972
|
+
await dumpPrompt(promptDumpDir, evalCase, promptInputs);
|
|
13973
|
+
}
|
|
13974
|
+
promptInputsList.push(promptInputs);
|
|
13975
|
+
}
|
|
13976
|
+
const batchRequests = evalCases.map((evalCase, index) => {
|
|
13977
|
+
const promptInputs = promptInputsList[index];
|
|
13978
|
+
return {
|
|
13979
|
+
prompt: promptInputs.request,
|
|
13980
|
+
guidelines: promptInputs.guidelines,
|
|
13981
|
+
guideline_patterns: evalCase.guideline_patterns,
|
|
13982
|
+
inputFiles: evalCase.file_paths,
|
|
13983
|
+
evalCaseId: evalCase.id,
|
|
13984
|
+
metadata: {
|
|
13985
|
+
systemPrompt: promptInputs.systemMessage ?? ""
|
|
13986
|
+
}
|
|
13987
|
+
};
|
|
13988
|
+
});
|
|
13989
|
+
const batchResponse = await provider.invokeBatch?.(batchRequests);
|
|
13990
|
+
if (!Array.isArray(batchResponse)) {
|
|
13991
|
+
throw new Error("Provider batching failed: invokeBatch did not return an array");
|
|
13992
|
+
}
|
|
13993
|
+
if (batchResponse.length !== evalCases.length) {
|
|
13994
|
+
throw new Error(
|
|
13995
|
+
`Provider batching failed: expected ${evalCases.length} responses, received ${batchResponse.length}`
|
|
13996
|
+
);
|
|
13997
|
+
}
|
|
13998
|
+
if (onProgress) {
|
|
13999
|
+
const startedAt = Date.now();
|
|
14000
|
+
for (let i6 = 0; i6 < evalCases.length; i6++) {
|
|
14001
|
+
await onProgress({
|
|
14002
|
+
workerId: 1,
|
|
14003
|
+
evalId: evalCases[i6].id,
|
|
14004
|
+
status: "running",
|
|
14005
|
+
startedAt
|
|
14006
|
+
});
|
|
14007
|
+
}
|
|
14008
|
+
}
|
|
14009
|
+
const results = [];
|
|
14010
|
+
for (let i6 = 0; i6 < evalCases.length; i6++) {
|
|
14011
|
+
const evalCase = evalCases[i6];
|
|
14012
|
+
const promptInputs = promptInputsList[i6];
|
|
14013
|
+
const providerResponse = batchResponse[i6];
|
|
14014
|
+
let result;
|
|
14015
|
+
try {
|
|
14016
|
+
result = await evaluateCandidate({
|
|
14017
|
+
evalCase,
|
|
14018
|
+
candidate: providerResponse.text ?? "",
|
|
14019
|
+
target,
|
|
14020
|
+
provider,
|
|
14021
|
+
evaluators: evaluatorRegistry,
|
|
14022
|
+
promptInputs,
|
|
14023
|
+
nowFn,
|
|
14024
|
+
attempt: 0,
|
|
14025
|
+
judgeProvider: await resolveJudgeProvider(target),
|
|
14026
|
+
agentTimeoutMs
|
|
14027
|
+
});
|
|
14028
|
+
} catch (error) {
|
|
14029
|
+
const errorResult = buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
|
|
14030
|
+
results.push(errorResult);
|
|
14031
|
+
if (onResult) {
|
|
14032
|
+
await onResult(errorResult);
|
|
14033
|
+
}
|
|
14034
|
+
if (onProgress) {
|
|
14035
|
+
await onProgress({
|
|
14036
|
+
workerId: 1,
|
|
14037
|
+
evalId: evalCase.id,
|
|
14038
|
+
status: "failed",
|
|
14039
|
+
completedAt: Date.now(),
|
|
14040
|
+
error: error instanceof Error ? error.message : String(error)
|
|
14041
|
+
});
|
|
14042
|
+
}
|
|
14043
|
+
continue;
|
|
14044
|
+
}
|
|
14045
|
+
results.push(result);
|
|
14046
|
+
if (onResult) {
|
|
14047
|
+
await onResult(result);
|
|
14048
|
+
}
|
|
14049
|
+
if (onProgress) {
|
|
14050
|
+
await onProgress({
|
|
14051
|
+
workerId: 1,
|
|
14052
|
+
evalId: evalCase.id,
|
|
14053
|
+
status: "completed",
|
|
14054
|
+
startedAt: 0,
|
|
14055
|
+
completedAt: Date.now()
|
|
14056
|
+
});
|
|
14057
|
+
}
|
|
14058
|
+
}
|
|
14059
|
+
return results;
|
|
14060
|
+
}
|
|
12601
14061
|
async function runEvalCase(options) {
|
|
12602
14062
|
const {
|
|
12603
14063
|
evalCase,
|
|
12604
14064
|
provider,
|
|
12605
14065
|
target,
|
|
12606
|
-
|
|
14066
|
+
evaluators,
|
|
12607
14067
|
now,
|
|
12608
14068
|
maxRetries,
|
|
12609
14069
|
agentTimeoutMs,
|
|
@@ -12658,27 +14118,49 @@ async function runEvalCase(options) {
|
|
|
12658
14118
|
if (cacheKey && cache && !cachedResponse) {
|
|
12659
14119
|
await cache.set(cacheKey, providerResponse);
|
|
12660
14120
|
}
|
|
12661
|
-
const graderKind = evalCase.grader ?? "heuristic";
|
|
12662
|
-
const activeGrader = graders[graderKind] ?? graders.heuristic;
|
|
12663
|
-
if (!activeGrader) {
|
|
12664
|
-
throw new Error(`No grader registered for kind '${graderKind}'`);
|
|
12665
|
-
}
|
|
12666
|
-
let grade;
|
|
12667
14121
|
try {
|
|
12668
|
-
|
|
12669
|
-
grade = await activeGrader.grade({
|
|
14122
|
+
return await evaluateCandidate({
|
|
12670
14123
|
evalCase,
|
|
12671
14124
|
candidate: providerResponse.text ?? "",
|
|
12672
14125
|
target,
|
|
12673
14126
|
provider,
|
|
12674
|
-
|
|
14127
|
+
evaluators,
|
|
12675
14128
|
promptInputs,
|
|
12676
|
-
|
|
12677
|
-
|
|
14129
|
+
nowFn,
|
|
14130
|
+
attempt,
|
|
14131
|
+
judgeProvider,
|
|
14132
|
+
agentTimeoutMs
|
|
12678
14133
|
});
|
|
12679
14134
|
} catch (error) {
|
|
12680
14135
|
return buildErrorResult(evalCase, target.name, nowFn(), error, promptInputs);
|
|
12681
14136
|
}
|
|
14137
|
+
}
|
|
14138
|
+
async function evaluateCandidate(options) {
|
|
14139
|
+
const {
|
|
14140
|
+
evalCase,
|
|
14141
|
+
candidate,
|
|
14142
|
+
target,
|
|
14143
|
+
provider,
|
|
14144
|
+
evaluators,
|
|
14145
|
+
promptInputs,
|
|
14146
|
+
nowFn,
|
|
14147
|
+
attempt,
|
|
14148
|
+
judgeProvider,
|
|
14149
|
+
agentTimeoutMs
|
|
14150
|
+
} = options;
|
|
14151
|
+
const gradeTimestamp = nowFn();
|
|
14152
|
+
const { score, evaluatorResults } = await runEvaluatorsForCase({
|
|
14153
|
+
evalCase,
|
|
14154
|
+
candidate,
|
|
14155
|
+
target,
|
|
14156
|
+
provider,
|
|
14157
|
+
evaluators,
|
|
14158
|
+
attempt,
|
|
14159
|
+
promptInputs,
|
|
14160
|
+
now: gradeTimestamp,
|
|
14161
|
+
judgeProvider,
|
|
14162
|
+
agentTimeoutMs
|
|
14163
|
+
});
|
|
12682
14164
|
const completedAt = nowFn();
|
|
12683
14165
|
const rawRequest = {
|
|
12684
14166
|
request: promptInputs.request,
|
|
@@ -12689,18 +14171,191 @@ async function runEvalCase(options) {
|
|
|
12689
14171
|
return {
|
|
12690
14172
|
eval_id: evalCase.id,
|
|
12691
14173
|
conversation_id: evalCase.conversation_id,
|
|
12692
|
-
score:
|
|
12693
|
-
hits:
|
|
12694
|
-
misses:
|
|
12695
|
-
model_answer:
|
|
12696
|
-
expected_aspect_count:
|
|
14174
|
+
score: score.score,
|
|
14175
|
+
hits: score.hits,
|
|
14176
|
+
misses: score.misses,
|
|
14177
|
+
model_answer: candidate,
|
|
14178
|
+
expected_aspect_count: score.expectedAspectCount,
|
|
12697
14179
|
target: target.name,
|
|
12698
14180
|
timestamp: completedAt.toISOString(),
|
|
12699
|
-
reasoning:
|
|
12700
|
-
raw_aspects:
|
|
14181
|
+
reasoning: score.reasoning,
|
|
14182
|
+
raw_aspects: score.rawAspects,
|
|
12701
14183
|
raw_request: rawRequest,
|
|
12702
|
-
|
|
14184
|
+
evaluator_raw_request: evaluatorResults ? void 0 : score.evaluatorRawRequest,
|
|
14185
|
+
evaluator_results: evaluatorResults
|
|
14186
|
+
};
|
|
14187
|
+
}
|
|
14188
|
+
async function runEvaluatorsForCase(options) {
|
|
14189
|
+
const { evalCase, candidate, target, provider, evaluators, attempt, promptInputs, now, judgeProvider, agentTimeoutMs } = options;
|
|
14190
|
+
if (evalCase.evaluators && evalCase.evaluators.length > 0) {
|
|
14191
|
+
return runEvaluatorList({
|
|
14192
|
+
evalCase,
|
|
14193
|
+
evaluators: evalCase.evaluators,
|
|
14194
|
+
candidate,
|
|
14195
|
+
target,
|
|
14196
|
+
provider,
|
|
14197
|
+
evaluatorRegistry: evaluators,
|
|
14198
|
+
attempt,
|
|
14199
|
+
promptInputs,
|
|
14200
|
+
now,
|
|
14201
|
+
judgeProvider,
|
|
14202
|
+
agentTimeoutMs
|
|
14203
|
+
});
|
|
14204
|
+
}
|
|
14205
|
+
const evaluatorKind = evalCase.evaluator ?? "llm_judge";
|
|
14206
|
+
const activeEvaluator = evaluators[evaluatorKind] ?? evaluators.llm_judge;
|
|
14207
|
+
if (!activeEvaluator) {
|
|
14208
|
+
throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
|
|
14209
|
+
}
|
|
14210
|
+
const score = await activeEvaluator.evaluate({
|
|
14211
|
+
evalCase,
|
|
14212
|
+
candidate,
|
|
14213
|
+
target,
|
|
14214
|
+
provider,
|
|
14215
|
+
attempt,
|
|
14216
|
+
promptInputs,
|
|
14217
|
+
now,
|
|
14218
|
+
judgeProvider
|
|
14219
|
+
});
|
|
14220
|
+
return { score };
|
|
14221
|
+
}
|
|
14222
|
+
async function runEvaluatorList(options) {
|
|
14223
|
+
const {
|
|
14224
|
+
evalCase,
|
|
14225
|
+
evaluators,
|
|
14226
|
+
candidate,
|
|
14227
|
+
target,
|
|
14228
|
+
provider,
|
|
14229
|
+
evaluatorRegistry,
|
|
14230
|
+
attempt,
|
|
14231
|
+
promptInputs,
|
|
14232
|
+
now,
|
|
14233
|
+
judgeProvider,
|
|
14234
|
+
agentTimeoutMs
|
|
14235
|
+
} = options;
|
|
14236
|
+
const scored = [];
|
|
14237
|
+
const evaluatorResults = [];
|
|
14238
|
+
for (const evaluator of evaluators ?? []) {
|
|
14239
|
+
try {
|
|
14240
|
+
if (evaluator.type === "llm_judge") {
|
|
14241
|
+
const score2 = await runLlmJudgeEvaluator({
|
|
14242
|
+
config: evaluator,
|
|
14243
|
+
evalCase,
|
|
14244
|
+
candidate,
|
|
14245
|
+
target,
|
|
14246
|
+
provider,
|
|
14247
|
+
evaluatorRegistry,
|
|
14248
|
+
attempt,
|
|
14249
|
+
promptInputs,
|
|
14250
|
+
now,
|
|
14251
|
+
judgeProvider
|
|
14252
|
+
});
|
|
14253
|
+
scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
|
|
14254
|
+
evaluatorResults.push({
|
|
14255
|
+
name: evaluator.name,
|
|
14256
|
+
type: evaluator.type,
|
|
14257
|
+
score: score2.score,
|
|
14258
|
+
hits: score2.hits,
|
|
14259
|
+
misses: score2.misses,
|
|
14260
|
+
reasoning: score2.reasoning,
|
|
14261
|
+
evaluator_raw_request: score2.evaluatorRawRequest
|
|
14262
|
+
});
|
|
14263
|
+
continue;
|
|
14264
|
+
}
|
|
14265
|
+
if (evaluator.type === "code") {
|
|
14266
|
+
const codeEvaluator = new CodeEvaluator({
|
|
14267
|
+
script: evaluator.script,
|
|
14268
|
+
cwd: evaluator.resolvedCwd ?? evaluator.cwd,
|
|
14269
|
+
agentTimeoutMs
|
|
14270
|
+
});
|
|
14271
|
+
const score2 = await codeEvaluator.evaluate({
|
|
14272
|
+
evalCase,
|
|
14273
|
+
candidate,
|
|
14274
|
+
target,
|
|
14275
|
+
provider,
|
|
14276
|
+
attempt,
|
|
14277
|
+
promptInputs,
|
|
14278
|
+
now
|
|
14279
|
+
});
|
|
14280
|
+
scored.push({ score: score2, name: evaluator.name, type: evaluator.type });
|
|
14281
|
+
evaluatorResults.push({
|
|
14282
|
+
name: evaluator.name,
|
|
14283
|
+
type: evaluator.type,
|
|
14284
|
+
score: score2.score,
|
|
14285
|
+
hits: score2.hits,
|
|
14286
|
+
misses: score2.misses,
|
|
14287
|
+
reasoning: score2.reasoning,
|
|
14288
|
+
evaluator_raw_request: score2.evaluatorRawRequest
|
|
14289
|
+
});
|
|
14290
|
+
continue;
|
|
14291
|
+
}
|
|
14292
|
+
} catch (error) {
|
|
14293
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14294
|
+
const fallbackScore = {
|
|
14295
|
+
score: 0,
|
|
14296
|
+
hits: [],
|
|
14297
|
+
misses: [`Evaluator '${evaluator.name}' failed: ${message}`],
|
|
14298
|
+
expectedAspectCount: 1,
|
|
14299
|
+
reasoning: message
|
|
14300
|
+
};
|
|
14301
|
+
scored.push({ score: fallbackScore, name: evaluator.name ?? "unknown", type: evaluator.type ?? "unknown" });
|
|
14302
|
+
evaluatorResults.push({
|
|
14303
|
+
name: evaluator.name ?? "unknown",
|
|
14304
|
+
type: evaluator.type ?? "unknown",
|
|
14305
|
+
score: 0,
|
|
14306
|
+
hits: [],
|
|
14307
|
+
misses: [`Evaluator '${evaluator.name ?? "unknown"}' failed: ${message}`],
|
|
14308
|
+
reasoning: message
|
|
14309
|
+
});
|
|
14310
|
+
}
|
|
14311
|
+
}
|
|
14312
|
+
const aggregateScore = scored.length > 0 ? scored.reduce((total, entry) => total + entry.score.score, 0) / scored.length : 0;
|
|
14313
|
+
const hits = scored.flatMap((entry) => entry.score.hits);
|
|
14314
|
+
const misses = scored.flatMap((entry) => entry.score.misses);
|
|
14315
|
+
const expectedAspectCount = scored.reduce((total, entry) => total + (entry.score.expectedAspectCount ?? 0), 0);
|
|
14316
|
+
const rawAspects = scored.flatMap((entry) => entry.score.rawAspects ?? []);
|
|
14317
|
+
const reasoningParts = scored.map((entry) => entry.score.reasoning ? `${entry.name}: ${entry.score.reasoning}` : void 0).filter(isNonEmptyString2);
|
|
14318
|
+
const reasoning = reasoningParts.length > 0 ? reasoningParts.join(" | ") : void 0;
|
|
14319
|
+
const score = {
|
|
14320
|
+
score: aggregateScore,
|
|
14321
|
+
hits,
|
|
14322
|
+
misses,
|
|
14323
|
+
expectedAspectCount,
|
|
14324
|
+
reasoning,
|
|
14325
|
+
rawAspects: rawAspects.length > 0 ? rawAspects : void 0
|
|
12703
14326
|
};
|
|
14327
|
+
return { score, evaluatorResults };
|
|
14328
|
+
}
|
|
14329
|
+
async function runLlmJudgeEvaluator(options) {
|
|
14330
|
+
const { config, evalCase, candidate, target, provider, evaluatorRegistry, attempt, promptInputs, now, judgeProvider } = options;
|
|
14331
|
+
const customPrompt = await resolveCustomPrompt(config);
|
|
14332
|
+
return evaluatorRegistry.llm_judge.evaluate({
|
|
14333
|
+
evalCase,
|
|
14334
|
+
candidate,
|
|
14335
|
+
target,
|
|
14336
|
+
provider,
|
|
14337
|
+
attempt,
|
|
14338
|
+
promptInputs,
|
|
14339
|
+
now,
|
|
14340
|
+
judgeProvider,
|
|
14341
|
+
systemPrompt: customPrompt,
|
|
14342
|
+
evaluator: config,
|
|
14343
|
+
judgeModel: config.model
|
|
14344
|
+
});
|
|
14345
|
+
}
|
|
14346
|
+
async function resolveCustomPrompt(config) {
|
|
14347
|
+
if (config.promptPath) {
|
|
14348
|
+
try {
|
|
14349
|
+
return await readFile4(config.promptPath, "utf8");
|
|
14350
|
+
} catch (error) {
|
|
14351
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14352
|
+
console.warn(`Could not read custom prompt at ${config.promptPath}: ${message}`);
|
|
14353
|
+
}
|
|
14354
|
+
}
|
|
14355
|
+
return config.prompt;
|
|
14356
|
+
}
|
|
14357
|
+
function isNonEmptyString2(value) {
|
|
14358
|
+
return typeof value === "string" && value.trim().length > 0;
|
|
12704
14359
|
}
|
|
12705
14360
|
function filterEvalCases(evalCases, evalId) {
|
|
12706
14361
|
if (!evalId) {
|
|
@@ -12708,9 +14363,8 @@ function filterEvalCases(evalCases, evalId) {
|
|
|
12708
14363
|
}
|
|
12709
14364
|
return evalCases.filter((evalCase) => evalCase.id === evalId);
|
|
12710
14365
|
}
|
|
12711
|
-
function
|
|
12712
|
-
const
|
|
12713
|
-
const llmJudge = overrides?.llm_judge ?? new QualityGrader({
|
|
14366
|
+
function buildEvaluatorRegistry(overrides, resolveJudgeProvider) {
|
|
14367
|
+
const llmJudge = overrides?.llm_judge ?? new LlmJudgeEvaluator({
|
|
12714
14368
|
resolveJudgeProvider: async (context2) => {
|
|
12715
14369
|
if (context2.judgeProvider) {
|
|
12716
14370
|
return context2.judgeProvider;
|
|
@@ -12720,15 +14374,14 @@ function buildGraderRegistry(overrides, resolveJudgeProvider) {
|
|
|
12720
14374
|
});
|
|
12721
14375
|
return {
|
|
12722
14376
|
...overrides,
|
|
12723
|
-
heuristic,
|
|
12724
14377
|
llm_judge: llmJudge
|
|
12725
14378
|
};
|
|
12726
14379
|
}
|
|
12727
14380
|
async function dumpPrompt(directory, evalCase, promptInputs) {
|
|
12728
14381
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
12729
14382
|
const filename = `${timestamp}_${sanitizeFilename(evalCase.id)}.json`;
|
|
12730
|
-
const filePath =
|
|
12731
|
-
await
|
|
14383
|
+
const filePath = path72.resolve(directory, filename);
|
|
14384
|
+
await mkdir22(path72.dirname(filePath), { recursive: true });
|
|
12732
14385
|
const payload = {
|
|
12733
14386
|
eval_id: evalCase.id,
|
|
12734
14387
|
request: promptInputs.request,
|
|
@@ -12745,7 +14398,7 @@ function sanitizeFilename(value) {
|
|
|
12745
14398
|
return sanitized.length > 0 ? sanitized : randomUUID2();
|
|
12746
14399
|
}
|
|
12747
14400
|
async function invokeProvider(provider, options) {
|
|
12748
|
-
const { evalCase,
|
|
14401
|
+
const { evalCase, promptInputs, attempt, agentTimeoutMs, signal } = options;
|
|
12749
14402
|
const controller = new AbortController();
|
|
12750
14403
|
const timeout = agentTimeoutMs ? setTimeout(() => controller.abort(), agentTimeoutMs) : void 0;
|
|
12751
14404
|
if (signal) {
|
|
@@ -12756,7 +14409,7 @@ async function invokeProvider(provider, options) {
|
|
|
12756
14409
|
prompt: promptInputs.request,
|
|
12757
14410
|
guidelines: promptInputs.guidelines,
|
|
12758
14411
|
guideline_patterns: evalCase.guideline_patterns,
|
|
12759
|
-
|
|
14412
|
+
inputFiles: evalCase.file_paths,
|
|
12760
14413
|
evalCaseId: evalCase.id,
|
|
12761
14414
|
attempt,
|
|
12762
14415
|
metadata: {
|
|
@@ -12825,19 +14478,19 @@ function createAgentKernel() {
|
|
|
12825
14478
|
// src/commands/eval/run-eval.ts
|
|
12826
14479
|
import { constants as constants6 } from "node:fs";
|
|
12827
14480
|
import { access as access6, mkdir as mkdir6 } from "node:fs/promises";
|
|
12828
|
-
import
|
|
14481
|
+
import path13 from "node:path";
|
|
12829
14482
|
import { pathToFileURL } from "node:url";
|
|
12830
14483
|
|
|
12831
14484
|
// src/commands/eval/env.ts
|
|
12832
14485
|
import { config as loadDotenv } from "dotenv";
|
|
12833
14486
|
import { constants as constants4 } from "node:fs";
|
|
12834
14487
|
import { access as access4 } from "node:fs/promises";
|
|
12835
|
-
import
|
|
14488
|
+
import path9 from "node:path";
|
|
12836
14489
|
function uniqueDirs(directories) {
|
|
12837
14490
|
const seen = /* @__PURE__ */ new Set();
|
|
12838
14491
|
const result = [];
|
|
12839
14492
|
for (const dir of directories) {
|
|
12840
|
-
const absolute =
|
|
14493
|
+
const absolute = path9.resolve(dir);
|
|
12841
14494
|
if (seen.has(absolute)) {
|
|
12842
14495
|
continue;
|
|
12843
14496
|
}
|
|
@@ -12856,14 +14509,14 @@ async function fileExists4(filePath) {
|
|
|
12856
14509
|
}
|
|
12857
14510
|
function collectAncestorDirectories(start, boundary) {
|
|
12858
14511
|
const directories = [];
|
|
12859
|
-
const boundaryDir =
|
|
12860
|
-
let current =
|
|
14512
|
+
const boundaryDir = path9.resolve(boundary);
|
|
14513
|
+
let current = path9.resolve(start);
|
|
12861
14514
|
while (current !== void 0) {
|
|
12862
14515
|
directories.push(current);
|
|
12863
14516
|
if (current === boundaryDir) {
|
|
12864
14517
|
break;
|
|
12865
14518
|
}
|
|
12866
|
-
const parent =
|
|
14519
|
+
const parent = path9.dirname(current);
|
|
12867
14520
|
if (parent === current) {
|
|
12868
14521
|
break;
|
|
12869
14522
|
}
|
|
@@ -12873,7 +14526,7 @@ function collectAncestorDirectories(start, boundary) {
|
|
|
12873
14526
|
}
|
|
12874
14527
|
async function loadEnvFromHierarchy(options) {
|
|
12875
14528
|
const { testFilePath, repoRoot, verbose } = options;
|
|
12876
|
-
const testDir =
|
|
14529
|
+
const testDir = path9.dirname(path9.resolve(testFilePath));
|
|
12877
14530
|
const cwd = process.cwd();
|
|
12878
14531
|
const searchDirs = uniqueDirs([
|
|
12879
14532
|
...collectAncestorDirectories(testDir, repoRoot),
|
|
@@ -12881,7 +14534,7 @@ async function loadEnvFromHierarchy(options) {
|
|
|
12881
14534
|
cwd
|
|
12882
14535
|
]);
|
|
12883
14536
|
for (const dir of searchDirs) {
|
|
12884
|
-
const candidate =
|
|
14537
|
+
const candidate = path9.join(dir, ".env");
|
|
12885
14538
|
if (await fileExists4(candidate)) {
|
|
12886
14539
|
loadDotenv({ path: candidate, override: false });
|
|
12887
14540
|
if (verbose) {
|
|
@@ -13105,7 +14758,7 @@ var Mutex = class {
|
|
|
13105
14758
|
// src/commands/eval/jsonl-writer.ts
|
|
13106
14759
|
import { createWriteStream } from "node:fs";
|
|
13107
14760
|
import { mkdir as mkdir4 } from "node:fs/promises";
|
|
13108
|
-
import
|
|
14761
|
+
import path10 from "node:path";
|
|
13109
14762
|
import { finished } from "node:stream/promises";
|
|
13110
14763
|
var JsonlWriter = class _JsonlWriter {
|
|
13111
14764
|
stream;
|
|
@@ -13115,7 +14768,7 @@ var JsonlWriter = class _JsonlWriter {
|
|
|
13115
14768
|
this.stream = stream;
|
|
13116
14769
|
}
|
|
13117
14770
|
static async open(filePath) {
|
|
13118
|
-
await mkdir4(
|
|
14771
|
+
await mkdir4(path10.dirname(filePath), { recursive: true });
|
|
13119
14772
|
const stream = createWriteStream(filePath, { flags: "w", encoding: "utf8" });
|
|
13120
14773
|
return new _JsonlWriter(stream);
|
|
13121
14774
|
}
|
|
@@ -13147,7 +14800,7 @@ var JsonlWriter = class _JsonlWriter {
|
|
|
13147
14800
|
// src/commands/eval/yaml-writer.ts
|
|
13148
14801
|
import { createWriteStream as createWriteStream2 } from "node:fs";
|
|
13149
14802
|
import { mkdir as mkdir5 } from "node:fs/promises";
|
|
13150
|
-
import
|
|
14803
|
+
import path11 from "node:path";
|
|
13151
14804
|
import { finished as finished2 } from "node:stream/promises";
|
|
13152
14805
|
import { stringify as stringifyYaml } from "yaml";
|
|
13153
14806
|
var YamlWriter = class _YamlWriter {
|
|
@@ -13159,7 +14812,7 @@ var YamlWriter = class _YamlWriter {
|
|
|
13159
14812
|
this.stream = stream;
|
|
13160
14813
|
}
|
|
13161
14814
|
static async open(filePath) {
|
|
13162
|
-
await mkdir5(
|
|
14815
|
+
await mkdir5(path11.dirname(filePath), { recursive: true });
|
|
13163
14816
|
const stream = createWriteStream2(filePath, { flags: "w", encoding: "utf8" });
|
|
13164
14817
|
return new _YamlWriter(stream);
|
|
13165
14818
|
}
|
|
@@ -13467,14 +15120,14 @@ function formatEvaluationSummary(summary) {
|
|
|
13467
15120
|
|
|
13468
15121
|
// src/commands/eval/targets.ts
|
|
13469
15122
|
import { constants as constants5 } from "node:fs";
|
|
13470
|
-
import { access as access5, readFile as
|
|
13471
|
-
import
|
|
15123
|
+
import { access as access5, readFile as readFile5 } from "node:fs/promises";
|
|
15124
|
+
import path12 from "node:path";
|
|
13472
15125
|
import { parse as parse4 } from "yaml";
|
|
13473
15126
|
var TARGET_FILE_CANDIDATES = [
|
|
13474
15127
|
"targets.yaml",
|
|
13475
15128
|
"targets.yml",
|
|
13476
|
-
|
|
13477
|
-
|
|
15129
|
+
path12.join(".agentv", "targets.yaml"),
|
|
15130
|
+
path12.join(".agentv", "targets.yml")
|
|
13478
15131
|
];
|
|
13479
15132
|
async function fileExists5(filePath) {
|
|
13480
15133
|
try {
|
|
@@ -13486,7 +15139,7 @@ async function fileExists5(filePath) {
|
|
|
13486
15139
|
}
|
|
13487
15140
|
async function readTestSuiteTarget(testFilePath) {
|
|
13488
15141
|
try {
|
|
13489
|
-
const raw = await
|
|
15142
|
+
const raw = await readFile5(path12.resolve(testFilePath), "utf8");
|
|
13490
15143
|
const parsed = parse4(raw);
|
|
13491
15144
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
13492
15145
|
const targetValue = parsed.target;
|
|
@@ -13501,12 +15154,12 @@ async function readTestSuiteTarget(testFilePath) {
|
|
|
13501
15154
|
async function discoverTargetsFile(options) {
|
|
13502
15155
|
const { explicitPath, testFilePath, repoRoot, cwd } = options;
|
|
13503
15156
|
if (explicitPath) {
|
|
13504
|
-
const resolvedExplicit =
|
|
15157
|
+
const resolvedExplicit = path12.resolve(explicitPath);
|
|
13505
15158
|
if (await fileExists5(resolvedExplicit)) {
|
|
13506
15159
|
return resolvedExplicit;
|
|
13507
15160
|
}
|
|
13508
15161
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
13509
|
-
const nested =
|
|
15162
|
+
const nested = path12.join(resolvedExplicit, candidate);
|
|
13510
15163
|
if (await fileExists5(nested)) {
|
|
13511
15164
|
return nested;
|
|
13512
15165
|
}
|
|
@@ -13514,13 +15167,13 @@ async function discoverTargetsFile(options) {
|
|
|
13514
15167
|
throw new Error(`targets.yaml not found at provided path: ${resolvedExplicit}`);
|
|
13515
15168
|
}
|
|
13516
15169
|
const directories = [...buildDirectoryChain(testFilePath, repoRoot)];
|
|
13517
|
-
const resolvedCwd =
|
|
15170
|
+
const resolvedCwd = path12.resolve(cwd);
|
|
13518
15171
|
if (!directories.includes(resolvedCwd)) {
|
|
13519
15172
|
directories.push(resolvedCwd);
|
|
13520
15173
|
}
|
|
13521
15174
|
for (const directory of directories) {
|
|
13522
15175
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
13523
|
-
const fullPath =
|
|
15176
|
+
const fullPath = path12.join(directory, candidate);
|
|
13524
15177
|
if (await fileExists5(fullPath)) {
|
|
13525
15178
|
return fullPath;
|
|
13526
15179
|
}
|
|
@@ -13645,15 +15298,15 @@ async function ensureFileExists(filePath, description) {
|
|
|
13645
15298
|
}
|
|
13646
15299
|
}
|
|
13647
15300
|
async function findRepoRoot(start) {
|
|
13648
|
-
const fallback =
|
|
15301
|
+
const fallback = path13.resolve(start);
|
|
13649
15302
|
let current = fallback;
|
|
13650
15303
|
while (current !== void 0) {
|
|
13651
|
-
const candidate =
|
|
15304
|
+
const candidate = path13.join(current, ".git");
|
|
13652
15305
|
try {
|
|
13653
15306
|
await access6(candidate, constants6.F_OK);
|
|
13654
15307
|
return current;
|
|
13655
15308
|
} catch {
|
|
13656
|
-
const parent =
|
|
15309
|
+
const parent = path13.dirname(current);
|
|
13657
15310
|
if (parent === current) {
|
|
13658
15311
|
break;
|
|
13659
15312
|
}
|
|
@@ -13663,21 +15316,21 @@ async function findRepoRoot(start) {
|
|
|
13663
15316
|
return fallback;
|
|
13664
15317
|
}
|
|
13665
15318
|
function buildDefaultOutputPath(testFilePath, cwd, format) {
|
|
13666
|
-
const testFileName =
|
|
15319
|
+
const testFileName = path13.basename(testFilePath);
|
|
13667
15320
|
const withoutExtension = testFileName.replace(/\.test\.ya?ml$/i, "").replace(/\.ya?ml$/i, "");
|
|
13668
15321
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
13669
15322
|
const baseName = withoutExtension.length > 0 ? withoutExtension : "results";
|
|
13670
15323
|
const extension = getDefaultExtension(format);
|
|
13671
|
-
return
|
|
15324
|
+
return path13.join(cwd, ".agentv", "results", `${baseName}_${timestamp}${extension}`);
|
|
13672
15325
|
}
|
|
13673
15326
|
function resolvePromptDirectory(option, cwd) {
|
|
13674
15327
|
if (option === void 0) {
|
|
13675
15328
|
return void 0;
|
|
13676
15329
|
}
|
|
13677
15330
|
if (typeof option === "string" && option.trim().length > 0) {
|
|
13678
|
-
return
|
|
15331
|
+
return path13.resolve(cwd, option);
|
|
13679
15332
|
}
|
|
13680
|
-
return
|
|
15333
|
+
return path13.join(cwd, ".agentv", "prompts");
|
|
13681
15334
|
}
|
|
13682
15335
|
function createEvaluationCache() {
|
|
13683
15336
|
const store = /* @__PURE__ */ new Map();
|
|
@@ -13693,7 +15346,7 @@ function createEvaluationCache() {
|
|
|
13693
15346
|
async function runEvalCommand(input) {
|
|
13694
15347
|
const options = normalizeOptions(input.rawOptions);
|
|
13695
15348
|
const cwd = process.cwd();
|
|
13696
|
-
const testFilePath =
|
|
15349
|
+
const testFilePath = path13.resolve(input.testFile);
|
|
13697
15350
|
await ensureFileExists(testFilePath, "Test file");
|
|
13698
15351
|
const repoRoot = await findRepoRoot(cwd);
|
|
13699
15352
|
if (options.verbose) {
|
|
@@ -13719,7 +15372,7 @@ async function runEvalCommand(input) {
|
|
|
13719
15372
|
const providerLabel = options.dryRun ? `${targetSelection.resolvedTarget.kind} (dry-run)` : targetSelection.resolvedTarget.kind;
|
|
13720
15373
|
const targetMessage = options.verbose ? `Using target (${targetSelection.targetSource}): ${targetSelection.targetName} [provider=${providerLabel}] via ${targetSelection.targetsFilePath}` : `Using target: ${targetSelection.targetName} [provider=${providerLabel}]`;
|
|
13721
15374
|
console.log(targetMessage);
|
|
13722
|
-
const outputPath = options.outPath ?
|
|
15375
|
+
const outputPath = options.outPath ? path13.resolve(options.outPath) : buildDefaultOutputPath(testFilePath, cwd, options.format);
|
|
13723
15376
|
console.log(`Output path: ${outputPath}`);
|
|
13724
15377
|
const promptDumpDir = resolvePromptDirectory(options.dumpPrompts, cwd);
|
|
13725
15378
|
if (promptDumpDir) {
|
|
@@ -13811,7 +15464,7 @@ async function resolveEvaluationRunner() {
|
|
|
13811
15464
|
if (!overridePath) {
|
|
13812
15465
|
return runEvaluation;
|
|
13813
15466
|
}
|
|
13814
|
-
const resolved =
|
|
15467
|
+
const resolved = path13.isAbsolute(overridePath) ? overridePath : path13.resolve(process.cwd(), overridePath);
|
|
13815
15468
|
const moduleUrl = pathToFileURL(resolved).href;
|
|
13816
15469
|
const mod = await import(moduleUrl);
|
|
13817
15470
|
const candidate = mod.runEvaluation;
|
|
@@ -13872,31 +15525,31 @@ function registerEvalCommand(program) {
|
|
|
13872
15525
|
|
|
13873
15526
|
// src/commands/init/index.ts
|
|
13874
15527
|
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
13875
|
-
import
|
|
15528
|
+
import path15 from "node:path";
|
|
13876
15529
|
|
|
13877
15530
|
// src/templates/index.ts
|
|
13878
15531
|
import { readFileSync } from "node:fs";
|
|
13879
|
-
import
|
|
15532
|
+
import path14 from "node:path";
|
|
13880
15533
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13881
15534
|
var TemplateManager = class {
|
|
13882
15535
|
static getTemplates() {
|
|
13883
|
-
const currentDir =
|
|
15536
|
+
const currentDir = path14.dirname(fileURLToPath2(import.meta.url));
|
|
13884
15537
|
let templatesDir;
|
|
13885
|
-
if (currentDir.includes(
|
|
13886
|
-
templatesDir =
|
|
15538
|
+
if (currentDir.includes(path14.sep + "dist")) {
|
|
15539
|
+
templatesDir = path14.join(currentDir, "templates");
|
|
13887
15540
|
} else {
|
|
13888
15541
|
templatesDir = currentDir;
|
|
13889
15542
|
}
|
|
13890
15543
|
const evalBuildPrompt = readFileSync(
|
|
13891
|
-
|
|
15544
|
+
path14.join(templatesDir, "eval-build.prompt.md"),
|
|
13892
15545
|
"utf-8"
|
|
13893
15546
|
);
|
|
13894
15547
|
const evalSchema = readFileSync(
|
|
13895
|
-
|
|
15548
|
+
path14.join(templatesDir, "eval-schema.json"),
|
|
13896
15549
|
"utf-8"
|
|
13897
15550
|
);
|
|
13898
15551
|
const configSchema = readFileSync(
|
|
13899
|
-
|
|
15552
|
+
path14.join(templatesDir, "config-schema.json"),
|
|
13900
15553
|
"utf-8"
|
|
13901
15554
|
);
|
|
13902
15555
|
return [
|
|
@@ -13918,28 +15571,37 @@ var TemplateManager = class {
|
|
|
13918
15571
|
|
|
13919
15572
|
// src/commands/init/index.ts
|
|
13920
15573
|
async function initCommand(options = {}) {
|
|
13921
|
-
const targetPath =
|
|
13922
|
-
const githubDir =
|
|
15574
|
+
const targetPath = path15.resolve(options.targetPath ?? ".");
|
|
15575
|
+
const githubDir = path15.join(targetPath, ".github");
|
|
13923
15576
|
if (!existsSync(githubDir)) {
|
|
13924
15577
|
mkdirSync(githubDir, { recursive: true });
|
|
13925
15578
|
}
|
|
13926
15579
|
const templates = TemplateManager.getTemplates();
|
|
13927
15580
|
for (const template of templates) {
|
|
13928
|
-
const targetFilePath =
|
|
13929
|
-
const targetDirPath =
|
|
15581
|
+
const targetFilePath = path15.join(githubDir, template.path);
|
|
15582
|
+
const targetDirPath = path15.dirname(targetFilePath);
|
|
13930
15583
|
if (!existsSync(targetDirPath)) {
|
|
13931
15584
|
mkdirSync(targetDirPath, { recursive: true });
|
|
13932
15585
|
}
|
|
13933
15586
|
writeFileSync(targetFilePath, template.content, "utf-8");
|
|
13934
|
-
console.log(`Created ${
|
|
15587
|
+
console.log(`Created ${path15.relative(targetPath, targetFilePath)}`);
|
|
13935
15588
|
}
|
|
13936
15589
|
console.log("\nAgentV initialized successfully!");
|
|
13937
15590
|
console.log(`
|
|
13938
|
-
Files installed to ${
|
|
15591
|
+
Files installed to ${path15.relative(targetPath, githubDir)}:`);
|
|
13939
15592
|
templates.forEach((t) => console.log(` - ${t.path}`));
|
|
13940
15593
|
console.log("\nYou can now create eval files using the schema and prompt templates.");
|
|
13941
15594
|
}
|
|
13942
15595
|
|
|
15596
|
+
// src/commands/status.ts
|
|
15597
|
+
function registerStatusCommand(program) {
|
|
15598
|
+
program.command("status").description("Show the latest AgentV kernel status").action(() => {
|
|
15599
|
+
const kernel = createAgentKernel();
|
|
15600
|
+
console.log(`Kernel status: ${kernel.status}`);
|
|
15601
|
+
});
|
|
15602
|
+
return program;
|
|
15603
|
+
}
|
|
15604
|
+
|
|
13943
15605
|
// src/commands/validate/format-output.ts
|
|
13944
15606
|
var ANSI_RED = "\x1B[31m";
|
|
13945
15607
|
var ANSI_YELLOW2 = "\x1B[33m";
|
|
@@ -14012,10 +15674,10 @@ function isTTY() {
|
|
|
14012
15674
|
}
|
|
14013
15675
|
|
|
14014
15676
|
// ../../packages/core/dist/evaluation/validation/index.js
|
|
14015
|
-
import { readFile as
|
|
15677
|
+
import { readFile as readFile6 } from "node:fs/promises";
|
|
14016
15678
|
import { parse as parse5 } from "yaml";
|
|
14017
15679
|
import { readFile as readFile23 } from "node:fs/promises";
|
|
14018
|
-
import
|
|
15680
|
+
import path16 from "node:path";
|
|
14019
15681
|
import { parse as parse23 } from "yaml";
|
|
14020
15682
|
import { readFile as readFile32 } from "node:fs/promises";
|
|
14021
15683
|
import path23 from "node:path";
|
|
@@ -14030,7 +15692,7 @@ var SCHEMA_TARGETS_V2 = "agentv-targets-v2";
|
|
|
14030
15692
|
var SCHEMA_CONFIG_V22 = "agentv-config-v2";
|
|
14031
15693
|
async function detectFileType(filePath) {
|
|
14032
15694
|
try {
|
|
14033
|
-
const content = await
|
|
15695
|
+
const content = await readFile6(filePath, "utf8");
|
|
14034
15696
|
const parsed = parse5(content);
|
|
14035
15697
|
if (typeof parsed !== "object" || parsed === null) {
|
|
14036
15698
|
return "unknown";
|
|
@@ -14060,7 +15722,7 @@ function isObject(value) {
|
|
|
14060
15722
|
}
|
|
14061
15723
|
async function validateEvalFile(filePath) {
|
|
14062
15724
|
const errors = [];
|
|
14063
|
-
const absolutePath =
|
|
15725
|
+
const absolutePath = path16.resolve(filePath);
|
|
14064
15726
|
let parsed;
|
|
14065
15727
|
try {
|
|
14066
15728
|
const content = await readFile23(absolutePath, "utf8");
|
|
@@ -14249,6 +15911,7 @@ function validateMessages(messages, location, filePath, errors) {
|
|
|
14249
15911
|
function isObject2(value) {
|
|
14250
15912
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
14251
15913
|
}
|
|
15914
|
+
var CLI_PLACEHOLDERS2 = /* @__PURE__ */ new Set(["PROMPT", "GUIDELINES", "EVAL_ID", "ATTEMPT", "FILES"]);
|
|
14252
15915
|
async function validateTargetsFile(filePath) {
|
|
14253
15916
|
const errors = [];
|
|
14254
15917
|
const absolutePath = path23.resolve(filePath);
|
|
@@ -14269,6 +15932,182 @@ async function validateTargetsFile(filePath) {
|
|
|
14269
15932
|
errors
|
|
14270
15933
|
};
|
|
14271
15934
|
}
|
|
15935
|
+
function validateCliSettings(settings, absolutePath2, location, errors2) {
|
|
15936
|
+
if (!isObject2(settings)) {
|
|
15937
|
+
errors2.push({
|
|
15938
|
+
severity: "error",
|
|
15939
|
+
filePath: absolutePath2,
|
|
15940
|
+
location,
|
|
15941
|
+
message: "CLI provider requires a 'settings' object"
|
|
15942
|
+
});
|
|
15943
|
+
return;
|
|
15944
|
+
}
|
|
15945
|
+
const commandTemplate = settings["command_template"] ?? settings["commandTemplate"];
|
|
15946
|
+
if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
|
|
15947
|
+
errors2.push({
|
|
15948
|
+
severity: "error",
|
|
15949
|
+
filePath: absolutePath2,
|
|
15950
|
+
location: `${location}.commandTemplate`,
|
|
15951
|
+
message: "CLI provider requires 'commandTemplate' as a non-empty string"
|
|
15952
|
+
});
|
|
15953
|
+
} else {
|
|
15954
|
+
recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
|
|
15955
|
+
}
|
|
15956
|
+
const attachmentsFormat = settings["attachments_format"] ?? settings["attachmentsFormat"];
|
|
15957
|
+
if (attachmentsFormat !== void 0 && typeof attachmentsFormat !== "string") {
|
|
15958
|
+
errors2.push({
|
|
15959
|
+
severity: "error",
|
|
15960
|
+
filePath: absolutePath2,
|
|
15961
|
+
location: `${location}.attachmentsFormat`,
|
|
15962
|
+
message: "'attachmentsFormat' must be a string when provided"
|
|
15963
|
+
});
|
|
15964
|
+
}
|
|
15965
|
+
const filesFormat = settings["files_format"] ?? settings["filesFormat"];
|
|
15966
|
+
if (filesFormat !== void 0 && typeof filesFormat !== "string") {
|
|
15967
|
+
errors2.push({
|
|
15968
|
+
severity: "error",
|
|
15969
|
+
filePath: absolutePath2,
|
|
15970
|
+
location: `${location}.filesFormat`,
|
|
15971
|
+
message: "'filesFormat' must be a string when provided"
|
|
15972
|
+
});
|
|
15973
|
+
}
|
|
15974
|
+
const cwd = settings["cwd"];
|
|
15975
|
+
if (cwd !== void 0 && typeof cwd !== "string") {
|
|
15976
|
+
errors2.push({
|
|
15977
|
+
severity: "error",
|
|
15978
|
+
filePath: absolutePath2,
|
|
15979
|
+
location: `${location}.cwd`,
|
|
15980
|
+
message: "'cwd' must be a string when provided"
|
|
15981
|
+
});
|
|
15982
|
+
}
|
|
15983
|
+
const timeoutSeconds = settings["timeout_seconds"] ?? settings["timeoutSeconds"];
|
|
15984
|
+
if (timeoutSeconds !== void 0) {
|
|
15985
|
+
const numericTimeout = Number(timeoutSeconds);
|
|
15986
|
+
if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
|
|
15987
|
+
errors2.push({
|
|
15988
|
+
severity: "error",
|
|
15989
|
+
filePath: absolutePath2,
|
|
15990
|
+
location: `${location}.timeoutSeconds`,
|
|
15991
|
+
message: "'timeoutSeconds' must be a positive number when provided"
|
|
15992
|
+
});
|
|
15993
|
+
}
|
|
15994
|
+
}
|
|
15995
|
+
const envOverrides = settings["env"];
|
|
15996
|
+
if (envOverrides !== void 0) {
|
|
15997
|
+
if (!isObject2(envOverrides)) {
|
|
15998
|
+
errors2.push({
|
|
15999
|
+
severity: "error",
|
|
16000
|
+
filePath: absolutePath2,
|
|
16001
|
+
location: `${location}.env`,
|
|
16002
|
+
message: "'env' must be an object with string values"
|
|
16003
|
+
});
|
|
16004
|
+
} else {
|
|
16005
|
+
for (const [key2, value] of Object.entries(envOverrides)) {
|
|
16006
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
16007
|
+
errors2.push({
|
|
16008
|
+
severity: "error",
|
|
16009
|
+
filePath: absolutePath2,
|
|
16010
|
+
location: `${location}.env.${key2}`,
|
|
16011
|
+
message: `Environment override '${key2}' must be a non-empty string`
|
|
16012
|
+
});
|
|
16013
|
+
}
|
|
16014
|
+
}
|
|
16015
|
+
}
|
|
16016
|
+
}
|
|
16017
|
+
const healthcheck = settings["healthcheck"];
|
|
16018
|
+
if (healthcheck !== void 0) {
|
|
16019
|
+
validateCliHealthcheck(healthcheck, absolutePath2, `${location}.healthcheck`, errors2);
|
|
16020
|
+
}
|
|
16021
|
+
}
|
|
16022
|
+
function validateCliHealthcheck(healthcheck, absolutePath2, location, errors2) {
|
|
16023
|
+
if (!isObject2(healthcheck)) {
|
|
16024
|
+
errors2.push({
|
|
16025
|
+
severity: "error",
|
|
16026
|
+
filePath: absolutePath2,
|
|
16027
|
+
location,
|
|
16028
|
+
message: "'healthcheck' must be an object when provided"
|
|
16029
|
+
});
|
|
16030
|
+
return;
|
|
16031
|
+
}
|
|
16032
|
+
const type = healthcheck["type"];
|
|
16033
|
+
if (type !== "http" && type !== "command") {
|
|
16034
|
+
errors2.push({
|
|
16035
|
+
severity: "error",
|
|
16036
|
+
filePath: absolutePath2,
|
|
16037
|
+
location: `${location}.type`,
|
|
16038
|
+
message: "healthcheck.type must be either 'http' or 'command'"
|
|
16039
|
+
});
|
|
16040
|
+
return;
|
|
16041
|
+
}
|
|
16042
|
+
const timeoutSeconds = healthcheck["timeout_seconds"] ?? healthcheck["timeoutSeconds"];
|
|
16043
|
+
if (timeoutSeconds !== void 0) {
|
|
16044
|
+
const numericTimeout = Number(timeoutSeconds);
|
|
16045
|
+
if (!Number.isFinite(numericTimeout) || numericTimeout <= 0) {
|
|
16046
|
+
errors2.push({
|
|
16047
|
+
severity: "error",
|
|
16048
|
+
filePath: absolutePath2,
|
|
16049
|
+
location: `${location}.timeoutSeconds`,
|
|
16050
|
+
message: "healthcheck.timeoutSeconds must be a positive number when provided"
|
|
16051
|
+
});
|
|
16052
|
+
}
|
|
16053
|
+
}
|
|
16054
|
+
if (type === "http") {
|
|
16055
|
+
const url = healthcheck["url"];
|
|
16056
|
+
if (typeof url !== "string" || url.trim().length === 0) {
|
|
16057
|
+
errors2.push({
|
|
16058
|
+
severity: "error",
|
|
16059
|
+
filePath: absolutePath2,
|
|
16060
|
+
location: `${location}.url`,
|
|
16061
|
+
message: "healthcheck.url must be a non-empty string for http checks"
|
|
16062
|
+
});
|
|
16063
|
+
}
|
|
16064
|
+
return;
|
|
16065
|
+
}
|
|
16066
|
+
const commandTemplate = healthcheck["command_template"] ?? healthcheck["commandTemplate"];
|
|
16067
|
+
if (typeof commandTemplate !== "string" || commandTemplate.trim().length === 0) {
|
|
16068
|
+
errors2.push({
|
|
16069
|
+
severity: "error",
|
|
16070
|
+
filePath: absolutePath2,
|
|
16071
|
+
location: `${location}.commandTemplate`,
|
|
16072
|
+
message: "healthcheck.commandTemplate must be a non-empty string for command checks"
|
|
16073
|
+
});
|
|
16074
|
+
} else {
|
|
16075
|
+
recordUnknownPlaceholders(commandTemplate, absolutePath2, `${location}.commandTemplate`, errors2);
|
|
16076
|
+
}
|
|
16077
|
+
const cwd = healthcheck["cwd"];
|
|
16078
|
+
if (cwd !== void 0 && typeof cwd !== "string") {
|
|
16079
|
+
errors2.push({
|
|
16080
|
+
severity: "error",
|
|
16081
|
+
filePath: absolutePath2,
|
|
16082
|
+
location: `${location}.cwd`,
|
|
16083
|
+
message: "healthcheck.cwd must be a string when provided"
|
|
16084
|
+
});
|
|
16085
|
+
}
|
|
16086
|
+
}
|
|
16087
|
+
function recordUnknownPlaceholders(template, absolutePath2, location, errors2) {
|
|
16088
|
+
const placeholders = extractPlaceholders(template);
|
|
16089
|
+
for (const placeholder of placeholders) {
|
|
16090
|
+
if (!CLI_PLACEHOLDERS2.has(placeholder)) {
|
|
16091
|
+
errors2.push({
|
|
16092
|
+
severity: "error",
|
|
16093
|
+
filePath: absolutePath2,
|
|
16094
|
+
location,
|
|
16095
|
+
message: `Unknown CLI placeholder '{${placeholder}}'. Supported placeholders: ${Array.from(CLI_PLACEHOLDERS2).join(", ")}`
|
|
16096
|
+
});
|
|
16097
|
+
}
|
|
16098
|
+
}
|
|
16099
|
+
}
|
|
16100
|
+
function extractPlaceholders(template) {
|
|
16101
|
+
const matches = template.matchAll(/\{([A-Z_]+)\}/g);
|
|
16102
|
+
const result = [];
|
|
16103
|
+
for (const match of matches) {
|
|
16104
|
+
const placeholder = match[1];
|
|
16105
|
+
if (placeholder) {
|
|
16106
|
+
result.push(placeholder);
|
|
16107
|
+
}
|
|
16108
|
+
}
|
|
16109
|
+
return result;
|
|
16110
|
+
}
|
|
14272
16111
|
if (!isObject2(parsed)) {
|
|
14273
16112
|
errors.push({
|
|
14274
16113
|
severity: "error",
|
|
@@ -14330,6 +16169,7 @@ async function validateTargetsFile(filePath) {
|
|
|
14330
16169
|
});
|
|
14331
16170
|
}
|
|
14332
16171
|
const provider = target["provider"];
|
|
16172
|
+
const providerValue = typeof provider === "string" ? provider.trim().toLowerCase() : void 0;
|
|
14333
16173
|
if (typeof provider !== "string" || provider.trim().length === 0) {
|
|
14334
16174
|
errors.push({
|
|
14335
16175
|
severity: "error",
|
|
@@ -14346,7 +16186,7 @@ async function validateTargetsFile(filePath) {
|
|
|
14346
16186
|
});
|
|
14347
16187
|
}
|
|
14348
16188
|
const settings = target["settings"];
|
|
14349
|
-
if (settings !== void 0 && !isObject2(settings)) {
|
|
16189
|
+
if (providerValue !== "cli" && settings !== void 0 && !isObject2(settings)) {
|
|
14350
16190
|
errors.push({
|
|
14351
16191
|
severity: "error",
|
|
14352
16192
|
filePath: absolutePath,
|
|
@@ -14354,6 +16194,9 @@ async function validateTargetsFile(filePath) {
|
|
|
14354
16194
|
message: "Invalid 'settings' field (must be an object)"
|
|
14355
16195
|
});
|
|
14356
16196
|
}
|
|
16197
|
+
if (providerValue === "cli") {
|
|
16198
|
+
validateCliSettings(settings, absolutePath, `${location}.settings`, errors);
|
|
16199
|
+
}
|
|
14357
16200
|
const judgeTarget = target["judge_target"];
|
|
14358
16201
|
if (judgeTarget !== void 0 && typeof judgeTarget !== "string") {
|
|
14359
16202
|
errors.push({
|
|
@@ -14558,7 +16401,7 @@ async function validateMessagesFileRefs(messages, location, searchRoots, filePat
|
|
|
14558
16401
|
// src/commands/validate/validate-files.ts
|
|
14559
16402
|
import { constants as constants7 } from "node:fs";
|
|
14560
16403
|
import { access as access7, readdir as readdir3, stat as stat3 } from "node:fs/promises";
|
|
14561
|
-
import
|
|
16404
|
+
import path17 from "node:path";
|
|
14562
16405
|
async function validateFiles(paths) {
|
|
14563
16406
|
const filePaths = await expandPaths(paths);
|
|
14564
16407
|
const results = [];
|
|
@@ -14576,7 +16419,7 @@ async function validateFiles(paths) {
|
|
|
14576
16419
|
};
|
|
14577
16420
|
}
|
|
14578
16421
|
async function validateSingleFile(filePath) {
|
|
14579
|
-
const absolutePath =
|
|
16422
|
+
const absolutePath = path17.resolve(filePath);
|
|
14580
16423
|
const fileType = await detectFileType(absolutePath);
|
|
14581
16424
|
if (fileType === "unknown") {
|
|
14582
16425
|
return {
|
|
@@ -14615,7 +16458,7 @@ async function validateSingleFile(filePath) {
|
|
|
14615
16458
|
async function expandPaths(paths) {
|
|
14616
16459
|
const expanded = [];
|
|
14617
16460
|
for (const inputPath of paths) {
|
|
14618
|
-
const absolutePath =
|
|
16461
|
+
const absolutePath = path17.resolve(inputPath);
|
|
14619
16462
|
try {
|
|
14620
16463
|
await access7(absolutePath, constants7.F_OK);
|
|
14621
16464
|
} catch {
|
|
@@ -14639,7 +16482,7 @@ async function findYamlFiles(dirPath) {
|
|
|
14639
16482
|
try {
|
|
14640
16483
|
const entries = await readdir3(dirPath, { withFileTypes: true });
|
|
14641
16484
|
for (const entry of entries) {
|
|
14642
|
-
const fullPath =
|
|
16485
|
+
const fullPath = path17.join(dirPath, entry.name);
|
|
14643
16486
|
if (entry.isDirectory()) {
|
|
14644
16487
|
if (entry.name === "node_modules" || entry.name.startsWith(".")) {
|
|
14645
16488
|
continue;
|
|
@@ -14656,7 +16499,7 @@ async function findYamlFiles(dirPath) {
|
|
|
14656
16499
|
return results;
|
|
14657
16500
|
}
|
|
14658
16501
|
function isYamlFile(filePath) {
|
|
14659
|
-
const ext =
|
|
16502
|
+
const ext = path17.extname(filePath).toLowerCase();
|
|
14660
16503
|
return ext === ".yaml" || ext === ".yml";
|
|
14661
16504
|
}
|
|
14662
16505
|
|
|
@@ -14685,15 +16528,6 @@ function registerValidateCommand(program) {
|
|
|
14685
16528
|
return program;
|
|
14686
16529
|
}
|
|
14687
16530
|
|
|
14688
|
-
// src/commands/status.ts
|
|
14689
|
-
function registerStatusCommand(program) {
|
|
14690
|
-
program.command("status").description("Show the latest AgentV kernel status").action(() => {
|
|
14691
|
-
const kernel = createAgentKernel();
|
|
14692
|
-
console.log(`Kernel status: ${kernel.status}`);
|
|
14693
|
-
});
|
|
14694
|
-
return program;
|
|
14695
|
-
}
|
|
14696
|
-
|
|
14697
16531
|
// src/index.ts
|
|
14698
16532
|
var packageJson = JSON.parse(readFileSync2(new URL("../package.json", import.meta.url), "utf8"));
|
|
14699
16533
|
function createProgram() {
|
|
@@ -14722,4 +16556,4 @@ export {
|
|
|
14722
16556
|
createProgram,
|
|
14723
16557
|
runCli
|
|
14724
16558
|
};
|
|
14725
|
-
//# sourceMappingURL=chunk-
|
|
16559
|
+
//# sourceMappingURL=chunk-THVRLL37.js.map
|