@agentv/core 2.5.2 → 2.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1117 -89
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1111 -88
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -3181,16 +3181,16 @@ async function runClaudeCodeWithTempFiles(options, stdoutFile, stderrFile, exitF
|
|
|
3181
3181
|
let lastStdoutSize = 0;
|
|
3182
3182
|
const readFileIfExists = async (filePath) => {
|
|
3183
3183
|
try {
|
|
3184
|
-
const { readFile:
|
|
3185
|
-
return await
|
|
3184
|
+
const { readFile: readFile10 } = await import("node:fs/promises");
|
|
3185
|
+
return await readFile10(filePath, "utf8");
|
|
3186
3186
|
} catch {
|
|
3187
3187
|
return "";
|
|
3188
3188
|
}
|
|
3189
3189
|
};
|
|
3190
3190
|
const fileExists4 = async (filePath) => {
|
|
3191
3191
|
try {
|
|
3192
|
-
const { access:
|
|
3193
|
-
await
|
|
3192
|
+
const { access: access5 } = await import("node:fs/promises");
|
|
3193
|
+
await access5(filePath);
|
|
3194
3194
|
return true;
|
|
3195
3195
|
} catch {
|
|
3196
3196
|
return false;
|
|
@@ -5396,14 +5396,1037 @@ async function defaultPiRunner(options) {
|
|
|
5396
5396
|
});
|
|
5397
5397
|
}
|
|
5398
5398
|
|
|
5399
|
-
// src/evaluation/providers/vscode.ts
|
|
5399
|
+
// src/evaluation/providers/vscode-provider.ts
|
|
5400
|
+
import path23 from "node:path";
|
|
5401
|
+
|
|
5402
|
+
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
5403
|
+
import { stat as stat3, writeFile as writeFile6 } from "node:fs/promises";
|
|
5404
|
+
import path21 from "node:path";
|
|
5405
|
+
|
|
5406
|
+
// src/evaluation/providers/vscode/utils/fs.ts
|
|
5407
|
+
import { constants as constants3 } from "node:fs";
|
|
5408
|
+
import { access as access3, mkdir as mkdir4, readdir, rm as rm4, stat } from "node:fs/promises";
|
|
5400
5409
|
import path13 from "node:path";
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5410
|
+
async function pathExists(target) {
|
|
5411
|
+
try {
|
|
5412
|
+
await access3(target, constants3.F_OK);
|
|
5413
|
+
return true;
|
|
5414
|
+
} catch {
|
|
5415
|
+
return false;
|
|
5416
|
+
}
|
|
5417
|
+
}
|
|
5418
|
+
async function ensureDir(target) {
|
|
5419
|
+
await mkdir4(target, { recursive: true });
|
|
5420
|
+
}
|
|
5421
|
+
async function readDirEntries(target) {
|
|
5422
|
+
const entries = await readdir(target, { withFileTypes: true });
|
|
5423
|
+
return entries.map((entry) => ({
|
|
5424
|
+
name: entry.name,
|
|
5425
|
+
absolutePath: path13.join(target, entry.name),
|
|
5426
|
+
isDirectory: entry.isDirectory()
|
|
5427
|
+
}));
|
|
5428
|
+
}
|
|
5429
|
+
async function removeIfExists(target) {
|
|
5430
|
+
try {
|
|
5431
|
+
await rm4(target, { force: true, recursive: false });
|
|
5432
|
+
} catch (error) {
|
|
5433
|
+
if (error.code !== "ENOENT") {
|
|
5434
|
+
throw error;
|
|
5435
|
+
}
|
|
5436
|
+
}
|
|
5437
|
+
}
|
|
5438
|
+
|
|
5439
|
+
// src/evaluation/providers/vscode/utils/path.ts
|
|
5440
|
+
import path14 from "node:path";
|
|
5441
|
+
function pathToFileUri2(filePath) {
|
|
5442
|
+
const absolutePath = path14.isAbsolute(filePath) ? filePath : path14.resolve(filePath);
|
|
5443
|
+
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
5444
|
+
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
5445
|
+
return `file:///${normalizedPath}`;
|
|
5446
|
+
}
|
|
5447
|
+
return `file://${normalizedPath}`;
|
|
5448
|
+
}
|
|
5449
|
+
|
|
5450
|
+
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
5451
|
+
import path15 from "node:path";
|
|
5452
|
+
|
|
5453
|
+
// src/evaluation/providers/vscode/utils/template.ts
|
|
5454
|
+
function renderTemplate2(content, variables) {
|
|
5455
|
+
if (!content) {
|
|
5456
|
+
return content;
|
|
5457
|
+
}
|
|
5458
|
+
const variableLookup = /* @__PURE__ */ new Map();
|
|
5459
|
+
for (const [key, value] of Object.entries(variables)) {
|
|
5460
|
+
variableLookup.set(key.toLowerCase(), value);
|
|
5461
|
+
}
|
|
5462
|
+
const referencedVariables = /* @__PURE__ */ new Set();
|
|
5463
|
+
const result = content.replace(/\{\{([a-zA-Z_]+)\}\}/gi, (match, variableName) => {
|
|
5464
|
+
const lowerCaseKey = variableName.toLowerCase();
|
|
5465
|
+
referencedVariables.add(lowerCaseKey);
|
|
5466
|
+
if (!variableLookup.has(lowerCaseKey)) {
|
|
5467
|
+
throw new Error(
|
|
5468
|
+
`Template variable '${variableName}' is not provided in the variables object`
|
|
5469
|
+
);
|
|
5470
|
+
}
|
|
5471
|
+
return variableLookup.get(lowerCaseKey);
|
|
5472
|
+
});
|
|
5473
|
+
return result;
|
|
5474
|
+
}
|
|
5475
|
+
|
|
5476
|
+
// src/evaluation/providers/vscode/dispatch/templates.ts
|
|
5477
|
+
var DEFAULT_REQUEST_TEMPLATE = `[[ ## task ## ]]
|
|
5478
|
+
|
|
5479
|
+
{{userQuery}}
|
|
5480
|
+
|
|
5481
|
+
[[ ## system_instructions ## ]]
|
|
5482
|
+
|
|
5483
|
+
**IMPORTANT**: Follow these exact steps:
|
|
5484
|
+
1. Create and write your complete response to: {{responseFileTmp}}
|
|
5485
|
+
2. When completely finished, run these PowerShell commands to signal completion:
|
|
5486
|
+
\`\`\`
|
|
5487
|
+
Move-Item -LiteralPath '{{responseFileTmp}}' -Destination '{{responseFileFinal}}'
|
|
5488
|
+
if (Test-Path subagent.lock) { del subagent.lock }
|
|
5489
|
+
\`\`\`
|
|
5490
|
+
|
|
5491
|
+
Do not proceed to step 2 until your response is completely written to the temporary file.
|
|
5492
|
+
`;
|
|
5493
|
+
var DEFAULT_BATCH_REQUEST_TEMPLATE = `[[ ## task ## ]]
|
|
5494
|
+
|
|
5495
|
+
{{userQuery}}
|
|
5496
|
+
|
|
5497
|
+
[[ ## system_instructions ## ]]
|
|
5498
|
+
|
|
5499
|
+
**IMPORTANT**: Follow these exact steps:
|
|
5500
|
+
1. Create and write your complete response to: {{responseFileTmp}}
|
|
5501
|
+
2. When completely finished and the response is stable, rename it to: {{responseFileFinal}}
|
|
5502
|
+
3. Do not unlock the workspace from this request; batch orchestration will handle unlocking after all responses are ready.
|
|
5503
|
+
`;
|
|
5504
|
+
var DEFAULT_BATCH_ORCHESTRATOR_TEMPLATE = `MANDATORY: Run #runSubagent tool in your Available Actions for each request file to process them in isolated contexts.
|
|
5505
|
+
DO NOT read the request files yourself - only pass the file paths to each subagent:
|
|
5506
|
+
|
|
5507
|
+
{{requestFiles}}
|
|
5508
|
+
|
|
5509
|
+
After ALL queries complete, verify all responses exist and unlock:
|
|
5510
|
+
|
|
5511
|
+
\`\`\`powershell
|
|
5512
|
+
$responses = @({{responseList}})
|
|
5513
|
+
$missing = $responses | Where-Object { -not (Test-Path "messages/$_") }
|
|
5514
|
+
if ($missing.Count -eq 0) { del subagent.lock }
|
|
5515
|
+
\`\`\`
|
|
5516
|
+
`;
|
|
5517
|
+
|
|
5518
|
+
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
5519
|
+
function loadDefaultRequestTemplate() {
|
|
5520
|
+
return DEFAULT_REQUEST_TEMPLATE;
|
|
5521
|
+
}
|
|
5522
|
+
function loadDefaultBatchRequestTemplate() {
|
|
5523
|
+
return DEFAULT_BATCH_REQUEST_TEMPLATE;
|
|
5524
|
+
}
|
|
5525
|
+
function loadDefaultBatchOrchestratorTemplate() {
|
|
5526
|
+
return DEFAULT_BATCH_ORCHESTRATOR_TEMPLATE;
|
|
5527
|
+
}
|
|
5528
|
+
function createRequestPrompt(userQuery, responseFileTmp, responseFileFinal, templateContent) {
|
|
5529
|
+
return renderTemplate2(templateContent, {
|
|
5530
|
+
userQuery,
|
|
5531
|
+
responseFileTmp,
|
|
5532
|
+
responseFileFinal
|
|
5533
|
+
});
|
|
5534
|
+
}
|
|
5535
|
+
function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal, templateContent) {
|
|
5536
|
+
return renderTemplate2(templateContent, {
|
|
5537
|
+
userQuery,
|
|
5538
|
+
responseFileTmp,
|
|
5539
|
+
responseFileFinal
|
|
5540
|
+
});
|
|
5541
|
+
}
|
|
5542
|
+
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
5543
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path15.basename(file)}`).join("\n");
|
|
5544
|
+
const responseList = responseFiles.map((file) => `"${path15.basename(file)}"`).join(", ");
|
|
5545
|
+
return renderTemplate2(templateContent, {
|
|
5546
|
+
requestFiles: requestLines,
|
|
5547
|
+
responseList
|
|
5548
|
+
});
|
|
5549
|
+
}
|
|
5550
|
+
|
|
5551
|
+
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
5552
|
+
import { readFile as readFile7 } from "node:fs/promises";
|
|
5553
|
+
import path16 from "node:path";
|
|
5554
|
+
|
|
5555
|
+
// src/evaluation/providers/vscode/utils/time.ts
|
|
5556
|
+
function sleep2(ms) {
|
|
5557
|
+
return new Promise((resolve) => {
|
|
5558
|
+
setTimeout(resolve, ms);
|
|
5559
|
+
});
|
|
5560
|
+
}
|
|
5561
|
+
|
|
5562
|
+
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
5563
|
+
async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, silent = false) {
|
|
5564
|
+
if (!silent) {
|
|
5565
|
+
console.error(`waiting for agent to finish: ${responseFileFinal}`);
|
|
5566
|
+
}
|
|
5567
|
+
try {
|
|
5568
|
+
while (!await pathExists(responseFileFinal)) {
|
|
5569
|
+
await sleep2(pollInterval);
|
|
5570
|
+
}
|
|
5571
|
+
} catch (error) {
|
|
5572
|
+
if (error.code === "ENOENT") {
|
|
5573
|
+
return false;
|
|
5574
|
+
}
|
|
5575
|
+
throw error;
|
|
5576
|
+
}
|
|
5577
|
+
let attempts = 0;
|
|
5578
|
+
const maxAttempts = 10;
|
|
5579
|
+
while (attempts < maxAttempts) {
|
|
5580
|
+
try {
|
|
5581
|
+
const content = await readFile7(responseFileFinal, { encoding: "utf8" });
|
|
5582
|
+
if (!silent) {
|
|
5583
|
+
process.stdout.write(`${content}
|
|
5584
|
+
`);
|
|
5585
|
+
}
|
|
5586
|
+
return true;
|
|
5587
|
+
} catch (error) {
|
|
5588
|
+
attempts += 1;
|
|
5589
|
+
if (error.code !== "EBUSY" || attempts >= maxAttempts) {
|
|
5590
|
+
if (!silent) {
|
|
5591
|
+
console.error(`error: failed to read agent response: ${error.message}`);
|
|
5592
|
+
}
|
|
5593
|
+
return false;
|
|
5594
|
+
}
|
|
5595
|
+
await sleep2(pollInterval);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
return false;
|
|
5599
|
+
}
|
|
5600
|
+
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
|
|
5601
|
+
if (!silent) {
|
|
5602
|
+
const fileList = responseFilesFinal.map((file) => path16.basename(file)).join(", ");
|
|
5603
|
+
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
5604
|
+
}
|
|
5605
|
+
try {
|
|
5606
|
+
const pending = new Set(responseFilesFinal);
|
|
5607
|
+
while (pending.size > 0) {
|
|
5608
|
+
for (const file of [...pending]) {
|
|
5609
|
+
if (await pathExists(file)) {
|
|
5610
|
+
pending.delete(file);
|
|
5611
|
+
}
|
|
5612
|
+
}
|
|
5613
|
+
if (pending.size > 0) {
|
|
5614
|
+
await sleep2(pollInterval);
|
|
5615
|
+
}
|
|
5616
|
+
}
|
|
5617
|
+
} catch (error) {
|
|
5618
|
+
if (error.code === "ENOENT") {
|
|
5619
|
+
return false;
|
|
5620
|
+
}
|
|
5621
|
+
throw error;
|
|
5622
|
+
}
|
|
5623
|
+
for (const file of responseFilesFinal) {
|
|
5624
|
+
let attempts = 0;
|
|
5625
|
+
const maxAttempts = 10;
|
|
5626
|
+
while (attempts < maxAttempts) {
|
|
5627
|
+
try {
|
|
5628
|
+
const content = await readFile7(file, { encoding: "utf8" });
|
|
5629
|
+
if (!silent) {
|
|
5630
|
+
process.stdout.write(`${content}
|
|
5631
|
+
`);
|
|
5632
|
+
}
|
|
5633
|
+
break;
|
|
5634
|
+
} catch (error) {
|
|
5635
|
+
attempts += 1;
|
|
5636
|
+
if (error.code !== "EBUSY" || attempts >= maxAttempts) {
|
|
5637
|
+
if (!silent) {
|
|
5638
|
+
console.error(`error: failed to read agent response: ${error.message}`);
|
|
5639
|
+
}
|
|
5640
|
+
return false;
|
|
5641
|
+
}
|
|
5642
|
+
await sleep2(pollInterval);
|
|
5643
|
+
}
|
|
5644
|
+
}
|
|
5645
|
+
}
|
|
5646
|
+
return true;
|
|
5647
|
+
}
|
|
5648
|
+
|
|
5649
|
+
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
5650
|
+
import { exec, spawn as spawn4 } from "node:child_process";
|
|
5651
|
+
import { mkdir as mkdir5, writeFile as writeFile4 } from "node:fs/promises";
|
|
5652
|
+
import path18 from "node:path";
|
|
5653
|
+
import { promisify as promisify3 } from "node:util";
|
|
5654
|
+
|
|
5655
|
+
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
5656
|
+
import os2 from "node:os";
|
|
5657
|
+
import path17 from "node:path";
|
|
5658
|
+
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
5659
|
+
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
5660
|
+
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
5661
|
+
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
5662
|
+
return path17.join(os2.homedir(), ".agentv", "subagents", folder);
|
|
5663
|
+
}
|
|
5664
|
+
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
5665
|
+
|
|
5666
|
+
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
5667
|
+
var execAsync3 = promisify3(exec);
|
|
5668
|
+
var DEFAULT_WAKEUP_CONTENT = `---
|
|
5669
|
+
description: 'Wake-up Signal'
|
|
5670
|
+
model: Grok Code Fast 1 (copilot)
|
|
5671
|
+
---`;
|
|
5672
|
+
async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
5673
|
+
try {
|
|
5674
|
+
const { stdout } = await execAsync3(`${vscodeCmd} --status`, {
|
|
5675
|
+
timeout: 1e4,
|
|
5676
|
+
windowsHide: true
|
|
5677
|
+
});
|
|
5678
|
+
return stdout.includes(workspaceName);
|
|
5679
|
+
} catch {
|
|
5680
|
+
return false;
|
|
5681
|
+
}
|
|
5682
|
+
}
|
|
5683
|
+
async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir, vscodeCmd, pollInterval = 1, timeout = 60) {
|
|
5684
|
+
const alreadyOpen = await checkWorkspaceOpened(workspaceName, vscodeCmd);
|
|
5685
|
+
if (alreadyOpen) {
|
|
5686
|
+
spawn4(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
5687
|
+
return true;
|
|
5688
|
+
}
|
|
5689
|
+
const aliveFile = path18.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
5690
|
+
await removeIfExists(aliveFile);
|
|
5691
|
+
const githubAgentsDir = path18.join(subagentDir, ".github", "agents");
|
|
5692
|
+
await mkdir5(githubAgentsDir, { recursive: true });
|
|
5693
|
+
const wakeupDst = path18.join(githubAgentsDir, "wakeup.md");
|
|
5694
|
+
await writeFile4(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
5695
|
+
spawn4(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
5696
|
+
await sleep2(100);
|
|
5697
|
+
const wakeupChatId = "wakeup";
|
|
5698
|
+
const chatArgs = [
|
|
5699
|
+
"-r",
|
|
5700
|
+
"chat",
|
|
5701
|
+
"-m",
|
|
5702
|
+
wakeupChatId,
|
|
5703
|
+
`create a file named .alive in the ${path18.basename(subagentDir)} folder`
|
|
5704
|
+
];
|
|
5705
|
+
spawn4(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5706
|
+
const start = Date.now();
|
|
5707
|
+
while (!await pathExists(aliveFile)) {
|
|
5708
|
+
if (Date.now() - start > timeout * 1e3) {
|
|
5709
|
+
console.error(`warning: Workspace readiness timeout after ${timeout}s`);
|
|
5710
|
+
return false;
|
|
5711
|
+
}
|
|
5712
|
+
await sleep2(pollInterval * 1e3);
|
|
5713
|
+
}
|
|
5714
|
+
return true;
|
|
5715
|
+
}
|
|
5716
|
+
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
5717
|
+
try {
|
|
5718
|
+
const workspacePath = path18.join(subagentDir, `${path18.basename(subagentDir)}.code-workspace`);
|
|
5719
|
+
const messagesDir = path18.join(subagentDir, "messages");
|
|
5720
|
+
await mkdir5(messagesDir, { recursive: true });
|
|
5721
|
+
const reqFile = path18.join(messagesDir, `${timestamp}_req.md`);
|
|
5722
|
+
await writeFile4(reqFile, requestInstructions, { encoding: "utf8" });
|
|
5723
|
+
const reqUri = pathToFileUri2(reqFile);
|
|
5724
|
+
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
5725
|
+
for (const attachment of attachmentPaths) {
|
|
5726
|
+
chatArgs.push("-a", attachment);
|
|
5727
|
+
}
|
|
5728
|
+
chatArgs.push("-a", reqFile);
|
|
5729
|
+
chatArgs.push(`Follow instructions in [${path18.basename(reqFile)}](${reqUri})`);
|
|
5730
|
+
const workspaceReady = await ensureWorkspaceFocused(
|
|
5731
|
+
workspacePath,
|
|
5732
|
+
path18.basename(subagentDir),
|
|
5733
|
+
subagentDir,
|
|
5734
|
+
vscodeCmd
|
|
5735
|
+
);
|
|
5736
|
+
if (!workspaceReady) {
|
|
5737
|
+
console.error("warning: Workspace may not be fully ready");
|
|
5738
|
+
}
|
|
5739
|
+
await sleep2(500);
|
|
5740
|
+
spawn4(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5741
|
+
return true;
|
|
5742
|
+
} catch (error) {
|
|
5743
|
+
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
5744
|
+
return false;
|
|
5745
|
+
}
|
|
5746
|
+
}
|
|
5747
|
+
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
5748
|
+
try {
|
|
5749
|
+
const workspacePath = path18.join(subagentDir, `${path18.basename(subagentDir)}.code-workspace`);
|
|
5750
|
+
const messagesDir = path18.join(subagentDir, "messages");
|
|
5751
|
+
await mkdir5(messagesDir, { recursive: true });
|
|
5752
|
+
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
5753
|
+
for (const attachment of attachmentPaths) {
|
|
5754
|
+
chatArgs.push("-a", attachment);
|
|
5755
|
+
}
|
|
5756
|
+
chatArgs.push(chatInstruction);
|
|
5757
|
+
const workspaceReady = await ensureWorkspaceFocused(
|
|
5758
|
+
workspacePath,
|
|
5759
|
+
path18.basename(subagentDir),
|
|
5760
|
+
subagentDir,
|
|
5761
|
+
vscodeCmd
|
|
5762
|
+
);
|
|
5763
|
+
if (!workspaceReady) {
|
|
5764
|
+
console.error("warning: Workspace may not be fully ready");
|
|
5765
|
+
}
|
|
5766
|
+
await sleep2(500);
|
|
5767
|
+
spawn4(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5768
|
+
return true;
|
|
5769
|
+
} catch (error) {
|
|
5770
|
+
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
5771
|
+
return false;
|
|
5772
|
+
}
|
|
5773
|
+
}
|
|
5774
|
+
|
|
5775
|
+
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
5776
|
+
import { copyFile, mkdir as mkdir6, readFile as readFile8, readdir as readdir2, stat as stat2, writeFile as writeFile5 } from "node:fs/promises";
|
|
5777
|
+
import path20 from "node:path";
|
|
5778
|
+
|
|
5779
|
+
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
5780
|
+
import path19 from "node:path";
|
|
5781
|
+
import JSON5 from "json5";
|
|
5782
|
+
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
5783
|
+
let workspace;
|
|
5784
|
+
try {
|
|
5785
|
+
workspace = JSON5.parse(workspaceContent);
|
|
5786
|
+
} catch (error) {
|
|
5787
|
+
throw new Error(`Invalid workspace JSON: ${error.message}`);
|
|
5788
|
+
}
|
|
5789
|
+
if (!workspace.folders) {
|
|
5790
|
+
throw new Error("Workspace file must contain a 'folders' array");
|
|
5791
|
+
}
|
|
5792
|
+
if (!Array.isArray(workspace.folders)) {
|
|
5793
|
+
throw new Error("Workspace 'folders' must be an array");
|
|
5794
|
+
}
|
|
5795
|
+
const transformedFolders = workspace.folders.map((folder) => {
|
|
5796
|
+
const folderPath = folder.path;
|
|
5797
|
+
if (path19.isAbsolute(folderPath)) {
|
|
5798
|
+
return folder;
|
|
5799
|
+
}
|
|
5800
|
+
const absolutePath = path19.resolve(templateDir, folderPath);
|
|
5801
|
+
return {
|
|
5802
|
+
...folder,
|
|
5803
|
+
path: absolutePath
|
|
5804
|
+
};
|
|
5805
|
+
});
|
|
5806
|
+
const updatedFolders = [{ path: "." }, ...transformedFolders];
|
|
5807
|
+
let transformedSettings = workspace.settings;
|
|
5808
|
+
if (workspace.settings) {
|
|
5809
|
+
transformedSettings = {
|
|
5810
|
+
...workspace.settings
|
|
5811
|
+
};
|
|
5812
|
+
const chatSettingsKeys = [
|
|
5813
|
+
"chat.promptFilesLocations",
|
|
5814
|
+
"chat.instructionsFilesLocations",
|
|
5815
|
+
"chat.modeFilesLocations"
|
|
5816
|
+
];
|
|
5817
|
+
for (const settingKey of chatSettingsKeys) {
|
|
5818
|
+
const locationMap = workspace.settings[settingKey];
|
|
5819
|
+
if (locationMap && typeof locationMap === "object") {
|
|
5820
|
+
const transformedMap = {};
|
|
5821
|
+
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
5822
|
+
const isAbsolute = path19.isAbsolute(locationPath);
|
|
5823
|
+
if (isAbsolute) {
|
|
5824
|
+
transformedMap[locationPath] = value;
|
|
5825
|
+
} else {
|
|
5826
|
+
const firstGlobIndex = locationPath.search(/[*]/);
|
|
5827
|
+
if (firstGlobIndex === -1) {
|
|
5828
|
+
const resolvedPath = path19.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
5829
|
+
transformedMap[resolvedPath] = value;
|
|
5830
|
+
} else {
|
|
5831
|
+
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
5832
|
+
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
5833
|
+
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
5834
|
+
const resolvedPath = (path19.resolve(templateDir, basePath) + patternPath).replace(
|
|
5835
|
+
/\\/g,
|
|
5836
|
+
"/"
|
|
5837
|
+
);
|
|
5838
|
+
transformedMap[resolvedPath] = value;
|
|
5839
|
+
}
|
|
5840
|
+
}
|
|
5841
|
+
}
|
|
5842
|
+
transformedSettings[settingKey] = transformedMap;
|
|
5843
|
+
}
|
|
5844
|
+
}
|
|
5845
|
+
}
|
|
5846
|
+
const transformedWorkspace = {
|
|
5847
|
+
...workspace,
|
|
5848
|
+
folders: updatedFolders,
|
|
5849
|
+
settings: transformedSettings
|
|
5850
|
+
};
|
|
5851
|
+
return JSON.stringify(transformedWorkspace, null, 2);
|
|
5852
|
+
}
|
|
5853
|
+
|
|
5854
|
+
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
5855
|
+
var DEFAULT_WORKSPACE_TEMPLATE = {
|
|
5856
|
+
folders: [
|
|
5857
|
+
{
|
|
5858
|
+
path: "."
|
|
5859
|
+
}
|
|
5860
|
+
]
|
|
5861
|
+
};
|
|
5862
|
+
function getSubagentRoot(vscodeCmd = "code") {
|
|
5863
|
+
return getDefaultSubagentRoot(vscodeCmd);
|
|
5864
|
+
}
|
|
5865
|
+
async function findUnlockedSubagent(subagentRoot) {
|
|
5866
|
+
if (!await pathExists(subagentRoot)) {
|
|
5867
|
+
return null;
|
|
5868
|
+
}
|
|
5869
|
+
const entries = await readDirEntries(subagentRoot);
|
|
5870
|
+
const subagents = entries.filter((entry) => entry.isDirectory && entry.name.startsWith("subagent-")).map((entry) => ({
|
|
5871
|
+
absolutePath: entry.absolutePath,
|
|
5872
|
+
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
5873
|
+
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
5874
|
+
for (const subagent of subagents) {
|
|
5875
|
+
const lockFile = path20.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
5876
|
+
if (!await pathExists(lockFile)) {
|
|
5877
|
+
return subagent.absolutePath;
|
|
5878
|
+
}
|
|
5879
|
+
}
|
|
5880
|
+
return null;
|
|
5881
|
+
}
|
|
5882
|
+
async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
5883
|
+
let workspaceContent;
|
|
5884
|
+
if (workspaceTemplate) {
|
|
5885
|
+
const workspaceSrc = path20.resolve(workspaceTemplate);
|
|
5886
|
+
if (!await pathExists(workspaceSrc)) {
|
|
5887
|
+
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
5888
|
+
}
|
|
5889
|
+
const stats = await stat2(workspaceSrc);
|
|
5890
|
+
if (!stats.isFile()) {
|
|
5891
|
+
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
5892
|
+
}
|
|
5893
|
+
const templateText = await readFile8(workspaceSrc, "utf8");
|
|
5894
|
+
workspaceContent = JSON.parse(templateText);
|
|
5895
|
+
} else {
|
|
5896
|
+
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
5897
|
+
}
|
|
5898
|
+
const workspaceName = `${path20.basename(subagentDir)}.code-workspace`;
|
|
5899
|
+
const workspaceDst = path20.join(subagentDir, workspaceName);
|
|
5900
|
+
const templateDir = workspaceTemplate ? path20.dirname(path20.resolve(workspaceTemplate)) : subagentDir;
|
|
5901
|
+
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
5902
|
+
const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
5903
|
+
await writeFile5(workspaceDst, transformedContent, "utf8");
|
|
5904
|
+
const messagesDir = path20.join(subagentDir, "messages");
|
|
5905
|
+
await mkdir6(messagesDir, { recursive: true });
|
|
5906
|
+
return { workspace: workspaceDst, messagesDir };
|
|
5907
|
+
}
|
|
5908
|
+
async function createSubagentLock(subagentDir) {
|
|
5909
|
+
const messagesDir = path20.join(subagentDir, "messages");
|
|
5910
|
+
if (await pathExists(messagesDir)) {
|
|
5911
|
+
const files = await readdir2(messagesDir);
|
|
5912
|
+
await Promise.all(
|
|
5913
|
+
files.map(async (file) => {
|
|
5914
|
+
const target = path20.join(messagesDir, file);
|
|
5915
|
+
await removeIfExists(target);
|
|
5916
|
+
})
|
|
5917
|
+
);
|
|
5918
|
+
}
|
|
5919
|
+
const githubAgentsDir = path20.join(subagentDir, ".github", "agents");
|
|
5920
|
+
if (await pathExists(githubAgentsDir)) {
|
|
5921
|
+
const agentFiles = await readdir2(githubAgentsDir);
|
|
5922
|
+
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
5923
|
+
await Promise.all(
|
|
5924
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path20.join(githubAgentsDir, file)))
|
|
5925
|
+
);
|
|
5926
|
+
}
|
|
5927
|
+
const lockFile = path20.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
5928
|
+
await writeFile5(lockFile, "", { encoding: "utf8" });
|
|
5929
|
+
return lockFile;
|
|
5930
|
+
}
|
|
5931
|
+
async function removeSubagentLock(subagentDir) {
|
|
5932
|
+
const lockFile = path20.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
5933
|
+
await removeIfExists(lockFile);
|
|
5934
|
+
}
|
|
5935
|
+
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
|
|
5936
|
+
if (dryRun) {
|
|
5937
|
+
return 0;
|
|
5938
|
+
}
|
|
5939
|
+
try {
|
|
5940
|
+
await copyAgentConfig(subagentDir, workspaceTemplate);
|
|
5941
|
+
} catch (error) {
|
|
5942
|
+
console.error(`error: ${error.message}`);
|
|
5943
|
+
return 1;
|
|
5944
|
+
}
|
|
5945
|
+
try {
|
|
5946
|
+
await createSubagentLock(subagentDir);
|
|
5947
|
+
} catch (error) {
|
|
5948
|
+
console.error(`error: Failed to create subagent lock: ${error.message}`);
|
|
5949
|
+
return 1;
|
|
5950
|
+
}
|
|
5951
|
+
if (promptFile) {
|
|
5952
|
+
const githubAgentsDir = path20.join(subagentDir, ".github", "agents");
|
|
5953
|
+
await mkdir6(githubAgentsDir, { recursive: true });
|
|
5954
|
+
const agentFile = path20.join(githubAgentsDir, `${chatId}.md`);
|
|
5955
|
+
try {
|
|
5956
|
+
await copyFile(promptFile, agentFile);
|
|
5957
|
+
} catch (error) {
|
|
5958
|
+
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
5959
|
+
return 1;
|
|
5960
|
+
}
|
|
5961
|
+
}
|
|
5962
|
+
return 0;
|
|
5963
|
+
}
|
|
5964
|
+
|
|
5965
|
+
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
5966
|
+
function generateTimestamp() {
|
|
5967
|
+
return (/* @__PURE__ */ new Date()).toISOString().replace(/[-:TZ.]/g, "").slice(0, 14);
|
|
5968
|
+
}
|
|
5969
|
+
async function resolvePromptFile(promptFile) {
|
|
5970
|
+
if (!promptFile) {
|
|
5971
|
+
return void 0;
|
|
5972
|
+
}
|
|
5973
|
+
const resolvedPrompt = path21.resolve(promptFile);
|
|
5974
|
+
if (!await pathExists(resolvedPrompt)) {
|
|
5975
|
+
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
5976
|
+
}
|
|
5977
|
+
const promptStats = await stat3(resolvedPrompt);
|
|
5978
|
+
if (!promptStats.isFile()) {
|
|
5979
|
+
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
5980
|
+
}
|
|
5981
|
+
return resolvedPrompt;
|
|
5982
|
+
}
|
|
5983
|
+
async function resolveAttachments(extraAttachments) {
|
|
5984
|
+
if (!extraAttachments) {
|
|
5985
|
+
return [];
|
|
5986
|
+
}
|
|
5987
|
+
const resolved = [];
|
|
5988
|
+
for (const attachment of extraAttachments) {
|
|
5989
|
+
const resolvedPath = path21.resolve(attachment);
|
|
5990
|
+
if (!await pathExists(resolvedPath)) {
|
|
5991
|
+
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
5992
|
+
}
|
|
5993
|
+
resolved.push(resolvedPath);
|
|
5994
|
+
}
|
|
5995
|
+
return resolved;
|
|
5996
|
+
}
|
|
5997
|
+
async function dispatchAgentSession(options) {
|
|
5998
|
+
const {
|
|
5999
|
+
userQuery,
|
|
6000
|
+
promptFile,
|
|
6001
|
+
requestTemplate,
|
|
6002
|
+
extraAttachments,
|
|
6003
|
+
workspaceTemplate,
|
|
6004
|
+
dryRun = false,
|
|
6005
|
+
wait = true,
|
|
6006
|
+
vscodeCmd = "code",
|
|
6007
|
+
subagentRoot,
|
|
6008
|
+
silent = false
|
|
6009
|
+
} = options;
|
|
6010
|
+
try {
|
|
6011
|
+
let resolvedPrompt;
|
|
6012
|
+
try {
|
|
6013
|
+
resolvedPrompt = await resolvePromptFile(promptFile);
|
|
6014
|
+
} catch (error) {
|
|
6015
|
+
return {
|
|
6016
|
+
exitCode: 1,
|
|
6017
|
+
error: error.message
|
|
6018
|
+
};
|
|
6019
|
+
}
|
|
6020
|
+
const templateContent = requestTemplate ?? loadDefaultRequestTemplate();
|
|
6021
|
+
const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
|
|
6022
|
+
const subagentDir = await findUnlockedSubagent(subagentRootPath);
|
|
6023
|
+
if (!subagentDir) {
|
|
6024
|
+
return {
|
|
6025
|
+
exitCode: 1,
|
|
6026
|
+
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
6027
|
+
};
|
|
6028
|
+
}
|
|
6029
|
+
const subagentName = path21.basename(subagentDir);
|
|
6030
|
+
const chatId = Math.random().toString(16).slice(2, 10);
|
|
6031
|
+
const preparationResult = await prepareSubagentDirectory(
|
|
6032
|
+
subagentDir,
|
|
6033
|
+
resolvedPrompt,
|
|
6034
|
+
chatId,
|
|
6035
|
+
workspaceTemplate,
|
|
6036
|
+
dryRun
|
|
6037
|
+
);
|
|
6038
|
+
if (preparationResult !== 0) {
|
|
6039
|
+
return {
|
|
6040
|
+
exitCode: preparationResult,
|
|
6041
|
+
subagentName,
|
|
6042
|
+
error: "Failed to prepare subagent workspace"
|
|
6043
|
+
};
|
|
6044
|
+
}
|
|
6045
|
+
let attachments;
|
|
6046
|
+
try {
|
|
6047
|
+
attachments = await resolveAttachments(extraAttachments);
|
|
6048
|
+
} catch (attachmentError) {
|
|
6049
|
+
return {
|
|
6050
|
+
exitCode: 1,
|
|
6051
|
+
subagentName,
|
|
6052
|
+
error: attachmentError.message
|
|
6053
|
+
};
|
|
6054
|
+
}
|
|
6055
|
+
const timestamp = generateTimestamp();
|
|
6056
|
+
const messagesDir = path21.join(subagentDir, "messages");
|
|
6057
|
+
const responseFileTmp = path21.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
6058
|
+
const responseFileFinal = path21.join(messagesDir, `${timestamp}_res.md`);
|
|
6059
|
+
const requestInstructions = createRequestPrompt(
|
|
6060
|
+
userQuery,
|
|
6061
|
+
responseFileTmp,
|
|
6062
|
+
responseFileFinal,
|
|
6063
|
+
templateContent
|
|
6064
|
+
);
|
|
6065
|
+
if (dryRun) {
|
|
6066
|
+
return {
|
|
6067
|
+
exitCode: 0,
|
|
6068
|
+
subagentName,
|
|
6069
|
+
responseFile: responseFileFinal,
|
|
6070
|
+
tempFile: responseFileTmp
|
|
6071
|
+
};
|
|
6072
|
+
}
|
|
6073
|
+
const launchSuccess = await launchVsCodeWithChat(
|
|
6074
|
+
subagentDir,
|
|
6075
|
+
chatId,
|
|
6076
|
+
attachments,
|
|
6077
|
+
requestInstructions,
|
|
6078
|
+
timestamp,
|
|
6079
|
+
vscodeCmd
|
|
6080
|
+
);
|
|
6081
|
+
if (!launchSuccess) {
|
|
6082
|
+
return {
|
|
6083
|
+
exitCode: 1,
|
|
6084
|
+
subagentName,
|
|
6085
|
+
responseFile: responseFileFinal,
|
|
6086
|
+
tempFile: responseFileTmp,
|
|
6087
|
+
error: "Failed to launch VS Code for subagent session"
|
|
6088
|
+
};
|
|
6089
|
+
}
|
|
6090
|
+
if (!wait) {
|
|
6091
|
+
return {
|
|
6092
|
+
exitCode: 0,
|
|
6093
|
+
subagentName,
|
|
6094
|
+
responseFile: responseFileFinal,
|
|
6095
|
+
tempFile: responseFileTmp
|
|
6096
|
+
};
|
|
6097
|
+
}
|
|
6098
|
+
const received = await waitForResponseOutput(responseFileFinal, 1e3, silent);
|
|
6099
|
+
if (!received) {
|
|
6100
|
+
return {
|
|
6101
|
+
exitCode: 1,
|
|
6102
|
+
subagentName,
|
|
6103
|
+
responseFile: responseFileFinal,
|
|
6104
|
+
tempFile: responseFileTmp,
|
|
6105
|
+
error: "Timed out waiting for agent response"
|
|
6106
|
+
};
|
|
6107
|
+
}
|
|
6108
|
+
await removeSubagentLock(subagentDir);
|
|
6109
|
+
return {
|
|
6110
|
+
exitCode: 0,
|
|
6111
|
+
subagentName,
|
|
6112
|
+
responseFile: responseFileFinal,
|
|
6113
|
+
tempFile: responseFileTmp
|
|
6114
|
+
};
|
|
6115
|
+
} catch (error) {
|
|
6116
|
+
return {
|
|
6117
|
+
exitCode: 1,
|
|
6118
|
+
error: error.message
|
|
6119
|
+
};
|
|
6120
|
+
}
|
|
6121
|
+
}
|
|
6122
|
+
async function dispatchBatchAgent(options) {
|
|
6123
|
+
const {
|
|
6124
|
+
userQueries,
|
|
6125
|
+
promptFile,
|
|
6126
|
+
requestTemplate,
|
|
6127
|
+
extraAttachments,
|
|
6128
|
+
workspaceTemplate,
|
|
6129
|
+
dryRun = false,
|
|
6130
|
+
wait = false,
|
|
6131
|
+
vscodeCmd = "code",
|
|
6132
|
+
subagentRoot,
|
|
6133
|
+
silent = false
|
|
6134
|
+
} = options;
|
|
6135
|
+
if (!userQueries || userQueries.length === 0) {
|
|
6136
|
+
return {
|
|
6137
|
+
exitCode: 1,
|
|
6138
|
+
requestFiles: [],
|
|
6139
|
+
queryCount: 0,
|
|
6140
|
+
error: "At least one query is required for batch dispatch"
|
|
6141
|
+
};
|
|
6142
|
+
}
|
|
6143
|
+
const queryCount = userQueries.length;
|
|
6144
|
+
let requestFiles = [];
|
|
6145
|
+
let responseFilesFinal = [];
|
|
6146
|
+
let subagentName;
|
|
6147
|
+
try {
|
|
6148
|
+
let resolvedPrompt;
|
|
6149
|
+
try {
|
|
6150
|
+
resolvedPrompt = await resolvePromptFile(promptFile);
|
|
6151
|
+
} catch (error) {
|
|
6152
|
+
return {
|
|
6153
|
+
exitCode: 1,
|
|
6154
|
+
requestFiles,
|
|
6155
|
+
queryCount,
|
|
6156
|
+
error: error.message
|
|
6157
|
+
};
|
|
6158
|
+
}
|
|
6159
|
+
const batchRequestTemplateContent = requestTemplate ?? loadDefaultBatchRequestTemplate();
|
|
6160
|
+
const orchestratorTemplateContent = loadDefaultBatchOrchestratorTemplate();
|
|
6161
|
+
const subagentRootPath = subagentRoot ?? getSubagentRoot(vscodeCmd);
|
|
6162
|
+
const subagentDir = await findUnlockedSubagent(subagentRootPath);
|
|
6163
|
+
if (!subagentDir) {
|
|
6164
|
+
return {
|
|
6165
|
+
exitCode: 1,
|
|
6166
|
+
requestFiles,
|
|
6167
|
+
queryCount,
|
|
6168
|
+
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
6169
|
+
};
|
|
6170
|
+
}
|
|
6171
|
+
subagentName = path21.basename(subagentDir);
|
|
6172
|
+
const chatId = Math.random().toString(16).slice(2, 10);
|
|
6173
|
+
const preparationResult = await prepareSubagentDirectory(
|
|
6174
|
+
subagentDir,
|
|
6175
|
+
resolvedPrompt,
|
|
6176
|
+
chatId,
|
|
6177
|
+
workspaceTemplate,
|
|
6178
|
+
dryRun
|
|
6179
|
+
);
|
|
6180
|
+
if (preparationResult !== 0) {
|
|
6181
|
+
return {
|
|
6182
|
+
exitCode: preparationResult,
|
|
6183
|
+
subagentName,
|
|
6184
|
+
requestFiles,
|
|
6185
|
+
queryCount,
|
|
6186
|
+
error: "Failed to prepare subagent workspace"
|
|
6187
|
+
};
|
|
6188
|
+
}
|
|
6189
|
+
let attachments;
|
|
6190
|
+
try {
|
|
6191
|
+
attachments = await resolveAttachments(extraAttachments);
|
|
6192
|
+
} catch (attachmentError) {
|
|
6193
|
+
return {
|
|
6194
|
+
exitCode: 1,
|
|
6195
|
+
subagentName,
|
|
6196
|
+
requestFiles,
|
|
6197
|
+
queryCount,
|
|
6198
|
+
error: attachmentError.message
|
|
6199
|
+
};
|
|
6200
|
+
}
|
|
6201
|
+
const timestamp = generateTimestamp();
|
|
6202
|
+
const messagesDir = path21.join(subagentDir, "messages");
|
|
6203
|
+
requestFiles = userQueries.map(
|
|
6204
|
+
(_, index) => path21.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
6205
|
+
);
|
|
6206
|
+
const responseTmpFiles = userQueries.map(
|
|
6207
|
+
(_, index) => path21.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
6208
|
+
);
|
|
6209
|
+
responseFilesFinal = userQueries.map(
|
|
6210
|
+
(_, index) => path21.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
6211
|
+
);
|
|
6212
|
+
const orchestratorFile = path21.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
6213
|
+
if (!dryRun) {
|
|
6214
|
+
await Promise.all(
|
|
6215
|
+
userQueries.map((query, index) => {
|
|
6216
|
+
const reqFile = requestFiles[index];
|
|
6217
|
+
const tmpFile = responseTmpFiles[index];
|
|
6218
|
+
const finalFile = responseFilesFinal[index];
|
|
6219
|
+
return writeFile6(
|
|
6220
|
+
reqFile,
|
|
6221
|
+
createBatchRequestPrompt(query, tmpFile, finalFile, batchRequestTemplateContent),
|
|
6222
|
+
{ encoding: "utf8" }
|
|
6223
|
+
);
|
|
6224
|
+
})
|
|
6225
|
+
);
|
|
6226
|
+
const orchestratorContent = createBatchOrchestratorPrompt(
|
|
6227
|
+
requestFiles,
|
|
6228
|
+
responseFilesFinal,
|
|
6229
|
+
orchestratorTemplateContent
|
|
6230
|
+
);
|
|
6231
|
+
await writeFile6(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
6232
|
+
}
|
|
6233
|
+
const chatAttachments = [orchestratorFile, ...attachments];
|
|
6234
|
+
const orchestratorUri = pathToFileUri2(orchestratorFile);
|
|
6235
|
+
const chatInstruction = `Follow instructions in [${timestamp}_orchestrator.md](${orchestratorUri}). Use #runSubagent tool.`;
|
|
6236
|
+
if (dryRun) {
|
|
6237
|
+
return {
|
|
6238
|
+
exitCode: 0,
|
|
6239
|
+
subagentName,
|
|
6240
|
+
requestFiles,
|
|
6241
|
+
responseFiles: wait ? responseFilesFinal : void 0,
|
|
6242
|
+
queryCount
|
|
6243
|
+
};
|
|
6244
|
+
}
|
|
6245
|
+
const launchSuccess = await launchVsCodeWithBatchChat(
|
|
6246
|
+
subagentDir,
|
|
6247
|
+
chatId,
|
|
6248
|
+
chatAttachments,
|
|
6249
|
+
chatInstruction,
|
|
6250
|
+
vscodeCmd
|
|
6251
|
+
);
|
|
6252
|
+
if (!launchSuccess) {
|
|
6253
|
+
return {
|
|
6254
|
+
exitCode: 1,
|
|
6255
|
+
subagentName,
|
|
6256
|
+
requestFiles,
|
|
6257
|
+
queryCount,
|
|
6258
|
+
error: "Failed to launch VS Code for batch dispatch"
|
|
6259
|
+
};
|
|
6260
|
+
}
|
|
6261
|
+
if (!wait) {
|
|
6262
|
+
return {
|
|
6263
|
+
exitCode: 0,
|
|
6264
|
+
subagentName,
|
|
6265
|
+
requestFiles,
|
|
6266
|
+
queryCount
|
|
6267
|
+
};
|
|
6268
|
+
}
|
|
6269
|
+
const responsesCompleted = await waitForBatchResponses(responseFilesFinal, 1e3, silent);
|
|
6270
|
+
if (!responsesCompleted) {
|
|
6271
|
+
return {
|
|
6272
|
+
exitCode: 1,
|
|
6273
|
+
subagentName,
|
|
6274
|
+
requestFiles,
|
|
6275
|
+
responseFiles: responseFilesFinal,
|
|
6276
|
+
queryCount,
|
|
6277
|
+
error: "Timed out waiting for batch responses"
|
|
6278
|
+
};
|
|
6279
|
+
}
|
|
6280
|
+
await removeSubagentLock(subagentDir);
|
|
6281
|
+
return {
|
|
6282
|
+
exitCode: 0,
|
|
6283
|
+
subagentName,
|
|
6284
|
+
requestFiles,
|
|
6285
|
+
responseFiles: responseFilesFinal,
|
|
6286
|
+
queryCount
|
|
6287
|
+
};
|
|
6288
|
+
} catch (error) {
|
|
6289
|
+
return {
|
|
6290
|
+
exitCode: 1,
|
|
6291
|
+
subagentName,
|
|
6292
|
+
requestFiles,
|
|
6293
|
+
responseFiles: responseFilesFinal.length > 0 ? responseFilesFinal : void 0,
|
|
6294
|
+
queryCount,
|
|
6295
|
+
error: error.message
|
|
6296
|
+
};
|
|
6297
|
+
}
|
|
6298
|
+
}
|
|
6299
|
+
|
|
6300
|
+
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
6301
|
+
import { writeFile as writeFile7 } from "node:fs/promises";
|
|
6302
|
+
import path22 from "node:path";
|
|
6303
|
+
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
6304
|
+
folders: [
|
|
6305
|
+
{
|
|
6306
|
+
path: "."
|
|
6307
|
+
}
|
|
6308
|
+
],
|
|
6309
|
+
settings: {
|
|
6310
|
+
"chat.modeFilesLocations": {
|
|
6311
|
+
".github/agents/**/*.md": true
|
|
6312
|
+
}
|
|
6313
|
+
}
|
|
6314
|
+
};
|
|
6315
|
+
var DEFAULT_WAKEUP_CONTENT2 = `---
|
|
6316
|
+
description: 'Wake-up Signal'
|
|
6317
|
+
tools: ['edit', 'runNotebooks', 'search', 'new', 'runCommands', 'runTasks', 'usages', 'vscodeAPI', 'problems', 'changes', 'testFailure', 'openSimpleBrowser', 'fetch', 'githubRepo']
|
|
6318
|
+
model: GPT-4.1 (copilot)
|
|
6319
|
+
---`;
|
|
6320
|
+
async function provisionSubagents(options) {
|
|
6321
|
+
const {
|
|
6322
|
+
targetRoot,
|
|
6323
|
+
subagents,
|
|
6324
|
+
lockName = DEFAULT_LOCK_NAME,
|
|
6325
|
+
force = false,
|
|
6326
|
+
dryRun = false,
|
|
6327
|
+
workspaceTemplate = DEFAULT_WORKSPACE_TEMPLATE2,
|
|
6328
|
+
wakeupContent = DEFAULT_WAKEUP_CONTENT2
|
|
6329
|
+
} = options;
|
|
6330
|
+
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
6331
|
+
throw new Error("subagents must be a positive integer");
|
|
6332
|
+
}
|
|
6333
|
+
const targetPath = path22.resolve(targetRoot);
|
|
6334
|
+
if (!dryRun) {
|
|
6335
|
+
await ensureDir(targetPath);
|
|
6336
|
+
}
|
|
6337
|
+
let highestNumber = 0;
|
|
6338
|
+
const lockedSubagents = /* @__PURE__ */ new Set();
|
|
6339
|
+
const existingSubagents = [];
|
|
6340
|
+
if (await pathExists(targetPath)) {
|
|
6341
|
+
const entries = await readDirEntries(targetPath);
|
|
6342
|
+
for (const entry of entries) {
|
|
6343
|
+
if (!entry.isDirectory || !entry.name.startsWith("subagent-")) {
|
|
6344
|
+
continue;
|
|
6345
|
+
}
|
|
6346
|
+
const suffix = entry.name.split("-")[1];
|
|
6347
|
+
if (!suffix) continue;
|
|
6348
|
+
const parsed = Number.parseInt(suffix, 10);
|
|
6349
|
+
if (!Number.isInteger(parsed)) {
|
|
6350
|
+
continue;
|
|
6351
|
+
}
|
|
6352
|
+
highestNumber = Math.max(highestNumber, parsed);
|
|
6353
|
+
const lockFile = path22.join(entry.absolutePath, lockName);
|
|
6354
|
+
const locked = await pathExists(lockFile);
|
|
6355
|
+
if (locked) {
|
|
6356
|
+
lockedSubagents.add(entry.absolutePath);
|
|
6357
|
+
}
|
|
6358
|
+
existingSubagents.push({ number: parsed, absolutePath: entry.absolutePath });
|
|
6359
|
+
}
|
|
6360
|
+
existingSubagents.sort((a, b) => a.number - b.number);
|
|
6361
|
+
}
|
|
6362
|
+
const created = [];
|
|
6363
|
+
const skippedExisting = [];
|
|
6364
|
+
let subagentsProvisioned = 0;
|
|
6365
|
+
for (const subagent of existingSubagents) {
|
|
6366
|
+
if (subagentsProvisioned >= subagents) {
|
|
6367
|
+
break;
|
|
6368
|
+
}
|
|
6369
|
+
const subagentDir = subagent.absolutePath;
|
|
6370
|
+
const githubAgentsDir = path22.join(subagentDir, ".github", "agents");
|
|
6371
|
+
const lockFile = path22.join(subagentDir, lockName);
|
|
6372
|
+
const workspaceDst = path22.join(subagentDir, `${path22.basename(subagentDir)}.code-workspace`);
|
|
6373
|
+
const wakeupDst = path22.join(githubAgentsDir, "wakeup.md");
|
|
6374
|
+
const isLocked = await pathExists(lockFile);
|
|
6375
|
+
if (isLocked && !force) {
|
|
6376
|
+
continue;
|
|
6377
|
+
}
|
|
6378
|
+
if (isLocked && force) {
|
|
6379
|
+
if (!dryRun) {
|
|
6380
|
+
await removeIfExists(lockFile);
|
|
6381
|
+
await ensureDir(githubAgentsDir);
|
|
6382
|
+
await writeFile7(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6383
|
+
await writeFile7(wakeupDst, wakeupContent, "utf8");
|
|
6384
|
+
}
|
|
6385
|
+
created.push(subagentDir);
|
|
6386
|
+
lockedSubagents.delete(subagentDir);
|
|
6387
|
+
subagentsProvisioned += 1;
|
|
6388
|
+
continue;
|
|
6389
|
+
}
|
|
6390
|
+
if (!isLocked && force) {
|
|
6391
|
+
if (!dryRun) {
|
|
6392
|
+
await ensureDir(githubAgentsDir);
|
|
6393
|
+
await writeFile7(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6394
|
+
await writeFile7(wakeupDst, wakeupContent, "utf8");
|
|
6395
|
+
}
|
|
6396
|
+
created.push(subagentDir);
|
|
6397
|
+
subagentsProvisioned += 1;
|
|
6398
|
+
continue;
|
|
6399
|
+
}
|
|
6400
|
+
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
6401
|
+
await ensureDir(githubAgentsDir);
|
|
6402
|
+
await writeFile7(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6403
|
+
await writeFile7(wakeupDst, wakeupContent, "utf8");
|
|
6404
|
+
}
|
|
6405
|
+
skippedExisting.push(subagentDir);
|
|
6406
|
+
subagentsProvisioned += 1;
|
|
6407
|
+
}
|
|
6408
|
+
let nextIndex = highestNumber;
|
|
6409
|
+
while (subagentsProvisioned < subagents) {
|
|
6410
|
+
nextIndex += 1;
|
|
6411
|
+
const subagentDir = path22.join(targetPath, `subagent-${nextIndex}`);
|
|
6412
|
+
const githubAgentsDir = path22.join(subagentDir, ".github", "agents");
|
|
6413
|
+
const workspaceDst = path22.join(subagentDir, `${path22.basename(subagentDir)}.code-workspace`);
|
|
6414
|
+
const wakeupDst = path22.join(githubAgentsDir, "wakeup.md");
|
|
6415
|
+
if (!dryRun) {
|
|
6416
|
+
await ensureDir(subagentDir);
|
|
6417
|
+
await ensureDir(githubAgentsDir);
|
|
6418
|
+
await writeFile7(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6419
|
+
await writeFile7(wakeupDst, wakeupContent, "utf8");
|
|
6420
|
+
}
|
|
6421
|
+
created.push(subagentDir);
|
|
6422
|
+
subagentsProvisioned += 1;
|
|
6423
|
+
}
|
|
6424
|
+
return {
|
|
6425
|
+
created,
|
|
6426
|
+
skippedExisting,
|
|
6427
|
+
skippedLocked: Array.from(lockedSubagents).sort()
|
|
6428
|
+
};
|
|
6429
|
+
}
|
|
5407
6430
|
|
|
5408
6431
|
// src/evaluation/providers/vscode-templates.ts
|
|
5409
6432
|
var AGENTV_REQUEST_TEMPLATE = `[[ ## task ## ]]
|
|
@@ -5440,7 +6463,7 @@ var AGENTV_BATCH_REQUEST_TEMPLATE = `[[ ## task ## ]]
|
|
|
5440
6463
|
3. Do not unlock the workspace from this request; batch orchestration will handle unlocking after all responses are ready.
|
|
5441
6464
|
`;
|
|
5442
6465
|
|
|
5443
|
-
// src/evaluation/providers/vscode.ts
|
|
6466
|
+
// src/evaluation/providers/vscode-provider.ts
|
|
5444
6467
|
var VSCodeProvider = class {
|
|
5445
6468
|
id;
|
|
5446
6469
|
kind;
|
|
@@ -5572,8 +6595,8 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
5572
6595
|
return "";
|
|
5573
6596
|
}
|
|
5574
6597
|
const buildList = (files) => files.map((absolutePath) => {
|
|
5575
|
-
const fileName =
|
|
5576
|
-
const fileUri =
|
|
6598
|
+
const fileName = path23.basename(absolutePath);
|
|
6599
|
+
const fileUri = pathToFileUri3(absolutePath);
|
|
5577
6600
|
return `* [${fileName}](${fileUri})`;
|
|
5578
6601
|
});
|
|
5579
6602
|
const sections = [];
|
|
@@ -5597,8 +6620,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
5597
6620
|
}
|
|
5598
6621
|
const unique = /* @__PURE__ */ new Map();
|
|
5599
6622
|
for (const attachment of attachments) {
|
|
5600
|
-
const absolutePath =
|
|
5601
|
-
const normalized = absolutePath.split(
|
|
6623
|
+
const absolutePath = path23.resolve(attachment);
|
|
6624
|
+
const normalized = absolutePath.split(path23.sep).join("/");
|
|
5602
6625
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
5603
6626
|
if (!unique.has(absolutePath)) {
|
|
5604
6627
|
unique.set(absolutePath, absolutePath);
|
|
@@ -5613,15 +6636,15 @@ function collectAttachmentFiles(attachments) {
|
|
|
5613
6636
|
}
|
|
5614
6637
|
const unique = /* @__PURE__ */ new Map();
|
|
5615
6638
|
for (const attachment of attachments) {
|
|
5616
|
-
const absolutePath =
|
|
6639
|
+
const absolutePath = path23.resolve(attachment);
|
|
5617
6640
|
if (!unique.has(absolutePath)) {
|
|
5618
6641
|
unique.set(absolutePath, absolutePath);
|
|
5619
6642
|
}
|
|
5620
6643
|
}
|
|
5621
6644
|
return Array.from(unique.values());
|
|
5622
6645
|
}
|
|
5623
|
-
function
|
|
5624
|
-
const absolutePath =
|
|
6646
|
+
function pathToFileUri3(filePath) {
|
|
6647
|
+
const absolutePath = path23.isAbsolute(filePath) ? filePath : path23.resolve(filePath);
|
|
5625
6648
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
5626
6649
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
5627
6650
|
return `file:///${normalizedPath}`;
|
|
@@ -5634,7 +6657,7 @@ function normalizeAttachments(attachments) {
|
|
|
5634
6657
|
}
|
|
5635
6658
|
const deduped = /* @__PURE__ */ new Set();
|
|
5636
6659
|
for (const attachment of attachments) {
|
|
5637
|
-
deduped.add(
|
|
6660
|
+
deduped.add(path23.resolve(attachment));
|
|
5638
6661
|
}
|
|
5639
6662
|
return Array.from(deduped);
|
|
5640
6663
|
}
|
|
@@ -5643,7 +6666,7 @@ function mergeAttachments(all) {
|
|
|
5643
6666
|
for (const list of all) {
|
|
5644
6667
|
if (!list) continue;
|
|
5645
6668
|
for (const inputFile of list) {
|
|
5646
|
-
deduped.add(
|
|
6669
|
+
deduped.add(path23.resolve(inputFile));
|
|
5647
6670
|
}
|
|
5648
6671
|
}
|
|
5649
6672
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -5690,9 +6713,9 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
5690
6713
|
}
|
|
5691
6714
|
|
|
5692
6715
|
// src/evaluation/providers/targets-file.ts
|
|
5693
|
-
import { constants as
|
|
5694
|
-
import { access as
|
|
5695
|
-
import
|
|
6716
|
+
import { constants as constants4 } from "node:fs";
|
|
6717
|
+
import { access as access4, readFile as readFile9 } from "node:fs/promises";
|
|
6718
|
+
import path24 from "node:path";
|
|
5696
6719
|
import { parse as parse3 } from "yaml";
|
|
5697
6720
|
function isRecord(value) {
|
|
5698
6721
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -5722,18 +6745,18 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
5722
6745
|
}
|
|
5723
6746
|
async function fileExists3(filePath) {
|
|
5724
6747
|
try {
|
|
5725
|
-
await
|
|
6748
|
+
await access4(filePath, constants4.F_OK);
|
|
5726
6749
|
return true;
|
|
5727
6750
|
} catch {
|
|
5728
6751
|
return false;
|
|
5729
6752
|
}
|
|
5730
6753
|
}
|
|
5731
6754
|
async function readTargetDefinitions(filePath) {
|
|
5732
|
-
const absolutePath =
|
|
6755
|
+
const absolutePath = path24.resolve(filePath);
|
|
5733
6756
|
if (!await fileExists3(absolutePath)) {
|
|
5734
6757
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
5735
6758
|
}
|
|
5736
|
-
const raw = await
|
|
6759
|
+
const raw = await readFile9(absolutePath, "utf8");
|
|
5737
6760
|
const parsed = parse3(raw);
|
|
5738
6761
|
if (!isRecord(parsed)) {
|
|
5739
6762
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -5897,10 +6920,10 @@ async function execFileWithStdinBun(argv, stdinPayload, options) {
|
|
|
5897
6920
|
}
|
|
5898
6921
|
}
|
|
5899
6922
|
async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
5900
|
-
const { spawn:
|
|
6923
|
+
const { spawn: spawn5 } = await import("node:child_process");
|
|
5901
6924
|
return new Promise((resolve, reject) => {
|
|
5902
6925
|
const [cmd, ...args] = argv;
|
|
5903
|
-
const child =
|
|
6926
|
+
const child = spawn5(cmd, args, {
|
|
5904
6927
|
cwd: options.cwd,
|
|
5905
6928
|
stdio: ["pipe", "pipe", "pipe"],
|
|
5906
6929
|
// Merge additional env vars with process.env
|
|
@@ -5940,21 +6963,21 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
5940
6963
|
});
|
|
5941
6964
|
}
|
|
5942
6965
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
5943
|
-
const { mkdir:
|
|
6966
|
+
const { mkdir: mkdir8, readFile: readFile10, rm: rm5, writeFile: writeFile8 } = await import("node:fs/promises");
|
|
5944
6967
|
const { tmpdir: tmpdir4 } = await import("node:os");
|
|
5945
|
-
const
|
|
6968
|
+
const path26 = await import("node:path");
|
|
5946
6969
|
const { randomUUID: randomUUID4 } = await import("node:crypto");
|
|
5947
|
-
const dir =
|
|
5948
|
-
await
|
|
5949
|
-
const stdinPath =
|
|
5950
|
-
const stdoutPath =
|
|
5951
|
-
const stderrPath =
|
|
5952
|
-
await
|
|
6970
|
+
const dir = path26.join(tmpdir4(), `agentv-exec-${randomUUID4()}`);
|
|
6971
|
+
await mkdir8(dir, { recursive: true });
|
|
6972
|
+
const stdinPath = path26.join(dir, "stdin.txt");
|
|
6973
|
+
const stdoutPath = path26.join(dir, "stdout.txt");
|
|
6974
|
+
const stderrPath = path26.join(dir, "stderr.txt");
|
|
6975
|
+
await writeFile8(stdinPath, stdinPayload, "utf8");
|
|
5953
6976
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
5954
|
-
const { spawn:
|
|
6977
|
+
const { spawn: spawn5 } = await import("node:child_process");
|
|
5955
6978
|
try {
|
|
5956
6979
|
const exitCode = await new Promise((resolve, reject) => {
|
|
5957
|
-
const child =
|
|
6980
|
+
const child = spawn5(wrappedCommand, {
|
|
5958
6981
|
shell: true,
|
|
5959
6982
|
cwd: options.cwd,
|
|
5960
6983
|
stdio: ["ignore", "ignore", "ignore"],
|
|
@@ -5978,11 +7001,11 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
5978
7001
|
resolve(code ?? 0);
|
|
5979
7002
|
});
|
|
5980
7003
|
});
|
|
5981
|
-
const stdout = (await
|
|
5982
|
-
const stderr = (await
|
|
7004
|
+
const stdout = (await readFile10(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
7005
|
+
const stderr = (await readFile10(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
5983
7006
|
return { stdout, stderr, exitCode };
|
|
5984
7007
|
} finally {
|
|
5985
|
-
await
|
|
7008
|
+
await rm5(dir, { recursive: true, force: true });
|
|
5986
7009
|
}
|
|
5987
7010
|
}
|
|
5988
7011
|
|
|
@@ -6251,7 +7274,7 @@ var CodeEvaluator = class {
|
|
|
6251
7274
|
outputMessages: context.outputMessages ?? null,
|
|
6252
7275
|
guidelineFiles: context.evalCase.guideline_paths,
|
|
6253
7276
|
inputFiles: context.evalCase.file_paths.filter(
|
|
6254
|
-
(
|
|
7277
|
+
(path26) => !context.evalCase.guideline_paths.includes(path26)
|
|
6255
7278
|
),
|
|
6256
7279
|
inputMessages: context.evalCase.input_messages,
|
|
6257
7280
|
traceSummary: context.traceSummary ?? null,
|
|
@@ -7171,115 +8194,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
7171
8194
|
* Evaluate a single field against the expected value.
|
|
7172
8195
|
*/
|
|
7173
8196
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
7174
|
-
const { path:
|
|
7175
|
-
const candidateValue = resolvePath(candidateData,
|
|
7176
|
-
const expectedValue = resolvePath(expectedData,
|
|
8197
|
+
const { path: path26, match, required = true, weight = 1 } = fieldConfig;
|
|
8198
|
+
const candidateValue = resolvePath(candidateData, path26);
|
|
8199
|
+
const expectedValue = resolvePath(expectedData, path26);
|
|
7177
8200
|
if (expectedValue === void 0) {
|
|
7178
8201
|
return {
|
|
7179
|
-
path:
|
|
8202
|
+
path: path26,
|
|
7180
8203
|
score: 1,
|
|
7181
8204
|
// No expected value means no comparison needed
|
|
7182
8205
|
weight,
|
|
7183
8206
|
hit: true,
|
|
7184
|
-
message: `${
|
|
8207
|
+
message: `${path26}: no expected value`
|
|
7185
8208
|
};
|
|
7186
8209
|
}
|
|
7187
8210
|
if (candidateValue === void 0) {
|
|
7188
8211
|
if (required) {
|
|
7189
8212
|
return {
|
|
7190
|
-
path:
|
|
8213
|
+
path: path26,
|
|
7191
8214
|
score: 0,
|
|
7192
8215
|
weight,
|
|
7193
8216
|
hit: false,
|
|
7194
|
-
message: `${
|
|
8217
|
+
message: `${path26} (required, missing)`
|
|
7195
8218
|
};
|
|
7196
8219
|
}
|
|
7197
8220
|
return {
|
|
7198
|
-
path:
|
|
8221
|
+
path: path26,
|
|
7199
8222
|
score: 1,
|
|
7200
8223
|
// Don't penalize missing optional fields
|
|
7201
8224
|
weight: 0,
|
|
7202
8225
|
// Zero weight means it won't affect the score
|
|
7203
8226
|
hit: true,
|
|
7204
|
-
message: `${
|
|
8227
|
+
message: `${path26}: optional field missing`
|
|
7205
8228
|
};
|
|
7206
8229
|
}
|
|
7207
8230
|
switch (match) {
|
|
7208
8231
|
case "exact":
|
|
7209
|
-
return this.compareExact(
|
|
8232
|
+
return this.compareExact(path26, candidateValue, expectedValue, weight);
|
|
7210
8233
|
case "numeric_tolerance":
|
|
7211
8234
|
return this.compareNumericTolerance(
|
|
7212
|
-
|
|
8235
|
+
path26,
|
|
7213
8236
|
candidateValue,
|
|
7214
8237
|
expectedValue,
|
|
7215
8238
|
fieldConfig,
|
|
7216
8239
|
weight
|
|
7217
8240
|
);
|
|
7218
8241
|
case "date":
|
|
7219
|
-
return this.compareDate(
|
|
8242
|
+
return this.compareDate(path26, candidateValue, expectedValue, fieldConfig, weight);
|
|
7220
8243
|
default:
|
|
7221
8244
|
return {
|
|
7222
|
-
path:
|
|
8245
|
+
path: path26,
|
|
7223
8246
|
score: 0,
|
|
7224
8247
|
weight,
|
|
7225
8248
|
hit: false,
|
|
7226
|
-
message: `${
|
|
8249
|
+
message: `${path26}: unknown match type "${match}"`
|
|
7227
8250
|
};
|
|
7228
8251
|
}
|
|
7229
8252
|
}
|
|
7230
8253
|
/**
|
|
7231
8254
|
* Exact equality comparison.
|
|
7232
8255
|
*/
|
|
7233
|
-
compareExact(
|
|
8256
|
+
compareExact(path26, candidateValue, expectedValue, weight) {
|
|
7234
8257
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
7235
8258
|
return {
|
|
7236
|
-
path:
|
|
8259
|
+
path: path26,
|
|
7237
8260
|
score: 1,
|
|
7238
8261
|
weight,
|
|
7239
8262
|
hit: true,
|
|
7240
|
-
message:
|
|
8263
|
+
message: path26
|
|
7241
8264
|
};
|
|
7242
8265
|
}
|
|
7243
8266
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
7244
8267
|
return {
|
|
7245
|
-
path:
|
|
8268
|
+
path: path26,
|
|
7246
8269
|
score: 0,
|
|
7247
8270
|
weight,
|
|
7248
8271
|
hit: false,
|
|
7249
|
-
message: `${
|
|
8272
|
+
message: `${path26} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
7250
8273
|
};
|
|
7251
8274
|
}
|
|
7252
8275
|
return {
|
|
7253
|
-
path:
|
|
8276
|
+
path: path26,
|
|
7254
8277
|
score: 0,
|
|
7255
8278
|
weight,
|
|
7256
8279
|
hit: false,
|
|
7257
|
-
message: `${
|
|
8280
|
+
message: `${path26} (value mismatch)`
|
|
7258
8281
|
};
|
|
7259
8282
|
}
|
|
7260
8283
|
/**
|
|
7261
8284
|
* Numeric comparison with absolute or relative tolerance.
|
|
7262
8285
|
*/
|
|
7263
|
-
compareNumericTolerance(
|
|
8286
|
+
compareNumericTolerance(path26, candidateValue, expectedValue, fieldConfig, weight) {
|
|
7264
8287
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
7265
8288
|
const candidateNum = toNumber(candidateValue);
|
|
7266
8289
|
const expectedNum = toNumber(expectedValue);
|
|
7267
8290
|
if (candidateNum === null || expectedNum === null) {
|
|
7268
8291
|
return {
|
|
7269
|
-
path:
|
|
8292
|
+
path: path26,
|
|
7270
8293
|
score: 0,
|
|
7271
8294
|
weight,
|
|
7272
8295
|
hit: false,
|
|
7273
|
-
message: `${
|
|
8296
|
+
message: `${path26} (non-numeric value)`
|
|
7274
8297
|
};
|
|
7275
8298
|
}
|
|
7276
8299
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
7277
8300
|
return {
|
|
7278
|
-
path:
|
|
8301
|
+
path: path26,
|
|
7279
8302
|
score: 0,
|
|
7280
8303
|
weight,
|
|
7281
8304
|
hit: false,
|
|
7282
|
-
message: `${
|
|
8305
|
+
message: `${path26} (invalid numeric value)`
|
|
7283
8306
|
};
|
|
7284
8307
|
}
|
|
7285
8308
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -7292,61 +8315,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
7292
8315
|
}
|
|
7293
8316
|
if (withinTolerance) {
|
|
7294
8317
|
return {
|
|
7295
|
-
path:
|
|
8318
|
+
path: path26,
|
|
7296
8319
|
score: 1,
|
|
7297
8320
|
weight,
|
|
7298
8321
|
hit: true,
|
|
7299
|
-
message: `${
|
|
8322
|
+
message: `${path26} (within tolerance: diff=${diff.toFixed(2)})`
|
|
7300
8323
|
};
|
|
7301
8324
|
}
|
|
7302
8325
|
return {
|
|
7303
|
-
path:
|
|
8326
|
+
path: path26,
|
|
7304
8327
|
score: 0,
|
|
7305
8328
|
weight,
|
|
7306
8329
|
hit: false,
|
|
7307
|
-
message: `${
|
|
8330
|
+
message: `${path26} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
7308
8331
|
};
|
|
7309
8332
|
}
|
|
7310
8333
|
/**
|
|
7311
8334
|
* Date comparison with format normalization.
|
|
7312
8335
|
*/
|
|
7313
|
-
compareDate(
|
|
8336
|
+
compareDate(path26, candidateValue, expectedValue, fieldConfig, weight) {
|
|
7314
8337
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
7315
8338
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
7316
8339
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
7317
8340
|
if (candidateDate === null) {
|
|
7318
8341
|
return {
|
|
7319
|
-
path:
|
|
8342
|
+
path: path26,
|
|
7320
8343
|
score: 0,
|
|
7321
8344
|
weight,
|
|
7322
8345
|
hit: false,
|
|
7323
|
-
message: `${
|
|
8346
|
+
message: `${path26} (unparseable candidate date)`
|
|
7324
8347
|
};
|
|
7325
8348
|
}
|
|
7326
8349
|
if (expectedDate === null) {
|
|
7327
8350
|
return {
|
|
7328
|
-
path:
|
|
8351
|
+
path: path26,
|
|
7329
8352
|
score: 0,
|
|
7330
8353
|
weight,
|
|
7331
8354
|
hit: false,
|
|
7332
|
-
message: `${
|
|
8355
|
+
message: `${path26} (unparseable expected date)`
|
|
7333
8356
|
};
|
|
7334
8357
|
}
|
|
7335
8358
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
7336
8359
|
return {
|
|
7337
|
-
path:
|
|
8360
|
+
path: path26,
|
|
7338
8361
|
score: 1,
|
|
7339
8362
|
weight,
|
|
7340
8363
|
hit: true,
|
|
7341
|
-
message:
|
|
8364
|
+
message: path26
|
|
7342
8365
|
};
|
|
7343
8366
|
}
|
|
7344
8367
|
return {
|
|
7345
|
-
path:
|
|
8368
|
+
path: path26,
|
|
7346
8369
|
score: 0,
|
|
7347
8370
|
weight,
|
|
7348
8371
|
hit: false,
|
|
7349
|
-
message: `${
|
|
8372
|
+
message: `${path26} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
7350
8373
|
};
|
|
7351
8374
|
}
|
|
7352
8375
|
/**
|
|
@@ -7386,11 +8409,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
7386
8409
|
};
|
|
7387
8410
|
}
|
|
7388
8411
|
};
|
|
7389
|
-
function resolvePath(obj,
|
|
7390
|
-
if (!
|
|
8412
|
+
function resolvePath(obj, path26) {
|
|
8413
|
+
if (!path26 || !obj) {
|
|
7391
8414
|
return void 0;
|
|
7392
8415
|
}
|
|
7393
|
-
const parts =
|
|
8416
|
+
const parts = path26.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
7394
8417
|
let current = obj;
|
|
7395
8418
|
for (const part of parts) {
|
|
7396
8419
|
if (current === null || current === void 0) {
|
|
@@ -7910,7 +8933,7 @@ var ToolTrajectoryEvaluator = class {
|
|
|
7910
8933
|
|
|
7911
8934
|
// src/evaluation/orchestrator.ts
|
|
7912
8935
|
import { createHash } from "node:crypto";
|
|
7913
|
-
import
|
|
8936
|
+
import path25 from "node:path";
|
|
7914
8937
|
import micromatch4 from "micromatch";
|
|
7915
8938
|
|
|
7916
8939
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -8713,7 +9736,7 @@ async function runEvaluatorList(options) {
|
|
|
8713
9736
|
});
|
|
8714
9737
|
}
|
|
8715
9738
|
if (evaluator.type === "composite") {
|
|
8716
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
9739
|
+
const evalFileDir = evalCase.guideline_paths[0] ? path25.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
8717
9740
|
const createEvaluator = (memberConfig) => {
|
|
8718
9741
|
switch (memberConfig.type) {
|
|
8719
9742
|
case "llm_judge":
|
|
@@ -9069,7 +10092,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
|
|
|
9069
10092
|
};
|
|
9070
10093
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
9071
10094
|
const scriptPath = script[script.length - 1];
|
|
9072
|
-
const cwd =
|
|
10095
|
+
const cwd = path25.dirname(scriptPath);
|
|
9073
10096
|
try {
|
|
9074
10097
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
9075
10098
|
const prompt = stdout.trim();
|