@agentv/core 2.5.5 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RP3M7COZ.js → chunk-LGQ5OPJD.js} +50 -1
- package/dist/chunk-LGQ5OPJD.js.map +1 -0
- package/dist/evaluation/validation/index.cjs +25 -0
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +25 -1
- package/dist/evaluation/validation/index.js.map +1 -1
- package/dist/index.cjs +836 -250
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +763 -227
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
- package/dist/chunk-RP3M7COZ.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
readTextFile,
|
|
11
11
|
resolveFileReference,
|
|
12
12
|
resolveTargetDefinition
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-LGQ5OPJD.js";
|
|
14
14
|
|
|
15
15
|
// src/evaluation/types.ts
|
|
16
16
|
var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
|
|
@@ -3222,8 +3222,8 @@ async function runClaudeCodeWithTempFiles(options, stdoutFile, stderrFile, exitF
|
|
|
3222
3222
|
};
|
|
3223
3223
|
const fileExists4 = async (filePath) => {
|
|
3224
3224
|
try {
|
|
3225
|
-
const { access:
|
|
3226
|
-
await
|
|
3225
|
+
const { access: access6 } = await import("node:fs/promises");
|
|
3226
|
+
await access6(filePath);
|
|
3227
3227
|
return true;
|
|
3228
3228
|
} catch {
|
|
3229
3229
|
return false;
|
|
@@ -4548,6 +4548,538 @@ function shouldShellExecute(executable) {
|
|
|
4548
4548
|
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
4549
4549
|
}
|
|
4550
4550
|
|
|
4551
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4552
|
+
import { exec as execCallback2, spawn as spawn3 } from "node:child_process";
|
|
4553
|
+
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
4554
|
+
import { constants as constants3, createWriteStream as createWriteStream3 } from "node:fs";
|
|
4555
|
+
import { access as access3, copyFile, mkdir as mkdir3, mkdtemp as mkdtemp3, rm as rm3, writeFile as writeFile3 } from "node:fs/promises";
|
|
4556
|
+
import { tmpdir as tmpdir3 } from "node:os";
|
|
4557
|
+
import path12 from "node:path";
|
|
4558
|
+
import { promisify as promisify3 } from "node:util";
|
|
4559
|
+
|
|
4560
|
+
// src/evaluation/providers/copilot-cli-log-tracker.ts
|
|
4561
|
+
var GLOBAL_LOGS_KEY3 = Symbol.for("agentv.copilotCliLogs");
|
|
4562
|
+
var GLOBAL_SUBSCRIBERS_KEY3 = Symbol.for("agentv.copilotCliLogSubscribers");
|
|
4563
|
+
function getCopilotCliLogStore() {
|
|
4564
|
+
const globalObject = globalThis;
|
|
4565
|
+
const existing = globalObject[GLOBAL_LOGS_KEY3];
|
|
4566
|
+
if (existing) {
|
|
4567
|
+
return existing;
|
|
4568
|
+
}
|
|
4569
|
+
const created = [];
|
|
4570
|
+
globalObject[GLOBAL_LOGS_KEY3] = created;
|
|
4571
|
+
return created;
|
|
4572
|
+
}
|
|
4573
|
+
function getSubscriberStore3() {
|
|
4574
|
+
const globalObject = globalThis;
|
|
4575
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY3];
|
|
4576
|
+
if (existing) {
|
|
4577
|
+
return existing;
|
|
4578
|
+
}
|
|
4579
|
+
const created = /* @__PURE__ */ new Set();
|
|
4580
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY3] = created;
|
|
4581
|
+
return created;
|
|
4582
|
+
}
|
|
4583
|
+
function notifySubscribers3(entry) {
|
|
4584
|
+
const subscribers = Array.from(getSubscriberStore3());
|
|
4585
|
+
for (const listener of subscribers) {
|
|
4586
|
+
try {
|
|
4587
|
+
listener(entry);
|
|
4588
|
+
} catch (error) {
|
|
4589
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4590
|
+
console.warn(`Copilot CLI log subscriber failed: ${message}`);
|
|
4591
|
+
}
|
|
4592
|
+
}
|
|
4593
|
+
}
|
|
4594
|
+
function recordCopilotCliLogEntry(entry) {
|
|
4595
|
+
getCopilotCliLogStore().push(entry);
|
|
4596
|
+
notifySubscribers3(entry);
|
|
4597
|
+
}
|
|
4598
|
+
function consumeCopilotCliLogEntries() {
|
|
4599
|
+
const store = getCopilotCliLogStore();
|
|
4600
|
+
if (store.length === 0) {
|
|
4601
|
+
return [];
|
|
4602
|
+
}
|
|
4603
|
+
return store.splice(0, store.length);
|
|
4604
|
+
}
|
|
4605
|
+
function subscribeToCopilotCliLogEntries(listener) {
|
|
4606
|
+
const store = getSubscriberStore3();
|
|
4607
|
+
store.add(listener);
|
|
4608
|
+
return () => {
|
|
4609
|
+
store.delete(listener);
|
|
4610
|
+
};
|
|
4611
|
+
}
|
|
4612
|
+
|
|
4613
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4614
|
+
var execAsync3 = promisify3(execCallback2);
|
|
4615
|
+
var WORKSPACE_PREFIX3 = "agentv-copilot-";
|
|
4616
|
+
var PROMPT_FILENAME3 = "prompt.md";
|
|
4617
|
+
var DEFAULT_SYSTEM_PROMPT4 = `**IMPORTANT**: Follow these instructions for your response:
|
|
4618
|
+
- Do NOT create any additional output files in the workspace.
|
|
4619
|
+
- All intended file outputs/changes MUST be written in your response.
|
|
4620
|
+
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
4621
|
+
This is required for evaluation scoring.`;
|
|
4622
|
+
async function copyInputFilesToWorkspace(workspaceRoot, inputFiles) {
|
|
4623
|
+
const usedNames = /* @__PURE__ */ new Map();
|
|
4624
|
+
const mappings = [];
|
|
4625
|
+
for (const originalPath of inputFiles) {
|
|
4626
|
+
const ext = path12.extname(originalPath);
|
|
4627
|
+
const stem = path12.basename(originalPath, ext);
|
|
4628
|
+
let relativeName;
|
|
4629
|
+
const baseKey = `${stem}${ext}`;
|
|
4630
|
+
const count = usedNames.get(baseKey) ?? 0;
|
|
4631
|
+
if (count === 0) {
|
|
4632
|
+
relativeName = baseKey;
|
|
4633
|
+
} else {
|
|
4634
|
+
relativeName = `${stem}_${count}${ext}`;
|
|
4635
|
+
}
|
|
4636
|
+
usedNames.set(baseKey, count + 1);
|
|
4637
|
+
const dest = path12.join(workspaceRoot, relativeName);
|
|
4638
|
+
await copyFile(originalPath, dest);
|
|
4639
|
+
mappings.push({ originalPath, workspaceRelativePath: relativeName });
|
|
4640
|
+
}
|
|
4641
|
+
return mappings;
|
|
4642
|
+
}
|
|
4643
|
+
function buildCopilotFilePrereadBlock(guidelineMappings, inputMappings) {
|
|
4644
|
+
if (guidelineMappings.length === 0 && inputMappings.length === 0) {
|
|
4645
|
+
return "";
|
|
4646
|
+
}
|
|
4647
|
+
const buildList = (mappings) => mappings.map((m) => `* ${m.workspaceRelativePath}`).join("\n");
|
|
4648
|
+
const sections = [];
|
|
4649
|
+
if (guidelineMappings.length > 0) {
|
|
4650
|
+
sections.push(`Read all guideline files:
|
|
4651
|
+
${buildList(guidelineMappings)}.`);
|
|
4652
|
+
}
|
|
4653
|
+
if (inputMappings.length > 0) {
|
|
4654
|
+
sections.push(`Read all input files:
|
|
4655
|
+
${buildList(inputMappings)}.`);
|
|
4656
|
+
}
|
|
4657
|
+
sections.push(
|
|
4658
|
+
"If any file is missing, fail with ERROR: missing-file <filename> and stop.",
|
|
4659
|
+
"Then apply system_instructions on the user query below."
|
|
4660
|
+
);
|
|
4661
|
+
return sections.join("\n");
|
|
4662
|
+
}
|
|
4663
|
+
var CopilotCliProvider = class {
|
|
4664
|
+
id;
|
|
4665
|
+
kind = "copilot-cli";
|
|
4666
|
+
targetName;
|
|
4667
|
+
supportsBatch = false;
|
|
4668
|
+
config;
|
|
4669
|
+
runCopilot;
|
|
4670
|
+
environmentCheck;
|
|
4671
|
+
resolvedExecutable;
|
|
4672
|
+
constructor(targetName, config, runner = defaultCopilotCliRunner) {
|
|
4673
|
+
this.id = `copilot-cli:${targetName}`;
|
|
4674
|
+
this.targetName = targetName;
|
|
4675
|
+
this.config = config;
|
|
4676
|
+
this.runCopilot = runner;
|
|
4677
|
+
}
|
|
4678
|
+
async invoke(request) {
|
|
4679
|
+
if (request.signal?.aborted) {
|
|
4680
|
+
throw new Error("Copilot CLI request was aborted before execution");
|
|
4681
|
+
}
|
|
4682
|
+
await this.ensureEnvironmentReady();
|
|
4683
|
+
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
4684
|
+
const workspaceRoot = await this.createWorkspace();
|
|
4685
|
+
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
4686
|
+
try {
|
|
4687
|
+
const copiedFiles = inputFiles ? await copyInputFilesToWorkspace(workspaceRoot, inputFiles) : [];
|
|
4688
|
+
const guidelineFileSet = new Set(
|
|
4689
|
+
collectGuidelineFiles(inputFiles, request.guideline_patterns)
|
|
4690
|
+
);
|
|
4691
|
+
const guidelineMappings = copiedFiles.filter((m) => guidelineFileSet.has(m.originalPath));
|
|
4692
|
+
const nonGuidelineMappings = copiedFiles.filter((m) => !guidelineFileSet.has(m.originalPath));
|
|
4693
|
+
const prereadBlock = buildCopilotFilePrereadBlock(guidelineMappings, nonGuidelineMappings);
|
|
4694
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT4;
|
|
4695
|
+
const promptParts = [systemPrompt];
|
|
4696
|
+
if (prereadBlock.length > 0) {
|
|
4697
|
+
promptParts.push("", prereadBlock);
|
|
4698
|
+
}
|
|
4699
|
+
promptParts.push("", "[[ ## user_query ## ]]", request.question.trim());
|
|
4700
|
+
const promptContent = promptParts.join("\n");
|
|
4701
|
+
const promptFile = path12.join(workspaceRoot, PROMPT_FILENAME3);
|
|
4702
|
+
await writeFile3(promptFile, promptContent, "utf8");
|
|
4703
|
+
const args = this.buildCopilotArgs(promptContent);
|
|
4704
|
+
const cwd = this.resolveCwd(workspaceRoot);
|
|
4705
|
+
const result = await this.executeCopilot(args, cwd, promptContent, request.signal, logger);
|
|
4706
|
+
if (result.timedOut) {
|
|
4707
|
+
throw new Error(
|
|
4708
|
+
`Copilot CLI timed out${formatTimeoutSuffix4(this.config.timeoutMs ?? void 0)}`
|
|
4709
|
+
);
|
|
4710
|
+
}
|
|
4711
|
+
if (result.exitCode !== 0) {
|
|
4712
|
+
const detail = pickDetail3(result.stderr, result.stdout);
|
|
4713
|
+
const prefix = `Copilot CLI exited with code ${result.exitCode}`;
|
|
4714
|
+
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
4715
|
+
}
|
|
4716
|
+
const assistantText = extractCopilotResponse(result.stdout);
|
|
4717
|
+
return {
|
|
4718
|
+
raw: {
|
|
4719
|
+
stdout: result.stdout,
|
|
4720
|
+
stderr: result.stderr,
|
|
4721
|
+
exitCode: result.exitCode,
|
|
4722
|
+
args,
|
|
4723
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4724
|
+
promptFile,
|
|
4725
|
+
workspace: workspaceRoot,
|
|
4726
|
+
inputFiles,
|
|
4727
|
+
copiedFiles,
|
|
4728
|
+
logFile: logger?.filePath
|
|
4729
|
+
},
|
|
4730
|
+
outputMessages: [{ role: "assistant", content: assistantText }]
|
|
4731
|
+
};
|
|
4732
|
+
} finally {
|
|
4733
|
+
await logger?.close();
|
|
4734
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
4735
|
+
}
|
|
4736
|
+
}
|
|
4737
|
+
async ensureEnvironmentReady() {
|
|
4738
|
+
if (!this.environmentCheck) {
|
|
4739
|
+
this.environmentCheck = this.validateEnvironment();
|
|
4740
|
+
}
|
|
4741
|
+
await this.environmentCheck;
|
|
4742
|
+
}
|
|
4743
|
+
async validateEnvironment() {
|
|
4744
|
+
this.resolvedExecutable = await locateExecutable2(this.config.executable);
|
|
4745
|
+
}
|
|
4746
|
+
resolveCwd(workspaceRoot) {
|
|
4747
|
+
if (!this.config.cwd) {
|
|
4748
|
+
return workspaceRoot;
|
|
4749
|
+
}
|
|
4750
|
+
return path12.resolve(this.config.cwd);
|
|
4751
|
+
}
|
|
4752
|
+
buildCopilotArgs(prompt) {
|
|
4753
|
+
const args = [];
|
|
4754
|
+
args.push("-s");
|
|
4755
|
+
args.push("--allow-all-tools");
|
|
4756
|
+
args.push("--no-color");
|
|
4757
|
+
if (this.config.model) {
|
|
4758
|
+
args.push("--model", this.config.model);
|
|
4759
|
+
}
|
|
4760
|
+
if (this.config.args && this.config.args.length > 0) {
|
|
4761
|
+
args.push(...this.config.args);
|
|
4762
|
+
}
|
|
4763
|
+
args.push("-p", prompt);
|
|
4764
|
+
return args;
|
|
4765
|
+
}
|
|
4766
|
+
async executeCopilot(args, cwd, promptContent, signal, logger) {
|
|
4767
|
+
try {
|
|
4768
|
+
return await this.runCopilot({
|
|
4769
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4770
|
+
args,
|
|
4771
|
+
cwd,
|
|
4772
|
+
prompt: promptContent,
|
|
4773
|
+
timeoutMs: this.config.timeoutMs,
|
|
4774
|
+
env: process.env,
|
|
4775
|
+
signal,
|
|
4776
|
+
onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : void 0,
|
|
4777
|
+
onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : void 0
|
|
4778
|
+
});
|
|
4779
|
+
} catch (error) {
|
|
4780
|
+
const err = error;
|
|
4781
|
+
if (err.code === "ENOENT") {
|
|
4782
|
+
throw new Error(
|
|
4783
|
+
`Copilot executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
|
|
4784
|
+
);
|
|
4785
|
+
}
|
|
4786
|
+
throw error;
|
|
4787
|
+
}
|
|
4788
|
+
}
|
|
4789
|
+
async createWorkspace() {
|
|
4790
|
+
return await mkdtemp3(path12.join(tmpdir3(), WORKSPACE_PREFIX3));
|
|
4791
|
+
}
|
|
4792
|
+
async cleanupWorkspace(workspaceRoot) {
|
|
4793
|
+
try {
|
|
4794
|
+
await rm3(workspaceRoot, { recursive: true, force: true });
|
|
4795
|
+
} catch {
|
|
4796
|
+
}
|
|
4797
|
+
}
|
|
4798
|
+
resolveLogDirectory() {
|
|
4799
|
+
const disabled = isCopilotLogStreamingDisabled();
|
|
4800
|
+
if (disabled) {
|
|
4801
|
+
return void 0;
|
|
4802
|
+
}
|
|
4803
|
+
if (this.config.logDir) {
|
|
4804
|
+
return path12.resolve(this.config.logDir);
|
|
4805
|
+
}
|
|
4806
|
+
return path12.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
4807
|
+
}
|
|
4808
|
+
async createStreamLogger(request) {
|
|
4809
|
+
const logDir = this.resolveLogDirectory();
|
|
4810
|
+
if (!logDir) {
|
|
4811
|
+
return void 0;
|
|
4812
|
+
}
|
|
4813
|
+
try {
|
|
4814
|
+
await mkdir3(logDir, { recursive: true });
|
|
4815
|
+
} catch (error) {
|
|
4816
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4817
|
+
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
4818
|
+
return void 0;
|
|
4819
|
+
}
|
|
4820
|
+
const filePath = path12.join(logDir, buildLogFilename3(request, this.targetName));
|
|
4821
|
+
try {
|
|
4822
|
+
const logger = await CopilotCliStreamLogger.create({
|
|
4823
|
+
filePath,
|
|
4824
|
+
targetName: this.targetName,
|
|
4825
|
+
evalCaseId: request.evalCaseId,
|
|
4826
|
+
attempt: request.attempt,
|
|
4827
|
+
format: this.config.logFormat ?? "summary"
|
|
4828
|
+
});
|
|
4829
|
+
recordCopilotCliLogEntry({
|
|
4830
|
+
filePath,
|
|
4831
|
+
targetName: this.targetName,
|
|
4832
|
+
evalCaseId: request.evalCaseId,
|
|
4833
|
+
attempt: request.attempt
|
|
4834
|
+
});
|
|
4835
|
+
return logger;
|
|
4836
|
+
} catch (error) {
|
|
4837
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4838
|
+
console.warn(`Skipping Copilot CLI stream logging for ${filePath}: ${message}`);
|
|
4839
|
+
return void 0;
|
|
4840
|
+
}
|
|
4841
|
+
}
|
|
4842
|
+
};
|
|
4843
|
+
var CopilotCliStreamLogger = class _CopilotCliStreamLogger {
|
|
4844
|
+
filePath;
|
|
4845
|
+
stream;
|
|
4846
|
+
startedAt = Date.now();
|
|
4847
|
+
stdoutBuffer = "";
|
|
4848
|
+
stderrBuffer = "";
|
|
4849
|
+
format;
|
|
4850
|
+
constructor(filePath, format) {
|
|
4851
|
+
this.filePath = filePath;
|
|
4852
|
+
this.format = format;
|
|
4853
|
+
this.stream = createWriteStream3(filePath, { flags: "a" });
|
|
4854
|
+
}
|
|
4855
|
+
static async create(options) {
|
|
4856
|
+
const logger = new _CopilotCliStreamLogger(options.filePath, options.format);
|
|
4857
|
+
const header = [
|
|
4858
|
+
"# Copilot CLI stream log",
|
|
4859
|
+
`# target: ${options.targetName}`,
|
|
4860
|
+
options.evalCaseId ? `# eval: ${options.evalCaseId}` : void 0,
|
|
4861
|
+
options.attempt !== void 0 ? `# attempt: ${options.attempt + 1}` : void 0,
|
|
4862
|
+
`# started: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
4863
|
+
""
|
|
4864
|
+
].filter((line) => Boolean(line));
|
|
4865
|
+
logger.writeLines(header);
|
|
4866
|
+
return logger;
|
|
4867
|
+
}
|
|
4868
|
+
handleStdoutChunk(chunk) {
|
|
4869
|
+
this.stdoutBuffer += chunk;
|
|
4870
|
+
this.flushBuffer("stdout");
|
|
4871
|
+
}
|
|
4872
|
+
handleStderrChunk(chunk) {
|
|
4873
|
+
this.stderrBuffer += chunk;
|
|
4874
|
+
this.flushBuffer("stderr");
|
|
4875
|
+
}
|
|
4876
|
+
async close() {
|
|
4877
|
+
this.flushBuffer("stdout");
|
|
4878
|
+
this.flushBuffer("stderr");
|
|
4879
|
+
this.flushRemainder();
|
|
4880
|
+
await new Promise((resolve, reject) => {
|
|
4881
|
+
this.stream.once("error", reject);
|
|
4882
|
+
this.stream.end(() => resolve());
|
|
4883
|
+
});
|
|
4884
|
+
}
|
|
4885
|
+
writeLines(lines) {
|
|
4886
|
+
for (const line of lines) {
|
|
4887
|
+
this.stream.write(`${line}
|
|
4888
|
+
`);
|
|
4889
|
+
}
|
|
4890
|
+
}
|
|
4891
|
+
flushBuffer(source) {
|
|
4892
|
+
const buffer = source === "stdout" ? this.stdoutBuffer : this.stderrBuffer;
|
|
4893
|
+
const lines = buffer.split(/\r?\n/);
|
|
4894
|
+
const remainder = lines.pop() ?? "";
|
|
4895
|
+
if (source === "stdout") {
|
|
4896
|
+
this.stdoutBuffer = remainder;
|
|
4897
|
+
} else {
|
|
4898
|
+
this.stderrBuffer = remainder;
|
|
4899
|
+
}
|
|
4900
|
+
for (const line of lines) {
|
|
4901
|
+
const formatted = this.formatLine(line, source);
|
|
4902
|
+
if (formatted) {
|
|
4903
|
+
this.stream.write(formatted);
|
|
4904
|
+
this.stream.write("\n");
|
|
4905
|
+
}
|
|
4906
|
+
}
|
|
4907
|
+
}
|
|
4908
|
+
formatLine(rawLine, source) {
|
|
4909
|
+
const trimmed = rawLine.trim();
|
|
4910
|
+
if (trimmed.length === 0) {
|
|
4911
|
+
return void 0;
|
|
4912
|
+
}
|
|
4913
|
+
const prefix = source === "stderr" ? "stderr: " : "";
|
|
4914
|
+
return `[+${formatElapsed3(this.startedAt)}] [${source}] ${prefix}${trimmed}`;
|
|
4915
|
+
}
|
|
4916
|
+
flushRemainder() {
|
|
4917
|
+
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
4918
|
+
if (stdoutRemainder.length > 0) {
|
|
4919
|
+
const formatted = this.formatLine(stdoutRemainder, "stdout");
|
|
4920
|
+
if (formatted) {
|
|
4921
|
+
this.stream.write(formatted);
|
|
4922
|
+
this.stream.write("\n");
|
|
4923
|
+
}
|
|
4924
|
+
}
|
|
4925
|
+
const stderrRemainder = this.stderrBuffer.trim();
|
|
4926
|
+
if (stderrRemainder.length > 0) {
|
|
4927
|
+
const formatted = this.formatLine(stderrRemainder, "stderr");
|
|
4928
|
+
if (formatted) {
|
|
4929
|
+
this.stream.write(formatted);
|
|
4930
|
+
this.stream.write("\n");
|
|
4931
|
+
}
|
|
4932
|
+
}
|
|
4933
|
+
this.stdoutBuffer = "";
|
|
4934
|
+
this.stderrBuffer = "";
|
|
4935
|
+
}
|
|
4936
|
+
};
|
|
4937
|
+
function isCopilotLogStreamingDisabled() {
|
|
4938
|
+
const envValue = process.env.AGENTV_COPILOT_STREAM_LOGS;
|
|
4939
|
+
if (!envValue) {
|
|
4940
|
+
return false;
|
|
4941
|
+
}
|
|
4942
|
+
const normalized = envValue.trim().toLowerCase();
|
|
4943
|
+
return normalized === "false" || normalized === "0" || normalized === "off";
|
|
4944
|
+
}
|
|
4945
|
+
function buildLogFilename3(request, targetName) {
|
|
4946
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
4947
|
+
const evalId = sanitizeForFilename3(request.evalCaseId ?? "copilot");
|
|
4948
|
+
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
4949
|
+
const target = sanitizeForFilename3(targetName);
|
|
4950
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${randomUUID3().slice(0, 8)}.log`;
|
|
4951
|
+
}
|
|
4952
|
+
function sanitizeForFilename3(value) {
|
|
4953
|
+
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
4954
|
+
return sanitized.length > 0 ? sanitized : "copilot";
|
|
4955
|
+
}
|
|
4956
|
+
function formatElapsed3(startedAt) {
|
|
4957
|
+
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
4958
|
+
const hours = Math.floor(elapsedSeconds / 3600);
|
|
4959
|
+
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
4960
|
+
const seconds = elapsedSeconds % 60;
|
|
4961
|
+
if (hours > 0) {
|
|
4962
|
+
return `${hours.toString().padStart(2, "0")}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
4963
|
+
}
|
|
4964
|
+
return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
4965
|
+
}
|
|
4966
|
+
var ANSI_ESCAPE_RE = /\x1B\[[0-9;]*[A-Za-z]/g;
|
|
4967
|
+
var ANSI_OSC_RE = /\x1B\][^\x07]*\x07/g;
|
|
4968
|
+
function stripAnsiEscapes(text) {
|
|
4969
|
+
return text.replace(ANSI_ESCAPE_RE, "").replace(ANSI_OSC_RE, "");
|
|
4970
|
+
}
|
|
4971
|
+
function extractCopilotResponse(stdout) {
|
|
4972
|
+
const cleaned = stripAnsiEscapes(stdout).trim();
|
|
4973
|
+
if (cleaned.length === 0) {
|
|
4974
|
+
throw new Error("Copilot CLI produced no output");
|
|
4975
|
+
}
|
|
4976
|
+
return cleaned;
|
|
4977
|
+
}
|
|
4978
|
+
function pickDetail3(stderr, stdout) {
|
|
4979
|
+
const errorText = stderr.trim();
|
|
4980
|
+
if (errorText.length > 0) {
|
|
4981
|
+
return errorText;
|
|
4982
|
+
}
|
|
4983
|
+
const stdoutText = stdout.trim();
|
|
4984
|
+
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
4985
|
+
}
|
|
4986
|
+
function formatTimeoutSuffix4(timeoutMs) {
|
|
4987
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
4988
|
+
return "";
|
|
4989
|
+
}
|
|
4990
|
+
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
4991
|
+
return ` after ${seconds}s`;
|
|
4992
|
+
}
|
|
4993
|
+
async function locateExecutable2(candidate) {
|
|
4994
|
+
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
4995
|
+
if (includesPathSeparator) {
|
|
4996
|
+
const resolved = path12.isAbsolute(candidate) ? candidate : path12.resolve(candidate);
|
|
4997
|
+
await access3(resolved, constants3.F_OK);
|
|
4998
|
+
return resolved;
|
|
4999
|
+
}
|
|
5000
|
+
const locator = process.platform === "win32" ? "where" : "which";
|
|
5001
|
+
try {
|
|
5002
|
+
const { stdout } = await execAsync3(`${locator} ${candidate}`);
|
|
5003
|
+
const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
5004
|
+
if (lines.length > 0 && lines[0]) {
|
|
5005
|
+
await access3(lines[0], constants3.F_OK);
|
|
5006
|
+
return lines[0];
|
|
5007
|
+
}
|
|
5008
|
+
} catch {
|
|
5009
|
+
}
|
|
5010
|
+
throw new Error(`Copilot executable '${candidate}' was not found on PATH`);
|
|
5011
|
+
}
|
|
5012
|
+
function shouldShellExecute2(executable) {
|
|
5013
|
+
if (process.platform !== "win32") {
|
|
5014
|
+
return false;
|
|
5015
|
+
}
|
|
5016
|
+
const lower = executable.toLowerCase();
|
|
5017
|
+
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
5018
|
+
}
|
|
5019
|
+
async function defaultCopilotCliRunner(options) {
|
|
5020
|
+
return await new Promise((resolve, reject) => {
|
|
5021
|
+
const child = spawn3(options.executable, options.args, {
|
|
5022
|
+
cwd: options.cwd,
|
|
5023
|
+
env: options.env,
|
|
5024
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
5025
|
+
shell: shouldShellExecute2(options.executable)
|
|
5026
|
+
});
|
|
5027
|
+
let stdout = "";
|
|
5028
|
+
let stderr = "";
|
|
5029
|
+
let timedOut = false;
|
|
5030
|
+
const onAbort = () => {
|
|
5031
|
+
child.kill("SIGTERM");
|
|
5032
|
+
};
|
|
5033
|
+
if (options.signal) {
|
|
5034
|
+
if (options.signal.aborted) {
|
|
5035
|
+
onAbort();
|
|
5036
|
+
} else {
|
|
5037
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
let timeoutHandle;
|
|
5041
|
+
if (options.timeoutMs && options.timeoutMs > 0) {
|
|
5042
|
+
timeoutHandle = setTimeout(() => {
|
|
5043
|
+
timedOut = true;
|
|
5044
|
+
child.kill("SIGTERM");
|
|
5045
|
+
}, options.timeoutMs);
|
|
5046
|
+
timeoutHandle.unref?.();
|
|
5047
|
+
}
|
|
5048
|
+
child.stdout.setEncoding("utf8");
|
|
5049
|
+
child.stdout.on("data", (chunk) => {
|
|
5050
|
+
stdout += chunk;
|
|
5051
|
+
options.onStdoutChunk?.(chunk);
|
|
5052
|
+
});
|
|
5053
|
+
child.stderr.setEncoding("utf8");
|
|
5054
|
+
child.stderr.on("data", (chunk) => {
|
|
5055
|
+
stderr += chunk;
|
|
5056
|
+
options.onStderrChunk?.(chunk);
|
|
5057
|
+
});
|
|
5058
|
+
child.stdin.end();
|
|
5059
|
+
const cleanup = () => {
|
|
5060
|
+
if (timeoutHandle) {
|
|
5061
|
+
clearTimeout(timeoutHandle);
|
|
5062
|
+
}
|
|
5063
|
+
if (options.signal) {
|
|
5064
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
5065
|
+
}
|
|
5066
|
+
};
|
|
5067
|
+
child.on("error", (error) => {
|
|
5068
|
+
cleanup();
|
|
5069
|
+
reject(error);
|
|
5070
|
+
});
|
|
5071
|
+
child.on("close", (code) => {
|
|
5072
|
+
cleanup();
|
|
5073
|
+
resolve({
|
|
5074
|
+
stdout,
|
|
5075
|
+
stderr,
|
|
5076
|
+
exitCode: typeof code === "number" ? code : -1,
|
|
5077
|
+
timedOut
|
|
5078
|
+
});
|
|
5079
|
+
});
|
|
5080
|
+
});
|
|
5081
|
+
}
|
|
5082
|
+
|
|
4551
5083
|
// src/evaluation/providers/mock.ts
|
|
4552
5084
|
var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
|
|
4553
5085
|
var MockProvider = class {
|
|
@@ -4751,38 +5283,38 @@ function extractToolCalls2(content) {
|
|
|
4751
5283
|
}
|
|
4752
5284
|
|
|
4753
5285
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
4754
|
-
import { spawn as
|
|
4755
|
-
import { randomUUID as
|
|
4756
|
-
import { createWriteStream as
|
|
4757
|
-
import { mkdir as
|
|
4758
|
-
import { tmpdir as
|
|
4759
|
-
import
|
|
5286
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
5287
|
+
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
5288
|
+
import { createWriteStream as createWriteStream4 } from "node:fs";
|
|
5289
|
+
import { mkdir as mkdir4, mkdtemp as mkdtemp4, rm as rm4, writeFile as writeFile4 } from "node:fs/promises";
|
|
5290
|
+
import { tmpdir as tmpdir4 } from "node:os";
|
|
5291
|
+
import path13 from "node:path";
|
|
4760
5292
|
|
|
4761
5293
|
// src/evaluation/providers/pi-log-tracker.ts
|
|
4762
|
-
var
|
|
4763
|
-
var
|
|
5294
|
+
var GLOBAL_LOGS_KEY4 = Symbol.for("agentv.piLogs");
|
|
5295
|
+
var GLOBAL_SUBSCRIBERS_KEY4 = Symbol.for("agentv.piLogSubscribers");
|
|
4764
5296
|
function getPiLogStore() {
|
|
4765
5297
|
const globalObject = globalThis;
|
|
4766
|
-
const existing = globalObject[
|
|
5298
|
+
const existing = globalObject[GLOBAL_LOGS_KEY4];
|
|
4767
5299
|
if (existing) {
|
|
4768
5300
|
return existing;
|
|
4769
5301
|
}
|
|
4770
5302
|
const created = [];
|
|
4771
|
-
globalObject[
|
|
5303
|
+
globalObject[GLOBAL_LOGS_KEY4] = created;
|
|
4772
5304
|
return created;
|
|
4773
5305
|
}
|
|
4774
|
-
function
|
|
5306
|
+
function getSubscriberStore4() {
|
|
4775
5307
|
const globalObject = globalThis;
|
|
4776
|
-
const existing = globalObject[
|
|
5308
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY4];
|
|
4777
5309
|
if (existing) {
|
|
4778
5310
|
return existing;
|
|
4779
5311
|
}
|
|
4780
5312
|
const created = /* @__PURE__ */ new Set();
|
|
4781
|
-
globalObject[
|
|
5313
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY4] = created;
|
|
4782
5314
|
return created;
|
|
4783
5315
|
}
|
|
4784
|
-
function
|
|
4785
|
-
const subscribers = Array.from(
|
|
5316
|
+
function notifySubscribers4(entry) {
|
|
5317
|
+
const subscribers = Array.from(getSubscriberStore4());
|
|
4786
5318
|
for (const listener of subscribers) {
|
|
4787
5319
|
try {
|
|
4788
5320
|
listener(entry);
|
|
@@ -4794,7 +5326,7 @@ function notifySubscribers3(entry) {
|
|
|
4794
5326
|
}
|
|
4795
5327
|
function recordPiLogEntry(entry) {
|
|
4796
5328
|
getPiLogStore().push(entry);
|
|
4797
|
-
|
|
5329
|
+
notifySubscribers4(entry);
|
|
4798
5330
|
}
|
|
4799
5331
|
function consumePiLogEntries() {
|
|
4800
5332
|
const store = getPiLogStore();
|
|
@@ -4804,7 +5336,7 @@ function consumePiLogEntries() {
|
|
|
4804
5336
|
return store.splice(0, store.length);
|
|
4805
5337
|
}
|
|
4806
5338
|
function subscribeToPiLogEntries(listener) {
|
|
4807
|
-
const store =
|
|
5339
|
+
const store = getSubscriberStore4();
|
|
4808
5340
|
store.add(listener);
|
|
4809
5341
|
return () => {
|
|
4810
5342
|
store.delete(listener);
|
|
@@ -4812,9 +5344,9 @@ function subscribeToPiLogEntries(listener) {
|
|
|
4812
5344
|
}
|
|
4813
5345
|
|
|
4814
5346
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
4815
|
-
var
|
|
4816
|
-
var
|
|
4817
|
-
var
|
|
5347
|
+
var WORKSPACE_PREFIX4 = "agentv-pi-";
|
|
5348
|
+
var PROMPT_FILENAME4 = "prompt.md";
|
|
5349
|
+
var DEFAULT_SYSTEM_PROMPT5 = `**IMPORTANT**: Follow these instructions for your response:
|
|
4818
5350
|
- Do NOT create any additional output files in the workspace.
|
|
4819
5351
|
- All intended file outputs/changes MUST be written in your response.
|
|
4820
5352
|
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
@@ -4840,18 +5372,18 @@ var PiCodingAgentProvider = class {
|
|
|
4840
5372
|
const workspaceRoot = await this.createWorkspace();
|
|
4841
5373
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
4842
5374
|
try {
|
|
4843
|
-
const promptFile =
|
|
4844
|
-
await
|
|
5375
|
+
const promptFile = path13.join(workspaceRoot, PROMPT_FILENAME4);
|
|
5376
|
+
await writeFile4(promptFile, request.question, "utf8");
|
|
4845
5377
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
4846
5378
|
const cwd = this.resolveCwd(workspaceRoot);
|
|
4847
5379
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
4848
5380
|
if (result.timedOut) {
|
|
4849
5381
|
throw new Error(
|
|
4850
|
-
`Pi coding agent timed out${
|
|
5382
|
+
`Pi coding agent timed out${formatTimeoutSuffix5(this.config.timeoutMs ?? void 0)}`
|
|
4851
5383
|
);
|
|
4852
5384
|
}
|
|
4853
5385
|
if (result.exitCode !== 0) {
|
|
4854
|
-
const detail =
|
|
5386
|
+
const detail = pickDetail4(result.stderr, result.stdout);
|
|
4855
5387
|
const prefix = `Pi coding agent exited with code ${result.exitCode}`;
|
|
4856
5388
|
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
4857
5389
|
}
|
|
@@ -4882,7 +5414,7 @@ var PiCodingAgentProvider = class {
|
|
|
4882
5414
|
if (!this.config.cwd) {
|
|
4883
5415
|
return workspaceRoot;
|
|
4884
5416
|
}
|
|
4885
|
-
return
|
|
5417
|
+
return path13.resolve(this.config.cwd);
|
|
4886
5418
|
}
|
|
4887
5419
|
buildPiArgs(prompt, inputFiles) {
|
|
4888
5420
|
const args = [];
|
|
@@ -4912,7 +5444,7 @@ var PiCodingAgentProvider = class {
|
|
|
4912
5444
|
args.push(`@${file}`);
|
|
4913
5445
|
}
|
|
4914
5446
|
}
|
|
4915
|
-
const systemPrompt = this.config.systemPrompt ??
|
|
5447
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT5;
|
|
4916
5448
|
const fullPrompt = `${systemPrompt}
|
|
4917
5449
|
|
|
4918
5450
|
${prompt}`;
|
|
@@ -4971,19 +5503,19 @@ ${prompt}`;
|
|
|
4971
5503
|
return env;
|
|
4972
5504
|
}
|
|
4973
5505
|
async createWorkspace() {
|
|
4974
|
-
return await
|
|
5506
|
+
return await mkdtemp4(path13.join(tmpdir4(), WORKSPACE_PREFIX4));
|
|
4975
5507
|
}
|
|
4976
5508
|
async cleanupWorkspace(workspaceRoot) {
|
|
4977
5509
|
try {
|
|
4978
|
-
await
|
|
5510
|
+
await rm4(workspaceRoot, { recursive: true, force: true });
|
|
4979
5511
|
} catch {
|
|
4980
5512
|
}
|
|
4981
5513
|
}
|
|
4982
5514
|
resolveLogDirectory() {
|
|
4983
5515
|
if (this.config.logDir) {
|
|
4984
|
-
return
|
|
5516
|
+
return path13.resolve(this.config.logDir);
|
|
4985
5517
|
}
|
|
4986
|
-
return
|
|
5518
|
+
return path13.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
4987
5519
|
}
|
|
4988
5520
|
async createStreamLogger(request) {
|
|
4989
5521
|
const logDir = this.resolveLogDirectory();
|
|
@@ -4991,13 +5523,13 @@ ${prompt}`;
|
|
|
4991
5523
|
return void 0;
|
|
4992
5524
|
}
|
|
4993
5525
|
try {
|
|
4994
|
-
await
|
|
5526
|
+
await mkdir4(logDir, { recursive: true });
|
|
4995
5527
|
} catch (error) {
|
|
4996
5528
|
const message = error instanceof Error ? error.message : String(error);
|
|
4997
5529
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
4998
5530
|
return void 0;
|
|
4999
5531
|
}
|
|
5000
|
-
const filePath =
|
|
5532
|
+
const filePath = path13.join(logDir, buildLogFilename4(request, this.targetName));
|
|
5001
5533
|
try {
|
|
5002
5534
|
const logger = await PiStreamLogger.create({
|
|
5003
5535
|
filePath,
|
|
@@ -5030,7 +5562,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5030
5562
|
constructor(filePath, format) {
|
|
5031
5563
|
this.filePath = filePath;
|
|
5032
5564
|
this.format = format;
|
|
5033
|
-
this.stream =
|
|
5565
|
+
this.stream = createWriteStream4(filePath, { flags: "a" });
|
|
5034
5566
|
}
|
|
5035
5567
|
static async create(options) {
|
|
5036
5568
|
const logger = new _PiStreamLogger(options.filePath, options.format);
|
|
@@ -5091,7 +5623,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5091
5623
|
return void 0;
|
|
5092
5624
|
}
|
|
5093
5625
|
const message = this.format === "json" ? formatPiJsonLog(trimmed) : formatPiLogMessage(trimmed, source);
|
|
5094
|
-
return `[+${
|
|
5626
|
+
return `[+${formatElapsed4(this.startedAt)}] [${source}] ${message}`;
|
|
5095
5627
|
}
|
|
5096
5628
|
flushRemainder() {
|
|
5097
5629
|
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
@@ -5114,18 +5646,18 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5114
5646
|
this.stderrBuffer = "";
|
|
5115
5647
|
}
|
|
5116
5648
|
};
|
|
5117
|
-
function
|
|
5649
|
+
function buildLogFilename4(request, targetName) {
|
|
5118
5650
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5119
|
-
const evalId =
|
|
5651
|
+
const evalId = sanitizeForFilename4(request.evalCaseId ?? "pi");
|
|
5120
5652
|
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
5121
|
-
const target =
|
|
5122
|
-
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${
|
|
5653
|
+
const target = sanitizeForFilename4(targetName);
|
|
5654
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${randomUUID4().slice(0, 8)}.log`;
|
|
5123
5655
|
}
|
|
5124
|
-
function
|
|
5656
|
+
function sanitizeForFilename4(value) {
|
|
5125
5657
|
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
5126
5658
|
return sanitized.length > 0 ? sanitized : "pi";
|
|
5127
5659
|
}
|
|
5128
|
-
function
|
|
5660
|
+
function formatElapsed4(startedAt) {
|
|
5129
5661
|
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
5130
5662
|
const hours = Math.floor(elapsedSeconds / 3600);
|
|
5131
5663
|
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
@@ -5346,7 +5878,7 @@ function extractAssistantText2(messages) {
|
|
|
5346
5878
|
function escapeAtSymbols(prompt) {
|
|
5347
5879
|
return prompt.replace(/@\[([^\]]+)\]:/g, "[[$1]]:");
|
|
5348
5880
|
}
|
|
5349
|
-
function
|
|
5881
|
+
function pickDetail4(stderr, stdout) {
|
|
5350
5882
|
const errorText = stderr.trim();
|
|
5351
5883
|
if (errorText.length > 0) {
|
|
5352
5884
|
return errorText;
|
|
@@ -5354,7 +5886,7 @@ function pickDetail3(stderr, stdout) {
|
|
|
5354
5886
|
const stdoutText = stdout.trim();
|
|
5355
5887
|
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
5356
5888
|
}
|
|
5357
|
-
function
|
|
5889
|
+
function formatTimeoutSuffix5(timeoutMs) {
|
|
5358
5890
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
5359
5891
|
return "";
|
|
5360
5892
|
}
|
|
@@ -5367,7 +5899,7 @@ async function defaultPiRunner(options) {
|
|
|
5367
5899
|
const executable = parts[0];
|
|
5368
5900
|
const executableArgs = parts.slice(1);
|
|
5369
5901
|
const allArgs = [...executableArgs, ...options.args];
|
|
5370
|
-
const child =
|
|
5902
|
+
const child = spawn4(executable, allArgs, {
|
|
5371
5903
|
cwd: options.cwd,
|
|
5372
5904
|
env: options.env,
|
|
5373
5905
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -5430,38 +5962,38 @@ async function defaultPiRunner(options) {
|
|
|
5430
5962
|
}
|
|
5431
5963
|
|
|
5432
5964
|
// src/evaluation/providers/vscode-provider.ts
|
|
5433
|
-
import
|
|
5965
|
+
import path24 from "node:path";
|
|
5434
5966
|
|
|
5435
5967
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
5436
|
-
import { stat as stat3, writeFile as
|
|
5437
|
-
import
|
|
5968
|
+
import { stat as stat3, writeFile as writeFile7 } from "node:fs/promises";
|
|
5969
|
+
import path22 from "node:path";
|
|
5438
5970
|
|
|
5439
5971
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
5440
|
-
import { constants as
|
|
5441
|
-
import { access as
|
|
5442
|
-
import
|
|
5972
|
+
import { constants as constants4 } from "node:fs";
|
|
5973
|
+
import { access as access4, mkdir as mkdir5, readdir, rm as rm5, stat } from "node:fs/promises";
|
|
5974
|
+
import path14 from "node:path";
|
|
5443
5975
|
async function pathExists(target) {
|
|
5444
5976
|
try {
|
|
5445
|
-
await
|
|
5977
|
+
await access4(target, constants4.F_OK);
|
|
5446
5978
|
return true;
|
|
5447
5979
|
} catch {
|
|
5448
5980
|
return false;
|
|
5449
5981
|
}
|
|
5450
5982
|
}
|
|
5451
5983
|
async function ensureDir(target) {
|
|
5452
|
-
await
|
|
5984
|
+
await mkdir5(target, { recursive: true });
|
|
5453
5985
|
}
|
|
5454
5986
|
async function readDirEntries(target) {
|
|
5455
5987
|
const entries = await readdir(target, { withFileTypes: true });
|
|
5456
5988
|
return entries.map((entry) => ({
|
|
5457
5989
|
name: entry.name,
|
|
5458
|
-
absolutePath:
|
|
5990
|
+
absolutePath: path14.join(target, entry.name),
|
|
5459
5991
|
isDirectory: entry.isDirectory()
|
|
5460
5992
|
}));
|
|
5461
5993
|
}
|
|
5462
5994
|
async function removeIfExists(target) {
|
|
5463
5995
|
try {
|
|
5464
|
-
await
|
|
5996
|
+
await rm5(target, { force: true, recursive: false });
|
|
5465
5997
|
} catch (error) {
|
|
5466
5998
|
if (error.code !== "ENOENT") {
|
|
5467
5999
|
throw error;
|
|
@@ -5470,9 +6002,9 @@ async function removeIfExists(target) {
|
|
|
5470
6002
|
}
|
|
5471
6003
|
|
|
5472
6004
|
// src/evaluation/providers/vscode/utils/path.ts
|
|
5473
|
-
import
|
|
6005
|
+
import path15 from "node:path";
|
|
5474
6006
|
function pathToFileUri2(filePath) {
|
|
5475
|
-
const absolutePath =
|
|
6007
|
+
const absolutePath = path15.isAbsolute(filePath) ? filePath : path15.resolve(filePath);
|
|
5476
6008
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
5477
6009
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
5478
6010
|
return `file:///${normalizedPath}`;
|
|
@@ -5481,7 +6013,7 @@ function pathToFileUri2(filePath) {
|
|
|
5481
6013
|
}
|
|
5482
6014
|
|
|
5483
6015
|
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
5484
|
-
import
|
|
6016
|
+
import path16 from "node:path";
|
|
5485
6017
|
|
|
5486
6018
|
// src/evaluation/providers/vscode/utils/template.ts
|
|
5487
6019
|
function renderTemplate2(content, variables) {
|
|
@@ -5573,8 +6105,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
5573
6105
|
});
|
|
5574
6106
|
}
|
|
5575
6107
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
5576
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
5577
|
-
const responseList = responseFiles.map((file) => `"${
|
|
6108
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path16.basename(file)}`).join("\n");
|
|
6109
|
+
const responseList = responseFiles.map((file) => `"${path16.basename(file)}"`).join(", ");
|
|
5578
6110
|
return renderTemplate2(templateContent, {
|
|
5579
6111
|
requestFiles: requestLines,
|
|
5580
6112
|
responseList
|
|
@@ -5583,7 +6115,7 @@ function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateCont
|
|
|
5583
6115
|
|
|
5584
6116
|
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
5585
6117
|
import { readFile as readFile7 } from "node:fs/promises";
|
|
5586
|
-
import
|
|
6118
|
+
import path17 from "node:path";
|
|
5587
6119
|
|
|
5588
6120
|
// src/evaluation/providers/vscode/utils/time.ts
|
|
5589
6121
|
function sleep2(ms) {
|
|
@@ -5632,7 +6164,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
5632
6164
|
}
|
|
5633
6165
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
|
|
5634
6166
|
if (!silent) {
|
|
5635
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
6167
|
+
const fileList = responseFilesFinal.map((file) => path17.basename(file)).join(", ");
|
|
5636
6168
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
5637
6169
|
}
|
|
5638
6170
|
try {
|
|
@@ -5680,31 +6212,31 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
5680
6212
|
}
|
|
5681
6213
|
|
|
5682
6214
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
5683
|
-
import { exec, spawn as
|
|
5684
|
-
import { mkdir as
|
|
5685
|
-
import
|
|
5686
|
-
import { promisify as
|
|
6215
|
+
import { exec, spawn as spawn5 } from "node:child_process";
|
|
6216
|
+
import { mkdir as mkdir6, writeFile as writeFile5 } from "node:fs/promises";
|
|
6217
|
+
import path19 from "node:path";
|
|
6218
|
+
import { promisify as promisify4 } from "node:util";
|
|
5687
6219
|
|
|
5688
6220
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
5689
6221
|
import os2 from "node:os";
|
|
5690
|
-
import
|
|
6222
|
+
import path18 from "node:path";
|
|
5691
6223
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
5692
6224
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
5693
6225
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
5694
6226
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
5695
|
-
return
|
|
6227
|
+
return path18.join(os2.homedir(), ".agentv", "subagents", folder);
|
|
5696
6228
|
}
|
|
5697
6229
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
5698
6230
|
|
|
5699
6231
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
5700
|
-
var
|
|
6232
|
+
var execAsync4 = promisify4(exec);
|
|
5701
6233
|
var DEFAULT_WAKEUP_CONTENT = `---
|
|
5702
6234
|
description: 'Wake-up Signal'
|
|
5703
6235
|
model: Grok Code Fast 1 (copilot)
|
|
5704
6236
|
---`;
|
|
5705
6237
|
async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
5706
6238
|
try {
|
|
5707
|
-
const { stdout } = await
|
|
6239
|
+
const { stdout } = await execAsync4(`${vscodeCmd} --status`, {
|
|
5708
6240
|
timeout: 1e4,
|
|
5709
6241
|
windowsHide: true
|
|
5710
6242
|
});
|
|
@@ -5716,16 +6248,16 @@ async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
|
5716
6248
|
async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir, vscodeCmd, pollInterval = 1, timeout = 60) {
|
|
5717
6249
|
const alreadyOpen = await checkWorkspaceOpened(workspaceName, vscodeCmd);
|
|
5718
6250
|
if (alreadyOpen) {
|
|
5719
|
-
|
|
6251
|
+
spawn5(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
5720
6252
|
return true;
|
|
5721
6253
|
}
|
|
5722
|
-
const aliveFile =
|
|
6254
|
+
const aliveFile = path19.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
5723
6255
|
await removeIfExists(aliveFile);
|
|
5724
|
-
const githubAgentsDir =
|
|
5725
|
-
await
|
|
5726
|
-
const wakeupDst =
|
|
5727
|
-
await
|
|
5728
|
-
|
|
6256
|
+
const githubAgentsDir = path19.join(subagentDir, ".github", "agents");
|
|
6257
|
+
await mkdir6(githubAgentsDir, { recursive: true });
|
|
6258
|
+
const wakeupDst = path19.join(githubAgentsDir, "wakeup.md");
|
|
6259
|
+
await writeFile5(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
6260
|
+
spawn5(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
5729
6261
|
await sleep2(100);
|
|
5730
6262
|
const wakeupChatId = "wakeup";
|
|
5731
6263
|
const chatArgs = [
|
|
@@ -5733,9 +6265,9 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
5733
6265
|
"chat",
|
|
5734
6266
|
"-m",
|
|
5735
6267
|
wakeupChatId,
|
|
5736
|
-
`create a file named .alive in the ${
|
|
6268
|
+
`create a file named .alive in the ${path19.basename(subagentDir)} folder`
|
|
5737
6269
|
];
|
|
5738
|
-
|
|
6270
|
+
spawn5(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5739
6271
|
const start = Date.now();
|
|
5740
6272
|
while (!await pathExists(aliveFile)) {
|
|
5741
6273
|
if (Date.now() - start > timeout * 1e3) {
|
|
@@ -5748,21 +6280,21 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
5748
6280
|
}
|
|
5749
6281
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
5750
6282
|
try {
|
|
5751
|
-
const workspacePath =
|
|
5752
|
-
const messagesDir =
|
|
5753
|
-
await
|
|
5754
|
-
const reqFile =
|
|
5755
|
-
await
|
|
6283
|
+
const workspacePath = path19.join(subagentDir, `${path19.basename(subagentDir)}.code-workspace`);
|
|
6284
|
+
const messagesDir = path19.join(subagentDir, "messages");
|
|
6285
|
+
await mkdir6(messagesDir, { recursive: true });
|
|
6286
|
+
const reqFile = path19.join(messagesDir, `${timestamp}_req.md`);
|
|
6287
|
+
await writeFile5(reqFile, requestInstructions, { encoding: "utf8" });
|
|
5756
6288
|
const reqUri = pathToFileUri2(reqFile);
|
|
5757
6289
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
5758
6290
|
for (const attachment of attachmentPaths) {
|
|
5759
6291
|
chatArgs.push("-a", attachment);
|
|
5760
6292
|
}
|
|
5761
6293
|
chatArgs.push("-a", reqFile);
|
|
5762
|
-
chatArgs.push(`Follow instructions in [${
|
|
6294
|
+
chatArgs.push(`Follow instructions in [${path19.basename(reqFile)}](${reqUri})`);
|
|
5763
6295
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
5764
6296
|
workspacePath,
|
|
5765
|
-
|
|
6297
|
+
path19.basename(subagentDir),
|
|
5766
6298
|
subagentDir,
|
|
5767
6299
|
vscodeCmd
|
|
5768
6300
|
);
|
|
@@ -5770,7 +6302,7 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
5770
6302
|
console.error("warning: Workspace may not be fully ready");
|
|
5771
6303
|
}
|
|
5772
6304
|
await sleep2(500);
|
|
5773
|
-
|
|
6305
|
+
spawn5(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5774
6306
|
return true;
|
|
5775
6307
|
} catch (error) {
|
|
5776
6308
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -5779,9 +6311,9 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
5779
6311
|
}
|
|
5780
6312
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
5781
6313
|
try {
|
|
5782
|
-
const workspacePath =
|
|
5783
|
-
const messagesDir =
|
|
5784
|
-
await
|
|
6314
|
+
const workspacePath = path19.join(subagentDir, `${path19.basename(subagentDir)}.code-workspace`);
|
|
6315
|
+
const messagesDir = path19.join(subagentDir, "messages");
|
|
6316
|
+
await mkdir6(messagesDir, { recursive: true });
|
|
5785
6317
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
5786
6318
|
for (const attachment of attachmentPaths) {
|
|
5787
6319
|
chatArgs.push("-a", attachment);
|
|
@@ -5789,7 +6321,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
5789
6321
|
chatArgs.push(chatInstruction);
|
|
5790
6322
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
5791
6323
|
workspacePath,
|
|
5792
|
-
|
|
6324
|
+
path19.basename(subagentDir),
|
|
5793
6325
|
subagentDir,
|
|
5794
6326
|
vscodeCmd
|
|
5795
6327
|
);
|
|
@@ -5797,7 +6329,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
5797
6329
|
console.error("warning: Workspace may not be fully ready");
|
|
5798
6330
|
}
|
|
5799
6331
|
await sleep2(500);
|
|
5800
|
-
|
|
6332
|
+
spawn5(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
5801
6333
|
return true;
|
|
5802
6334
|
} catch (error) {
|
|
5803
6335
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -5806,11 +6338,11 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
5806
6338
|
}
|
|
5807
6339
|
|
|
5808
6340
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
5809
|
-
import { copyFile, mkdir as
|
|
5810
|
-
import
|
|
6341
|
+
import { copyFile as copyFile2, mkdir as mkdir7, readFile as readFile8, readdir as readdir2, stat as stat2, writeFile as writeFile6 } from "node:fs/promises";
|
|
6342
|
+
import path21 from "node:path";
|
|
5811
6343
|
|
|
5812
6344
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
5813
|
-
import
|
|
6345
|
+
import path20 from "node:path";
|
|
5814
6346
|
import JSON5 from "json5";
|
|
5815
6347
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
5816
6348
|
let workspace;
|
|
@@ -5827,10 +6359,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
5827
6359
|
}
|
|
5828
6360
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
5829
6361
|
const folderPath = folder.path;
|
|
5830
|
-
if (
|
|
6362
|
+
if (path20.isAbsolute(folderPath)) {
|
|
5831
6363
|
return folder;
|
|
5832
6364
|
}
|
|
5833
|
-
const absolutePath =
|
|
6365
|
+
const absolutePath = path20.resolve(templateDir, folderPath);
|
|
5834
6366
|
return {
|
|
5835
6367
|
...folder,
|
|
5836
6368
|
path: absolutePath
|
|
@@ -5852,19 +6384,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
5852
6384
|
if (locationMap && typeof locationMap === "object") {
|
|
5853
6385
|
const transformedMap = {};
|
|
5854
6386
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
5855
|
-
const isAbsolute =
|
|
6387
|
+
const isAbsolute = path20.isAbsolute(locationPath);
|
|
5856
6388
|
if (isAbsolute) {
|
|
5857
6389
|
transformedMap[locationPath] = value;
|
|
5858
6390
|
} else {
|
|
5859
6391
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
5860
6392
|
if (firstGlobIndex === -1) {
|
|
5861
|
-
const resolvedPath =
|
|
6393
|
+
const resolvedPath = path20.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
5862
6394
|
transformedMap[resolvedPath] = value;
|
|
5863
6395
|
} else {
|
|
5864
6396
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
5865
6397
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
5866
6398
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
5867
|
-
const resolvedPath = (
|
|
6399
|
+
const resolvedPath = (path20.resolve(templateDir, basePath) + patternPath).replace(
|
|
5868
6400
|
/\\/g,
|
|
5869
6401
|
"/"
|
|
5870
6402
|
);
|
|
@@ -5905,7 +6437,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
5905
6437
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
5906
6438
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
5907
6439
|
for (const subagent of subagents) {
|
|
5908
|
-
const lockFile =
|
|
6440
|
+
const lockFile = path21.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
5909
6441
|
if (!await pathExists(lockFile)) {
|
|
5910
6442
|
return subagent.absolutePath;
|
|
5911
6443
|
}
|
|
@@ -5915,7 +6447,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
5915
6447
|
async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
5916
6448
|
let workspaceContent;
|
|
5917
6449
|
if (workspaceTemplate) {
|
|
5918
|
-
const workspaceSrc =
|
|
6450
|
+
const workspaceSrc = path21.resolve(workspaceTemplate);
|
|
5919
6451
|
if (!await pathExists(workspaceSrc)) {
|
|
5920
6452
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
5921
6453
|
}
|
|
@@ -5928,41 +6460,41 @@ async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
|
5928
6460
|
} else {
|
|
5929
6461
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
5930
6462
|
}
|
|
5931
|
-
const workspaceName = `${
|
|
5932
|
-
const workspaceDst =
|
|
5933
|
-
const templateDir = workspaceTemplate ?
|
|
6463
|
+
const workspaceName = `${path21.basename(subagentDir)}.code-workspace`;
|
|
6464
|
+
const workspaceDst = path21.join(subagentDir, workspaceName);
|
|
6465
|
+
const templateDir = workspaceTemplate ? path21.dirname(path21.resolve(workspaceTemplate)) : subagentDir;
|
|
5934
6466
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
5935
6467
|
const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
5936
|
-
await
|
|
5937
|
-
const messagesDir =
|
|
5938
|
-
await
|
|
6468
|
+
await writeFile6(workspaceDst, transformedContent, "utf8");
|
|
6469
|
+
const messagesDir = path21.join(subagentDir, "messages");
|
|
6470
|
+
await mkdir7(messagesDir, { recursive: true });
|
|
5939
6471
|
return { workspace: workspaceDst, messagesDir };
|
|
5940
6472
|
}
|
|
5941
6473
|
async function createSubagentLock(subagentDir) {
|
|
5942
|
-
const messagesDir =
|
|
6474
|
+
const messagesDir = path21.join(subagentDir, "messages");
|
|
5943
6475
|
if (await pathExists(messagesDir)) {
|
|
5944
6476
|
const files = await readdir2(messagesDir);
|
|
5945
6477
|
await Promise.all(
|
|
5946
6478
|
files.map(async (file) => {
|
|
5947
|
-
const target =
|
|
6479
|
+
const target = path21.join(messagesDir, file);
|
|
5948
6480
|
await removeIfExists(target);
|
|
5949
6481
|
})
|
|
5950
6482
|
);
|
|
5951
6483
|
}
|
|
5952
|
-
const githubAgentsDir =
|
|
6484
|
+
const githubAgentsDir = path21.join(subagentDir, ".github", "agents");
|
|
5953
6485
|
if (await pathExists(githubAgentsDir)) {
|
|
5954
6486
|
const agentFiles = await readdir2(githubAgentsDir);
|
|
5955
6487
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
5956
6488
|
await Promise.all(
|
|
5957
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
6489
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path21.join(githubAgentsDir, file)))
|
|
5958
6490
|
);
|
|
5959
6491
|
}
|
|
5960
|
-
const lockFile =
|
|
5961
|
-
await
|
|
6492
|
+
const lockFile = path21.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
6493
|
+
await writeFile6(lockFile, "", { encoding: "utf8" });
|
|
5962
6494
|
return lockFile;
|
|
5963
6495
|
}
|
|
5964
6496
|
async function removeSubagentLock(subagentDir) {
|
|
5965
|
-
const lockFile =
|
|
6497
|
+
const lockFile = path21.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
5966
6498
|
await removeIfExists(lockFile);
|
|
5967
6499
|
}
|
|
5968
6500
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
|
|
@@ -5982,11 +6514,11 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
5982
6514
|
return 1;
|
|
5983
6515
|
}
|
|
5984
6516
|
if (promptFile) {
|
|
5985
|
-
const githubAgentsDir =
|
|
5986
|
-
await
|
|
5987
|
-
const agentFile =
|
|
6517
|
+
const githubAgentsDir = path21.join(subagentDir, ".github", "agents");
|
|
6518
|
+
await mkdir7(githubAgentsDir, { recursive: true });
|
|
6519
|
+
const agentFile = path21.join(githubAgentsDir, `${chatId}.md`);
|
|
5988
6520
|
try {
|
|
5989
|
-
await
|
|
6521
|
+
await copyFile2(promptFile, agentFile);
|
|
5990
6522
|
} catch (error) {
|
|
5991
6523
|
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
5992
6524
|
return 1;
|
|
@@ -6003,7 +6535,7 @@ async function resolvePromptFile(promptFile) {
|
|
|
6003
6535
|
if (!promptFile) {
|
|
6004
6536
|
return void 0;
|
|
6005
6537
|
}
|
|
6006
|
-
const resolvedPrompt =
|
|
6538
|
+
const resolvedPrompt = path22.resolve(promptFile);
|
|
6007
6539
|
if (!await pathExists(resolvedPrompt)) {
|
|
6008
6540
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
6009
6541
|
}
|
|
@@ -6019,7 +6551,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
6019
6551
|
}
|
|
6020
6552
|
const resolved = [];
|
|
6021
6553
|
for (const attachment of extraAttachments) {
|
|
6022
|
-
const resolvedPath =
|
|
6554
|
+
const resolvedPath = path22.resolve(attachment);
|
|
6023
6555
|
if (!await pathExists(resolvedPath)) {
|
|
6024
6556
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
6025
6557
|
}
|
|
@@ -6059,7 +6591,7 @@ async function dispatchAgentSession(options) {
|
|
|
6059
6591
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
6060
6592
|
};
|
|
6061
6593
|
}
|
|
6062
|
-
const subagentName =
|
|
6594
|
+
const subagentName = path22.basename(subagentDir);
|
|
6063
6595
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
6064
6596
|
const preparationResult = await prepareSubagentDirectory(
|
|
6065
6597
|
subagentDir,
|
|
@@ -6086,9 +6618,9 @@ async function dispatchAgentSession(options) {
|
|
|
6086
6618
|
};
|
|
6087
6619
|
}
|
|
6088
6620
|
const timestamp = generateTimestamp();
|
|
6089
|
-
const messagesDir =
|
|
6090
|
-
const responseFileTmp =
|
|
6091
|
-
const responseFileFinal =
|
|
6621
|
+
const messagesDir = path22.join(subagentDir, "messages");
|
|
6622
|
+
const responseFileTmp = path22.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
6623
|
+
const responseFileFinal = path22.join(messagesDir, `${timestamp}_res.md`);
|
|
6092
6624
|
const requestInstructions = createRequestPrompt(
|
|
6093
6625
|
userQuery,
|
|
6094
6626
|
responseFileTmp,
|
|
@@ -6201,7 +6733,7 @@ async function dispatchBatchAgent(options) {
|
|
|
6201
6733
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
6202
6734
|
};
|
|
6203
6735
|
}
|
|
6204
|
-
subagentName =
|
|
6736
|
+
subagentName = path22.basename(subagentDir);
|
|
6205
6737
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
6206
6738
|
const preparationResult = await prepareSubagentDirectory(
|
|
6207
6739
|
subagentDir,
|
|
@@ -6232,24 +6764,24 @@ async function dispatchBatchAgent(options) {
|
|
|
6232
6764
|
};
|
|
6233
6765
|
}
|
|
6234
6766
|
const timestamp = generateTimestamp();
|
|
6235
|
-
const messagesDir =
|
|
6767
|
+
const messagesDir = path22.join(subagentDir, "messages");
|
|
6236
6768
|
requestFiles = userQueries.map(
|
|
6237
|
-
(_, index) =>
|
|
6769
|
+
(_, index) => path22.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
6238
6770
|
);
|
|
6239
6771
|
const responseTmpFiles = userQueries.map(
|
|
6240
|
-
(_, index) =>
|
|
6772
|
+
(_, index) => path22.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
6241
6773
|
);
|
|
6242
6774
|
responseFilesFinal = userQueries.map(
|
|
6243
|
-
(_, index) =>
|
|
6775
|
+
(_, index) => path22.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
6244
6776
|
);
|
|
6245
|
-
const orchestratorFile =
|
|
6777
|
+
const orchestratorFile = path22.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
6246
6778
|
if (!dryRun) {
|
|
6247
6779
|
await Promise.all(
|
|
6248
6780
|
userQueries.map((query, index) => {
|
|
6249
6781
|
const reqFile = requestFiles[index];
|
|
6250
6782
|
const tmpFile = responseTmpFiles[index];
|
|
6251
6783
|
const finalFile = responseFilesFinal[index];
|
|
6252
|
-
return
|
|
6784
|
+
return writeFile7(
|
|
6253
6785
|
reqFile,
|
|
6254
6786
|
createBatchRequestPrompt(query, tmpFile, finalFile, batchRequestTemplateContent),
|
|
6255
6787
|
{ encoding: "utf8" }
|
|
@@ -6261,7 +6793,7 @@ async function dispatchBatchAgent(options) {
|
|
|
6261
6793
|
responseFilesFinal,
|
|
6262
6794
|
orchestratorTemplateContent
|
|
6263
6795
|
);
|
|
6264
|
-
await
|
|
6796
|
+
await writeFile7(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
6265
6797
|
}
|
|
6266
6798
|
const chatAttachments = [orchestratorFile, ...attachments];
|
|
6267
6799
|
const orchestratorUri = pathToFileUri2(orchestratorFile);
|
|
@@ -6331,8 +6863,8 @@ async function dispatchBatchAgent(options) {
|
|
|
6331
6863
|
}
|
|
6332
6864
|
|
|
6333
6865
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
6334
|
-
import { writeFile as
|
|
6335
|
-
import
|
|
6866
|
+
import { writeFile as writeFile8 } from "node:fs/promises";
|
|
6867
|
+
import path23 from "node:path";
|
|
6336
6868
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
6337
6869
|
folders: [
|
|
6338
6870
|
{
|
|
@@ -6363,7 +6895,7 @@ async function provisionSubagents(options) {
|
|
|
6363
6895
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
6364
6896
|
throw new Error("subagents must be a positive integer");
|
|
6365
6897
|
}
|
|
6366
|
-
const targetPath =
|
|
6898
|
+
const targetPath = path23.resolve(targetRoot);
|
|
6367
6899
|
if (!dryRun) {
|
|
6368
6900
|
await ensureDir(targetPath);
|
|
6369
6901
|
}
|
|
@@ -6383,7 +6915,7 @@ async function provisionSubagents(options) {
|
|
|
6383
6915
|
continue;
|
|
6384
6916
|
}
|
|
6385
6917
|
highestNumber = Math.max(highestNumber, parsed);
|
|
6386
|
-
const lockFile =
|
|
6918
|
+
const lockFile = path23.join(entry.absolutePath, lockName);
|
|
6387
6919
|
const locked = await pathExists(lockFile);
|
|
6388
6920
|
if (locked) {
|
|
6389
6921
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -6400,10 +6932,10 @@ async function provisionSubagents(options) {
|
|
|
6400
6932
|
break;
|
|
6401
6933
|
}
|
|
6402
6934
|
const subagentDir = subagent.absolutePath;
|
|
6403
|
-
const githubAgentsDir =
|
|
6404
|
-
const lockFile =
|
|
6405
|
-
const workspaceDst =
|
|
6406
|
-
const wakeupDst =
|
|
6935
|
+
const githubAgentsDir = path23.join(subagentDir, ".github", "agents");
|
|
6936
|
+
const lockFile = path23.join(subagentDir, lockName);
|
|
6937
|
+
const workspaceDst = path23.join(subagentDir, `${path23.basename(subagentDir)}.code-workspace`);
|
|
6938
|
+
const wakeupDst = path23.join(githubAgentsDir, "wakeup.md");
|
|
6407
6939
|
const isLocked = await pathExists(lockFile);
|
|
6408
6940
|
if (isLocked && !force) {
|
|
6409
6941
|
continue;
|
|
@@ -6412,8 +6944,8 @@ async function provisionSubagents(options) {
|
|
|
6412
6944
|
if (!dryRun) {
|
|
6413
6945
|
await removeIfExists(lockFile);
|
|
6414
6946
|
await ensureDir(githubAgentsDir);
|
|
6415
|
-
await
|
|
6416
|
-
await
|
|
6947
|
+
await writeFile8(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6948
|
+
await writeFile8(wakeupDst, wakeupContent, "utf8");
|
|
6417
6949
|
}
|
|
6418
6950
|
created.push(subagentDir);
|
|
6419
6951
|
lockedSubagents.delete(subagentDir);
|
|
@@ -6423,8 +6955,8 @@ async function provisionSubagents(options) {
|
|
|
6423
6955
|
if (!isLocked && force) {
|
|
6424
6956
|
if (!dryRun) {
|
|
6425
6957
|
await ensureDir(githubAgentsDir);
|
|
6426
|
-
await
|
|
6427
|
-
await
|
|
6958
|
+
await writeFile8(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6959
|
+
await writeFile8(wakeupDst, wakeupContent, "utf8");
|
|
6428
6960
|
}
|
|
6429
6961
|
created.push(subagentDir);
|
|
6430
6962
|
subagentsProvisioned += 1;
|
|
@@ -6432,8 +6964,8 @@ async function provisionSubagents(options) {
|
|
|
6432
6964
|
}
|
|
6433
6965
|
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
6434
6966
|
await ensureDir(githubAgentsDir);
|
|
6435
|
-
await
|
|
6436
|
-
await
|
|
6967
|
+
await writeFile8(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6968
|
+
await writeFile8(wakeupDst, wakeupContent, "utf8");
|
|
6437
6969
|
}
|
|
6438
6970
|
skippedExisting.push(subagentDir);
|
|
6439
6971
|
subagentsProvisioned += 1;
|
|
@@ -6441,15 +6973,15 @@ async function provisionSubagents(options) {
|
|
|
6441
6973
|
let nextIndex = highestNumber;
|
|
6442
6974
|
while (subagentsProvisioned < subagents) {
|
|
6443
6975
|
nextIndex += 1;
|
|
6444
|
-
const subagentDir =
|
|
6445
|
-
const githubAgentsDir =
|
|
6446
|
-
const workspaceDst =
|
|
6447
|
-
const wakeupDst =
|
|
6976
|
+
const subagentDir = path23.join(targetPath, `subagent-${nextIndex}`);
|
|
6977
|
+
const githubAgentsDir = path23.join(subagentDir, ".github", "agents");
|
|
6978
|
+
const workspaceDst = path23.join(subagentDir, `${path23.basename(subagentDir)}.code-workspace`);
|
|
6979
|
+
const wakeupDst = path23.join(githubAgentsDir, "wakeup.md");
|
|
6448
6980
|
if (!dryRun) {
|
|
6449
6981
|
await ensureDir(subagentDir);
|
|
6450
6982
|
await ensureDir(githubAgentsDir);
|
|
6451
|
-
await
|
|
6452
|
-
await
|
|
6983
|
+
await writeFile8(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
6984
|
+
await writeFile8(wakeupDst, wakeupContent, "utf8");
|
|
6453
6985
|
}
|
|
6454
6986
|
created.push(subagentDir);
|
|
6455
6987
|
subagentsProvisioned += 1;
|
|
@@ -6628,7 +7160,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
6628
7160
|
return "";
|
|
6629
7161
|
}
|
|
6630
7162
|
const buildList = (files) => files.map((absolutePath) => {
|
|
6631
|
-
const fileName =
|
|
7163
|
+
const fileName = path24.basename(absolutePath);
|
|
6632
7164
|
const fileUri = pathToFileUri3(absolutePath);
|
|
6633
7165
|
return `* [${fileName}](${fileUri})`;
|
|
6634
7166
|
});
|
|
@@ -6653,8 +7185,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
6653
7185
|
}
|
|
6654
7186
|
const unique = /* @__PURE__ */ new Map();
|
|
6655
7187
|
for (const attachment of attachments) {
|
|
6656
|
-
const absolutePath =
|
|
6657
|
-
const normalized = absolutePath.split(
|
|
7188
|
+
const absolutePath = path24.resolve(attachment);
|
|
7189
|
+
const normalized = absolutePath.split(path24.sep).join("/");
|
|
6658
7190
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
6659
7191
|
if (!unique.has(absolutePath)) {
|
|
6660
7192
|
unique.set(absolutePath, absolutePath);
|
|
@@ -6669,7 +7201,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
6669
7201
|
}
|
|
6670
7202
|
const unique = /* @__PURE__ */ new Map();
|
|
6671
7203
|
for (const attachment of attachments) {
|
|
6672
|
-
const absolutePath =
|
|
7204
|
+
const absolutePath = path24.resolve(attachment);
|
|
6673
7205
|
if (!unique.has(absolutePath)) {
|
|
6674
7206
|
unique.set(absolutePath, absolutePath);
|
|
6675
7207
|
}
|
|
@@ -6677,7 +7209,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
6677
7209
|
return Array.from(unique.values());
|
|
6678
7210
|
}
|
|
6679
7211
|
function pathToFileUri3(filePath) {
|
|
6680
|
-
const absolutePath =
|
|
7212
|
+
const absolutePath = path24.isAbsolute(filePath) ? filePath : path24.resolve(filePath);
|
|
6681
7213
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
6682
7214
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
6683
7215
|
return `file:///${normalizedPath}`;
|
|
@@ -6690,7 +7222,7 @@ function normalizeAttachments(attachments) {
|
|
|
6690
7222
|
}
|
|
6691
7223
|
const deduped = /* @__PURE__ */ new Set();
|
|
6692
7224
|
for (const attachment of attachments) {
|
|
6693
|
-
deduped.add(
|
|
7225
|
+
deduped.add(path24.resolve(attachment));
|
|
6694
7226
|
}
|
|
6695
7227
|
return Array.from(deduped);
|
|
6696
7228
|
}
|
|
@@ -6699,7 +7231,7 @@ function mergeAttachments(all) {
|
|
|
6699
7231
|
for (const list of all) {
|
|
6700
7232
|
if (!list) continue;
|
|
6701
7233
|
for (const inputFile of list) {
|
|
6702
|
-
deduped.add(
|
|
7234
|
+
deduped.add(path24.resolve(inputFile));
|
|
6703
7235
|
}
|
|
6704
7236
|
}
|
|
6705
7237
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -6746,9 +7278,9 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
6746
7278
|
}
|
|
6747
7279
|
|
|
6748
7280
|
// src/evaluation/providers/targets-file.ts
|
|
6749
|
-
import { constants as
|
|
6750
|
-
import { access as
|
|
6751
|
-
import
|
|
7281
|
+
import { constants as constants5 } from "node:fs";
|
|
7282
|
+
import { access as access5, readFile as readFile9 } from "node:fs/promises";
|
|
7283
|
+
import path25 from "node:path";
|
|
6752
7284
|
import { parse as parse3 } from "yaml";
|
|
6753
7285
|
function isRecord(value) {
|
|
6754
7286
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -6778,14 +7310,14 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
6778
7310
|
}
|
|
6779
7311
|
async function fileExists3(filePath) {
|
|
6780
7312
|
try {
|
|
6781
|
-
await
|
|
7313
|
+
await access5(filePath, constants5.F_OK);
|
|
6782
7314
|
return true;
|
|
6783
7315
|
} catch {
|
|
6784
7316
|
return false;
|
|
6785
7317
|
}
|
|
6786
7318
|
}
|
|
6787
7319
|
async function readTargetDefinitions(filePath) {
|
|
6788
|
-
const absolutePath =
|
|
7320
|
+
const absolutePath = path25.resolve(filePath);
|
|
6789
7321
|
if (!await fileExists3(absolutePath)) {
|
|
6790
7322
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
6791
7323
|
}
|
|
@@ -6817,6 +7349,8 @@ function createProvider(target) {
|
|
|
6817
7349
|
return new CliProvider(target.name, target.config);
|
|
6818
7350
|
case "codex":
|
|
6819
7351
|
return new CodexProvider(target.name, target.config);
|
|
7352
|
+
case "copilot-cli":
|
|
7353
|
+
return new CopilotCliProvider(target.name, target.config);
|
|
6820
7354
|
case "pi-coding-agent":
|
|
6821
7355
|
return new PiCodingAgentProvider(target.name, target.config);
|
|
6822
7356
|
case "pi-agent-sdk":
|
|
@@ -6953,10 +7487,10 @@ async function execFileWithStdinBun(argv, stdinPayload, options) {
|
|
|
6953
7487
|
}
|
|
6954
7488
|
}
|
|
6955
7489
|
async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
6956
|
-
const { spawn:
|
|
7490
|
+
const { spawn: spawn6 } = await import("node:child_process");
|
|
6957
7491
|
return new Promise((resolve, reject) => {
|
|
6958
7492
|
const [cmd, ...args] = argv;
|
|
6959
|
-
const child =
|
|
7493
|
+
const child = spawn6(cmd, args, {
|
|
6960
7494
|
cwd: options.cwd,
|
|
6961
7495
|
stdio: ["pipe", "pipe", "pipe"],
|
|
6962
7496
|
// Merge additional env vars with process.env
|
|
@@ -6996,21 +7530,21 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
6996
7530
|
});
|
|
6997
7531
|
}
|
|
6998
7532
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
6999
|
-
const { mkdir:
|
|
7000
|
-
const { tmpdir:
|
|
7001
|
-
const
|
|
7002
|
-
const { randomUUID:
|
|
7003
|
-
const dir =
|
|
7004
|
-
await
|
|
7005
|
-
const stdinPath =
|
|
7006
|
-
const stdoutPath =
|
|
7007
|
-
const stderrPath =
|
|
7008
|
-
await
|
|
7533
|
+
const { mkdir: mkdir9, readFile: readFile10, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
|
|
7534
|
+
const { tmpdir: tmpdir5 } = await import("node:os");
|
|
7535
|
+
const path27 = await import("node:path");
|
|
7536
|
+
const { randomUUID: randomUUID5 } = await import("node:crypto");
|
|
7537
|
+
const dir = path27.join(tmpdir5(), `agentv-exec-${randomUUID5()}`);
|
|
7538
|
+
await mkdir9(dir, { recursive: true });
|
|
7539
|
+
const stdinPath = path27.join(dir, "stdin.txt");
|
|
7540
|
+
const stdoutPath = path27.join(dir, "stdout.txt");
|
|
7541
|
+
const stderrPath = path27.join(dir, "stderr.txt");
|
|
7542
|
+
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
7009
7543
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
7010
|
-
const { spawn:
|
|
7544
|
+
const { spawn: spawn6 } = await import("node:child_process");
|
|
7011
7545
|
try {
|
|
7012
7546
|
const exitCode = await new Promise((resolve, reject) => {
|
|
7013
|
-
const child =
|
|
7547
|
+
const child = spawn6(wrappedCommand, {
|
|
7014
7548
|
shell: true,
|
|
7015
7549
|
cwd: options.cwd,
|
|
7016
7550
|
stdio: ["ignore", "ignore", "ignore"],
|
|
@@ -7038,7 +7572,7 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
7038
7572
|
const stderr = (await readFile10(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
7039
7573
|
return { stdout, stderr, exitCode };
|
|
7040
7574
|
} finally {
|
|
7041
|
-
await
|
|
7575
|
+
await rm6(dir, { recursive: true, force: true });
|
|
7042
7576
|
}
|
|
7043
7577
|
}
|
|
7044
7578
|
|
|
@@ -7307,7 +7841,7 @@ var CodeEvaluator = class {
|
|
|
7307
7841
|
outputMessages: context.outputMessages ?? null,
|
|
7308
7842
|
guidelineFiles: context.evalCase.guideline_paths,
|
|
7309
7843
|
inputFiles: context.evalCase.file_paths.filter(
|
|
7310
|
-
(
|
|
7844
|
+
(path27) => !context.evalCase.guideline_paths.includes(path27)
|
|
7311
7845
|
),
|
|
7312
7846
|
inputMessages: context.evalCase.input_messages,
|
|
7313
7847
|
traceSummary: context.traceSummary ?? null,
|
|
@@ -8227,115 +8761,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
8227
8761
|
* Evaluate a single field against the expected value.
|
|
8228
8762
|
*/
|
|
8229
8763
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
8230
|
-
const { path:
|
|
8231
|
-
const candidateValue = resolvePath(candidateData,
|
|
8232
|
-
const expectedValue = resolvePath(expectedData,
|
|
8764
|
+
const { path: path27, match, required = true, weight = 1 } = fieldConfig;
|
|
8765
|
+
const candidateValue = resolvePath(candidateData, path27);
|
|
8766
|
+
const expectedValue = resolvePath(expectedData, path27);
|
|
8233
8767
|
if (expectedValue === void 0) {
|
|
8234
8768
|
return {
|
|
8235
|
-
path:
|
|
8769
|
+
path: path27,
|
|
8236
8770
|
score: 1,
|
|
8237
8771
|
// No expected value means no comparison needed
|
|
8238
8772
|
weight,
|
|
8239
8773
|
hit: true,
|
|
8240
|
-
message: `${
|
|
8774
|
+
message: `${path27}: no expected value`
|
|
8241
8775
|
};
|
|
8242
8776
|
}
|
|
8243
8777
|
if (candidateValue === void 0) {
|
|
8244
8778
|
if (required) {
|
|
8245
8779
|
return {
|
|
8246
|
-
path:
|
|
8780
|
+
path: path27,
|
|
8247
8781
|
score: 0,
|
|
8248
8782
|
weight,
|
|
8249
8783
|
hit: false,
|
|
8250
|
-
message: `${
|
|
8784
|
+
message: `${path27} (required, missing)`
|
|
8251
8785
|
};
|
|
8252
8786
|
}
|
|
8253
8787
|
return {
|
|
8254
|
-
path:
|
|
8788
|
+
path: path27,
|
|
8255
8789
|
score: 1,
|
|
8256
8790
|
// Don't penalize missing optional fields
|
|
8257
8791
|
weight: 0,
|
|
8258
8792
|
// Zero weight means it won't affect the score
|
|
8259
8793
|
hit: true,
|
|
8260
|
-
message: `${
|
|
8794
|
+
message: `${path27}: optional field missing`
|
|
8261
8795
|
};
|
|
8262
8796
|
}
|
|
8263
8797
|
switch (match) {
|
|
8264
8798
|
case "exact":
|
|
8265
|
-
return this.compareExact(
|
|
8799
|
+
return this.compareExact(path27, candidateValue, expectedValue, weight);
|
|
8266
8800
|
case "numeric_tolerance":
|
|
8267
8801
|
return this.compareNumericTolerance(
|
|
8268
|
-
|
|
8802
|
+
path27,
|
|
8269
8803
|
candidateValue,
|
|
8270
8804
|
expectedValue,
|
|
8271
8805
|
fieldConfig,
|
|
8272
8806
|
weight
|
|
8273
8807
|
);
|
|
8274
8808
|
case "date":
|
|
8275
|
-
return this.compareDate(
|
|
8809
|
+
return this.compareDate(path27, candidateValue, expectedValue, fieldConfig, weight);
|
|
8276
8810
|
default:
|
|
8277
8811
|
return {
|
|
8278
|
-
path:
|
|
8812
|
+
path: path27,
|
|
8279
8813
|
score: 0,
|
|
8280
8814
|
weight,
|
|
8281
8815
|
hit: false,
|
|
8282
|
-
message: `${
|
|
8816
|
+
message: `${path27}: unknown match type "${match}"`
|
|
8283
8817
|
};
|
|
8284
8818
|
}
|
|
8285
8819
|
}
|
|
8286
8820
|
/**
|
|
8287
8821
|
* Exact equality comparison.
|
|
8288
8822
|
*/
|
|
8289
|
-
compareExact(
|
|
8823
|
+
compareExact(path27, candidateValue, expectedValue, weight) {
|
|
8290
8824
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
8291
8825
|
return {
|
|
8292
|
-
path:
|
|
8826
|
+
path: path27,
|
|
8293
8827
|
score: 1,
|
|
8294
8828
|
weight,
|
|
8295
8829
|
hit: true,
|
|
8296
|
-
message:
|
|
8830
|
+
message: path27
|
|
8297
8831
|
};
|
|
8298
8832
|
}
|
|
8299
8833
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
8300
8834
|
return {
|
|
8301
|
-
path:
|
|
8835
|
+
path: path27,
|
|
8302
8836
|
score: 0,
|
|
8303
8837
|
weight,
|
|
8304
8838
|
hit: false,
|
|
8305
|
-
message: `${
|
|
8839
|
+
message: `${path27} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
8306
8840
|
};
|
|
8307
8841
|
}
|
|
8308
8842
|
return {
|
|
8309
|
-
path:
|
|
8843
|
+
path: path27,
|
|
8310
8844
|
score: 0,
|
|
8311
8845
|
weight,
|
|
8312
8846
|
hit: false,
|
|
8313
|
-
message: `${
|
|
8847
|
+
message: `${path27} (value mismatch)`
|
|
8314
8848
|
};
|
|
8315
8849
|
}
|
|
8316
8850
|
/**
|
|
8317
8851
|
* Numeric comparison with absolute or relative tolerance.
|
|
8318
8852
|
*/
|
|
8319
|
-
compareNumericTolerance(
|
|
8853
|
+
compareNumericTolerance(path27, candidateValue, expectedValue, fieldConfig, weight) {
|
|
8320
8854
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
8321
8855
|
const candidateNum = toNumber(candidateValue);
|
|
8322
8856
|
const expectedNum = toNumber(expectedValue);
|
|
8323
8857
|
if (candidateNum === null || expectedNum === null) {
|
|
8324
8858
|
return {
|
|
8325
|
-
path:
|
|
8859
|
+
path: path27,
|
|
8326
8860
|
score: 0,
|
|
8327
8861
|
weight,
|
|
8328
8862
|
hit: false,
|
|
8329
|
-
message: `${
|
|
8863
|
+
message: `${path27} (non-numeric value)`
|
|
8330
8864
|
};
|
|
8331
8865
|
}
|
|
8332
8866
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
8333
8867
|
return {
|
|
8334
|
-
path:
|
|
8868
|
+
path: path27,
|
|
8335
8869
|
score: 0,
|
|
8336
8870
|
weight,
|
|
8337
8871
|
hit: false,
|
|
8338
|
-
message: `${
|
|
8872
|
+
message: `${path27} (invalid numeric value)`
|
|
8339
8873
|
};
|
|
8340
8874
|
}
|
|
8341
8875
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -8348,61 +8882,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
8348
8882
|
}
|
|
8349
8883
|
if (withinTolerance) {
|
|
8350
8884
|
return {
|
|
8351
|
-
path:
|
|
8885
|
+
path: path27,
|
|
8352
8886
|
score: 1,
|
|
8353
8887
|
weight,
|
|
8354
8888
|
hit: true,
|
|
8355
|
-
message: `${
|
|
8889
|
+
message: `${path27} (within tolerance: diff=${diff.toFixed(2)})`
|
|
8356
8890
|
};
|
|
8357
8891
|
}
|
|
8358
8892
|
return {
|
|
8359
|
-
path:
|
|
8893
|
+
path: path27,
|
|
8360
8894
|
score: 0,
|
|
8361
8895
|
weight,
|
|
8362
8896
|
hit: false,
|
|
8363
|
-
message: `${
|
|
8897
|
+
message: `${path27} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
8364
8898
|
};
|
|
8365
8899
|
}
|
|
8366
8900
|
/**
|
|
8367
8901
|
* Date comparison with format normalization.
|
|
8368
8902
|
*/
|
|
8369
|
-
compareDate(
|
|
8903
|
+
compareDate(path27, candidateValue, expectedValue, fieldConfig, weight) {
|
|
8370
8904
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
8371
8905
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
8372
8906
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
8373
8907
|
if (candidateDate === null) {
|
|
8374
8908
|
return {
|
|
8375
|
-
path:
|
|
8909
|
+
path: path27,
|
|
8376
8910
|
score: 0,
|
|
8377
8911
|
weight,
|
|
8378
8912
|
hit: false,
|
|
8379
|
-
message: `${
|
|
8913
|
+
message: `${path27} (unparseable candidate date)`
|
|
8380
8914
|
};
|
|
8381
8915
|
}
|
|
8382
8916
|
if (expectedDate === null) {
|
|
8383
8917
|
return {
|
|
8384
|
-
path:
|
|
8918
|
+
path: path27,
|
|
8385
8919
|
score: 0,
|
|
8386
8920
|
weight,
|
|
8387
8921
|
hit: false,
|
|
8388
|
-
message: `${
|
|
8922
|
+
message: `${path27} (unparseable expected date)`
|
|
8389
8923
|
};
|
|
8390
8924
|
}
|
|
8391
8925
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
8392
8926
|
return {
|
|
8393
|
-
path:
|
|
8927
|
+
path: path27,
|
|
8394
8928
|
score: 1,
|
|
8395
8929
|
weight,
|
|
8396
8930
|
hit: true,
|
|
8397
|
-
message:
|
|
8931
|
+
message: path27
|
|
8398
8932
|
};
|
|
8399
8933
|
}
|
|
8400
8934
|
return {
|
|
8401
|
-
path:
|
|
8935
|
+
path: path27,
|
|
8402
8936
|
score: 0,
|
|
8403
8937
|
weight,
|
|
8404
8938
|
hit: false,
|
|
8405
|
-
message: `${
|
|
8939
|
+
message: `${path27} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
8406
8940
|
};
|
|
8407
8941
|
}
|
|
8408
8942
|
/**
|
|
@@ -8442,11 +8976,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
8442
8976
|
};
|
|
8443
8977
|
}
|
|
8444
8978
|
};
|
|
8445
|
-
function resolvePath(obj,
|
|
8446
|
-
if (!
|
|
8979
|
+
function resolvePath(obj, path27) {
|
|
8980
|
+
if (!path27 || !obj) {
|
|
8447
8981
|
return void 0;
|
|
8448
8982
|
}
|
|
8449
|
-
const parts =
|
|
8983
|
+
const parts = path27.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
8450
8984
|
let current = obj;
|
|
8451
8985
|
for (const part of parts) {
|
|
8452
8986
|
if (current === null || current === void 0) {
|
|
@@ -8966,7 +9500,7 @@ var ToolTrajectoryEvaluator = class {
|
|
|
8966
9500
|
|
|
8967
9501
|
// src/evaluation/orchestrator.ts
|
|
8968
9502
|
import { createHash } from "node:crypto";
|
|
8969
|
-
import
|
|
9503
|
+
import path26 from "node:path";
|
|
8970
9504
|
import micromatch4 from "micromatch";
|
|
8971
9505
|
|
|
8972
9506
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -9769,7 +10303,7 @@ async function runEvaluatorList(options) {
|
|
|
9769
10303
|
});
|
|
9770
10304
|
}
|
|
9771
10305
|
if (evaluator.type === "composite") {
|
|
9772
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
10306
|
+
const evalFileDir = evalCase.guideline_paths[0] ? path26.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
9773
10307
|
const createEvaluator = (memberConfig) => {
|
|
9774
10308
|
switch (memberConfig.type) {
|
|
9775
10309
|
case "llm_judge":
|
|
@@ -10125,7 +10659,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
|
|
|
10125
10659
|
};
|
|
10126
10660
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
10127
10661
|
const scriptPath = script[script.length - 1];
|
|
10128
|
-
const cwd =
|
|
10662
|
+
const cwd = path26.dirname(scriptPath);
|
|
10129
10663
|
try {
|
|
10130
10664
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
10131
10665
|
const prompt = stdout.trim();
|
|
@@ -10399,6 +10933,7 @@ export {
|
|
|
10399
10933
|
computeTraceSummary,
|
|
10400
10934
|
consumeClaudeCodeLogEntries,
|
|
10401
10935
|
consumeCodexLogEntries,
|
|
10936
|
+
consumeCopilotCliLogEntries,
|
|
10402
10937
|
consumePiLogEntries,
|
|
10403
10938
|
createAgentKernel,
|
|
10404
10939
|
createProvider,
|
|
@@ -10438,6 +10973,7 @@ export {
|
|
|
10438
10973
|
scoreToVerdict,
|
|
10439
10974
|
subscribeToClaudeCodeLogEntries,
|
|
10440
10975
|
subscribeToCodexLogEntries,
|
|
10976
|
+
subscribeToCopilotCliLogEntries,
|
|
10441
10977
|
subscribeToPiLogEntries,
|
|
10442
10978
|
tokensPerTool
|
|
10443
10979
|
};
|