@agentv/core 2.5.5 → 2.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RP3M7COZ.js → chunk-LGQ5OPJD.js} +50 -1
- package/dist/chunk-LGQ5OPJD.js.map +1 -0
- package/dist/evaluation/validation/index.cjs +25 -0
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +25 -1
- package/dist/evaluation/validation/index.js.map +1 -1
- package/dist/index.cjs +836 -250
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +763 -227
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/dist/chunk-RP3M7COZ.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -49,6 +49,7 @@ __export(index_exports, {
|
|
|
49
49
|
computeTraceSummary: () => computeTraceSummary,
|
|
50
50
|
consumeClaudeCodeLogEntries: () => consumeClaudeCodeLogEntries,
|
|
51
51
|
consumeCodexLogEntries: () => consumeCodexLogEntries,
|
|
52
|
+
consumeCopilotCliLogEntries: () => consumeCopilotCliLogEntries,
|
|
52
53
|
consumePiLogEntries: () => consumePiLogEntries,
|
|
53
54
|
createAgentKernel: () => createAgentKernel,
|
|
54
55
|
createProvider: () => createProvider,
|
|
@@ -88,6 +89,7 @@ __export(index_exports, {
|
|
|
88
89
|
scoreToVerdict: () => scoreToVerdict,
|
|
89
90
|
subscribeToClaudeCodeLogEntries: () => subscribeToClaudeCodeLogEntries,
|
|
90
91
|
subscribeToCodexLogEntries: () => subscribeToCodexLogEntries,
|
|
92
|
+
subscribeToCopilotCliLogEntries: () => subscribeToCopilotCliLogEntries,
|
|
91
93
|
subscribeToPiLogEntries: () => subscribeToPiLogEntries,
|
|
92
94
|
tokensPerTool: () => tokensPerTool
|
|
93
95
|
});
|
|
@@ -3418,8 +3420,8 @@ async function runClaudeCodeWithTempFiles(options, stdoutFile, stderrFile, exitF
|
|
|
3418
3420
|
};
|
|
3419
3421
|
const fileExists4 = async (filePath) => {
|
|
3420
3422
|
try {
|
|
3421
|
-
const { access:
|
|
3422
|
-
await
|
|
3423
|
+
const { access: access7 } = await import("fs/promises");
|
|
3424
|
+
await access7(filePath);
|
|
3423
3425
|
return true;
|
|
3424
3426
|
} catch {
|
|
3425
3427
|
return false;
|
|
@@ -4744,6 +4746,538 @@ function shouldShellExecute(executable) {
|
|
|
4744
4746
|
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
4745
4747
|
}
|
|
4746
4748
|
|
|
4749
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4750
|
+
var import_node_child_process4 = require("child_process");
|
|
4751
|
+
var import_node_crypto3 = require("crypto");
|
|
4752
|
+
var import_node_fs5 = require("fs");
|
|
4753
|
+
var import_promises12 = require("fs/promises");
|
|
4754
|
+
var import_node_os4 = require("os");
|
|
4755
|
+
var import_node_path13 = __toESM(require("path"), 1);
|
|
4756
|
+
var import_node_util3 = require("util");
|
|
4757
|
+
|
|
4758
|
+
// src/evaluation/providers/copilot-cli-log-tracker.ts
|
|
4759
|
+
var GLOBAL_LOGS_KEY3 = Symbol.for("agentv.copilotCliLogs");
|
|
4760
|
+
var GLOBAL_SUBSCRIBERS_KEY3 = Symbol.for("agentv.copilotCliLogSubscribers");
|
|
4761
|
+
function getCopilotCliLogStore() {
|
|
4762
|
+
const globalObject = globalThis;
|
|
4763
|
+
const existing = globalObject[GLOBAL_LOGS_KEY3];
|
|
4764
|
+
if (existing) {
|
|
4765
|
+
return existing;
|
|
4766
|
+
}
|
|
4767
|
+
const created = [];
|
|
4768
|
+
globalObject[GLOBAL_LOGS_KEY3] = created;
|
|
4769
|
+
return created;
|
|
4770
|
+
}
|
|
4771
|
+
function getSubscriberStore3() {
|
|
4772
|
+
const globalObject = globalThis;
|
|
4773
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY3];
|
|
4774
|
+
if (existing) {
|
|
4775
|
+
return existing;
|
|
4776
|
+
}
|
|
4777
|
+
const created = /* @__PURE__ */ new Set();
|
|
4778
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY3] = created;
|
|
4779
|
+
return created;
|
|
4780
|
+
}
|
|
4781
|
+
function notifySubscribers3(entry) {
|
|
4782
|
+
const subscribers = Array.from(getSubscriberStore3());
|
|
4783
|
+
for (const listener of subscribers) {
|
|
4784
|
+
try {
|
|
4785
|
+
listener(entry);
|
|
4786
|
+
} catch (error) {
|
|
4787
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4788
|
+
console.warn(`Copilot CLI log subscriber failed: ${message}`);
|
|
4789
|
+
}
|
|
4790
|
+
}
|
|
4791
|
+
}
|
|
4792
|
+
function recordCopilotCliLogEntry(entry) {
|
|
4793
|
+
getCopilotCliLogStore().push(entry);
|
|
4794
|
+
notifySubscribers3(entry);
|
|
4795
|
+
}
|
|
4796
|
+
function consumeCopilotCliLogEntries() {
|
|
4797
|
+
const store = getCopilotCliLogStore();
|
|
4798
|
+
if (store.length === 0) {
|
|
4799
|
+
return [];
|
|
4800
|
+
}
|
|
4801
|
+
return store.splice(0, store.length);
|
|
4802
|
+
}
|
|
4803
|
+
function subscribeToCopilotCliLogEntries(listener) {
|
|
4804
|
+
const store = getSubscriberStore3();
|
|
4805
|
+
store.add(listener);
|
|
4806
|
+
return () => {
|
|
4807
|
+
store.delete(listener);
|
|
4808
|
+
};
|
|
4809
|
+
}
|
|
4810
|
+
|
|
4811
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4812
|
+
var execAsync3 = (0, import_node_util3.promisify)(import_node_child_process4.exec);
|
|
4813
|
+
var WORKSPACE_PREFIX3 = "agentv-copilot-";
|
|
4814
|
+
var PROMPT_FILENAME3 = "prompt.md";
|
|
4815
|
+
var DEFAULT_SYSTEM_PROMPT4 = `**IMPORTANT**: Follow these instructions for your response:
|
|
4816
|
+
- Do NOT create any additional output files in the workspace.
|
|
4817
|
+
- All intended file outputs/changes MUST be written in your response.
|
|
4818
|
+
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
4819
|
+
This is required for evaluation scoring.`;
|
|
4820
|
+
async function copyInputFilesToWorkspace(workspaceRoot, inputFiles) {
|
|
4821
|
+
const usedNames = /* @__PURE__ */ new Map();
|
|
4822
|
+
const mappings = [];
|
|
4823
|
+
for (const originalPath of inputFiles) {
|
|
4824
|
+
const ext = import_node_path13.default.extname(originalPath);
|
|
4825
|
+
const stem = import_node_path13.default.basename(originalPath, ext);
|
|
4826
|
+
let relativeName;
|
|
4827
|
+
const baseKey = `${stem}${ext}`;
|
|
4828
|
+
const count = usedNames.get(baseKey) ?? 0;
|
|
4829
|
+
if (count === 0) {
|
|
4830
|
+
relativeName = baseKey;
|
|
4831
|
+
} else {
|
|
4832
|
+
relativeName = `${stem}_${count}${ext}`;
|
|
4833
|
+
}
|
|
4834
|
+
usedNames.set(baseKey, count + 1);
|
|
4835
|
+
const dest = import_node_path13.default.join(workspaceRoot, relativeName);
|
|
4836
|
+
await (0, import_promises12.copyFile)(originalPath, dest);
|
|
4837
|
+
mappings.push({ originalPath, workspaceRelativePath: relativeName });
|
|
4838
|
+
}
|
|
4839
|
+
return mappings;
|
|
4840
|
+
}
|
|
4841
|
+
function buildCopilotFilePrereadBlock(guidelineMappings, inputMappings) {
|
|
4842
|
+
if (guidelineMappings.length === 0 && inputMappings.length === 0) {
|
|
4843
|
+
return "";
|
|
4844
|
+
}
|
|
4845
|
+
const buildList = (mappings) => mappings.map((m) => `* ${m.workspaceRelativePath}`).join("\n");
|
|
4846
|
+
const sections = [];
|
|
4847
|
+
if (guidelineMappings.length > 0) {
|
|
4848
|
+
sections.push(`Read all guideline files:
|
|
4849
|
+
${buildList(guidelineMappings)}.`);
|
|
4850
|
+
}
|
|
4851
|
+
if (inputMappings.length > 0) {
|
|
4852
|
+
sections.push(`Read all input files:
|
|
4853
|
+
${buildList(inputMappings)}.`);
|
|
4854
|
+
}
|
|
4855
|
+
sections.push(
|
|
4856
|
+
"If any file is missing, fail with ERROR: missing-file <filename> and stop.",
|
|
4857
|
+
"Then apply system_instructions on the user query below."
|
|
4858
|
+
);
|
|
4859
|
+
return sections.join("\n");
|
|
4860
|
+
}
|
|
4861
|
+
var CopilotCliProvider = class {
|
|
4862
|
+
id;
|
|
4863
|
+
kind = "copilot-cli";
|
|
4864
|
+
targetName;
|
|
4865
|
+
supportsBatch = false;
|
|
4866
|
+
config;
|
|
4867
|
+
runCopilot;
|
|
4868
|
+
environmentCheck;
|
|
4869
|
+
resolvedExecutable;
|
|
4870
|
+
constructor(targetName, config, runner = defaultCopilotCliRunner) {
|
|
4871
|
+
this.id = `copilot-cli:${targetName}`;
|
|
4872
|
+
this.targetName = targetName;
|
|
4873
|
+
this.config = config;
|
|
4874
|
+
this.runCopilot = runner;
|
|
4875
|
+
}
|
|
4876
|
+
async invoke(request) {
|
|
4877
|
+
if (request.signal?.aborted) {
|
|
4878
|
+
throw new Error("Copilot CLI request was aborted before execution");
|
|
4879
|
+
}
|
|
4880
|
+
await this.ensureEnvironmentReady();
|
|
4881
|
+
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
4882
|
+
const workspaceRoot = await this.createWorkspace();
|
|
4883
|
+
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
4884
|
+
try {
|
|
4885
|
+
const copiedFiles = inputFiles ? await copyInputFilesToWorkspace(workspaceRoot, inputFiles) : [];
|
|
4886
|
+
const guidelineFileSet = new Set(
|
|
4887
|
+
collectGuidelineFiles(inputFiles, request.guideline_patterns)
|
|
4888
|
+
);
|
|
4889
|
+
const guidelineMappings = copiedFiles.filter((m) => guidelineFileSet.has(m.originalPath));
|
|
4890
|
+
const nonGuidelineMappings = copiedFiles.filter((m) => !guidelineFileSet.has(m.originalPath));
|
|
4891
|
+
const prereadBlock = buildCopilotFilePrereadBlock(guidelineMappings, nonGuidelineMappings);
|
|
4892
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT4;
|
|
4893
|
+
const promptParts = [systemPrompt];
|
|
4894
|
+
if (prereadBlock.length > 0) {
|
|
4895
|
+
promptParts.push("", prereadBlock);
|
|
4896
|
+
}
|
|
4897
|
+
promptParts.push("", "[[ ## user_query ## ]]", request.question.trim());
|
|
4898
|
+
const promptContent = promptParts.join("\n");
|
|
4899
|
+
const promptFile = import_node_path13.default.join(workspaceRoot, PROMPT_FILENAME3);
|
|
4900
|
+
await (0, import_promises12.writeFile)(promptFile, promptContent, "utf8");
|
|
4901
|
+
const args = this.buildCopilotArgs(promptContent);
|
|
4902
|
+
const cwd = this.resolveCwd(workspaceRoot);
|
|
4903
|
+
const result = await this.executeCopilot(args, cwd, promptContent, request.signal, logger);
|
|
4904
|
+
if (result.timedOut) {
|
|
4905
|
+
throw new Error(
|
|
4906
|
+
`Copilot CLI timed out${formatTimeoutSuffix4(this.config.timeoutMs ?? void 0)}`
|
|
4907
|
+
);
|
|
4908
|
+
}
|
|
4909
|
+
if (result.exitCode !== 0) {
|
|
4910
|
+
const detail = pickDetail3(result.stderr, result.stdout);
|
|
4911
|
+
const prefix = `Copilot CLI exited with code ${result.exitCode}`;
|
|
4912
|
+
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
4913
|
+
}
|
|
4914
|
+
const assistantText = extractCopilotResponse(result.stdout);
|
|
4915
|
+
return {
|
|
4916
|
+
raw: {
|
|
4917
|
+
stdout: result.stdout,
|
|
4918
|
+
stderr: result.stderr,
|
|
4919
|
+
exitCode: result.exitCode,
|
|
4920
|
+
args,
|
|
4921
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4922
|
+
promptFile,
|
|
4923
|
+
workspace: workspaceRoot,
|
|
4924
|
+
inputFiles,
|
|
4925
|
+
copiedFiles,
|
|
4926
|
+
logFile: logger?.filePath
|
|
4927
|
+
},
|
|
4928
|
+
outputMessages: [{ role: "assistant", content: assistantText }]
|
|
4929
|
+
};
|
|
4930
|
+
} finally {
|
|
4931
|
+
await logger?.close();
|
|
4932
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
4933
|
+
}
|
|
4934
|
+
}
|
|
4935
|
+
async ensureEnvironmentReady() {
|
|
4936
|
+
if (!this.environmentCheck) {
|
|
4937
|
+
this.environmentCheck = this.validateEnvironment();
|
|
4938
|
+
}
|
|
4939
|
+
await this.environmentCheck;
|
|
4940
|
+
}
|
|
4941
|
+
async validateEnvironment() {
|
|
4942
|
+
this.resolvedExecutable = await locateExecutable2(this.config.executable);
|
|
4943
|
+
}
|
|
4944
|
+
resolveCwd(workspaceRoot) {
|
|
4945
|
+
if (!this.config.cwd) {
|
|
4946
|
+
return workspaceRoot;
|
|
4947
|
+
}
|
|
4948
|
+
return import_node_path13.default.resolve(this.config.cwd);
|
|
4949
|
+
}
|
|
4950
|
+
buildCopilotArgs(prompt) {
|
|
4951
|
+
const args = [];
|
|
4952
|
+
args.push("-s");
|
|
4953
|
+
args.push("--allow-all-tools");
|
|
4954
|
+
args.push("--no-color");
|
|
4955
|
+
if (this.config.model) {
|
|
4956
|
+
args.push("--model", this.config.model);
|
|
4957
|
+
}
|
|
4958
|
+
if (this.config.args && this.config.args.length > 0) {
|
|
4959
|
+
args.push(...this.config.args);
|
|
4960
|
+
}
|
|
4961
|
+
args.push("-p", prompt);
|
|
4962
|
+
return args;
|
|
4963
|
+
}
|
|
4964
|
+
async executeCopilot(args, cwd, promptContent, signal, logger) {
|
|
4965
|
+
try {
|
|
4966
|
+
return await this.runCopilot({
|
|
4967
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4968
|
+
args,
|
|
4969
|
+
cwd,
|
|
4970
|
+
prompt: promptContent,
|
|
4971
|
+
timeoutMs: this.config.timeoutMs,
|
|
4972
|
+
env: process.env,
|
|
4973
|
+
signal,
|
|
4974
|
+
onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : void 0,
|
|
4975
|
+
onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : void 0
|
|
4976
|
+
});
|
|
4977
|
+
} catch (error) {
|
|
4978
|
+
const err = error;
|
|
4979
|
+
if (err.code === "ENOENT") {
|
|
4980
|
+
throw new Error(
|
|
4981
|
+
`Copilot executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
|
|
4982
|
+
);
|
|
4983
|
+
}
|
|
4984
|
+
throw error;
|
|
4985
|
+
}
|
|
4986
|
+
}
|
|
4987
|
+
async createWorkspace() {
|
|
4988
|
+
return await (0, import_promises12.mkdtemp)(import_node_path13.default.join((0, import_node_os4.tmpdir)(), WORKSPACE_PREFIX3));
|
|
4989
|
+
}
|
|
4990
|
+
async cleanupWorkspace(workspaceRoot) {
|
|
4991
|
+
try {
|
|
4992
|
+
await (0, import_promises12.rm)(workspaceRoot, { recursive: true, force: true });
|
|
4993
|
+
} catch {
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
resolveLogDirectory() {
|
|
4997
|
+
const disabled = isCopilotLogStreamingDisabled();
|
|
4998
|
+
if (disabled) {
|
|
4999
|
+
return void 0;
|
|
5000
|
+
}
|
|
5001
|
+
if (this.config.logDir) {
|
|
5002
|
+
return import_node_path13.default.resolve(this.config.logDir);
|
|
5003
|
+
}
|
|
5004
|
+
return import_node_path13.default.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
5005
|
+
}
|
|
5006
|
+
async createStreamLogger(request) {
|
|
5007
|
+
const logDir = this.resolveLogDirectory();
|
|
5008
|
+
if (!logDir) {
|
|
5009
|
+
return void 0;
|
|
5010
|
+
}
|
|
5011
|
+
try {
|
|
5012
|
+
await (0, import_promises12.mkdir)(logDir, { recursive: true });
|
|
5013
|
+
} catch (error) {
|
|
5014
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5015
|
+
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
5016
|
+
return void 0;
|
|
5017
|
+
}
|
|
5018
|
+
const filePath = import_node_path13.default.join(logDir, buildLogFilename3(request, this.targetName));
|
|
5019
|
+
try {
|
|
5020
|
+
const logger = await CopilotCliStreamLogger.create({
|
|
5021
|
+
filePath,
|
|
5022
|
+
targetName: this.targetName,
|
|
5023
|
+
evalCaseId: request.evalCaseId,
|
|
5024
|
+
attempt: request.attempt,
|
|
5025
|
+
format: this.config.logFormat ?? "summary"
|
|
5026
|
+
});
|
|
5027
|
+
recordCopilotCliLogEntry({
|
|
5028
|
+
filePath,
|
|
5029
|
+
targetName: this.targetName,
|
|
5030
|
+
evalCaseId: request.evalCaseId,
|
|
5031
|
+
attempt: request.attempt
|
|
5032
|
+
});
|
|
5033
|
+
return logger;
|
|
5034
|
+
} catch (error) {
|
|
5035
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5036
|
+
console.warn(`Skipping Copilot CLI stream logging for ${filePath}: ${message}`);
|
|
5037
|
+
return void 0;
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
};
|
|
5041
|
+
var CopilotCliStreamLogger = class _CopilotCliStreamLogger {
|
|
5042
|
+
filePath;
|
|
5043
|
+
stream;
|
|
5044
|
+
startedAt = Date.now();
|
|
5045
|
+
stdoutBuffer = "";
|
|
5046
|
+
stderrBuffer = "";
|
|
5047
|
+
format;
|
|
5048
|
+
constructor(filePath, format) {
|
|
5049
|
+
this.filePath = filePath;
|
|
5050
|
+
this.format = format;
|
|
5051
|
+
this.stream = (0, import_node_fs5.createWriteStream)(filePath, { flags: "a" });
|
|
5052
|
+
}
|
|
5053
|
+
static async create(options) {
|
|
5054
|
+
const logger = new _CopilotCliStreamLogger(options.filePath, options.format);
|
|
5055
|
+
const header = [
|
|
5056
|
+
"# Copilot CLI stream log",
|
|
5057
|
+
`# target: ${options.targetName}`,
|
|
5058
|
+
options.evalCaseId ? `# eval: ${options.evalCaseId}` : void 0,
|
|
5059
|
+
options.attempt !== void 0 ? `# attempt: ${options.attempt + 1}` : void 0,
|
|
5060
|
+
`# started: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
5061
|
+
""
|
|
5062
|
+
].filter((line) => Boolean(line));
|
|
5063
|
+
logger.writeLines(header);
|
|
5064
|
+
return logger;
|
|
5065
|
+
}
|
|
5066
|
+
handleStdoutChunk(chunk) {
|
|
5067
|
+
this.stdoutBuffer += chunk;
|
|
5068
|
+
this.flushBuffer("stdout");
|
|
5069
|
+
}
|
|
5070
|
+
handleStderrChunk(chunk) {
|
|
5071
|
+
this.stderrBuffer += chunk;
|
|
5072
|
+
this.flushBuffer("stderr");
|
|
5073
|
+
}
|
|
5074
|
+
async close() {
|
|
5075
|
+
this.flushBuffer("stdout");
|
|
5076
|
+
this.flushBuffer("stderr");
|
|
5077
|
+
this.flushRemainder();
|
|
5078
|
+
await new Promise((resolve, reject) => {
|
|
5079
|
+
this.stream.once("error", reject);
|
|
5080
|
+
this.stream.end(() => resolve());
|
|
5081
|
+
});
|
|
5082
|
+
}
|
|
5083
|
+
writeLines(lines) {
|
|
5084
|
+
for (const line of lines) {
|
|
5085
|
+
this.stream.write(`${line}
|
|
5086
|
+
`);
|
|
5087
|
+
}
|
|
5088
|
+
}
|
|
5089
|
+
flushBuffer(source) {
|
|
5090
|
+
const buffer = source === "stdout" ? this.stdoutBuffer : this.stderrBuffer;
|
|
5091
|
+
const lines = buffer.split(/\r?\n/);
|
|
5092
|
+
const remainder = lines.pop() ?? "";
|
|
5093
|
+
if (source === "stdout") {
|
|
5094
|
+
this.stdoutBuffer = remainder;
|
|
5095
|
+
} else {
|
|
5096
|
+
this.stderrBuffer = remainder;
|
|
5097
|
+
}
|
|
5098
|
+
for (const line of lines) {
|
|
5099
|
+
const formatted = this.formatLine(line, source);
|
|
5100
|
+
if (formatted) {
|
|
5101
|
+
this.stream.write(formatted);
|
|
5102
|
+
this.stream.write("\n");
|
|
5103
|
+
}
|
|
5104
|
+
}
|
|
5105
|
+
}
|
|
5106
|
+
formatLine(rawLine, source) {
|
|
5107
|
+
const trimmed = rawLine.trim();
|
|
5108
|
+
if (trimmed.length === 0) {
|
|
5109
|
+
return void 0;
|
|
5110
|
+
}
|
|
5111
|
+
const prefix = source === "stderr" ? "stderr: " : "";
|
|
5112
|
+
return `[+${formatElapsed3(this.startedAt)}] [${source}] ${prefix}${trimmed}`;
|
|
5113
|
+
}
|
|
5114
|
+
flushRemainder() {
|
|
5115
|
+
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
5116
|
+
if (stdoutRemainder.length > 0) {
|
|
5117
|
+
const formatted = this.formatLine(stdoutRemainder, "stdout");
|
|
5118
|
+
if (formatted) {
|
|
5119
|
+
this.stream.write(formatted);
|
|
5120
|
+
this.stream.write("\n");
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
5123
|
+
const stderrRemainder = this.stderrBuffer.trim();
|
|
5124
|
+
if (stderrRemainder.length > 0) {
|
|
5125
|
+
const formatted = this.formatLine(stderrRemainder, "stderr");
|
|
5126
|
+
if (formatted) {
|
|
5127
|
+
this.stream.write(formatted);
|
|
5128
|
+
this.stream.write("\n");
|
|
5129
|
+
}
|
|
5130
|
+
}
|
|
5131
|
+
this.stdoutBuffer = "";
|
|
5132
|
+
this.stderrBuffer = "";
|
|
5133
|
+
}
|
|
5134
|
+
};
|
|
5135
|
+
function isCopilotLogStreamingDisabled() {
|
|
5136
|
+
const envValue = process.env.AGENTV_COPILOT_STREAM_LOGS;
|
|
5137
|
+
if (!envValue) {
|
|
5138
|
+
return false;
|
|
5139
|
+
}
|
|
5140
|
+
const normalized = envValue.trim().toLowerCase();
|
|
5141
|
+
return normalized === "false" || normalized === "0" || normalized === "off";
|
|
5142
|
+
}
|
|
5143
|
+
function buildLogFilename3(request, targetName) {
|
|
5144
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5145
|
+
const evalId = sanitizeForFilename3(request.evalCaseId ?? "copilot");
|
|
5146
|
+
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
5147
|
+
const target = sanitizeForFilename3(targetName);
|
|
5148
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0, import_node_crypto3.randomUUID)().slice(0, 8)}.log`;
|
|
5149
|
+
}
|
|
5150
|
+
function sanitizeForFilename3(value) {
|
|
5151
|
+
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
5152
|
+
return sanitized.length > 0 ? sanitized : "copilot";
|
|
5153
|
+
}
|
|
5154
|
+
function formatElapsed3(startedAt) {
|
|
5155
|
+
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
5156
|
+
const hours = Math.floor(elapsedSeconds / 3600);
|
|
5157
|
+
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
5158
|
+
const seconds = elapsedSeconds % 60;
|
|
5159
|
+
if (hours > 0) {
|
|
5160
|
+
return `${hours.toString().padStart(2, "0")}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
5161
|
+
}
|
|
5162
|
+
return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
5163
|
+
}
|
|
5164
|
+
var ANSI_ESCAPE_RE = /\x1B\[[0-9;]*[A-Za-z]/g;
|
|
5165
|
+
var ANSI_OSC_RE = /\x1B\][^\x07]*\x07/g;
|
|
5166
|
+
function stripAnsiEscapes(text) {
|
|
5167
|
+
return text.replace(ANSI_ESCAPE_RE, "").replace(ANSI_OSC_RE, "");
|
|
5168
|
+
}
|
|
5169
|
+
function extractCopilotResponse(stdout) {
|
|
5170
|
+
const cleaned = stripAnsiEscapes(stdout).trim();
|
|
5171
|
+
if (cleaned.length === 0) {
|
|
5172
|
+
throw new Error("Copilot CLI produced no output");
|
|
5173
|
+
}
|
|
5174
|
+
return cleaned;
|
|
5175
|
+
}
|
|
5176
|
+
function pickDetail3(stderr, stdout) {
|
|
5177
|
+
const errorText = stderr.trim();
|
|
5178
|
+
if (errorText.length > 0) {
|
|
5179
|
+
return errorText;
|
|
5180
|
+
}
|
|
5181
|
+
const stdoutText = stdout.trim();
|
|
5182
|
+
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
5183
|
+
}
|
|
5184
|
+
function formatTimeoutSuffix4(timeoutMs) {
|
|
5185
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
5186
|
+
return "";
|
|
5187
|
+
}
|
|
5188
|
+
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
5189
|
+
return ` after ${seconds}s`;
|
|
5190
|
+
}
|
|
5191
|
+
async function locateExecutable2(candidate) {
|
|
5192
|
+
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
5193
|
+
if (includesPathSeparator) {
|
|
5194
|
+
const resolved = import_node_path13.default.isAbsolute(candidate) ? candidate : import_node_path13.default.resolve(candidate);
|
|
5195
|
+
await (0, import_promises12.access)(resolved, import_node_fs5.constants.F_OK);
|
|
5196
|
+
return resolved;
|
|
5197
|
+
}
|
|
5198
|
+
const locator = process.platform === "win32" ? "where" : "which";
|
|
5199
|
+
try {
|
|
5200
|
+
const { stdout } = await execAsync3(`${locator} ${candidate}`);
|
|
5201
|
+
const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
5202
|
+
if (lines.length > 0 && lines[0]) {
|
|
5203
|
+
await (0, import_promises12.access)(lines[0], import_node_fs5.constants.F_OK);
|
|
5204
|
+
return lines[0];
|
|
5205
|
+
}
|
|
5206
|
+
} catch {
|
|
5207
|
+
}
|
|
5208
|
+
throw new Error(`Copilot executable '${candidate}' was not found on PATH`);
|
|
5209
|
+
}
|
|
5210
|
+
function shouldShellExecute2(executable) {
|
|
5211
|
+
if (process.platform !== "win32") {
|
|
5212
|
+
return false;
|
|
5213
|
+
}
|
|
5214
|
+
const lower = executable.toLowerCase();
|
|
5215
|
+
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
5216
|
+
}
|
|
5217
|
+
async function defaultCopilotCliRunner(options) {
|
|
5218
|
+
return await new Promise((resolve, reject) => {
|
|
5219
|
+
const child = (0, import_node_child_process4.spawn)(options.executable, options.args, {
|
|
5220
|
+
cwd: options.cwd,
|
|
5221
|
+
env: options.env,
|
|
5222
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
5223
|
+
shell: shouldShellExecute2(options.executable)
|
|
5224
|
+
});
|
|
5225
|
+
let stdout = "";
|
|
5226
|
+
let stderr = "";
|
|
5227
|
+
let timedOut = false;
|
|
5228
|
+
const onAbort = () => {
|
|
5229
|
+
child.kill("SIGTERM");
|
|
5230
|
+
};
|
|
5231
|
+
if (options.signal) {
|
|
5232
|
+
if (options.signal.aborted) {
|
|
5233
|
+
onAbort();
|
|
5234
|
+
} else {
|
|
5235
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
5236
|
+
}
|
|
5237
|
+
}
|
|
5238
|
+
let timeoutHandle;
|
|
5239
|
+
if (options.timeoutMs && options.timeoutMs > 0) {
|
|
5240
|
+
timeoutHandle = setTimeout(() => {
|
|
5241
|
+
timedOut = true;
|
|
5242
|
+
child.kill("SIGTERM");
|
|
5243
|
+
}, options.timeoutMs);
|
|
5244
|
+
timeoutHandle.unref?.();
|
|
5245
|
+
}
|
|
5246
|
+
child.stdout.setEncoding("utf8");
|
|
5247
|
+
child.stdout.on("data", (chunk) => {
|
|
5248
|
+
stdout += chunk;
|
|
5249
|
+
options.onStdoutChunk?.(chunk);
|
|
5250
|
+
});
|
|
5251
|
+
child.stderr.setEncoding("utf8");
|
|
5252
|
+
child.stderr.on("data", (chunk) => {
|
|
5253
|
+
stderr += chunk;
|
|
5254
|
+
options.onStderrChunk?.(chunk);
|
|
5255
|
+
});
|
|
5256
|
+
child.stdin.end();
|
|
5257
|
+
const cleanup = () => {
|
|
5258
|
+
if (timeoutHandle) {
|
|
5259
|
+
clearTimeout(timeoutHandle);
|
|
5260
|
+
}
|
|
5261
|
+
if (options.signal) {
|
|
5262
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
5263
|
+
}
|
|
5264
|
+
};
|
|
5265
|
+
child.on("error", (error) => {
|
|
5266
|
+
cleanup();
|
|
5267
|
+
reject(error);
|
|
5268
|
+
});
|
|
5269
|
+
child.on("close", (code) => {
|
|
5270
|
+
cleanup();
|
|
5271
|
+
resolve({
|
|
5272
|
+
stdout,
|
|
5273
|
+
stderr,
|
|
5274
|
+
exitCode: typeof code === "number" ? code : -1,
|
|
5275
|
+
timedOut
|
|
5276
|
+
});
|
|
5277
|
+
});
|
|
5278
|
+
});
|
|
5279
|
+
}
|
|
5280
|
+
|
|
4747
5281
|
// src/evaluation/providers/mock.ts
|
|
4748
5282
|
var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
|
|
4749
5283
|
var MockProvider = class {
|
|
@@ -4947,38 +5481,38 @@ function extractToolCalls2(content) {
|
|
|
4947
5481
|
}
|
|
4948
5482
|
|
|
4949
5483
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
4950
|
-
var
|
|
4951
|
-
var
|
|
4952
|
-
var
|
|
4953
|
-
var
|
|
4954
|
-
var
|
|
4955
|
-
var
|
|
5484
|
+
var import_node_child_process5 = require("child_process");
|
|
5485
|
+
var import_node_crypto4 = require("crypto");
|
|
5486
|
+
var import_node_fs6 = require("fs");
|
|
5487
|
+
var import_promises13 = require("fs/promises");
|
|
5488
|
+
var import_node_os5 = require("os");
|
|
5489
|
+
var import_node_path14 = __toESM(require("path"), 1);
|
|
4956
5490
|
|
|
4957
5491
|
// src/evaluation/providers/pi-log-tracker.ts
|
|
4958
|
-
var
|
|
4959
|
-
var
|
|
5492
|
+
var GLOBAL_LOGS_KEY4 = Symbol.for("agentv.piLogs");
|
|
5493
|
+
var GLOBAL_SUBSCRIBERS_KEY4 = Symbol.for("agentv.piLogSubscribers");
|
|
4960
5494
|
function getPiLogStore() {
|
|
4961
5495
|
const globalObject = globalThis;
|
|
4962
|
-
const existing = globalObject[
|
|
5496
|
+
const existing = globalObject[GLOBAL_LOGS_KEY4];
|
|
4963
5497
|
if (existing) {
|
|
4964
5498
|
return existing;
|
|
4965
5499
|
}
|
|
4966
5500
|
const created = [];
|
|
4967
|
-
globalObject[
|
|
5501
|
+
globalObject[GLOBAL_LOGS_KEY4] = created;
|
|
4968
5502
|
return created;
|
|
4969
5503
|
}
|
|
4970
|
-
function
|
|
5504
|
+
function getSubscriberStore4() {
|
|
4971
5505
|
const globalObject = globalThis;
|
|
4972
|
-
const existing = globalObject[
|
|
5506
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY4];
|
|
4973
5507
|
if (existing) {
|
|
4974
5508
|
return existing;
|
|
4975
5509
|
}
|
|
4976
5510
|
const created = /* @__PURE__ */ new Set();
|
|
4977
|
-
globalObject[
|
|
5511
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY4] = created;
|
|
4978
5512
|
return created;
|
|
4979
5513
|
}
|
|
4980
|
-
function
|
|
4981
|
-
const subscribers = Array.from(
|
|
5514
|
+
function notifySubscribers4(entry) {
|
|
5515
|
+
const subscribers = Array.from(getSubscriberStore4());
|
|
4982
5516
|
for (const listener of subscribers) {
|
|
4983
5517
|
try {
|
|
4984
5518
|
listener(entry);
|
|
@@ -4990,7 +5524,7 @@ function notifySubscribers3(entry) {
|
|
|
4990
5524
|
}
|
|
4991
5525
|
function recordPiLogEntry(entry) {
|
|
4992
5526
|
getPiLogStore().push(entry);
|
|
4993
|
-
|
|
5527
|
+
notifySubscribers4(entry);
|
|
4994
5528
|
}
|
|
4995
5529
|
function consumePiLogEntries() {
|
|
4996
5530
|
const store = getPiLogStore();
|
|
@@ -5000,7 +5534,7 @@ function consumePiLogEntries() {
|
|
|
5000
5534
|
return store.splice(0, store.length);
|
|
5001
5535
|
}
|
|
5002
5536
|
function subscribeToPiLogEntries(listener) {
|
|
5003
|
-
const store =
|
|
5537
|
+
const store = getSubscriberStore4();
|
|
5004
5538
|
store.add(listener);
|
|
5005
5539
|
return () => {
|
|
5006
5540
|
store.delete(listener);
|
|
@@ -5008,9 +5542,9 @@ function subscribeToPiLogEntries(listener) {
|
|
|
5008
5542
|
}
|
|
5009
5543
|
|
|
5010
5544
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
5011
|
-
var
|
|
5012
|
-
var
|
|
5013
|
-
var
|
|
5545
|
+
var WORKSPACE_PREFIX4 = "agentv-pi-";
|
|
5546
|
+
var PROMPT_FILENAME4 = "prompt.md";
|
|
5547
|
+
var DEFAULT_SYSTEM_PROMPT5 = `**IMPORTANT**: Follow these instructions for your response:
|
|
5014
5548
|
- Do NOT create any additional output files in the workspace.
|
|
5015
5549
|
- All intended file outputs/changes MUST be written in your response.
|
|
5016
5550
|
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
@@ -5036,18 +5570,18 @@ var PiCodingAgentProvider = class {
|
|
|
5036
5570
|
const workspaceRoot = await this.createWorkspace();
|
|
5037
5571
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
5038
5572
|
try {
|
|
5039
|
-
const promptFile =
|
|
5040
|
-
await (0,
|
|
5573
|
+
const promptFile = import_node_path14.default.join(workspaceRoot, PROMPT_FILENAME4);
|
|
5574
|
+
await (0, import_promises13.writeFile)(promptFile, request.question, "utf8");
|
|
5041
5575
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
5042
5576
|
const cwd = this.resolveCwd(workspaceRoot);
|
|
5043
5577
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
5044
5578
|
if (result.timedOut) {
|
|
5045
5579
|
throw new Error(
|
|
5046
|
-
`Pi coding agent timed out${
|
|
5580
|
+
`Pi coding agent timed out${formatTimeoutSuffix5(this.config.timeoutMs ?? void 0)}`
|
|
5047
5581
|
);
|
|
5048
5582
|
}
|
|
5049
5583
|
if (result.exitCode !== 0) {
|
|
5050
|
-
const detail =
|
|
5584
|
+
const detail = pickDetail4(result.stderr, result.stdout);
|
|
5051
5585
|
const prefix = `Pi coding agent exited with code ${result.exitCode}`;
|
|
5052
5586
|
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
5053
5587
|
}
|
|
@@ -5078,7 +5612,7 @@ var PiCodingAgentProvider = class {
|
|
|
5078
5612
|
if (!this.config.cwd) {
|
|
5079
5613
|
return workspaceRoot;
|
|
5080
5614
|
}
|
|
5081
|
-
return
|
|
5615
|
+
return import_node_path14.default.resolve(this.config.cwd);
|
|
5082
5616
|
}
|
|
5083
5617
|
buildPiArgs(prompt, inputFiles) {
|
|
5084
5618
|
const args = [];
|
|
@@ -5108,7 +5642,7 @@ var PiCodingAgentProvider = class {
|
|
|
5108
5642
|
args.push(`@${file}`);
|
|
5109
5643
|
}
|
|
5110
5644
|
}
|
|
5111
|
-
const systemPrompt = this.config.systemPrompt ??
|
|
5645
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT5;
|
|
5112
5646
|
const fullPrompt = `${systemPrompt}
|
|
5113
5647
|
|
|
5114
5648
|
${prompt}`;
|
|
@@ -5167,19 +5701,19 @@ ${prompt}`;
|
|
|
5167
5701
|
return env;
|
|
5168
5702
|
}
|
|
5169
5703
|
async createWorkspace() {
|
|
5170
|
-
return await (0,
|
|
5704
|
+
return await (0, import_promises13.mkdtemp)(import_node_path14.default.join((0, import_node_os5.tmpdir)(), WORKSPACE_PREFIX4));
|
|
5171
5705
|
}
|
|
5172
5706
|
async cleanupWorkspace(workspaceRoot) {
|
|
5173
5707
|
try {
|
|
5174
|
-
await (0,
|
|
5708
|
+
await (0, import_promises13.rm)(workspaceRoot, { recursive: true, force: true });
|
|
5175
5709
|
} catch {
|
|
5176
5710
|
}
|
|
5177
5711
|
}
|
|
5178
5712
|
resolveLogDirectory() {
|
|
5179
5713
|
if (this.config.logDir) {
|
|
5180
|
-
return
|
|
5714
|
+
return import_node_path14.default.resolve(this.config.logDir);
|
|
5181
5715
|
}
|
|
5182
|
-
return
|
|
5716
|
+
return import_node_path14.default.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
5183
5717
|
}
|
|
5184
5718
|
async createStreamLogger(request) {
|
|
5185
5719
|
const logDir = this.resolveLogDirectory();
|
|
@@ -5187,13 +5721,13 @@ ${prompt}`;
|
|
|
5187
5721
|
return void 0;
|
|
5188
5722
|
}
|
|
5189
5723
|
try {
|
|
5190
|
-
await (0,
|
|
5724
|
+
await (0, import_promises13.mkdir)(logDir, { recursive: true });
|
|
5191
5725
|
} catch (error) {
|
|
5192
5726
|
const message = error instanceof Error ? error.message : String(error);
|
|
5193
5727
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
5194
5728
|
return void 0;
|
|
5195
5729
|
}
|
|
5196
|
-
const filePath =
|
|
5730
|
+
const filePath = import_node_path14.default.join(logDir, buildLogFilename4(request, this.targetName));
|
|
5197
5731
|
try {
|
|
5198
5732
|
const logger = await PiStreamLogger.create({
|
|
5199
5733
|
filePath,
|
|
@@ -5226,7 +5760,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5226
5760
|
constructor(filePath, format) {
|
|
5227
5761
|
this.filePath = filePath;
|
|
5228
5762
|
this.format = format;
|
|
5229
|
-
this.stream = (0,
|
|
5763
|
+
this.stream = (0, import_node_fs6.createWriteStream)(filePath, { flags: "a" });
|
|
5230
5764
|
}
|
|
5231
5765
|
static async create(options) {
|
|
5232
5766
|
const logger = new _PiStreamLogger(options.filePath, options.format);
|
|
@@ -5287,7 +5821,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5287
5821
|
return void 0;
|
|
5288
5822
|
}
|
|
5289
5823
|
const message = this.format === "json" ? formatPiJsonLog(trimmed) : formatPiLogMessage(trimmed, source);
|
|
5290
|
-
return `[+${
|
|
5824
|
+
return `[+${formatElapsed4(this.startedAt)}] [${source}] ${message}`;
|
|
5291
5825
|
}
|
|
5292
5826
|
flushRemainder() {
|
|
5293
5827
|
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
@@ -5310,18 +5844,18 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5310
5844
|
this.stderrBuffer = "";
|
|
5311
5845
|
}
|
|
5312
5846
|
};
|
|
5313
|
-
function
|
|
5847
|
+
function buildLogFilename4(request, targetName) {
|
|
5314
5848
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5315
|
-
const evalId =
|
|
5849
|
+
const evalId = sanitizeForFilename4(request.evalCaseId ?? "pi");
|
|
5316
5850
|
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
5317
|
-
const target =
|
|
5318
|
-
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0,
|
|
5851
|
+
const target = sanitizeForFilename4(targetName);
|
|
5852
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0, import_node_crypto4.randomUUID)().slice(0, 8)}.log`;
|
|
5319
5853
|
}
|
|
5320
|
-
function
|
|
5854
|
+
function sanitizeForFilename4(value) {
|
|
5321
5855
|
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
5322
5856
|
return sanitized.length > 0 ? sanitized : "pi";
|
|
5323
5857
|
}
|
|
5324
|
-
function
|
|
5858
|
+
function formatElapsed4(startedAt) {
|
|
5325
5859
|
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
5326
5860
|
const hours = Math.floor(elapsedSeconds / 3600);
|
|
5327
5861
|
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
@@ -5542,7 +6076,7 @@ function extractAssistantText2(messages) {
|
|
|
5542
6076
|
function escapeAtSymbols(prompt) {
|
|
5543
6077
|
return prompt.replace(/@\[([^\]]+)\]:/g, "[[$1]]:");
|
|
5544
6078
|
}
|
|
5545
|
-
function
|
|
6079
|
+
function pickDetail4(stderr, stdout) {
|
|
5546
6080
|
const errorText = stderr.trim();
|
|
5547
6081
|
if (errorText.length > 0) {
|
|
5548
6082
|
return errorText;
|
|
@@ -5550,7 +6084,7 @@ function pickDetail3(stderr, stdout) {
|
|
|
5550
6084
|
const stdoutText = stdout.trim();
|
|
5551
6085
|
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
5552
6086
|
}
|
|
5553
|
-
function
|
|
6087
|
+
function formatTimeoutSuffix5(timeoutMs) {
|
|
5554
6088
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
5555
6089
|
return "";
|
|
5556
6090
|
}
|
|
@@ -5563,7 +6097,7 @@ async function defaultPiRunner(options) {
|
|
|
5563
6097
|
const executable = parts[0];
|
|
5564
6098
|
const executableArgs = parts.slice(1);
|
|
5565
6099
|
const allArgs = [...executableArgs, ...options.args];
|
|
5566
|
-
const child = (0,
|
|
6100
|
+
const child = (0, import_node_child_process5.spawn)(executable, allArgs, {
|
|
5567
6101
|
cwd: options.cwd,
|
|
5568
6102
|
env: options.env,
|
|
5569
6103
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -5626,7 +6160,7 @@ async function defaultPiRunner(options) {
|
|
|
5626
6160
|
}
|
|
5627
6161
|
|
|
5628
6162
|
// src/evaluation/providers/targets.ts
|
|
5629
|
-
var
|
|
6163
|
+
var import_node_path15 = __toESM(require("path"), 1);
|
|
5630
6164
|
var import_zod2 = require("zod");
|
|
5631
6165
|
var CliHealthcheckHttpInputSchema = import_zod2.z.object({
|
|
5632
6166
|
type: import_zod2.z.literal("http"),
|
|
@@ -5732,11 +6266,11 @@ function normalizeCliHealthcheck(input, env, targetName, evalFilePath) {
|
|
|
5732
6266
|
allowLiteral: true,
|
|
5733
6267
|
optionalEnv: true
|
|
5734
6268
|
});
|
|
5735
|
-
if (cwd && evalFilePath && !
|
|
5736
|
-
cwd =
|
|
6269
|
+
if (cwd && evalFilePath && !import_node_path15.default.isAbsolute(cwd)) {
|
|
6270
|
+
cwd = import_node_path15.default.resolve(import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath)), cwd);
|
|
5737
6271
|
}
|
|
5738
6272
|
if (!cwd && evalFilePath) {
|
|
5739
|
-
cwd =
|
|
6273
|
+
cwd = import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath));
|
|
5740
6274
|
}
|
|
5741
6275
|
return {
|
|
5742
6276
|
type: "command",
|
|
@@ -5763,11 +6297,11 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
|
|
|
5763
6297
|
allowLiteral: true,
|
|
5764
6298
|
optionalEnv: true
|
|
5765
6299
|
});
|
|
5766
|
-
if (cwd && evalFilePath && !
|
|
5767
|
-
cwd =
|
|
6300
|
+
if (cwd && evalFilePath && !import_node_path15.default.isAbsolute(cwd)) {
|
|
6301
|
+
cwd = import_node_path15.default.resolve(import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath)), cwd);
|
|
5768
6302
|
}
|
|
5769
6303
|
if (!cwd && evalFilePath) {
|
|
5770
|
-
cwd =
|
|
6304
|
+
cwd = import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath));
|
|
5771
6305
|
}
|
|
5772
6306
|
const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds;
|
|
5773
6307
|
const timeoutMs = timeoutSeconds !== void 0 ? Math.floor(timeoutSeconds * 1e3) : void 0;
|
|
@@ -5891,6 +6425,15 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
|
|
|
5891
6425
|
providerBatching,
|
|
5892
6426
|
config: resolveCodexConfig(parsed, env)
|
|
5893
6427
|
};
|
|
6428
|
+
case "copilot-cli":
|
|
6429
|
+
return {
|
|
6430
|
+
kind: "copilot-cli",
|
|
6431
|
+
name: parsed.name,
|
|
6432
|
+
judgeTarget: parsed.judge_target,
|
|
6433
|
+
workers: parsed.workers,
|
|
6434
|
+
providerBatching,
|
|
6435
|
+
config: resolveCopilotConfig(parsed, env)
|
|
6436
|
+
};
|
|
5894
6437
|
case "pi":
|
|
5895
6438
|
case "pi-coding-agent":
|
|
5896
6439
|
return {
|
|
@@ -6067,6 +6610,44 @@ function normalizeCodexLogFormat(value) {
|
|
|
6067
6610
|
}
|
|
6068
6611
|
throw new Error("codex log format must be 'summary' or 'json'");
|
|
6069
6612
|
}
|
|
6613
|
+
function resolveCopilotConfig(target, env) {
|
|
6614
|
+
const executableSource = target.executable ?? target.command ?? target.binary;
|
|
6615
|
+
const modelSource = target.model;
|
|
6616
|
+
const argsSource = target.args ?? target.arguments;
|
|
6617
|
+
const cwdSource = target.cwd;
|
|
6618
|
+
const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
|
|
6619
|
+
const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
|
|
6620
|
+
const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat;
|
|
6621
|
+
const systemPromptSource = target.system_prompt ?? target.systemPrompt;
|
|
6622
|
+
const executable = resolveOptionalString(executableSource, env, `${target.name} copilot executable`, {
|
|
6623
|
+
allowLiteral: true,
|
|
6624
|
+
optionalEnv: true
|
|
6625
|
+
}) ?? "copilot";
|
|
6626
|
+
const model = resolveOptionalString(modelSource, env, `${target.name} copilot model`, {
|
|
6627
|
+
allowLiteral: true,
|
|
6628
|
+
optionalEnv: true
|
|
6629
|
+
});
|
|
6630
|
+
const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot args`);
|
|
6631
|
+
const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot cwd`, {
|
|
6632
|
+
allowLiteral: true,
|
|
6633
|
+
optionalEnv: true
|
|
6634
|
+
});
|
|
6635
|
+
const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} copilot timeout`);
|
|
6636
|
+
const logDir = resolveOptionalString(logDirSource, env, `${target.name} copilot log directory`, {
|
|
6637
|
+
allowLiteral: true,
|
|
6638
|
+
optionalEnv: true
|
|
6639
|
+
});
|
|
6640
|
+
const logFormat = normalizeCopilotLogFormat(logFormatSource);
|
|
6641
|
+
const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
|
|
6642
|
+
return { executable, model, args, cwd, timeoutMs, logDir, logFormat, systemPrompt };
|
|
6643
|
+
}
|
|
6644
|
+
function normalizeCopilotLogFormat(value) {
|
|
6645
|
+
if (value === void 0 || value === null) return void 0;
|
|
6646
|
+
if (typeof value !== "string") throw new Error("copilot log format must be 'summary' or 'json'");
|
|
6647
|
+
const normalized = value.trim().toLowerCase();
|
|
6648
|
+
if (normalized === "json" || normalized === "summary") return normalized;
|
|
6649
|
+
throw new Error("copilot log format must be 'summary' or 'json'");
|
|
6650
|
+
}
|
|
6070
6651
|
function resolvePiCodingAgentConfig(target, env) {
|
|
6071
6652
|
const executableSource = target.executable ?? target.command ?? target.binary;
|
|
6072
6653
|
const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
|
|
@@ -6272,8 +6853,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
6272
6853
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
6273
6854
|
if (!parseResult.success) {
|
|
6274
6855
|
const firstError = parseResult.error.errors[0];
|
|
6275
|
-
const
|
|
6276
|
-
const prefix =
|
|
6856
|
+
const path29 = firstError?.path.join(".") || "";
|
|
6857
|
+
const prefix = path29 ? `${target.name} ${path29}: ` : `${target.name}: `;
|
|
6277
6858
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
6278
6859
|
}
|
|
6279
6860
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -6461,38 +7042,38 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
6461
7042
|
}
|
|
6462
7043
|
|
|
6463
7044
|
// src/evaluation/providers/vscode-provider.ts
|
|
6464
|
-
var
|
|
7045
|
+
var import_node_path26 = __toESM(require("path"), 1);
|
|
6465
7046
|
|
|
6466
7047
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
6467
|
-
var
|
|
6468
|
-
var
|
|
7048
|
+
var import_promises18 = require("fs/promises");
|
|
7049
|
+
var import_node_path24 = __toESM(require("path"), 1);
|
|
6469
7050
|
|
|
6470
7051
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
6471
|
-
var
|
|
6472
|
-
var
|
|
6473
|
-
var
|
|
7052
|
+
var import_node_fs7 = require("fs");
|
|
7053
|
+
var import_promises14 = require("fs/promises");
|
|
7054
|
+
var import_node_path16 = __toESM(require("path"), 1);
|
|
6474
7055
|
async function pathExists(target) {
|
|
6475
7056
|
try {
|
|
6476
|
-
await (0,
|
|
7057
|
+
await (0, import_promises14.access)(target, import_node_fs7.constants.F_OK);
|
|
6477
7058
|
return true;
|
|
6478
7059
|
} catch {
|
|
6479
7060
|
return false;
|
|
6480
7061
|
}
|
|
6481
7062
|
}
|
|
6482
7063
|
async function ensureDir(target) {
|
|
6483
|
-
await (0,
|
|
7064
|
+
await (0, import_promises14.mkdir)(target, { recursive: true });
|
|
6484
7065
|
}
|
|
6485
7066
|
async function readDirEntries(target) {
|
|
6486
|
-
const entries = await (0,
|
|
7067
|
+
const entries = await (0, import_promises14.readdir)(target, { withFileTypes: true });
|
|
6487
7068
|
return entries.map((entry) => ({
|
|
6488
7069
|
name: entry.name,
|
|
6489
|
-
absolutePath:
|
|
7070
|
+
absolutePath: import_node_path16.default.join(target, entry.name),
|
|
6490
7071
|
isDirectory: entry.isDirectory()
|
|
6491
7072
|
}));
|
|
6492
7073
|
}
|
|
6493
7074
|
async function removeIfExists(target) {
|
|
6494
7075
|
try {
|
|
6495
|
-
await (0,
|
|
7076
|
+
await (0, import_promises14.rm)(target, { force: true, recursive: false });
|
|
6496
7077
|
} catch (error) {
|
|
6497
7078
|
if (error.code !== "ENOENT") {
|
|
6498
7079
|
throw error;
|
|
@@ -6501,9 +7082,9 @@ async function removeIfExists(target) {
|
|
|
6501
7082
|
}
|
|
6502
7083
|
|
|
6503
7084
|
// src/evaluation/providers/vscode/utils/path.ts
|
|
6504
|
-
var
|
|
7085
|
+
var import_node_path17 = __toESM(require("path"), 1);
|
|
6505
7086
|
function pathToFileUri2(filePath) {
|
|
6506
|
-
const absolutePath =
|
|
7087
|
+
const absolutePath = import_node_path17.default.isAbsolute(filePath) ? filePath : import_node_path17.default.resolve(filePath);
|
|
6507
7088
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
6508
7089
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
6509
7090
|
return `file:///${normalizedPath}`;
|
|
@@ -6512,7 +7093,7 @@ function pathToFileUri2(filePath) {
|
|
|
6512
7093
|
}
|
|
6513
7094
|
|
|
6514
7095
|
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
6515
|
-
var
|
|
7096
|
+
var import_node_path18 = __toESM(require("path"), 1);
|
|
6516
7097
|
|
|
6517
7098
|
// src/evaluation/providers/vscode/utils/template.ts
|
|
6518
7099
|
function renderTemplate2(content, variables) {
|
|
@@ -6604,8 +7185,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
6604
7185
|
});
|
|
6605
7186
|
}
|
|
6606
7187
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
6607
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
6608
|
-
const responseList = responseFiles.map((file) => `"${
|
|
7188
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${import_node_path18.default.basename(file)}`).join("\n");
|
|
7189
|
+
const responseList = responseFiles.map((file) => `"${import_node_path18.default.basename(file)}"`).join(", ");
|
|
6609
7190
|
return renderTemplate2(templateContent, {
|
|
6610
7191
|
requestFiles: requestLines,
|
|
6611
7192
|
responseList
|
|
@@ -6613,8 +7194,8 @@ function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateCont
|
|
|
6613
7194
|
}
|
|
6614
7195
|
|
|
6615
7196
|
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
6616
|
-
var
|
|
6617
|
-
var
|
|
7197
|
+
var import_promises15 = require("fs/promises");
|
|
7198
|
+
var import_node_path19 = __toESM(require("path"), 1);
|
|
6618
7199
|
|
|
6619
7200
|
// src/evaluation/providers/vscode/utils/time.ts
|
|
6620
7201
|
function sleep2(ms) {
|
|
@@ -6642,7 +7223,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
6642
7223
|
const maxAttempts = 10;
|
|
6643
7224
|
while (attempts < maxAttempts) {
|
|
6644
7225
|
try {
|
|
6645
|
-
const content = await (0,
|
|
7226
|
+
const content = await (0, import_promises15.readFile)(responseFileFinal, { encoding: "utf8" });
|
|
6646
7227
|
if (!silent) {
|
|
6647
7228
|
process.stdout.write(`${content}
|
|
6648
7229
|
`);
|
|
@@ -6663,7 +7244,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
6663
7244
|
}
|
|
6664
7245
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
|
|
6665
7246
|
if (!silent) {
|
|
6666
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
7247
|
+
const fileList = responseFilesFinal.map((file) => import_node_path19.default.basename(file)).join(", ");
|
|
6667
7248
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
6668
7249
|
}
|
|
6669
7250
|
try {
|
|
@@ -6689,7 +7270,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
6689
7270
|
const maxAttempts = 10;
|
|
6690
7271
|
while (attempts < maxAttempts) {
|
|
6691
7272
|
try {
|
|
6692
|
-
const content = await (0,
|
|
7273
|
+
const content = await (0, import_promises15.readFile)(file, { encoding: "utf8" });
|
|
6693
7274
|
if (!silent) {
|
|
6694
7275
|
process.stdout.write(`${content}
|
|
6695
7276
|
`);
|
|
@@ -6711,31 +7292,31 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
6711
7292
|
}
|
|
6712
7293
|
|
|
6713
7294
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
6714
|
-
var
|
|
6715
|
-
var
|
|
6716
|
-
var
|
|
6717
|
-
var
|
|
7295
|
+
var import_node_child_process6 = require("child_process");
|
|
7296
|
+
var import_promises16 = require("fs/promises");
|
|
7297
|
+
var import_node_path21 = __toESM(require("path"), 1);
|
|
7298
|
+
var import_node_util4 = require("util");
|
|
6718
7299
|
|
|
6719
7300
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
6720
|
-
var
|
|
6721
|
-
var
|
|
7301
|
+
var import_node_os6 = __toESM(require("os"), 1);
|
|
7302
|
+
var import_node_path20 = __toESM(require("path"), 1);
|
|
6722
7303
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
6723
7304
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
6724
7305
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
6725
7306
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
6726
|
-
return
|
|
7307
|
+
return import_node_path20.default.join(import_node_os6.default.homedir(), ".agentv", "subagents", folder);
|
|
6727
7308
|
}
|
|
6728
7309
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
6729
7310
|
|
|
6730
7311
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
6731
|
-
var
|
|
7312
|
+
var execAsync4 = (0, import_node_util4.promisify)(import_node_child_process6.exec);
|
|
6732
7313
|
var DEFAULT_WAKEUP_CONTENT = `---
|
|
6733
7314
|
description: 'Wake-up Signal'
|
|
6734
7315
|
model: Grok Code Fast 1 (copilot)
|
|
6735
7316
|
---`;
|
|
6736
7317
|
async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
6737
7318
|
try {
|
|
6738
|
-
const { stdout } = await
|
|
7319
|
+
const { stdout } = await execAsync4(`${vscodeCmd} --status`, {
|
|
6739
7320
|
timeout: 1e4,
|
|
6740
7321
|
windowsHide: true
|
|
6741
7322
|
});
|
|
@@ -6747,16 +7328,16 @@ async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
|
6747
7328
|
async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir, vscodeCmd, pollInterval = 1, timeout = 60) {
|
|
6748
7329
|
const alreadyOpen = await checkWorkspaceOpened(workspaceName, vscodeCmd);
|
|
6749
7330
|
if (alreadyOpen) {
|
|
6750
|
-
(0,
|
|
7331
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
6751
7332
|
return true;
|
|
6752
7333
|
}
|
|
6753
|
-
const aliveFile =
|
|
7334
|
+
const aliveFile = import_node_path21.default.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
6754
7335
|
await removeIfExists(aliveFile);
|
|
6755
|
-
const githubAgentsDir =
|
|
6756
|
-
await (0,
|
|
6757
|
-
const wakeupDst =
|
|
6758
|
-
await (0,
|
|
6759
|
-
(0,
|
|
7336
|
+
const githubAgentsDir = import_node_path21.default.join(subagentDir, ".github", "agents");
|
|
7337
|
+
await (0, import_promises16.mkdir)(githubAgentsDir, { recursive: true });
|
|
7338
|
+
const wakeupDst = import_node_path21.default.join(githubAgentsDir, "wakeup.md");
|
|
7339
|
+
await (0, import_promises16.writeFile)(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
7340
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
6760
7341
|
await sleep2(100);
|
|
6761
7342
|
const wakeupChatId = "wakeup";
|
|
6762
7343
|
const chatArgs = [
|
|
@@ -6764,9 +7345,9 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
6764
7345
|
"chat",
|
|
6765
7346
|
"-m",
|
|
6766
7347
|
wakeupChatId,
|
|
6767
|
-
`create a file named .alive in the ${
|
|
7348
|
+
`create a file named .alive in the ${import_node_path21.default.basename(subagentDir)} folder`
|
|
6768
7349
|
];
|
|
6769
|
-
(0,
|
|
7350
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6770
7351
|
const start = Date.now();
|
|
6771
7352
|
while (!await pathExists(aliveFile)) {
|
|
6772
7353
|
if (Date.now() - start > timeout * 1e3) {
|
|
@@ -6779,21 +7360,21 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
6779
7360
|
}
|
|
6780
7361
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
6781
7362
|
try {
|
|
6782
|
-
const workspacePath =
|
|
6783
|
-
const messagesDir =
|
|
6784
|
-
await (0,
|
|
6785
|
-
const reqFile =
|
|
6786
|
-
await (0,
|
|
7363
|
+
const workspacePath = import_node_path21.default.join(subagentDir, `${import_node_path21.default.basename(subagentDir)}.code-workspace`);
|
|
7364
|
+
const messagesDir = import_node_path21.default.join(subagentDir, "messages");
|
|
7365
|
+
await (0, import_promises16.mkdir)(messagesDir, { recursive: true });
|
|
7366
|
+
const reqFile = import_node_path21.default.join(messagesDir, `${timestamp}_req.md`);
|
|
7367
|
+
await (0, import_promises16.writeFile)(reqFile, requestInstructions, { encoding: "utf8" });
|
|
6787
7368
|
const reqUri = pathToFileUri2(reqFile);
|
|
6788
7369
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
6789
7370
|
for (const attachment of attachmentPaths) {
|
|
6790
7371
|
chatArgs.push("-a", attachment);
|
|
6791
7372
|
}
|
|
6792
7373
|
chatArgs.push("-a", reqFile);
|
|
6793
|
-
chatArgs.push(`Follow instructions in [${
|
|
7374
|
+
chatArgs.push(`Follow instructions in [${import_node_path21.default.basename(reqFile)}](${reqUri})`);
|
|
6794
7375
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
6795
7376
|
workspacePath,
|
|
6796
|
-
|
|
7377
|
+
import_node_path21.default.basename(subagentDir),
|
|
6797
7378
|
subagentDir,
|
|
6798
7379
|
vscodeCmd
|
|
6799
7380
|
);
|
|
@@ -6801,7 +7382,7 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
6801
7382
|
console.error("warning: Workspace may not be fully ready");
|
|
6802
7383
|
}
|
|
6803
7384
|
await sleep2(500);
|
|
6804
|
-
(0,
|
|
7385
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6805
7386
|
return true;
|
|
6806
7387
|
} catch (error) {
|
|
6807
7388
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -6810,9 +7391,9 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
6810
7391
|
}
|
|
6811
7392
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
6812
7393
|
try {
|
|
6813
|
-
const workspacePath =
|
|
6814
|
-
const messagesDir =
|
|
6815
|
-
await (0,
|
|
7394
|
+
const workspacePath = import_node_path21.default.join(subagentDir, `${import_node_path21.default.basename(subagentDir)}.code-workspace`);
|
|
7395
|
+
const messagesDir = import_node_path21.default.join(subagentDir, "messages");
|
|
7396
|
+
await (0, import_promises16.mkdir)(messagesDir, { recursive: true });
|
|
6816
7397
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
6817
7398
|
for (const attachment of attachmentPaths) {
|
|
6818
7399
|
chatArgs.push("-a", attachment);
|
|
@@ -6820,7 +7401,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6820
7401
|
chatArgs.push(chatInstruction);
|
|
6821
7402
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
6822
7403
|
workspacePath,
|
|
6823
|
-
|
|
7404
|
+
import_node_path21.default.basename(subagentDir),
|
|
6824
7405
|
subagentDir,
|
|
6825
7406
|
vscodeCmd
|
|
6826
7407
|
);
|
|
@@ -6828,7 +7409,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6828
7409
|
console.error("warning: Workspace may not be fully ready");
|
|
6829
7410
|
}
|
|
6830
7411
|
await sleep2(500);
|
|
6831
|
-
(0,
|
|
7412
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6832
7413
|
return true;
|
|
6833
7414
|
} catch (error) {
|
|
6834
7415
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -6837,11 +7418,11 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6837
7418
|
}
|
|
6838
7419
|
|
|
6839
7420
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
6840
|
-
var
|
|
6841
|
-
var
|
|
7421
|
+
var import_promises17 = require("fs/promises");
|
|
7422
|
+
var import_node_path23 = __toESM(require("path"), 1);
|
|
6842
7423
|
|
|
6843
7424
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
6844
|
-
var
|
|
7425
|
+
var import_node_path22 = __toESM(require("path"), 1);
|
|
6845
7426
|
var import_json5 = __toESM(require("json5"), 1);
|
|
6846
7427
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
6847
7428
|
let workspace;
|
|
@@ -6858,10 +7439,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
6858
7439
|
}
|
|
6859
7440
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
6860
7441
|
const folderPath = folder.path;
|
|
6861
|
-
if (
|
|
7442
|
+
if (import_node_path22.default.isAbsolute(folderPath)) {
|
|
6862
7443
|
return folder;
|
|
6863
7444
|
}
|
|
6864
|
-
const absolutePath =
|
|
7445
|
+
const absolutePath = import_node_path22.default.resolve(templateDir, folderPath);
|
|
6865
7446
|
return {
|
|
6866
7447
|
...folder,
|
|
6867
7448
|
path: absolutePath
|
|
@@ -6883,19 +7464,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
6883
7464
|
if (locationMap && typeof locationMap === "object") {
|
|
6884
7465
|
const transformedMap = {};
|
|
6885
7466
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
6886
|
-
const isAbsolute =
|
|
7467
|
+
const isAbsolute = import_node_path22.default.isAbsolute(locationPath);
|
|
6887
7468
|
if (isAbsolute) {
|
|
6888
7469
|
transformedMap[locationPath] = value;
|
|
6889
7470
|
} else {
|
|
6890
7471
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
6891
7472
|
if (firstGlobIndex === -1) {
|
|
6892
|
-
const resolvedPath =
|
|
7473
|
+
const resolvedPath = import_node_path22.default.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
6893
7474
|
transformedMap[resolvedPath] = value;
|
|
6894
7475
|
} else {
|
|
6895
7476
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
6896
7477
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
6897
7478
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
6898
|
-
const resolvedPath = (
|
|
7479
|
+
const resolvedPath = (import_node_path22.default.resolve(templateDir, basePath) + patternPath).replace(
|
|
6899
7480
|
/\\/g,
|
|
6900
7481
|
"/"
|
|
6901
7482
|
);
|
|
@@ -6936,7 +7517,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
6936
7517
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
6937
7518
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
6938
7519
|
for (const subagent of subagents) {
|
|
6939
|
-
const lockFile =
|
|
7520
|
+
const lockFile = import_node_path23.default.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
6940
7521
|
if (!await pathExists(lockFile)) {
|
|
6941
7522
|
return subagent.absolutePath;
|
|
6942
7523
|
}
|
|
@@ -6946,54 +7527,54 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
6946
7527
|
async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
6947
7528
|
let workspaceContent;
|
|
6948
7529
|
if (workspaceTemplate) {
|
|
6949
|
-
const workspaceSrc =
|
|
7530
|
+
const workspaceSrc = import_node_path23.default.resolve(workspaceTemplate);
|
|
6950
7531
|
if (!await pathExists(workspaceSrc)) {
|
|
6951
7532
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
6952
7533
|
}
|
|
6953
|
-
const stats = await (0,
|
|
7534
|
+
const stats = await (0, import_promises17.stat)(workspaceSrc);
|
|
6954
7535
|
if (!stats.isFile()) {
|
|
6955
7536
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
6956
7537
|
}
|
|
6957
|
-
const templateText = await (0,
|
|
7538
|
+
const templateText = await (0, import_promises17.readFile)(workspaceSrc, "utf8");
|
|
6958
7539
|
workspaceContent = JSON.parse(templateText);
|
|
6959
7540
|
} else {
|
|
6960
7541
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
6961
7542
|
}
|
|
6962
|
-
const workspaceName = `${
|
|
6963
|
-
const workspaceDst =
|
|
6964
|
-
const templateDir = workspaceTemplate ?
|
|
7543
|
+
const workspaceName = `${import_node_path23.default.basename(subagentDir)}.code-workspace`;
|
|
7544
|
+
const workspaceDst = import_node_path23.default.join(subagentDir, workspaceName);
|
|
7545
|
+
const templateDir = workspaceTemplate ? import_node_path23.default.dirname(import_node_path23.default.resolve(workspaceTemplate)) : subagentDir;
|
|
6965
7546
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
6966
7547
|
const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
6967
|
-
await (0,
|
|
6968
|
-
const messagesDir =
|
|
6969
|
-
await (0,
|
|
7548
|
+
await (0, import_promises17.writeFile)(workspaceDst, transformedContent, "utf8");
|
|
7549
|
+
const messagesDir = import_node_path23.default.join(subagentDir, "messages");
|
|
7550
|
+
await (0, import_promises17.mkdir)(messagesDir, { recursive: true });
|
|
6970
7551
|
return { workspace: workspaceDst, messagesDir };
|
|
6971
7552
|
}
|
|
6972
7553
|
async function createSubagentLock(subagentDir) {
|
|
6973
|
-
const messagesDir =
|
|
7554
|
+
const messagesDir = import_node_path23.default.join(subagentDir, "messages");
|
|
6974
7555
|
if (await pathExists(messagesDir)) {
|
|
6975
|
-
const files = await (0,
|
|
7556
|
+
const files = await (0, import_promises17.readdir)(messagesDir);
|
|
6976
7557
|
await Promise.all(
|
|
6977
7558
|
files.map(async (file) => {
|
|
6978
|
-
const target =
|
|
7559
|
+
const target = import_node_path23.default.join(messagesDir, file);
|
|
6979
7560
|
await removeIfExists(target);
|
|
6980
7561
|
})
|
|
6981
7562
|
);
|
|
6982
7563
|
}
|
|
6983
|
-
const githubAgentsDir =
|
|
7564
|
+
const githubAgentsDir = import_node_path23.default.join(subagentDir, ".github", "agents");
|
|
6984
7565
|
if (await pathExists(githubAgentsDir)) {
|
|
6985
|
-
const agentFiles = await (0,
|
|
7566
|
+
const agentFiles = await (0, import_promises17.readdir)(githubAgentsDir);
|
|
6986
7567
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
6987
7568
|
await Promise.all(
|
|
6988
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
7569
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(import_node_path23.default.join(githubAgentsDir, file)))
|
|
6989
7570
|
);
|
|
6990
7571
|
}
|
|
6991
|
-
const lockFile =
|
|
6992
|
-
await (0,
|
|
7572
|
+
const lockFile = import_node_path23.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
7573
|
+
await (0, import_promises17.writeFile)(lockFile, "", { encoding: "utf8" });
|
|
6993
7574
|
return lockFile;
|
|
6994
7575
|
}
|
|
6995
7576
|
async function removeSubagentLock(subagentDir) {
|
|
6996
|
-
const lockFile =
|
|
7577
|
+
const lockFile = import_node_path23.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
6997
7578
|
await removeIfExists(lockFile);
|
|
6998
7579
|
}
|
|
6999
7580
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
|
|
@@ -7013,11 +7594,11 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
7013
7594
|
return 1;
|
|
7014
7595
|
}
|
|
7015
7596
|
if (promptFile) {
|
|
7016
|
-
const githubAgentsDir =
|
|
7017
|
-
await (0,
|
|
7018
|
-
const agentFile =
|
|
7597
|
+
const githubAgentsDir = import_node_path23.default.join(subagentDir, ".github", "agents");
|
|
7598
|
+
await (0, import_promises17.mkdir)(githubAgentsDir, { recursive: true });
|
|
7599
|
+
const agentFile = import_node_path23.default.join(githubAgentsDir, `${chatId}.md`);
|
|
7019
7600
|
try {
|
|
7020
|
-
await (0,
|
|
7601
|
+
await (0, import_promises17.copyFile)(promptFile, agentFile);
|
|
7021
7602
|
} catch (error) {
|
|
7022
7603
|
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
7023
7604
|
return 1;
|
|
@@ -7034,11 +7615,11 @@ async function resolvePromptFile(promptFile) {
|
|
|
7034
7615
|
if (!promptFile) {
|
|
7035
7616
|
return void 0;
|
|
7036
7617
|
}
|
|
7037
|
-
const resolvedPrompt =
|
|
7618
|
+
const resolvedPrompt = import_node_path24.default.resolve(promptFile);
|
|
7038
7619
|
if (!await pathExists(resolvedPrompt)) {
|
|
7039
7620
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
7040
7621
|
}
|
|
7041
|
-
const promptStats = await (0,
|
|
7622
|
+
const promptStats = await (0, import_promises18.stat)(resolvedPrompt);
|
|
7042
7623
|
if (!promptStats.isFile()) {
|
|
7043
7624
|
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
7044
7625
|
}
|
|
@@ -7050,7 +7631,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
7050
7631
|
}
|
|
7051
7632
|
const resolved = [];
|
|
7052
7633
|
for (const attachment of extraAttachments) {
|
|
7053
|
-
const resolvedPath =
|
|
7634
|
+
const resolvedPath = import_node_path24.default.resolve(attachment);
|
|
7054
7635
|
if (!await pathExists(resolvedPath)) {
|
|
7055
7636
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
7056
7637
|
}
|
|
@@ -7090,7 +7671,7 @@ async function dispatchAgentSession(options) {
|
|
|
7090
7671
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
7091
7672
|
};
|
|
7092
7673
|
}
|
|
7093
|
-
const subagentName =
|
|
7674
|
+
const subagentName = import_node_path24.default.basename(subagentDir);
|
|
7094
7675
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
7095
7676
|
const preparationResult = await prepareSubagentDirectory(
|
|
7096
7677
|
subagentDir,
|
|
@@ -7117,9 +7698,9 @@ async function dispatchAgentSession(options) {
|
|
|
7117
7698
|
};
|
|
7118
7699
|
}
|
|
7119
7700
|
const timestamp = generateTimestamp();
|
|
7120
|
-
const messagesDir =
|
|
7121
|
-
const responseFileTmp =
|
|
7122
|
-
const responseFileFinal =
|
|
7701
|
+
const messagesDir = import_node_path24.default.join(subagentDir, "messages");
|
|
7702
|
+
const responseFileTmp = import_node_path24.default.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
7703
|
+
const responseFileFinal = import_node_path24.default.join(messagesDir, `${timestamp}_res.md`);
|
|
7123
7704
|
const requestInstructions = createRequestPrompt(
|
|
7124
7705
|
userQuery,
|
|
7125
7706
|
responseFileTmp,
|
|
@@ -7232,7 +7813,7 @@ async function dispatchBatchAgent(options) {
|
|
|
7232
7813
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
7233
7814
|
};
|
|
7234
7815
|
}
|
|
7235
|
-
subagentName =
|
|
7816
|
+
subagentName = import_node_path24.default.basename(subagentDir);
|
|
7236
7817
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
7237
7818
|
const preparationResult = await prepareSubagentDirectory(
|
|
7238
7819
|
subagentDir,
|
|
@@ -7263,24 +7844,24 @@ async function dispatchBatchAgent(options) {
|
|
|
7263
7844
|
};
|
|
7264
7845
|
}
|
|
7265
7846
|
const timestamp = generateTimestamp();
|
|
7266
|
-
const messagesDir =
|
|
7847
|
+
const messagesDir = import_node_path24.default.join(subagentDir, "messages");
|
|
7267
7848
|
requestFiles = userQueries.map(
|
|
7268
|
-
(_, index) =>
|
|
7849
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
7269
7850
|
);
|
|
7270
7851
|
const responseTmpFiles = userQueries.map(
|
|
7271
|
-
(_, index) =>
|
|
7852
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
7272
7853
|
);
|
|
7273
7854
|
responseFilesFinal = userQueries.map(
|
|
7274
|
-
(_, index) =>
|
|
7855
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
7275
7856
|
);
|
|
7276
|
-
const orchestratorFile =
|
|
7857
|
+
const orchestratorFile = import_node_path24.default.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
7277
7858
|
if (!dryRun) {
|
|
7278
7859
|
await Promise.all(
|
|
7279
7860
|
userQueries.map((query, index) => {
|
|
7280
7861
|
const reqFile = requestFiles[index];
|
|
7281
7862
|
const tmpFile = responseTmpFiles[index];
|
|
7282
7863
|
const finalFile = responseFilesFinal[index];
|
|
7283
|
-
return (0,
|
|
7864
|
+
return (0, import_promises18.writeFile)(
|
|
7284
7865
|
reqFile,
|
|
7285
7866
|
createBatchRequestPrompt(query, tmpFile, finalFile, batchRequestTemplateContent),
|
|
7286
7867
|
{ encoding: "utf8" }
|
|
@@ -7292,7 +7873,7 @@ async function dispatchBatchAgent(options) {
|
|
|
7292
7873
|
responseFilesFinal,
|
|
7293
7874
|
orchestratorTemplateContent
|
|
7294
7875
|
);
|
|
7295
|
-
await (0,
|
|
7876
|
+
await (0, import_promises18.writeFile)(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
7296
7877
|
}
|
|
7297
7878
|
const chatAttachments = [orchestratorFile, ...attachments];
|
|
7298
7879
|
const orchestratorUri = pathToFileUri2(orchestratorFile);
|
|
@@ -7362,8 +7943,8 @@ async function dispatchBatchAgent(options) {
|
|
|
7362
7943
|
}
|
|
7363
7944
|
|
|
7364
7945
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
7365
|
-
var
|
|
7366
|
-
var
|
|
7946
|
+
var import_promises19 = require("fs/promises");
|
|
7947
|
+
var import_node_path25 = __toESM(require("path"), 1);
|
|
7367
7948
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
7368
7949
|
folders: [
|
|
7369
7950
|
{
|
|
@@ -7394,7 +7975,7 @@ async function provisionSubagents(options) {
|
|
|
7394
7975
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
7395
7976
|
throw new Error("subagents must be a positive integer");
|
|
7396
7977
|
}
|
|
7397
|
-
const targetPath =
|
|
7978
|
+
const targetPath = import_node_path25.default.resolve(targetRoot);
|
|
7398
7979
|
if (!dryRun) {
|
|
7399
7980
|
await ensureDir(targetPath);
|
|
7400
7981
|
}
|
|
@@ -7414,7 +7995,7 @@ async function provisionSubagents(options) {
|
|
|
7414
7995
|
continue;
|
|
7415
7996
|
}
|
|
7416
7997
|
highestNumber = Math.max(highestNumber, parsed);
|
|
7417
|
-
const lockFile =
|
|
7998
|
+
const lockFile = import_node_path25.default.join(entry.absolutePath, lockName);
|
|
7418
7999
|
const locked = await pathExists(lockFile);
|
|
7419
8000
|
if (locked) {
|
|
7420
8001
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -7431,10 +8012,10 @@ async function provisionSubagents(options) {
|
|
|
7431
8012
|
break;
|
|
7432
8013
|
}
|
|
7433
8014
|
const subagentDir = subagent.absolutePath;
|
|
7434
|
-
const githubAgentsDir =
|
|
7435
|
-
const lockFile =
|
|
7436
|
-
const workspaceDst =
|
|
7437
|
-
const wakeupDst =
|
|
8015
|
+
const githubAgentsDir = import_node_path25.default.join(subagentDir, ".github", "agents");
|
|
8016
|
+
const lockFile = import_node_path25.default.join(subagentDir, lockName);
|
|
8017
|
+
const workspaceDst = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
8018
|
+
const wakeupDst = import_node_path25.default.join(githubAgentsDir, "wakeup.md");
|
|
7438
8019
|
const isLocked = await pathExists(lockFile);
|
|
7439
8020
|
if (isLocked && !force) {
|
|
7440
8021
|
continue;
|
|
@@ -7443,8 +8024,8 @@ async function provisionSubagents(options) {
|
|
|
7443
8024
|
if (!dryRun) {
|
|
7444
8025
|
await removeIfExists(lockFile);
|
|
7445
8026
|
await ensureDir(githubAgentsDir);
|
|
7446
|
-
await (0,
|
|
7447
|
-
await (0,
|
|
8027
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8028
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7448
8029
|
}
|
|
7449
8030
|
created.push(subagentDir);
|
|
7450
8031
|
lockedSubagents.delete(subagentDir);
|
|
@@ -7454,8 +8035,8 @@ async function provisionSubagents(options) {
|
|
|
7454
8035
|
if (!isLocked && force) {
|
|
7455
8036
|
if (!dryRun) {
|
|
7456
8037
|
await ensureDir(githubAgentsDir);
|
|
7457
|
-
await (0,
|
|
7458
|
-
await (0,
|
|
8038
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8039
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7459
8040
|
}
|
|
7460
8041
|
created.push(subagentDir);
|
|
7461
8042
|
subagentsProvisioned += 1;
|
|
@@ -7463,8 +8044,8 @@ async function provisionSubagents(options) {
|
|
|
7463
8044
|
}
|
|
7464
8045
|
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
7465
8046
|
await ensureDir(githubAgentsDir);
|
|
7466
|
-
await (0,
|
|
7467
|
-
await (0,
|
|
8047
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8048
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7468
8049
|
}
|
|
7469
8050
|
skippedExisting.push(subagentDir);
|
|
7470
8051
|
subagentsProvisioned += 1;
|
|
@@ -7472,15 +8053,15 @@ async function provisionSubagents(options) {
|
|
|
7472
8053
|
let nextIndex = highestNumber;
|
|
7473
8054
|
while (subagentsProvisioned < subagents) {
|
|
7474
8055
|
nextIndex += 1;
|
|
7475
|
-
const subagentDir =
|
|
7476
|
-
const githubAgentsDir =
|
|
7477
|
-
const workspaceDst =
|
|
7478
|
-
const wakeupDst =
|
|
8056
|
+
const subagentDir = import_node_path25.default.join(targetPath, `subagent-${nextIndex}`);
|
|
8057
|
+
const githubAgentsDir = import_node_path25.default.join(subagentDir, ".github", "agents");
|
|
8058
|
+
const workspaceDst = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
8059
|
+
const wakeupDst = import_node_path25.default.join(githubAgentsDir, "wakeup.md");
|
|
7479
8060
|
if (!dryRun) {
|
|
7480
8061
|
await ensureDir(subagentDir);
|
|
7481
8062
|
await ensureDir(githubAgentsDir);
|
|
7482
|
-
await (0,
|
|
7483
|
-
await (0,
|
|
8063
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8064
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7484
8065
|
}
|
|
7485
8066
|
created.push(subagentDir);
|
|
7486
8067
|
subagentsProvisioned += 1;
|
|
@@ -7659,7 +8240,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
7659
8240
|
return "";
|
|
7660
8241
|
}
|
|
7661
8242
|
const buildList = (files) => files.map((absolutePath) => {
|
|
7662
|
-
const fileName =
|
|
8243
|
+
const fileName = import_node_path26.default.basename(absolutePath);
|
|
7663
8244
|
const fileUri = pathToFileUri3(absolutePath);
|
|
7664
8245
|
return `* [${fileName}](${fileUri})`;
|
|
7665
8246
|
});
|
|
@@ -7684,8 +8265,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
7684
8265
|
}
|
|
7685
8266
|
const unique = /* @__PURE__ */ new Map();
|
|
7686
8267
|
for (const attachment of attachments) {
|
|
7687
|
-
const absolutePath =
|
|
7688
|
-
const normalized = absolutePath.split(
|
|
8268
|
+
const absolutePath = import_node_path26.default.resolve(attachment);
|
|
8269
|
+
const normalized = absolutePath.split(import_node_path26.default.sep).join("/");
|
|
7689
8270
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
7690
8271
|
if (!unique.has(absolutePath)) {
|
|
7691
8272
|
unique.set(absolutePath, absolutePath);
|
|
@@ -7700,7 +8281,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
7700
8281
|
}
|
|
7701
8282
|
const unique = /* @__PURE__ */ new Map();
|
|
7702
8283
|
for (const attachment of attachments) {
|
|
7703
|
-
const absolutePath =
|
|
8284
|
+
const absolutePath = import_node_path26.default.resolve(attachment);
|
|
7704
8285
|
if (!unique.has(absolutePath)) {
|
|
7705
8286
|
unique.set(absolutePath, absolutePath);
|
|
7706
8287
|
}
|
|
@@ -7708,7 +8289,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
7708
8289
|
return Array.from(unique.values());
|
|
7709
8290
|
}
|
|
7710
8291
|
function pathToFileUri3(filePath) {
|
|
7711
|
-
const absolutePath =
|
|
8292
|
+
const absolutePath = import_node_path26.default.isAbsolute(filePath) ? filePath : import_node_path26.default.resolve(filePath);
|
|
7712
8293
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
7713
8294
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
7714
8295
|
return `file:///${normalizedPath}`;
|
|
@@ -7721,7 +8302,7 @@ function normalizeAttachments(attachments) {
|
|
|
7721
8302
|
}
|
|
7722
8303
|
const deduped = /* @__PURE__ */ new Set();
|
|
7723
8304
|
for (const attachment of attachments) {
|
|
7724
|
-
deduped.add(
|
|
8305
|
+
deduped.add(import_node_path26.default.resolve(attachment));
|
|
7725
8306
|
}
|
|
7726
8307
|
return Array.from(deduped);
|
|
7727
8308
|
}
|
|
@@ -7730,7 +8311,7 @@ function mergeAttachments(all) {
|
|
|
7730
8311
|
for (const list of all) {
|
|
7731
8312
|
if (!list) continue;
|
|
7732
8313
|
for (const inputFile of list) {
|
|
7733
|
-
deduped.add(
|
|
8314
|
+
deduped.add(import_node_path26.default.resolve(inputFile));
|
|
7734
8315
|
}
|
|
7735
8316
|
}
|
|
7736
8317
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -7777,9 +8358,9 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
7777
8358
|
}
|
|
7778
8359
|
|
|
7779
8360
|
// src/evaluation/providers/targets-file.ts
|
|
7780
|
-
var
|
|
7781
|
-
var
|
|
7782
|
-
var
|
|
8361
|
+
var import_node_fs8 = require("fs");
|
|
8362
|
+
var import_promises20 = require("fs/promises");
|
|
8363
|
+
var import_node_path27 = __toESM(require("path"), 1);
|
|
7783
8364
|
var import_yaml4 = require("yaml");
|
|
7784
8365
|
function isRecord(value) {
|
|
7785
8366
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -7809,18 +8390,18 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
7809
8390
|
}
|
|
7810
8391
|
async function fileExists3(filePath) {
|
|
7811
8392
|
try {
|
|
7812
|
-
await (0,
|
|
8393
|
+
await (0, import_promises20.access)(filePath, import_node_fs8.constants.F_OK);
|
|
7813
8394
|
return true;
|
|
7814
8395
|
} catch {
|
|
7815
8396
|
return false;
|
|
7816
8397
|
}
|
|
7817
8398
|
}
|
|
7818
8399
|
async function readTargetDefinitions(filePath) {
|
|
7819
|
-
const absolutePath =
|
|
8400
|
+
const absolutePath = import_node_path27.default.resolve(filePath);
|
|
7820
8401
|
if (!await fileExists3(absolutePath)) {
|
|
7821
8402
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
7822
8403
|
}
|
|
7823
|
-
const raw = await (0,
|
|
8404
|
+
const raw = await (0, import_promises20.readFile)(absolutePath, "utf8");
|
|
7824
8405
|
const parsed = (0, import_yaml4.parse)(raw);
|
|
7825
8406
|
if (!isRecord(parsed)) {
|
|
7826
8407
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -7848,6 +8429,8 @@ function createProvider(target) {
|
|
|
7848
8429
|
return new CliProvider(target.name, target.config);
|
|
7849
8430
|
case "codex":
|
|
7850
8431
|
return new CodexProvider(target.name, target.config);
|
|
8432
|
+
case "copilot-cli":
|
|
8433
|
+
return new CopilotCliProvider(target.name, target.config);
|
|
7851
8434
|
case "pi-coding-agent":
|
|
7852
8435
|
return new PiCodingAgentProvider(target.name, target.config);
|
|
7853
8436
|
case "pi-agent-sdk":
|
|
@@ -7984,10 +8567,10 @@ async function execFileWithStdinBun(argv, stdinPayload, options) {
|
|
|
7984
8567
|
}
|
|
7985
8568
|
}
|
|
7986
8569
|
async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
7987
|
-
const { spawn:
|
|
8570
|
+
const { spawn: spawn6 } = await import("child_process");
|
|
7988
8571
|
return new Promise((resolve, reject) => {
|
|
7989
8572
|
const [cmd, ...args] = argv;
|
|
7990
|
-
const child =
|
|
8573
|
+
const child = spawn6(cmd, args, {
|
|
7991
8574
|
cwd: options.cwd,
|
|
7992
8575
|
stdio: ["pipe", "pipe", "pipe"],
|
|
7993
8576
|
// Merge additional env vars with process.env
|
|
@@ -8027,21 +8610,21 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
8027
8610
|
});
|
|
8028
8611
|
}
|
|
8029
8612
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
8030
|
-
const { mkdir:
|
|
8031
|
-
const { tmpdir:
|
|
8032
|
-
const
|
|
8033
|
-
const { randomUUID:
|
|
8034
|
-
const dir =
|
|
8035
|
-
await
|
|
8036
|
-
const stdinPath =
|
|
8037
|
-
const stdoutPath =
|
|
8038
|
-
const stderrPath =
|
|
8039
|
-
await
|
|
8613
|
+
const { mkdir: mkdir9, readFile: readFile11, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
|
|
8614
|
+
const { tmpdir: tmpdir5 } = await import("os");
|
|
8615
|
+
const path29 = await import("path");
|
|
8616
|
+
const { randomUUID: randomUUID5 } = await import("crypto");
|
|
8617
|
+
const dir = path29.join(tmpdir5(), `agentv-exec-${randomUUID5()}`);
|
|
8618
|
+
await mkdir9(dir, { recursive: true });
|
|
8619
|
+
const stdinPath = path29.join(dir, "stdin.txt");
|
|
8620
|
+
const stdoutPath = path29.join(dir, "stdout.txt");
|
|
8621
|
+
const stderrPath = path29.join(dir, "stderr.txt");
|
|
8622
|
+
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
8040
8623
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
8041
|
-
const { spawn:
|
|
8624
|
+
const { spawn: spawn6 } = await import("child_process");
|
|
8042
8625
|
try {
|
|
8043
8626
|
const exitCode = await new Promise((resolve, reject) => {
|
|
8044
|
-
const child =
|
|
8627
|
+
const child = spawn6(wrappedCommand, {
|
|
8045
8628
|
shell: true,
|
|
8046
8629
|
cwd: options.cwd,
|
|
8047
8630
|
stdio: ["ignore", "ignore", "ignore"],
|
|
@@ -8069,17 +8652,17 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
8069
8652
|
const stderr = (await readFile11(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
8070
8653
|
return { stdout, stderr, exitCode };
|
|
8071
8654
|
} finally {
|
|
8072
|
-
await
|
|
8655
|
+
await rm6(dir, { recursive: true, force: true });
|
|
8073
8656
|
}
|
|
8074
8657
|
}
|
|
8075
8658
|
|
|
8076
8659
|
// src/runtime/target-proxy.ts
|
|
8077
|
-
var
|
|
8660
|
+
var import_node_crypto5 = require("crypto");
|
|
8078
8661
|
var import_node_http = require("http");
|
|
8079
8662
|
var DEFAULT_MAX_CALLS = 50;
|
|
8080
8663
|
async function createTargetProxy(options) {
|
|
8081
8664
|
const { defaultProvider, targetResolver, availableTargets, maxCalls } = options;
|
|
8082
|
-
const token = (0,
|
|
8665
|
+
const token = (0, import_node_crypto5.randomBytes)(32).toString("hex");
|
|
8083
8666
|
let callCount = 0;
|
|
8084
8667
|
let isShutdown = false;
|
|
8085
8668
|
const targetsList = availableTargets ?? [defaultProvider.targetName];
|
|
@@ -8338,7 +8921,7 @@ var CodeEvaluator = class {
|
|
|
8338
8921
|
outputMessages: context.outputMessages ?? null,
|
|
8339
8922
|
guidelineFiles: context.evalCase.guideline_paths,
|
|
8340
8923
|
inputFiles: context.evalCase.file_paths.filter(
|
|
8341
|
-
(
|
|
8924
|
+
(path29) => !context.evalCase.guideline_paths.includes(path29)
|
|
8342
8925
|
),
|
|
8343
8926
|
inputMessages: context.evalCase.input_messages,
|
|
8344
8927
|
traceSummary: context.traceSummary ?? null,
|
|
@@ -8454,6 +9037,7 @@ var import_ai3 = require("ai");
|
|
|
8454
9037
|
// src/evaluation/providers/types.ts
|
|
8455
9038
|
var AGENT_PROVIDER_KINDS = [
|
|
8456
9039
|
"codex",
|
|
9040
|
+
"copilot-cli",
|
|
8457
9041
|
"pi-coding-agent",
|
|
8458
9042
|
"claude-code",
|
|
8459
9043
|
"vscode",
|
|
@@ -9285,115 +9869,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
9285
9869
|
* Evaluate a single field against the expected value.
|
|
9286
9870
|
*/
|
|
9287
9871
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
9288
|
-
const { path:
|
|
9289
|
-
const candidateValue = resolvePath(candidateData,
|
|
9290
|
-
const expectedValue = resolvePath(expectedData,
|
|
9872
|
+
const { path: path29, match, required = true, weight = 1 } = fieldConfig;
|
|
9873
|
+
const candidateValue = resolvePath(candidateData, path29);
|
|
9874
|
+
const expectedValue = resolvePath(expectedData, path29);
|
|
9291
9875
|
if (expectedValue === void 0) {
|
|
9292
9876
|
return {
|
|
9293
|
-
path:
|
|
9877
|
+
path: path29,
|
|
9294
9878
|
score: 1,
|
|
9295
9879
|
// No expected value means no comparison needed
|
|
9296
9880
|
weight,
|
|
9297
9881
|
hit: true,
|
|
9298
|
-
message: `${
|
|
9882
|
+
message: `${path29}: no expected value`
|
|
9299
9883
|
};
|
|
9300
9884
|
}
|
|
9301
9885
|
if (candidateValue === void 0) {
|
|
9302
9886
|
if (required) {
|
|
9303
9887
|
return {
|
|
9304
|
-
path:
|
|
9888
|
+
path: path29,
|
|
9305
9889
|
score: 0,
|
|
9306
9890
|
weight,
|
|
9307
9891
|
hit: false,
|
|
9308
|
-
message: `${
|
|
9892
|
+
message: `${path29} (required, missing)`
|
|
9309
9893
|
};
|
|
9310
9894
|
}
|
|
9311
9895
|
return {
|
|
9312
|
-
path:
|
|
9896
|
+
path: path29,
|
|
9313
9897
|
score: 1,
|
|
9314
9898
|
// Don't penalize missing optional fields
|
|
9315
9899
|
weight: 0,
|
|
9316
9900
|
// Zero weight means it won't affect the score
|
|
9317
9901
|
hit: true,
|
|
9318
|
-
message: `${
|
|
9902
|
+
message: `${path29}: optional field missing`
|
|
9319
9903
|
};
|
|
9320
9904
|
}
|
|
9321
9905
|
switch (match) {
|
|
9322
9906
|
case "exact":
|
|
9323
|
-
return this.compareExact(
|
|
9907
|
+
return this.compareExact(path29, candidateValue, expectedValue, weight);
|
|
9324
9908
|
case "numeric_tolerance":
|
|
9325
9909
|
return this.compareNumericTolerance(
|
|
9326
|
-
|
|
9910
|
+
path29,
|
|
9327
9911
|
candidateValue,
|
|
9328
9912
|
expectedValue,
|
|
9329
9913
|
fieldConfig,
|
|
9330
9914
|
weight
|
|
9331
9915
|
);
|
|
9332
9916
|
case "date":
|
|
9333
|
-
return this.compareDate(
|
|
9917
|
+
return this.compareDate(path29, candidateValue, expectedValue, fieldConfig, weight);
|
|
9334
9918
|
default:
|
|
9335
9919
|
return {
|
|
9336
|
-
path:
|
|
9920
|
+
path: path29,
|
|
9337
9921
|
score: 0,
|
|
9338
9922
|
weight,
|
|
9339
9923
|
hit: false,
|
|
9340
|
-
message: `${
|
|
9924
|
+
message: `${path29}: unknown match type "${match}"`
|
|
9341
9925
|
};
|
|
9342
9926
|
}
|
|
9343
9927
|
}
|
|
9344
9928
|
/**
|
|
9345
9929
|
* Exact equality comparison.
|
|
9346
9930
|
*/
|
|
9347
|
-
compareExact(
|
|
9931
|
+
compareExact(path29, candidateValue, expectedValue, weight) {
|
|
9348
9932
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
9349
9933
|
return {
|
|
9350
|
-
path:
|
|
9934
|
+
path: path29,
|
|
9351
9935
|
score: 1,
|
|
9352
9936
|
weight,
|
|
9353
9937
|
hit: true,
|
|
9354
|
-
message:
|
|
9938
|
+
message: path29
|
|
9355
9939
|
};
|
|
9356
9940
|
}
|
|
9357
9941
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
9358
9942
|
return {
|
|
9359
|
-
path:
|
|
9943
|
+
path: path29,
|
|
9360
9944
|
score: 0,
|
|
9361
9945
|
weight,
|
|
9362
9946
|
hit: false,
|
|
9363
|
-
message: `${
|
|
9947
|
+
message: `${path29} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
9364
9948
|
};
|
|
9365
9949
|
}
|
|
9366
9950
|
return {
|
|
9367
|
-
path:
|
|
9951
|
+
path: path29,
|
|
9368
9952
|
score: 0,
|
|
9369
9953
|
weight,
|
|
9370
9954
|
hit: false,
|
|
9371
|
-
message: `${
|
|
9955
|
+
message: `${path29} (value mismatch)`
|
|
9372
9956
|
};
|
|
9373
9957
|
}
|
|
9374
9958
|
/**
|
|
9375
9959
|
* Numeric comparison with absolute or relative tolerance.
|
|
9376
9960
|
*/
|
|
9377
|
-
compareNumericTolerance(
|
|
9961
|
+
compareNumericTolerance(path29, candidateValue, expectedValue, fieldConfig, weight) {
|
|
9378
9962
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
9379
9963
|
const candidateNum = toNumber(candidateValue);
|
|
9380
9964
|
const expectedNum = toNumber(expectedValue);
|
|
9381
9965
|
if (candidateNum === null || expectedNum === null) {
|
|
9382
9966
|
return {
|
|
9383
|
-
path:
|
|
9967
|
+
path: path29,
|
|
9384
9968
|
score: 0,
|
|
9385
9969
|
weight,
|
|
9386
9970
|
hit: false,
|
|
9387
|
-
message: `${
|
|
9971
|
+
message: `${path29} (non-numeric value)`
|
|
9388
9972
|
};
|
|
9389
9973
|
}
|
|
9390
9974
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
9391
9975
|
return {
|
|
9392
|
-
path:
|
|
9976
|
+
path: path29,
|
|
9393
9977
|
score: 0,
|
|
9394
9978
|
weight,
|
|
9395
9979
|
hit: false,
|
|
9396
|
-
message: `${
|
|
9980
|
+
message: `${path29} (invalid numeric value)`
|
|
9397
9981
|
};
|
|
9398
9982
|
}
|
|
9399
9983
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -9406,61 +9990,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
9406
9990
|
}
|
|
9407
9991
|
if (withinTolerance) {
|
|
9408
9992
|
return {
|
|
9409
|
-
path:
|
|
9993
|
+
path: path29,
|
|
9410
9994
|
score: 1,
|
|
9411
9995
|
weight,
|
|
9412
9996
|
hit: true,
|
|
9413
|
-
message: `${
|
|
9997
|
+
message: `${path29} (within tolerance: diff=${diff.toFixed(2)})`
|
|
9414
9998
|
};
|
|
9415
9999
|
}
|
|
9416
10000
|
return {
|
|
9417
|
-
path:
|
|
10001
|
+
path: path29,
|
|
9418
10002
|
score: 0,
|
|
9419
10003
|
weight,
|
|
9420
10004
|
hit: false,
|
|
9421
|
-
message: `${
|
|
10005
|
+
message: `${path29} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
9422
10006
|
};
|
|
9423
10007
|
}
|
|
9424
10008
|
/**
|
|
9425
10009
|
* Date comparison with format normalization.
|
|
9426
10010
|
*/
|
|
9427
|
-
compareDate(
|
|
10011
|
+
compareDate(path29, candidateValue, expectedValue, fieldConfig, weight) {
|
|
9428
10012
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
9429
10013
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
9430
10014
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
9431
10015
|
if (candidateDate === null) {
|
|
9432
10016
|
return {
|
|
9433
|
-
path:
|
|
10017
|
+
path: path29,
|
|
9434
10018
|
score: 0,
|
|
9435
10019
|
weight,
|
|
9436
10020
|
hit: false,
|
|
9437
|
-
message: `${
|
|
10021
|
+
message: `${path29} (unparseable candidate date)`
|
|
9438
10022
|
};
|
|
9439
10023
|
}
|
|
9440
10024
|
if (expectedDate === null) {
|
|
9441
10025
|
return {
|
|
9442
|
-
path:
|
|
10026
|
+
path: path29,
|
|
9443
10027
|
score: 0,
|
|
9444
10028
|
weight,
|
|
9445
10029
|
hit: false,
|
|
9446
|
-
message: `${
|
|
10030
|
+
message: `${path29} (unparseable expected date)`
|
|
9447
10031
|
};
|
|
9448
10032
|
}
|
|
9449
10033
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
9450
10034
|
return {
|
|
9451
|
-
path:
|
|
10035
|
+
path: path29,
|
|
9452
10036
|
score: 1,
|
|
9453
10037
|
weight,
|
|
9454
10038
|
hit: true,
|
|
9455
|
-
message:
|
|
10039
|
+
message: path29
|
|
9456
10040
|
};
|
|
9457
10041
|
}
|
|
9458
10042
|
return {
|
|
9459
|
-
path:
|
|
10043
|
+
path: path29,
|
|
9460
10044
|
score: 0,
|
|
9461
10045
|
weight,
|
|
9462
10046
|
hit: false,
|
|
9463
|
-
message: `${
|
|
10047
|
+
message: `${path29} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
9464
10048
|
};
|
|
9465
10049
|
}
|
|
9466
10050
|
/**
|
|
@@ -9500,11 +10084,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
9500
10084
|
};
|
|
9501
10085
|
}
|
|
9502
10086
|
};
|
|
9503
|
-
function resolvePath(obj,
|
|
9504
|
-
if (!
|
|
10087
|
+
function resolvePath(obj, path29) {
|
|
10088
|
+
if (!path29 || !obj) {
|
|
9505
10089
|
return void 0;
|
|
9506
10090
|
}
|
|
9507
|
-
const parts =
|
|
10091
|
+
const parts = path29.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
9508
10092
|
let current = obj;
|
|
9509
10093
|
for (const part of parts) {
|
|
9510
10094
|
if (current === null || current === void 0) {
|
|
@@ -10023,8 +10607,8 @@ var ToolTrajectoryEvaluator = class {
|
|
|
10023
10607
|
};
|
|
10024
10608
|
|
|
10025
10609
|
// src/evaluation/orchestrator.ts
|
|
10026
|
-
var
|
|
10027
|
-
var
|
|
10610
|
+
var import_node_crypto6 = require("crypto");
|
|
10611
|
+
var import_node_path28 = __toESM(require("path"), 1);
|
|
10028
10612
|
var import_micromatch4 = __toESM(require("micromatch"), 1);
|
|
10029
10613
|
|
|
10030
10614
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -10827,7 +11411,7 @@ async function runEvaluatorList(options) {
|
|
|
10827
11411
|
});
|
|
10828
11412
|
}
|
|
10829
11413
|
if (evaluator.type === "composite") {
|
|
10830
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
11414
|
+
const evalFileDir = evalCase.guideline_paths[0] ? import_node_path28.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
10831
11415
|
const createEvaluator = (memberConfig) => {
|
|
10832
11416
|
switch (memberConfig.type) {
|
|
10833
11417
|
case "llm_judge":
|
|
@@ -11183,7 +11767,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
|
|
|
11183
11767
|
};
|
|
11184
11768
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
11185
11769
|
const scriptPath = script[script.length - 1];
|
|
11186
|
-
const cwd =
|
|
11770
|
+
const cwd = import_node_path28.default.dirname(scriptPath);
|
|
11187
11771
|
try {
|
|
11188
11772
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
11189
11773
|
const prompt = stdout.trim();
|
|
@@ -11297,7 +11881,7 @@ function extractProviderError(response) {
|
|
|
11297
11881
|
return trimmed.length > 0 ? trimmed : void 0;
|
|
11298
11882
|
}
|
|
11299
11883
|
function createCacheKey(provider, target, evalCase, promptInputs) {
|
|
11300
|
-
const hash = (0,
|
|
11884
|
+
const hash = (0, import_node_crypto6.createHash)("sha256");
|
|
11301
11885
|
hash.update(provider.id);
|
|
11302
11886
|
hash.update(target.name);
|
|
11303
11887
|
hash.update(evalCase.id);
|
|
@@ -11458,6 +12042,7 @@ function createAgentKernel() {
|
|
|
11458
12042
|
computeTraceSummary,
|
|
11459
12043
|
consumeClaudeCodeLogEntries,
|
|
11460
12044
|
consumeCodexLogEntries,
|
|
12045
|
+
consumeCopilotCliLogEntries,
|
|
11461
12046
|
consumePiLogEntries,
|
|
11462
12047
|
createAgentKernel,
|
|
11463
12048
|
createProvider,
|
|
@@ -11497,6 +12082,7 @@ function createAgentKernel() {
|
|
|
11497
12082
|
scoreToVerdict,
|
|
11498
12083
|
subscribeToClaudeCodeLogEntries,
|
|
11499
12084
|
subscribeToCodexLogEntries,
|
|
12085
|
+
subscribeToCopilotCliLogEntries,
|
|
11500
12086
|
subscribeToPiLogEntries,
|
|
11501
12087
|
tokensPerTool
|
|
11502
12088
|
});
|