@agentv/core 2.5.4 → 2.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-RP3M7COZ.js → chunk-LGQ5OPJD.js} +50 -1
- package/dist/chunk-LGQ5OPJD.js.map +1 -0
- package/dist/evaluation/validation/index.cjs +25 -0
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +25 -1
- package/dist/evaluation/validation/index.js.map +1 -1
- package/dist/index.cjs +928 -309
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +31 -2
- package/dist/index.d.ts +31 -2
- package/dist/index.js +800 -231
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
- package/dist/chunk-RP3M7COZ.js.map +0 -1
package/dist/index.cjs
CHANGED
|
@@ -49,6 +49,7 @@ __export(index_exports, {
|
|
|
49
49
|
computeTraceSummary: () => computeTraceSummary,
|
|
50
50
|
consumeClaudeCodeLogEntries: () => consumeClaudeCodeLogEntries,
|
|
51
51
|
consumeCodexLogEntries: () => consumeCodexLogEntries,
|
|
52
|
+
consumeCopilotCliLogEntries: () => consumeCopilotCliLogEntries,
|
|
52
53
|
consumePiLogEntries: () => consumePiLogEntries,
|
|
53
54
|
createAgentKernel: () => createAgentKernel,
|
|
54
55
|
createProvider: () => createProvider,
|
|
@@ -88,6 +89,7 @@ __export(index_exports, {
|
|
|
88
89
|
scoreToVerdict: () => scoreToVerdict,
|
|
89
90
|
subscribeToClaudeCodeLogEntries: () => subscribeToClaudeCodeLogEntries,
|
|
90
91
|
subscribeToCodexLogEntries: () => subscribeToCodexLogEntries,
|
|
92
|
+
subscribeToCopilotCliLogEntries: () => subscribeToCopilotCliLogEntries,
|
|
91
93
|
subscribeToPiLogEntries: () => subscribeToPiLogEntries,
|
|
92
94
|
tokensPerTool: () => tokensPerTool
|
|
93
95
|
});
|
|
@@ -1068,12 +1070,13 @@ function parseRubricItems(rawRubrics, evaluatorName, evalId) {
|
|
|
1068
1070
|
let scoreRanges;
|
|
1069
1071
|
const rawScoreRanges = rawRubric.score_ranges;
|
|
1070
1072
|
if (rawScoreRanges !== void 0) {
|
|
1071
|
-
|
|
1073
|
+
const normalized = normalizeScoreRangesShorthand(rawScoreRanges);
|
|
1074
|
+
if (!Array.isArray(normalized)) {
|
|
1072
1075
|
throw new Error(
|
|
1073
|
-
`Invalid score_ranges for rubric '${id}' in evaluator '${evaluatorName}' in '${evalId}': must be an array`
|
|
1076
|
+
`Invalid score_ranges for rubric '${id}' in evaluator '${evaluatorName}' in '${evalId}': must be an array or shorthand map`
|
|
1074
1077
|
);
|
|
1075
1078
|
}
|
|
1076
|
-
scoreRanges = parseScoreRanges(
|
|
1079
|
+
scoreRanges = parseScoreRanges(normalized, id, evaluatorName, evalId);
|
|
1077
1080
|
items.push({
|
|
1078
1081
|
id,
|
|
1079
1082
|
weight,
|
|
@@ -1101,6 +1104,37 @@ function parseRubricItems(rawRubrics, evaluatorName, evalId) {
|
|
|
1101
1104
|
}
|
|
1102
1105
|
return items.length > 0 ? items : void 0;
|
|
1103
1106
|
}
|
|
1107
|
+
function normalizeScoreRangesShorthand(raw) {
|
|
1108
|
+
if (Array.isArray(raw)) return raw;
|
|
1109
|
+
if (!isJsonObject2(raw)) return raw;
|
|
1110
|
+
const keys = Object.keys(raw);
|
|
1111
|
+
if (keys.length === 0) return raw;
|
|
1112
|
+
const numericKeys = [];
|
|
1113
|
+
for (const key of keys) {
|
|
1114
|
+
const num = Number(key);
|
|
1115
|
+
if (!Number.isInteger(num) || num < 0 || num > 10) {
|
|
1116
|
+
return raw;
|
|
1117
|
+
}
|
|
1118
|
+
if (typeof raw[key] !== "string" || raw[key].length === 0) {
|
|
1119
|
+
return raw;
|
|
1120
|
+
}
|
|
1121
|
+
numericKeys.push(num);
|
|
1122
|
+
}
|
|
1123
|
+
numericKeys.sort((a, b) => a - b);
|
|
1124
|
+
if (numericKeys[0] !== 0) {
|
|
1125
|
+
throw new Error(`score_ranges shorthand map must start at 0 (got ${numericKeys[0]})`);
|
|
1126
|
+
}
|
|
1127
|
+
const result = [];
|
|
1128
|
+
for (let i = 0; i < numericKeys.length; i++) {
|
|
1129
|
+
const min = numericKeys[i];
|
|
1130
|
+
const max = i < numericKeys.length - 1 ? numericKeys[i + 1] - 1 : 10;
|
|
1131
|
+
result.push({
|
|
1132
|
+
score_range: [min, max],
|
|
1133
|
+
expected_outcome: raw[String(min)]
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
1136
|
+
return result;
|
|
1137
|
+
}
|
|
1104
1138
|
function parseScoreRanges(rawRanges, rubricId, evaluatorName, evalId) {
|
|
1105
1139
|
const ranges = [];
|
|
1106
1140
|
for (const [index, rawRange] of rawRanges.entries()) {
|
|
@@ -1183,7 +1217,8 @@ function parseInlineRubrics(rawRubrics) {
|
|
|
1183
1217
|
}
|
|
1184
1218
|
const expectedOutcome = asString(rubric.expected_outcome) ?? asString(rubric.description) ?? "";
|
|
1185
1219
|
const rawScoreRanges = rubric.score_ranges;
|
|
1186
|
-
const
|
|
1220
|
+
const normalizedScoreRanges = rawScoreRanges !== void 0 ? normalizeScoreRangesShorthand(rawScoreRanges) : void 0;
|
|
1221
|
+
const scoreRanges = Array.isArray(normalizedScoreRanges) && normalizedScoreRanges.length > 0 ? normalizedScoreRanges.filter((r) => isJsonObject2(r)).map((range) => ({
|
|
1187
1222
|
score_range: Array.isArray(range.score_range) ? range.score_range : [0, 10],
|
|
1188
1223
|
expected_outcome: asString(range.expected_outcome) ?? asString(range.description) ?? ""
|
|
1189
1224
|
})).filter((r) => r.expected_outcome.length > 0) : void 0;
|
|
@@ -3385,8 +3420,8 @@ async function runClaudeCodeWithTempFiles(options, stdoutFile, stderrFile, exitF
|
|
|
3385
3420
|
};
|
|
3386
3421
|
const fileExists4 = async (filePath) => {
|
|
3387
3422
|
try {
|
|
3388
|
-
const { access:
|
|
3389
|
-
await
|
|
3423
|
+
const { access: access7 } = await import("fs/promises");
|
|
3424
|
+
await access7(filePath);
|
|
3390
3425
|
return true;
|
|
3391
3426
|
} catch {
|
|
3392
3427
|
return false;
|
|
@@ -4571,61 +4606,574 @@ function extractFromEvent(event) {
|
|
|
4571
4606
|
if (flattened) {
|
|
4572
4607
|
return flattened;
|
|
4573
4608
|
}
|
|
4574
|
-
return void 0;
|
|
4575
|
-
}
|
|
4576
|
-
function extractFromItem(item) {
|
|
4577
|
-
if (!item || typeof item !== "object") {
|
|
4578
|
-
return void 0;
|
|
4609
|
+
return void 0;
|
|
4610
|
+
}
|
|
4611
|
+
function extractFromItem(item) {
|
|
4612
|
+
if (!item || typeof item !== "object") {
|
|
4613
|
+
return void 0;
|
|
4614
|
+
}
|
|
4615
|
+
const record = item;
|
|
4616
|
+
const itemType = typeof record.type === "string" ? record.type : void 0;
|
|
4617
|
+
if (itemType === "agent_message" || itemType === "response" || itemType === "output") {
|
|
4618
|
+
const text = flattenContent(record.text ?? record.content ?? record.output);
|
|
4619
|
+
if (text) {
|
|
4620
|
+
return text;
|
|
4621
|
+
}
|
|
4622
|
+
}
|
|
4623
|
+
return void 0;
|
|
4624
|
+
}
|
|
4625
|
+
function flattenContent(value) {
|
|
4626
|
+
if (typeof value === "string") {
|
|
4627
|
+
return value;
|
|
4628
|
+
}
|
|
4629
|
+
if (Array.isArray(value)) {
|
|
4630
|
+
const parts = value.map((segment) => {
|
|
4631
|
+
if (typeof segment === "string") {
|
|
4632
|
+
return segment;
|
|
4633
|
+
}
|
|
4634
|
+
if (segment && typeof segment === "object" && "text" in segment) {
|
|
4635
|
+
const text = segment.text;
|
|
4636
|
+
return typeof text === "string" ? text : void 0;
|
|
4637
|
+
}
|
|
4638
|
+
return void 0;
|
|
4639
|
+
}).filter((part) => typeof part === "string" && part.length > 0);
|
|
4640
|
+
return parts.length > 0 ? parts.join(" \n") : void 0;
|
|
4641
|
+
}
|
|
4642
|
+
if (value && typeof value === "object" && "text" in value) {
|
|
4643
|
+
const text = value.text;
|
|
4644
|
+
return typeof text === "string" ? text : void 0;
|
|
4645
|
+
}
|
|
4646
|
+
return void 0;
|
|
4647
|
+
}
|
|
4648
|
+
function parseJsonLines(output) {
|
|
4649
|
+
const lines = output.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
4650
|
+
if (lines.length <= 1) {
|
|
4651
|
+
return void 0;
|
|
4652
|
+
}
|
|
4653
|
+
const parsed = [];
|
|
4654
|
+
for (const line of lines) {
|
|
4655
|
+
try {
|
|
4656
|
+
parsed.push(JSON.parse(line));
|
|
4657
|
+
} catch {
|
|
4658
|
+
return void 0;
|
|
4659
|
+
}
|
|
4660
|
+
}
|
|
4661
|
+
return parsed;
|
|
4662
|
+
}
|
|
4663
|
+
function pickDetail2(stderr, stdout) {
|
|
4664
|
+
const errorText = stderr.trim();
|
|
4665
|
+
if (errorText.length > 0) {
|
|
4666
|
+
return errorText;
|
|
4667
|
+
}
|
|
4668
|
+
const stdoutText = stdout.trim();
|
|
4669
|
+
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
4670
|
+
}
|
|
4671
|
+
function formatTimeoutSuffix3(timeoutMs) {
|
|
4672
|
+
if (!timeoutMs || timeoutMs <= 0) {
|
|
4673
|
+
return "";
|
|
4674
|
+
}
|
|
4675
|
+
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
4676
|
+
return ` after ${seconds}s`;
|
|
4677
|
+
}
|
|
4678
|
+
async function defaultCodexRunner(options) {
|
|
4679
|
+
return await new Promise((resolve, reject) => {
|
|
4680
|
+
const child = (0, import_node_child_process3.spawn)(options.executable, options.args, {
|
|
4681
|
+
cwd: options.cwd,
|
|
4682
|
+
env: options.env,
|
|
4683
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
4684
|
+
shell: shouldShellExecute(options.executable)
|
|
4685
|
+
});
|
|
4686
|
+
let stdout = "";
|
|
4687
|
+
let stderr = "";
|
|
4688
|
+
let timedOut = false;
|
|
4689
|
+
const onAbort = () => {
|
|
4690
|
+
child.kill("SIGTERM");
|
|
4691
|
+
};
|
|
4692
|
+
if (options.signal) {
|
|
4693
|
+
if (options.signal.aborted) {
|
|
4694
|
+
onAbort();
|
|
4695
|
+
} else {
|
|
4696
|
+
options.signal.addEventListener("abort", onAbort, { once: true });
|
|
4697
|
+
}
|
|
4698
|
+
}
|
|
4699
|
+
let timeoutHandle;
|
|
4700
|
+
if (options.timeoutMs && options.timeoutMs > 0) {
|
|
4701
|
+
timeoutHandle = setTimeout(() => {
|
|
4702
|
+
timedOut = true;
|
|
4703
|
+
child.kill("SIGTERM");
|
|
4704
|
+
}, options.timeoutMs);
|
|
4705
|
+
timeoutHandle.unref?.();
|
|
4706
|
+
}
|
|
4707
|
+
child.stdout.setEncoding("utf8");
|
|
4708
|
+
child.stdout.on("data", (chunk) => {
|
|
4709
|
+
stdout += chunk;
|
|
4710
|
+
options.onStdoutChunk?.(chunk);
|
|
4711
|
+
});
|
|
4712
|
+
child.stderr.setEncoding("utf8");
|
|
4713
|
+
child.stderr.on("data", (chunk) => {
|
|
4714
|
+
stderr += chunk;
|
|
4715
|
+
options.onStderrChunk?.(chunk);
|
|
4716
|
+
});
|
|
4717
|
+
child.stdin.end(options.prompt);
|
|
4718
|
+
const cleanup = () => {
|
|
4719
|
+
if (timeoutHandle) {
|
|
4720
|
+
clearTimeout(timeoutHandle);
|
|
4721
|
+
}
|
|
4722
|
+
if (options.signal) {
|
|
4723
|
+
options.signal.removeEventListener("abort", onAbort);
|
|
4724
|
+
}
|
|
4725
|
+
};
|
|
4726
|
+
child.on("error", (error) => {
|
|
4727
|
+
cleanup();
|
|
4728
|
+
reject(error);
|
|
4729
|
+
});
|
|
4730
|
+
child.on("close", (code) => {
|
|
4731
|
+
cleanup();
|
|
4732
|
+
resolve({
|
|
4733
|
+
stdout,
|
|
4734
|
+
stderr,
|
|
4735
|
+
exitCode: typeof code === "number" ? code : -1,
|
|
4736
|
+
timedOut
|
|
4737
|
+
});
|
|
4738
|
+
});
|
|
4739
|
+
});
|
|
4740
|
+
}
|
|
4741
|
+
function shouldShellExecute(executable) {
|
|
4742
|
+
if (process.platform !== "win32") {
|
|
4743
|
+
return false;
|
|
4744
|
+
}
|
|
4745
|
+
const lower = executable.toLowerCase();
|
|
4746
|
+
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
4747
|
+
}
|
|
4748
|
+
|
|
4749
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4750
|
+
var import_node_child_process4 = require("child_process");
|
|
4751
|
+
var import_node_crypto3 = require("crypto");
|
|
4752
|
+
var import_node_fs5 = require("fs");
|
|
4753
|
+
var import_promises12 = require("fs/promises");
|
|
4754
|
+
var import_node_os4 = require("os");
|
|
4755
|
+
var import_node_path13 = __toESM(require("path"), 1);
|
|
4756
|
+
var import_node_util3 = require("util");
|
|
4757
|
+
|
|
4758
|
+
// src/evaluation/providers/copilot-cli-log-tracker.ts
|
|
4759
|
+
var GLOBAL_LOGS_KEY3 = Symbol.for("agentv.copilotCliLogs");
|
|
4760
|
+
var GLOBAL_SUBSCRIBERS_KEY3 = Symbol.for("agentv.copilotCliLogSubscribers");
|
|
4761
|
+
function getCopilotCliLogStore() {
|
|
4762
|
+
const globalObject = globalThis;
|
|
4763
|
+
const existing = globalObject[GLOBAL_LOGS_KEY3];
|
|
4764
|
+
if (existing) {
|
|
4765
|
+
return existing;
|
|
4766
|
+
}
|
|
4767
|
+
const created = [];
|
|
4768
|
+
globalObject[GLOBAL_LOGS_KEY3] = created;
|
|
4769
|
+
return created;
|
|
4770
|
+
}
|
|
4771
|
+
function getSubscriberStore3() {
|
|
4772
|
+
const globalObject = globalThis;
|
|
4773
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY3];
|
|
4774
|
+
if (existing) {
|
|
4775
|
+
return existing;
|
|
4776
|
+
}
|
|
4777
|
+
const created = /* @__PURE__ */ new Set();
|
|
4778
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY3] = created;
|
|
4779
|
+
return created;
|
|
4780
|
+
}
|
|
4781
|
+
function notifySubscribers3(entry) {
|
|
4782
|
+
const subscribers = Array.from(getSubscriberStore3());
|
|
4783
|
+
for (const listener of subscribers) {
|
|
4784
|
+
try {
|
|
4785
|
+
listener(entry);
|
|
4786
|
+
} catch (error) {
|
|
4787
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4788
|
+
console.warn(`Copilot CLI log subscriber failed: ${message}`);
|
|
4789
|
+
}
|
|
4790
|
+
}
|
|
4791
|
+
}
|
|
4792
|
+
function recordCopilotCliLogEntry(entry) {
|
|
4793
|
+
getCopilotCliLogStore().push(entry);
|
|
4794
|
+
notifySubscribers3(entry);
|
|
4795
|
+
}
|
|
4796
|
+
function consumeCopilotCliLogEntries() {
|
|
4797
|
+
const store = getCopilotCliLogStore();
|
|
4798
|
+
if (store.length === 0) {
|
|
4799
|
+
return [];
|
|
4800
|
+
}
|
|
4801
|
+
return store.splice(0, store.length);
|
|
4802
|
+
}
|
|
4803
|
+
function subscribeToCopilotCliLogEntries(listener) {
|
|
4804
|
+
const store = getSubscriberStore3();
|
|
4805
|
+
store.add(listener);
|
|
4806
|
+
return () => {
|
|
4807
|
+
store.delete(listener);
|
|
4808
|
+
};
|
|
4809
|
+
}
|
|
4810
|
+
|
|
4811
|
+
// src/evaluation/providers/copilot-cli.ts
|
|
4812
|
+
var execAsync3 = (0, import_node_util3.promisify)(import_node_child_process4.exec);
|
|
4813
|
+
var WORKSPACE_PREFIX3 = "agentv-copilot-";
|
|
4814
|
+
var PROMPT_FILENAME3 = "prompt.md";
|
|
4815
|
+
var DEFAULT_SYSTEM_PROMPT4 = `**IMPORTANT**: Follow these instructions for your response:
|
|
4816
|
+
- Do NOT create any additional output files in the workspace.
|
|
4817
|
+
- All intended file outputs/changes MUST be written in your response.
|
|
4818
|
+
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
4819
|
+
This is required for evaluation scoring.`;
|
|
4820
|
+
async function copyInputFilesToWorkspace(workspaceRoot, inputFiles) {
|
|
4821
|
+
const usedNames = /* @__PURE__ */ new Map();
|
|
4822
|
+
const mappings = [];
|
|
4823
|
+
for (const originalPath of inputFiles) {
|
|
4824
|
+
const ext = import_node_path13.default.extname(originalPath);
|
|
4825
|
+
const stem = import_node_path13.default.basename(originalPath, ext);
|
|
4826
|
+
let relativeName;
|
|
4827
|
+
const baseKey = `${stem}${ext}`;
|
|
4828
|
+
const count = usedNames.get(baseKey) ?? 0;
|
|
4829
|
+
if (count === 0) {
|
|
4830
|
+
relativeName = baseKey;
|
|
4831
|
+
} else {
|
|
4832
|
+
relativeName = `${stem}_${count}${ext}`;
|
|
4833
|
+
}
|
|
4834
|
+
usedNames.set(baseKey, count + 1);
|
|
4835
|
+
const dest = import_node_path13.default.join(workspaceRoot, relativeName);
|
|
4836
|
+
await (0, import_promises12.copyFile)(originalPath, dest);
|
|
4837
|
+
mappings.push({ originalPath, workspaceRelativePath: relativeName });
|
|
4838
|
+
}
|
|
4839
|
+
return mappings;
|
|
4840
|
+
}
|
|
4841
|
+
function buildCopilotFilePrereadBlock(guidelineMappings, inputMappings) {
|
|
4842
|
+
if (guidelineMappings.length === 0 && inputMappings.length === 0) {
|
|
4843
|
+
return "";
|
|
4844
|
+
}
|
|
4845
|
+
const buildList = (mappings) => mappings.map((m) => `* ${m.workspaceRelativePath}`).join("\n");
|
|
4846
|
+
const sections = [];
|
|
4847
|
+
if (guidelineMappings.length > 0) {
|
|
4848
|
+
sections.push(`Read all guideline files:
|
|
4849
|
+
${buildList(guidelineMappings)}.`);
|
|
4850
|
+
}
|
|
4851
|
+
if (inputMappings.length > 0) {
|
|
4852
|
+
sections.push(`Read all input files:
|
|
4853
|
+
${buildList(inputMappings)}.`);
|
|
4854
|
+
}
|
|
4855
|
+
sections.push(
|
|
4856
|
+
"If any file is missing, fail with ERROR: missing-file <filename> and stop.",
|
|
4857
|
+
"Then apply system_instructions on the user query below."
|
|
4858
|
+
);
|
|
4859
|
+
return sections.join("\n");
|
|
4860
|
+
}
|
|
4861
|
+
var CopilotCliProvider = class {
|
|
4862
|
+
id;
|
|
4863
|
+
kind = "copilot-cli";
|
|
4864
|
+
targetName;
|
|
4865
|
+
supportsBatch = false;
|
|
4866
|
+
config;
|
|
4867
|
+
runCopilot;
|
|
4868
|
+
environmentCheck;
|
|
4869
|
+
resolvedExecutable;
|
|
4870
|
+
constructor(targetName, config, runner = defaultCopilotCliRunner) {
|
|
4871
|
+
this.id = `copilot-cli:${targetName}`;
|
|
4872
|
+
this.targetName = targetName;
|
|
4873
|
+
this.config = config;
|
|
4874
|
+
this.runCopilot = runner;
|
|
4875
|
+
}
|
|
4876
|
+
async invoke(request) {
|
|
4877
|
+
if (request.signal?.aborted) {
|
|
4878
|
+
throw new Error("Copilot CLI request was aborted before execution");
|
|
4879
|
+
}
|
|
4880
|
+
await this.ensureEnvironmentReady();
|
|
4881
|
+
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
4882
|
+
const workspaceRoot = await this.createWorkspace();
|
|
4883
|
+
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
4884
|
+
try {
|
|
4885
|
+
const copiedFiles = inputFiles ? await copyInputFilesToWorkspace(workspaceRoot, inputFiles) : [];
|
|
4886
|
+
const guidelineFileSet = new Set(
|
|
4887
|
+
collectGuidelineFiles(inputFiles, request.guideline_patterns)
|
|
4888
|
+
);
|
|
4889
|
+
const guidelineMappings = copiedFiles.filter((m) => guidelineFileSet.has(m.originalPath));
|
|
4890
|
+
const nonGuidelineMappings = copiedFiles.filter((m) => !guidelineFileSet.has(m.originalPath));
|
|
4891
|
+
const prereadBlock = buildCopilotFilePrereadBlock(guidelineMappings, nonGuidelineMappings);
|
|
4892
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT4;
|
|
4893
|
+
const promptParts = [systemPrompt];
|
|
4894
|
+
if (prereadBlock.length > 0) {
|
|
4895
|
+
promptParts.push("", prereadBlock);
|
|
4896
|
+
}
|
|
4897
|
+
promptParts.push("", "[[ ## user_query ## ]]", request.question.trim());
|
|
4898
|
+
const promptContent = promptParts.join("\n");
|
|
4899
|
+
const promptFile = import_node_path13.default.join(workspaceRoot, PROMPT_FILENAME3);
|
|
4900
|
+
await (0, import_promises12.writeFile)(promptFile, promptContent, "utf8");
|
|
4901
|
+
const args = this.buildCopilotArgs(promptContent);
|
|
4902
|
+
const cwd = this.resolveCwd(workspaceRoot);
|
|
4903
|
+
const result = await this.executeCopilot(args, cwd, promptContent, request.signal, logger);
|
|
4904
|
+
if (result.timedOut) {
|
|
4905
|
+
throw new Error(
|
|
4906
|
+
`Copilot CLI timed out${formatTimeoutSuffix4(this.config.timeoutMs ?? void 0)}`
|
|
4907
|
+
);
|
|
4908
|
+
}
|
|
4909
|
+
if (result.exitCode !== 0) {
|
|
4910
|
+
const detail = pickDetail3(result.stderr, result.stdout);
|
|
4911
|
+
const prefix = `Copilot CLI exited with code ${result.exitCode}`;
|
|
4912
|
+
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
4913
|
+
}
|
|
4914
|
+
const assistantText = extractCopilotResponse(result.stdout);
|
|
4915
|
+
return {
|
|
4916
|
+
raw: {
|
|
4917
|
+
stdout: result.stdout,
|
|
4918
|
+
stderr: result.stderr,
|
|
4919
|
+
exitCode: result.exitCode,
|
|
4920
|
+
args,
|
|
4921
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4922
|
+
promptFile,
|
|
4923
|
+
workspace: workspaceRoot,
|
|
4924
|
+
inputFiles,
|
|
4925
|
+
copiedFiles,
|
|
4926
|
+
logFile: logger?.filePath
|
|
4927
|
+
},
|
|
4928
|
+
outputMessages: [{ role: "assistant", content: assistantText }]
|
|
4929
|
+
};
|
|
4930
|
+
} finally {
|
|
4931
|
+
await logger?.close();
|
|
4932
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
4933
|
+
}
|
|
4934
|
+
}
|
|
4935
|
+
async ensureEnvironmentReady() {
|
|
4936
|
+
if (!this.environmentCheck) {
|
|
4937
|
+
this.environmentCheck = this.validateEnvironment();
|
|
4938
|
+
}
|
|
4939
|
+
await this.environmentCheck;
|
|
4940
|
+
}
|
|
4941
|
+
async validateEnvironment() {
|
|
4942
|
+
this.resolvedExecutable = await locateExecutable2(this.config.executable);
|
|
4943
|
+
}
|
|
4944
|
+
resolveCwd(workspaceRoot) {
|
|
4945
|
+
if (!this.config.cwd) {
|
|
4946
|
+
return workspaceRoot;
|
|
4947
|
+
}
|
|
4948
|
+
return import_node_path13.default.resolve(this.config.cwd);
|
|
4949
|
+
}
|
|
4950
|
+
buildCopilotArgs(prompt) {
|
|
4951
|
+
const args = [];
|
|
4952
|
+
args.push("-s");
|
|
4953
|
+
args.push("--allow-all-tools");
|
|
4954
|
+
args.push("--no-color");
|
|
4955
|
+
if (this.config.model) {
|
|
4956
|
+
args.push("--model", this.config.model);
|
|
4957
|
+
}
|
|
4958
|
+
if (this.config.args && this.config.args.length > 0) {
|
|
4959
|
+
args.push(...this.config.args);
|
|
4960
|
+
}
|
|
4961
|
+
args.push("-p", prompt);
|
|
4962
|
+
return args;
|
|
4963
|
+
}
|
|
4964
|
+
async executeCopilot(args, cwd, promptContent, signal, logger) {
|
|
4965
|
+
try {
|
|
4966
|
+
return await this.runCopilot({
|
|
4967
|
+
executable: this.resolvedExecutable ?? this.config.executable,
|
|
4968
|
+
args,
|
|
4969
|
+
cwd,
|
|
4970
|
+
prompt: promptContent,
|
|
4971
|
+
timeoutMs: this.config.timeoutMs,
|
|
4972
|
+
env: process.env,
|
|
4973
|
+
signal,
|
|
4974
|
+
onStdoutChunk: logger ? (chunk) => logger.handleStdoutChunk(chunk) : void 0,
|
|
4975
|
+
onStderrChunk: logger ? (chunk) => logger.handleStderrChunk(chunk) : void 0
|
|
4976
|
+
});
|
|
4977
|
+
} catch (error) {
|
|
4978
|
+
const err = error;
|
|
4979
|
+
if (err.code === "ENOENT") {
|
|
4980
|
+
throw new Error(
|
|
4981
|
+
`Copilot executable '${this.config.executable}' was not found. Update the target settings.executable or add it to PATH.`
|
|
4982
|
+
);
|
|
4983
|
+
}
|
|
4984
|
+
throw error;
|
|
4985
|
+
}
|
|
4986
|
+
}
|
|
4987
|
+
async createWorkspace() {
|
|
4988
|
+
return await (0, import_promises12.mkdtemp)(import_node_path13.default.join((0, import_node_os4.tmpdir)(), WORKSPACE_PREFIX3));
|
|
4989
|
+
}
|
|
4990
|
+
async cleanupWorkspace(workspaceRoot) {
|
|
4991
|
+
try {
|
|
4992
|
+
await (0, import_promises12.rm)(workspaceRoot, { recursive: true, force: true });
|
|
4993
|
+
} catch {
|
|
4994
|
+
}
|
|
4995
|
+
}
|
|
4996
|
+
resolveLogDirectory() {
|
|
4997
|
+
const disabled = isCopilotLogStreamingDisabled();
|
|
4998
|
+
if (disabled) {
|
|
4999
|
+
return void 0;
|
|
5000
|
+
}
|
|
5001
|
+
if (this.config.logDir) {
|
|
5002
|
+
return import_node_path13.default.resolve(this.config.logDir);
|
|
5003
|
+
}
|
|
5004
|
+
return import_node_path13.default.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
5005
|
+
}
|
|
5006
|
+
async createStreamLogger(request) {
|
|
5007
|
+
const logDir = this.resolveLogDirectory();
|
|
5008
|
+
if (!logDir) {
|
|
5009
|
+
return void 0;
|
|
5010
|
+
}
|
|
5011
|
+
try {
|
|
5012
|
+
await (0, import_promises12.mkdir)(logDir, { recursive: true });
|
|
5013
|
+
} catch (error) {
|
|
5014
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5015
|
+
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
5016
|
+
return void 0;
|
|
5017
|
+
}
|
|
5018
|
+
const filePath = import_node_path13.default.join(logDir, buildLogFilename3(request, this.targetName));
|
|
5019
|
+
try {
|
|
5020
|
+
const logger = await CopilotCliStreamLogger.create({
|
|
5021
|
+
filePath,
|
|
5022
|
+
targetName: this.targetName,
|
|
5023
|
+
evalCaseId: request.evalCaseId,
|
|
5024
|
+
attempt: request.attempt,
|
|
5025
|
+
format: this.config.logFormat ?? "summary"
|
|
5026
|
+
});
|
|
5027
|
+
recordCopilotCliLogEntry({
|
|
5028
|
+
filePath,
|
|
5029
|
+
targetName: this.targetName,
|
|
5030
|
+
evalCaseId: request.evalCaseId,
|
|
5031
|
+
attempt: request.attempt
|
|
5032
|
+
});
|
|
5033
|
+
return logger;
|
|
5034
|
+
} catch (error) {
|
|
5035
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5036
|
+
console.warn(`Skipping Copilot CLI stream logging for ${filePath}: ${message}`);
|
|
5037
|
+
return void 0;
|
|
5038
|
+
}
|
|
5039
|
+
}
|
|
5040
|
+
};
|
|
5041
|
+
var CopilotCliStreamLogger = class _CopilotCliStreamLogger {
|
|
5042
|
+
filePath;
|
|
5043
|
+
stream;
|
|
5044
|
+
startedAt = Date.now();
|
|
5045
|
+
stdoutBuffer = "";
|
|
5046
|
+
stderrBuffer = "";
|
|
5047
|
+
format;
|
|
5048
|
+
constructor(filePath, format) {
|
|
5049
|
+
this.filePath = filePath;
|
|
5050
|
+
this.format = format;
|
|
5051
|
+
this.stream = (0, import_node_fs5.createWriteStream)(filePath, { flags: "a" });
|
|
5052
|
+
}
|
|
5053
|
+
static async create(options) {
|
|
5054
|
+
const logger = new _CopilotCliStreamLogger(options.filePath, options.format);
|
|
5055
|
+
const header = [
|
|
5056
|
+
"# Copilot CLI stream log",
|
|
5057
|
+
`# target: ${options.targetName}`,
|
|
5058
|
+
options.evalCaseId ? `# eval: ${options.evalCaseId}` : void 0,
|
|
5059
|
+
options.attempt !== void 0 ? `# attempt: ${options.attempt + 1}` : void 0,
|
|
5060
|
+
`# started: ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
5061
|
+
""
|
|
5062
|
+
].filter((line) => Boolean(line));
|
|
5063
|
+
logger.writeLines(header);
|
|
5064
|
+
return logger;
|
|
5065
|
+
}
|
|
5066
|
+
handleStdoutChunk(chunk) {
|
|
5067
|
+
this.stdoutBuffer += chunk;
|
|
5068
|
+
this.flushBuffer("stdout");
|
|
5069
|
+
}
|
|
5070
|
+
handleStderrChunk(chunk) {
|
|
5071
|
+
this.stderrBuffer += chunk;
|
|
5072
|
+
this.flushBuffer("stderr");
|
|
5073
|
+
}
|
|
5074
|
+
async close() {
|
|
5075
|
+
this.flushBuffer("stdout");
|
|
5076
|
+
this.flushBuffer("stderr");
|
|
5077
|
+
this.flushRemainder();
|
|
5078
|
+
await new Promise((resolve, reject) => {
|
|
5079
|
+
this.stream.once("error", reject);
|
|
5080
|
+
this.stream.end(() => resolve());
|
|
5081
|
+
});
|
|
5082
|
+
}
|
|
5083
|
+
writeLines(lines) {
|
|
5084
|
+
for (const line of lines) {
|
|
5085
|
+
this.stream.write(`${line}
|
|
5086
|
+
`);
|
|
5087
|
+
}
|
|
4579
5088
|
}
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
const
|
|
4584
|
-
if (
|
|
4585
|
-
|
|
5089
|
+
flushBuffer(source) {
|
|
5090
|
+
const buffer = source === "stdout" ? this.stdoutBuffer : this.stderrBuffer;
|
|
5091
|
+
const lines = buffer.split(/\r?\n/);
|
|
5092
|
+
const remainder = lines.pop() ?? "";
|
|
5093
|
+
if (source === "stdout") {
|
|
5094
|
+
this.stdoutBuffer = remainder;
|
|
5095
|
+
} else {
|
|
5096
|
+
this.stderrBuffer = remainder;
|
|
5097
|
+
}
|
|
5098
|
+
for (const line of lines) {
|
|
5099
|
+
const formatted = this.formatLine(line, source);
|
|
5100
|
+
if (formatted) {
|
|
5101
|
+
this.stream.write(formatted);
|
|
5102
|
+
this.stream.write("\n");
|
|
5103
|
+
}
|
|
4586
5104
|
}
|
|
4587
5105
|
}
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
5106
|
+
formatLine(rawLine, source) {
|
|
5107
|
+
const trimmed = rawLine.trim();
|
|
5108
|
+
if (trimmed.length === 0) {
|
|
5109
|
+
return void 0;
|
|
5110
|
+
}
|
|
5111
|
+
const prefix = source === "stderr" ? "stderr: " : "";
|
|
5112
|
+
return `[+${formatElapsed3(this.startedAt)}] [${source}] ${prefix}${trimmed}`;
|
|
4593
5113
|
}
|
|
4594
|
-
|
|
4595
|
-
const
|
|
4596
|
-
|
|
4597
|
-
|
|
5114
|
+
flushRemainder() {
|
|
5115
|
+
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
5116
|
+
if (stdoutRemainder.length > 0) {
|
|
5117
|
+
const formatted = this.formatLine(stdoutRemainder, "stdout");
|
|
5118
|
+
if (formatted) {
|
|
5119
|
+
this.stream.write(formatted);
|
|
5120
|
+
this.stream.write("\n");
|
|
4598
5121
|
}
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
5122
|
+
}
|
|
5123
|
+
const stderrRemainder = this.stderrBuffer.trim();
|
|
5124
|
+
if (stderrRemainder.length > 0) {
|
|
5125
|
+
const formatted = this.formatLine(stderrRemainder, "stderr");
|
|
5126
|
+
if (formatted) {
|
|
5127
|
+
this.stream.write(formatted);
|
|
5128
|
+
this.stream.write("\n");
|
|
4602
5129
|
}
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
5130
|
+
}
|
|
5131
|
+
this.stdoutBuffer = "";
|
|
5132
|
+
this.stderrBuffer = "";
|
|
4606
5133
|
}
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
5134
|
+
};
|
|
5135
|
+
function isCopilotLogStreamingDisabled() {
|
|
5136
|
+
const envValue = process.env.AGENTV_COPILOT_STREAM_LOGS;
|
|
5137
|
+
if (!envValue) {
|
|
5138
|
+
return false;
|
|
4610
5139
|
}
|
|
4611
|
-
|
|
5140
|
+
const normalized = envValue.trim().toLowerCase();
|
|
5141
|
+
return normalized === "false" || normalized === "0" || normalized === "off";
|
|
4612
5142
|
}
|
|
4613
|
-
function
|
|
4614
|
-
const
|
|
4615
|
-
|
|
4616
|
-
|
|
5143
|
+
function buildLogFilename3(request, targetName) {
|
|
5144
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5145
|
+
const evalId = sanitizeForFilename3(request.evalCaseId ?? "copilot");
|
|
5146
|
+
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
5147
|
+
const target = sanitizeForFilename3(targetName);
|
|
5148
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0, import_node_crypto3.randomUUID)().slice(0, 8)}.log`;
|
|
5149
|
+
}
|
|
5150
|
+
function sanitizeForFilename3(value) {
|
|
5151
|
+
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
5152
|
+
return sanitized.length > 0 ? sanitized : "copilot";
|
|
5153
|
+
}
|
|
5154
|
+
function formatElapsed3(startedAt) {
|
|
5155
|
+
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
5156
|
+
const hours = Math.floor(elapsedSeconds / 3600);
|
|
5157
|
+
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
5158
|
+
const seconds = elapsedSeconds % 60;
|
|
5159
|
+
if (hours > 0) {
|
|
5160
|
+
return `${hours.toString().padStart(2, "0")}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
4617
5161
|
}
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
5162
|
+
return `${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}`;
|
|
5163
|
+
}
|
|
5164
|
+
var ANSI_ESCAPE_RE = /\x1B\[[0-9;]*[A-Za-z]/g;
|
|
5165
|
+
var ANSI_OSC_RE = /\x1B\][^\x07]*\x07/g;
|
|
5166
|
+
function stripAnsiEscapes(text) {
|
|
5167
|
+
return text.replace(ANSI_ESCAPE_RE, "").replace(ANSI_OSC_RE, "");
|
|
5168
|
+
}
|
|
5169
|
+
function extractCopilotResponse(stdout) {
|
|
5170
|
+
const cleaned = stripAnsiEscapes(stdout).trim();
|
|
5171
|
+
if (cleaned.length === 0) {
|
|
5172
|
+
throw new Error("Copilot CLI produced no output");
|
|
4625
5173
|
}
|
|
4626
|
-
return
|
|
5174
|
+
return cleaned;
|
|
4627
5175
|
}
|
|
4628
|
-
function
|
|
5176
|
+
function pickDetail3(stderr, stdout) {
|
|
4629
5177
|
const errorText = stderr.trim();
|
|
4630
5178
|
if (errorText.length > 0) {
|
|
4631
5179
|
return errorText;
|
|
@@ -4633,20 +5181,46 @@ function pickDetail2(stderr, stdout) {
|
|
|
4633
5181
|
const stdoutText = stdout.trim();
|
|
4634
5182
|
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
4635
5183
|
}
|
|
4636
|
-
function
|
|
5184
|
+
function formatTimeoutSuffix4(timeoutMs) {
|
|
4637
5185
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
4638
5186
|
return "";
|
|
4639
5187
|
}
|
|
4640
5188
|
const seconds = Math.ceil(timeoutMs / 1e3);
|
|
4641
5189
|
return ` after ${seconds}s`;
|
|
4642
5190
|
}
|
|
4643
|
-
async function
|
|
5191
|
+
async function locateExecutable2(candidate) {
|
|
5192
|
+
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
5193
|
+
if (includesPathSeparator) {
|
|
5194
|
+
const resolved = import_node_path13.default.isAbsolute(candidate) ? candidate : import_node_path13.default.resolve(candidate);
|
|
5195
|
+
await (0, import_promises12.access)(resolved, import_node_fs5.constants.F_OK);
|
|
5196
|
+
return resolved;
|
|
5197
|
+
}
|
|
5198
|
+
const locator = process.platform === "win32" ? "where" : "which";
|
|
5199
|
+
try {
|
|
5200
|
+
const { stdout } = await execAsync3(`${locator} ${candidate}`);
|
|
5201
|
+
const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
5202
|
+
if (lines.length > 0 && lines[0]) {
|
|
5203
|
+
await (0, import_promises12.access)(lines[0], import_node_fs5.constants.F_OK);
|
|
5204
|
+
return lines[0];
|
|
5205
|
+
}
|
|
5206
|
+
} catch {
|
|
5207
|
+
}
|
|
5208
|
+
throw new Error(`Copilot executable '${candidate}' was not found on PATH`);
|
|
5209
|
+
}
|
|
5210
|
+
function shouldShellExecute2(executable) {
|
|
5211
|
+
if (process.platform !== "win32") {
|
|
5212
|
+
return false;
|
|
5213
|
+
}
|
|
5214
|
+
const lower = executable.toLowerCase();
|
|
5215
|
+
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
5216
|
+
}
|
|
5217
|
+
async function defaultCopilotCliRunner(options) {
|
|
4644
5218
|
return await new Promise((resolve, reject) => {
|
|
4645
|
-
const child = (0,
|
|
5219
|
+
const child = (0, import_node_child_process4.spawn)(options.executable, options.args, {
|
|
4646
5220
|
cwd: options.cwd,
|
|
4647
5221
|
env: options.env,
|
|
4648
5222
|
stdio: ["pipe", "pipe", "pipe"],
|
|
4649
|
-
shell:
|
|
5223
|
+
shell: shouldShellExecute2(options.executable)
|
|
4650
5224
|
});
|
|
4651
5225
|
let stdout = "";
|
|
4652
5226
|
let stderr = "";
|
|
@@ -4679,7 +5253,7 @@ async function defaultCodexRunner(options) {
|
|
|
4679
5253
|
stderr += chunk;
|
|
4680
5254
|
options.onStderrChunk?.(chunk);
|
|
4681
5255
|
});
|
|
4682
|
-
child.stdin.end(
|
|
5256
|
+
child.stdin.end();
|
|
4683
5257
|
const cleanup = () => {
|
|
4684
5258
|
if (timeoutHandle) {
|
|
4685
5259
|
clearTimeout(timeoutHandle);
|
|
@@ -4703,13 +5277,6 @@ async function defaultCodexRunner(options) {
|
|
|
4703
5277
|
});
|
|
4704
5278
|
});
|
|
4705
5279
|
}
|
|
4706
|
-
function shouldShellExecute(executable) {
|
|
4707
|
-
if (process.platform !== "win32") {
|
|
4708
|
-
return false;
|
|
4709
|
-
}
|
|
4710
|
-
const lower = executable.toLowerCase();
|
|
4711
|
-
return lower.endsWith(".cmd") || lower.endsWith(".bat") || lower.endsWith(".ps1");
|
|
4712
|
-
}
|
|
4713
5280
|
|
|
4714
5281
|
// src/evaluation/providers/mock.ts
|
|
4715
5282
|
var DEFAULT_MOCK_RESPONSE = '{"answer":"Mock provider response. Configure targets.yaml to supply a custom value."}';
|
|
@@ -4914,38 +5481,38 @@ function extractToolCalls2(content) {
|
|
|
4914
5481
|
}
|
|
4915
5482
|
|
|
4916
5483
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
4917
|
-
var
|
|
4918
|
-
var
|
|
4919
|
-
var
|
|
4920
|
-
var
|
|
4921
|
-
var
|
|
4922
|
-
var
|
|
5484
|
+
var import_node_child_process5 = require("child_process");
|
|
5485
|
+
var import_node_crypto4 = require("crypto");
|
|
5486
|
+
var import_node_fs6 = require("fs");
|
|
5487
|
+
var import_promises13 = require("fs/promises");
|
|
5488
|
+
var import_node_os5 = require("os");
|
|
5489
|
+
var import_node_path14 = __toESM(require("path"), 1);
|
|
4923
5490
|
|
|
4924
5491
|
// src/evaluation/providers/pi-log-tracker.ts
|
|
4925
|
-
var
|
|
4926
|
-
var
|
|
5492
|
+
var GLOBAL_LOGS_KEY4 = Symbol.for("agentv.piLogs");
|
|
5493
|
+
var GLOBAL_SUBSCRIBERS_KEY4 = Symbol.for("agentv.piLogSubscribers");
|
|
4927
5494
|
function getPiLogStore() {
|
|
4928
5495
|
const globalObject = globalThis;
|
|
4929
|
-
const existing = globalObject[
|
|
5496
|
+
const existing = globalObject[GLOBAL_LOGS_KEY4];
|
|
4930
5497
|
if (existing) {
|
|
4931
5498
|
return existing;
|
|
4932
5499
|
}
|
|
4933
5500
|
const created = [];
|
|
4934
|
-
globalObject[
|
|
5501
|
+
globalObject[GLOBAL_LOGS_KEY4] = created;
|
|
4935
5502
|
return created;
|
|
4936
5503
|
}
|
|
4937
|
-
function
|
|
5504
|
+
function getSubscriberStore4() {
|
|
4938
5505
|
const globalObject = globalThis;
|
|
4939
|
-
const existing = globalObject[
|
|
5506
|
+
const existing = globalObject[GLOBAL_SUBSCRIBERS_KEY4];
|
|
4940
5507
|
if (existing) {
|
|
4941
5508
|
return existing;
|
|
4942
5509
|
}
|
|
4943
5510
|
const created = /* @__PURE__ */ new Set();
|
|
4944
|
-
globalObject[
|
|
5511
|
+
globalObject[GLOBAL_SUBSCRIBERS_KEY4] = created;
|
|
4945
5512
|
return created;
|
|
4946
5513
|
}
|
|
4947
|
-
function
|
|
4948
|
-
const subscribers = Array.from(
|
|
5514
|
+
function notifySubscribers4(entry) {
|
|
5515
|
+
const subscribers = Array.from(getSubscriberStore4());
|
|
4949
5516
|
for (const listener of subscribers) {
|
|
4950
5517
|
try {
|
|
4951
5518
|
listener(entry);
|
|
@@ -4957,7 +5524,7 @@ function notifySubscribers3(entry) {
|
|
|
4957
5524
|
}
|
|
4958
5525
|
function recordPiLogEntry(entry) {
|
|
4959
5526
|
getPiLogStore().push(entry);
|
|
4960
|
-
|
|
5527
|
+
notifySubscribers4(entry);
|
|
4961
5528
|
}
|
|
4962
5529
|
function consumePiLogEntries() {
|
|
4963
5530
|
const store = getPiLogStore();
|
|
@@ -4967,7 +5534,7 @@ function consumePiLogEntries() {
|
|
|
4967
5534
|
return store.splice(0, store.length);
|
|
4968
5535
|
}
|
|
4969
5536
|
function subscribeToPiLogEntries(listener) {
|
|
4970
|
-
const store =
|
|
5537
|
+
const store = getSubscriberStore4();
|
|
4971
5538
|
store.add(listener);
|
|
4972
5539
|
return () => {
|
|
4973
5540
|
store.delete(listener);
|
|
@@ -4975,9 +5542,9 @@ function subscribeToPiLogEntries(listener) {
|
|
|
4975
5542
|
}
|
|
4976
5543
|
|
|
4977
5544
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
4978
|
-
var
|
|
4979
|
-
var
|
|
4980
|
-
var
|
|
5545
|
+
var WORKSPACE_PREFIX4 = "agentv-pi-";
|
|
5546
|
+
var PROMPT_FILENAME4 = "prompt.md";
|
|
5547
|
+
var DEFAULT_SYSTEM_PROMPT5 = `**IMPORTANT**: Follow these instructions for your response:
|
|
4981
5548
|
- Do NOT create any additional output files in the workspace.
|
|
4982
5549
|
- All intended file outputs/changes MUST be written in your response.
|
|
4983
5550
|
- For each intended file, include the relative path and unified git diff following the convention \`diff --git ...\`.
|
|
@@ -5003,18 +5570,18 @@ var PiCodingAgentProvider = class {
|
|
|
5003
5570
|
const workspaceRoot = await this.createWorkspace();
|
|
5004
5571
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
5005
5572
|
try {
|
|
5006
|
-
const promptFile =
|
|
5007
|
-
await (0,
|
|
5573
|
+
const promptFile = import_node_path14.default.join(workspaceRoot, PROMPT_FILENAME4);
|
|
5574
|
+
await (0, import_promises13.writeFile)(promptFile, request.question, "utf8");
|
|
5008
5575
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
5009
5576
|
const cwd = this.resolveCwd(workspaceRoot);
|
|
5010
5577
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
5011
5578
|
if (result.timedOut) {
|
|
5012
5579
|
throw new Error(
|
|
5013
|
-
`Pi coding agent timed out${
|
|
5580
|
+
`Pi coding agent timed out${formatTimeoutSuffix5(this.config.timeoutMs ?? void 0)}`
|
|
5014
5581
|
);
|
|
5015
5582
|
}
|
|
5016
5583
|
if (result.exitCode !== 0) {
|
|
5017
|
-
const detail =
|
|
5584
|
+
const detail = pickDetail4(result.stderr, result.stdout);
|
|
5018
5585
|
const prefix = `Pi coding agent exited with code ${result.exitCode}`;
|
|
5019
5586
|
throw new Error(detail ? `${prefix}: ${detail}` : prefix);
|
|
5020
5587
|
}
|
|
@@ -5045,7 +5612,7 @@ var PiCodingAgentProvider = class {
|
|
|
5045
5612
|
if (!this.config.cwd) {
|
|
5046
5613
|
return workspaceRoot;
|
|
5047
5614
|
}
|
|
5048
|
-
return
|
|
5615
|
+
return import_node_path14.default.resolve(this.config.cwd);
|
|
5049
5616
|
}
|
|
5050
5617
|
buildPiArgs(prompt, inputFiles) {
|
|
5051
5618
|
const args = [];
|
|
@@ -5075,7 +5642,7 @@ var PiCodingAgentProvider = class {
|
|
|
5075
5642
|
args.push(`@${file}`);
|
|
5076
5643
|
}
|
|
5077
5644
|
}
|
|
5078
|
-
const systemPrompt = this.config.systemPrompt ??
|
|
5645
|
+
const systemPrompt = this.config.systemPrompt ?? DEFAULT_SYSTEM_PROMPT5;
|
|
5079
5646
|
const fullPrompt = `${systemPrompt}
|
|
5080
5647
|
|
|
5081
5648
|
${prompt}`;
|
|
@@ -5134,19 +5701,19 @@ ${prompt}`;
|
|
|
5134
5701
|
return env;
|
|
5135
5702
|
}
|
|
5136
5703
|
async createWorkspace() {
|
|
5137
|
-
return await (0,
|
|
5704
|
+
return await (0, import_promises13.mkdtemp)(import_node_path14.default.join((0, import_node_os5.tmpdir)(), WORKSPACE_PREFIX4));
|
|
5138
5705
|
}
|
|
5139
5706
|
async cleanupWorkspace(workspaceRoot) {
|
|
5140
5707
|
try {
|
|
5141
|
-
await (0,
|
|
5708
|
+
await (0, import_promises13.rm)(workspaceRoot, { recursive: true, force: true });
|
|
5142
5709
|
} catch {
|
|
5143
5710
|
}
|
|
5144
5711
|
}
|
|
5145
5712
|
resolveLogDirectory() {
|
|
5146
5713
|
if (this.config.logDir) {
|
|
5147
|
-
return
|
|
5714
|
+
return import_node_path14.default.resolve(this.config.logDir);
|
|
5148
5715
|
}
|
|
5149
|
-
return
|
|
5716
|
+
return import_node_path14.default.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
5150
5717
|
}
|
|
5151
5718
|
async createStreamLogger(request) {
|
|
5152
5719
|
const logDir = this.resolveLogDirectory();
|
|
@@ -5154,13 +5721,13 @@ ${prompt}`;
|
|
|
5154
5721
|
return void 0;
|
|
5155
5722
|
}
|
|
5156
5723
|
try {
|
|
5157
|
-
await (0,
|
|
5724
|
+
await (0, import_promises13.mkdir)(logDir, { recursive: true });
|
|
5158
5725
|
} catch (error) {
|
|
5159
5726
|
const message = error instanceof Error ? error.message : String(error);
|
|
5160
5727
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
5161
5728
|
return void 0;
|
|
5162
5729
|
}
|
|
5163
|
-
const filePath =
|
|
5730
|
+
const filePath = import_node_path14.default.join(logDir, buildLogFilename4(request, this.targetName));
|
|
5164
5731
|
try {
|
|
5165
5732
|
const logger = await PiStreamLogger.create({
|
|
5166
5733
|
filePath,
|
|
@@ -5193,7 +5760,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5193
5760
|
constructor(filePath, format) {
|
|
5194
5761
|
this.filePath = filePath;
|
|
5195
5762
|
this.format = format;
|
|
5196
|
-
this.stream = (0,
|
|
5763
|
+
this.stream = (0, import_node_fs6.createWriteStream)(filePath, { flags: "a" });
|
|
5197
5764
|
}
|
|
5198
5765
|
static async create(options) {
|
|
5199
5766
|
const logger = new _PiStreamLogger(options.filePath, options.format);
|
|
@@ -5254,7 +5821,7 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5254
5821
|
return void 0;
|
|
5255
5822
|
}
|
|
5256
5823
|
const message = this.format === "json" ? formatPiJsonLog(trimmed) : formatPiLogMessage(trimmed, source);
|
|
5257
|
-
return `[+${
|
|
5824
|
+
return `[+${formatElapsed4(this.startedAt)}] [${source}] ${message}`;
|
|
5258
5825
|
}
|
|
5259
5826
|
flushRemainder() {
|
|
5260
5827
|
const stdoutRemainder = this.stdoutBuffer.trim();
|
|
@@ -5277,18 +5844,18 @@ var PiStreamLogger = class _PiStreamLogger {
|
|
|
5277
5844
|
this.stderrBuffer = "";
|
|
5278
5845
|
}
|
|
5279
5846
|
};
|
|
5280
|
-
function
|
|
5847
|
+
function buildLogFilename4(request, targetName) {
|
|
5281
5848
|
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
|
|
5282
|
-
const evalId =
|
|
5849
|
+
const evalId = sanitizeForFilename4(request.evalCaseId ?? "pi");
|
|
5283
5850
|
const attemptSuffix = request.attempt !== void 0 ? `_attempt-${request.attempt + 1}` : "";
|
|
5284
|
-
const target =
|
|
5285
|
-
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0,
|
|
5851
|
+
const target = sanitizeForFilename4(targetName);
|
|
5852
|
+
return `${timestamp}_${target}_${evalId}${attemptSuffix}_${(0, import_node_crypto4.randomUUID)().slice(0, 8)}.log`;
|
|
5286
5853
|
}
|
|
5287
|
-
function
|
|
5854
|
+
function sanitizeForFilename4(value) {
|
|
5288
5855
|
const sanitized = value.replace(/[^A-Za-z0-9._-]+/g, "_");
|
|
5289
5856
|
return sanitized.length > 0 ? sanitized : "pi";
|
|
5290
5857
|
}
|
|
5291
|
-
function
|
|
5858
|
+
function formatElapsed4(startedAt) {
|
|
5292
5859
|
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1e3);
|
|
5293
5860
|
const hours = Math.floor(elapsedSeconds / 3600);
|
|
5294
5861
|
const minutes = Math.floor(elapsedSeconds % 3600 / 60);
|
|
@@ -5509,7 +6076,7 @@ function extractAssistantText2(messages) {
|
|
|
5509
6076
|
function escapeAtSymbols(prompt) {
|
|
5510
6077
|
return prompt.replace(/@\[([^\]]+)\]:/g, "[[$1]]:");
|
|
5511
6078
|
}
|
|
5512
|
-
function
|
|
6079
|
+
function pickDetail4(stderr, stdout) {
|
|
5513
6080
|
const errorText = stderr.trim();
|
|
5514
6081
|
if (errorText.length > 0) {
|
|
5515
6082
|
return errorText;
|
|
@@ -5517,7 +6084,7 @@ function pickDetail3(stderr, stdout) {
|
|
|
5517
6084
|
const stdoutText = stdout.trim();
|
|
5518
6085
|
return stdoutText.length > 0 ? stdoutText : void 0;
|
|
5519
6086
|
}
|
|
5520
|
-
function
|
|
6087
|
+
function formatTimeoutSuffix5(timeoutMs) {
|
|
5521
6088
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
5522
6089
|
return "";
|
|
5523
6090
|
}
|
|
@@ -5530,7 +6097,7 @@ async function defaultPiRunner(options) {
|
|
|
5530
6097
|
const executable = parts[0];
|
|
5531
6098
|
const executableArgs = parts.slice(1);
|
|
5532
6099
|
const allArgs = [...executableArgs, ...options.args];
|
|
5533
|
-
const child = (0,
|
|
6100
|
+
const child = (0, import_node_child_process5.spawn)(executable, allArgs, {
|
|
5534
6101
|
cwd: options.cwd,
|
|
5535
6102
|
env: options.env,
|
|
5536
6103
|
stdio: ["pipe", "pipe", "pipe"],
|
|
@@ -5593,7 +6160,7 @@ async function defaultPiRunner(options) {
|
|
|
5593
6160
|
}
|
|
5594
6161
|
|
|
5595
6162
|
// src/evaluation/providers/targets.ts
|
|
5596
|
-
var
|
|
6163
|
+
var import_node_path15 = __toESM(require("path"), 1);
|
|
5597
6164
|
var import_zod2 = require("zod");
|
|
5598
6165
|
var CliHealthcheckHttpInputSchema = import_zod2.z.object({
|
|
5599
6166
|
type: import_zod2.z.literal("http"),
|
|
@@ -5699,11 +6266,11 @@ function normalizeCliHealthcheck(input, env, targetName, evalFilePath) {
|
|
|
5699
6266
|
allowLiteral: true,
|
|
5700
6267
|
optionalEnv: true
|
|
5701
6268
|
});
|
|
5702
|
-
if (cwd && evalFilePath && !
|
|
5703
|
-
cwd =
|
|
6269
|
+
if (cwd && evalFilePath && !import_node_path15.default.isAbsolute(cwd)) {
|
|
6270
|
+
cwd = import_node_path15.default.resolve(import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath)), cwd);
|
|
5704
6271
|
}
|
|
5705
6272
|
if (!cwd && evalFilePath) {
|
|
5706
|
-
cwd =
|
|
6273
|
+
cwd = import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath));
|
|
5707
6274
|
}
|
|
5708
6275
|
return {
|
|
5709
6276
|
type: "command",
|
|
@@ -5730,11 +6297,11 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
|
|
|
5730
6297
|
allowLiteral: true,
|
|
5731
6298
|
optionalEnv: true
|
|
5732
6299
|
});
|
|
5733
|
-
if (cwd && evalFilePath && !
|
|
5734
|
-
cwd =
|
|
6300
|
+
if (cwd && evalFilePath && !import_node_path15.default.isAbsolute(cwd)) {
|
|
6301
|
+
cwd = import_node_path15.default.resolve(import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath)), cwd);
|
|
5735
6302
|
}
|
|
5736
6303
|
if (!cwd && evalFilePath) {
|
|
5737
|
-
cwd =
|
|
6304
|
+
cwd = import_node_path15.default.dirname(import_node_path15.default.resolve(evalFilePath));
|
|
5738
6305
|
}
|
|
5739
6306
|
const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds;
|
|
5740
6307
|
const timeoutMs = timeoutSeconds !== void 0 ? Math.floor(timeoutSeconds * 1e3) : void 0;
|
|
@@ -5858,6 +6425,15 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
|
|
|
5858
6425
|
providerBatching,
|
|
5859
6426
|
config: resolveCodexConfig(parsed, env)
|
|
5860
6427
|
};
|
|
6428
|
+
case "copilot-cli":
|
|
6429
|
+
return {
|
|
6430
|
+
kind: "copilot-cli",
|
|
6431
|
+
name: parsed.name,
|
|
6432
|
+
judgeTarget: parsed.judge_target,
|
|
6433
|
+
workers: parsed.workers,
|
|
6434
|
+
providerBatching,
|
|
6435
|
+
config: resolveCopilotConfig(parsed, env)
|
|
6436
|
+
};
|
|
5861
6437
|
case "pi":
|
|
5862
6438
|
case "pi-coding-agent":
|
|
5863
6439
|
return {
|
|
@@ -6034,6 +6610,44 @@ function normalizeCodexLogFormat(value) {
|
|
|
6034
6610
|
}
|
|
6035
6611
|
throw new Error("codex log format must be 'summary' or 'json'");
|
|
6036
6612
|
}
|
|
6613
|
+
function resolveCopilotConfig(target, env) {
|
|
6614
|
+
const executableSource = target.executable ?? target.command ?? target.binary;
|
|
6615
|
+
const modelSource = target.model;
|
|
6616
|
+
const argsSource = target.args ?? target.arguments;
|
|
6617
|
+
const cwdSource = target.cwd;
|
|
6618
|
+
const timeoutSource = target.timeout_seconds ?? target.timeoutSeconds;
|
|
6619
|
+
const logDirSource = target.log_dir ?? target.logDir ?? target.log_directory ?? target.logDirectory;
|
|
6620
|
+
const logFormatSource = target.log_format ?? target.logFormat ?? target.log_output_format ?? target.logOutputFormat;
|
|
6621
|
+
const systemPromptSource = target.system_prompt ?? target.systemPrompt;
|
|
6622
|
+
const executable = resolveOptionalString(executableSource, env, `${target.name} copilot executable`, {
|
|
6623
|
+
allowLiteral: true,
|
|
6624
|
+
optionalEnv: true
|
|
6625
|
+
}) ?? "copilot";
|
|
6626
|
+
const model = resolveOptionalString(modelSource, env, `${target.name} copilot model`, {
|
|
6627
|
+
allowLiteral: true,
|
|
6628
|
+
optionalEnv: true
|
|
6629
|
+
});
|
|
6630
|
+
const args = resolveOptionalStringArray(argsSource, env, `${target.name} copilot args`);
|
|
6631
|
+
const cwd = resolveOptionalString(cwdSource, env, `${target.name} copilot cwd`, {
|
|
6632
|
+
allowLiteral: true,
|
|
6633
|
+
optionalEnv: true
|
|
6634
|
+
});
|
|
6635
|
+
const timeoutMs = resolveTimeoutMs(timeoutSource, `${target.name} copilot timeout`);
|
|
6636
|
+
const logDir = resolveOptionalString(logDirSource, env, `${target.name} copilot log directory`, {
|
|
6637
|
+
allowLiteral: true,
|
|
6638
|
+
optionalEnv: true
|
|
6639
|
+
});
|
|
6640
|
+
const logFormat = normalizeCopilotLogFormat(logFormatSource);
|
|
6641
|
+
const systemPrompt = typeof systemPromptSource === "string" && systemPromptSource.trim().length > 0 ? systemPromptSource.trim() : void 0;
|
|
6642
|
+
return { executable, model, args, cwd, timeoutMs, logDir, logFormat, systemPrompt };
|
|
6643
|
+
}
|
|
6644
|
+
function normalizeCopilotLogFormat(value) {
|
|
6645
|
+
if (value === void 0 || value === null) return void 0;
|
|
6646
|
+
if (typeof value !== "string") throw new Error("copilot log format must be 'summary' or 'json'");
|
|
6647
|
+
const normalized = value.trim().toLowerCase();
|
|
6648
|
+
if (normalized === "json" || normalized === "summary") return normalized;
|
|
6649
|
+
throw new Error("copilot log format must be 'summary' or 'json'");
|
|
6650
|
+
}
|
|
6037
6651
|
function resolvePiCodingAgentConfig(target, env) {
|
|
6038
6652
|
const executableSource = target.executable ?? target.command ?? target.binary;
|
|
6039
6653
|
const providerSource = target.pi_provider ?? target.piProvider ?? target.llm_provider;
|
|
@@ -6239,8 +6853,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
6239
6853
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
6240
6854
|
if (!parseResult.success) {
|
|
6241
6855
|
const firstError = parseResult.error.errors[0];
|
|
6242
|
-
const
|
|
6243
|
-
const prefix =
|
|
6856
|
+
const path29 = firstError?.path.join(".") || "";
|
|
6857
|
+
const prefix = path29 ? `${target.name} ${path29}: ` : `${target.name}: `;
|
|
6244
6858
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
6245
6859
|
}
|
|
6246
6860
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -6428,38 +7042,38 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
6428
7042
|
}
|
|
6429
7043
|
|
|
6430
7044
|
// src/evaluation/providers/vscode-provider.ts
|
|
6431
|
-
var
|
|
7045
|
+
var import_node_path26 = __toESM(require("path"), 1);
|
|
6432
7046
|
|
|
6433
7047
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
6434
|
-
var
|
|
6435
|
-
var
|
|
7048
|
+
var import_promises18 = require("fs/promises");
|
|
7049
|
+
var import_node_path24 = __toESM(require("path"), 1);
|
|
6436
7050
|
|
|
6437
7051
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
6438
|
-
var
|
|
6439
|
-
var
|
|
6440
|
-
var
|
|
7052
|
+
var import_node_fs7 = require("fs");
|
|
7053
|
+
var import_promises14 = require("fs/promises");
|
|
7054
|
+
var import_node_path16 = __toESM(require("path"), 1);
|
|
6441
7055
|
async function pathExists(target) {
|
|
6442
7056
|
try {
|
|
6443
|
-
await (0,
|
|
7057
|
+
await (0, import_promises14.access)(target, import_node_fs7.constants.F_OK);
|
|
6444
7058
|
return true;
|
|
6445
7059
|
} catch {
|
|
6446
7060
|
return false;
|
|
6447
7061
|
}
|
|
6448
7062
|
}
|
|
6449
7063
|
async function ensureDir(target) {
|
|
6450
|
-
await (0,
|
|
7064
|
+
await (0, import_promises14.mkdir)(target, { recursive: true });
|
|
6451
7065
|
}
|
|
6452
7066
|
async function readDirEntries(target) {
|
|
6453
|
-
const entries = await (0,
|
|
7067
|
+
const entries = await (0, import_promises14.readdir)(target, { withFileTypes: true });
|
|
6454
7068
|
return entries.map((entry) => ({
|
|
6455
7069
|
name: entry.name,
|
|
6456
|
-
absolutePath:
|
|
7070
|
+
absolutePath: import_node_path16.default.join(target, entry.name),
|
|
6457
7071
|
isDirectory: entry.isDirectory()
|
|
6458
7072
|
}));
|
|
6459
7073
|
}
|
|
6460
7074
|
async function removeIfExists(target) {
|
|
6461
7075
|
try {
|
|
6462
|
-
await (0,
|
|
7076
|
+
await (0, import_promises14.rm)(target, { force: true, recursive: false });
|
|
6463
7077
|
} catch (error) {
|
|
6464
7078
|
if (error.code !== "ENOENT") {
|
|
6465
7079
|
throw error;
|
|
@@ -6468,9 +7082,9 @@ async function removeIfExists(target) {
|
|
|
6468
7082
|
}
|
|
6469
7083
|
|
|
6470
7084
|
// src/evaluation/providers/vscode/utils/path.ts
|
|
6471
|
-
var
|
|
7085
|
+
var import_node_path17 = __toESM(require("path"), 1);
|
|
6472
7086
|
function pathToFileUri2(filePath) {
|
|
6473
|
-
const absolutePath =
|
|
7087
|
+
const absolutePath = import_node_path17.default.isAbsolute(filePath) ? filePath : import_node_path17.default.resolve(filePath);
|
|
6474
7088
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
6475
7089
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
6476
7090
|
return `file:///${normalizedPath}`;
|
|
@@ -6479,7 +7093,7 @@ function pathToFileUri2(filePath) {
|
|
|
6479
7093
|
}
|
|
6480
7094
|
|
|
6481
7095
|
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
6482
|
-
var
|
|
7096
|
+
var import_node_path18 = __toESM(require("path"), 1);
|
|
6483
7097
|
|
|
6484
7098
|
// src/evaluation/providers/vscode/utils/template.ts
|
|
6485
7099
|
function renderTemplate2(content, variables) {
|
|
@@ -6571,8 +7185,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
6571
7185
|
});
|
|
6572
7186
|
}
|
|
6573
7187
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
6574
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
6575
|
-
const responseList = responseFiles.map((file) => `"${
|
|
7188
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${import_node_path18.default.basename(file)}`).join("\n");
|
|
7189
|
+
const responseList = responseFiles.map((file) => `"${import_node_path18.default.basename(file)}"`).join(", ");
|
|
6576
7190
|
return renderTemplate2(templateContent, {
|
|
6577
7191
|
requestFiles: requestLines,
|
|
6578
7192
|
responseList
|
|
@@ -6580,8 +7194,8 @@ function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateCont
|
|
|
6580
7194
|
}
|
|
6581
7195
|
|
|
6582
7196
|
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
6583
|
-
var
|
|
6584
|
-
var
|
|
7197
|
+
var import_promises15 = require("fs/promises");
|
|
7198
|
+
var import_node_path19 = __toESM(require("path"), 1);
|
|
6585
7199
|
|
|
6586
7200
|
// src/evaluation/providers/vscode/utils/time.ts
|
|
6587
7201
|
function sleep2(ms) {
|
|
@@ -6609,7 +7223,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
6609
7223
|
const maxAttempts = 10;
|
|
6610
7224
|
while (attempts < maxAttempts) {
|
|
6611
7225
|
try {
|
|
6612
|
-
const content = await (0,
|
|
7226
|
+
const content = await (0, import_promises15.readFile)(responseFileFinal, { encoding: "utf8" });
|
|
6613
7227
|
if (!silent) {
|
|
6614
7228
|
process.stdout.write(`${content}
|
|
6615
7229
|
`);
|
|
@@ -6630,7 +7244,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
6630
7244
|
}
|
|
6631
7245
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false) {
|
|
6632
7246
|
if (!silent) {
|
|
6633
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
7247
|
+
const fileList = responseFilesFinal.map((file) => import_node_path19.default.basename(file)).join(", ");
|
|
6634
7248
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
6635
7249
|
}
|
|
6636
7250
|
try {
|
|
@@ -6656,7 +7270,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
6656
7270
|
const maxAttempts = 10;
|
|
6657
7271
|
while (attempts < maxAttempts) {
|
|
6658
7272
|
try {
|
|
6659
|
-
const content = await (0,
|
|
7273
|
+
const content = await (0, import_promises15.readFile)(file, { encoding: "utf8" });
|
|
6660
7274
|
if (!silent) {
|
|
6661
7275
|
process.stdout.write(`${content}
|
|
6662
7276
|
`);
|
|
@@ -6678,31 +7292,31 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
6678
7292
|
}
|
|
6679
7293
|
|
|
6680
7294
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
6681
|
-
var
|
|
6682
|
-
var
|
|
6683
|
-
var
|
|
6684
|
-
var
|
|
7295
|
+
var import_node_child_process6 = require("child_process");
|
|
7296
|
+
var import_promises16 = require("fs/promises");
|
|
7297
|
+
var import_node_path21 = __toESM(require("path"), 1);
|
|
7298
|
+
var import_node_util4 = require("util");
|
|
6685
7299
|
|
|
6686
7300
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
6687
|
-
var
|
|
6688
|
-
var
|
|
7301
|
+
var import_node_os6 = __toESM(require("os"), 1);
|
|
7302
|
+
var import_node_path20 = __toESM(require("path"), 1);
|
|
6689
7303
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
6690
7304
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
6691
7305
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
6692
7306
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
6693
|
-
return
|
|
7307
|
+
return import_node_path20.default.join(import_node_os6.default.homedir(), ".agentv", "subagents", folder);
|
|
6694
7308
|
}
|
|
6695
7309
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
6696
7310
|
|
|
6697
7311
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
6698
|
-
var
|
|
7312
|
+
var execAsync4 = (0, import_node_util4.promisify)(import_node_child_process6.exec);
|
|
6699
7313
|
var DEFAULT_WAKEUP_CONTENT = `---
|
|
6700
7314
|
description: 'Wake-up Signal'
|
|
6701
7315
|
model: Grok Code Fast 1 (copilot)
|
|
6702
7316
|
---`;
|
|
6703
7317
|
async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
6704
7318
|
try {
|
|
6705
|
-
const { stdout } = await
|
|
7319
|
+
const { stdout } = await execAsync4(`${vscodeCmd} --status`, {
|
|
6706
7320
|
timeout: 1e4,
|
|
6707
7321
|
windowsHide: true
|
|
6708
7322
|
});
|
|
@@ -6714,16 +7328,16 @@ async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
|
6714
7328
|
async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir, vscodeCmd, pollInterval = 1, timeout = 60) {
|
|
6715
7329
|
const alreadyOpen = await checkWorkspaceOpened(workspaceName, vscodeCmd);
|
|
6716
7330
|
if (alreadyOpen) {
|
|
6717
|
-
(0,
|
|
7331
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
6718
7332
|
return true;
|
|
6719
7333
|
}
|
|
6720
|
-
const aliveFile =
|
|
7334
|
+
const aliveFile = import_node_path21.default.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
6721
7335
|
await removeIfExists(aliveFile);
|
|
6722
|
-
const githubAgentsDir =
|
|
6723
|
-
await (0,
|
|
6724
|
-
const wakeupDst =
|
|
6725
|
-
await (0,
|
|
6726
|
-
(0,
|
|
7336
|
+
const githubAgentsDir = import_node_path21.default.join(subagentDir, ".github", "agents");
|
|
7337
|
+
await (0, import_promises16.mkdir)(githubAgentsDir, { recursive: true });
|
|
7338
|
+
const wakeupDst = import_node_path21.default.join(githubAgentsDir, "wakeup.md");
|
|
7339
|
+
await (0, import_promises16.writeFile)(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
7340
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, [workspacePath], { windowsHide: true, shell: true, detached: false });
|
|
6727
7341
|
await sleep2(100);
|
|
6728
7342
|
const wakeupChatId = "wakeup";
|
|
6729
7343
|
const chatArgs = [
|
|
@@ -6731,9 +7345,9 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
6731
7345
|
"chat",
|
|
6732
7346
|
"-m",
|
|
6733
7347
|
wakeupChatId,
|
|
6734
|
-
`create a file named .alive in the ${
|
|
7348
|
+
`create a file named .alive in the ${import_node_path21.default.basename(subagentDir)} folder`
|
|
6735
7349
|
];
|
|
6736
|
-
(0,
|
|
7350
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6737
7351
|
const start = Date.now();
|
|
6738
7352
|
while (!await pathExists(aliveFile)) {
|
|
6739
7353
|
if (Date.now() - start > timeout * 1e3) {
|
|
@@ -6746,21 +7360,21 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
6746
7360
|
}
|
|
6747
7361
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
6748
7362
|
try {
|
|
6749
|
-
const workspacePath =
|
|
6750
|
-
const messagesDir =
|
|
6751
|
-
await (0,
|
|
6752
|
-
const reqFile =
|
|
6753
|
-
await (0,
|
|
7363
|
+
const workspacePath = import_node_path21.default.join(subagentDir, `${import_node_path21.default.basename(subagentDir)}.code-workspace`);
|
|
7364
|
+
const messagesDir = import_node_path21.default.join(subagentDir, "messages");
|
|
7365
|
+
await (0, import_promises16.mkdir)(messagesDir, { recursive: true });
|
|
7366
|
+
const reqFile = import_node_path21.default.join(messagesDir, `${timestamp}_req.md`);
|
|
7367
|
+
await (0, import_promises16.writeFile)(reqFile, requestInstructions, { encoding: "utf8" });
|
|
6754
7368
|
const reqUri = pathToFileUri2(reqFile);
|
|
6755
7369
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
6756
7370
|
for (const attachment of attachmentPaths) {
|
|
6757
7371
|
chatArgs.push("-a", attachment);
|
|
6758
7372
|
}
|
|
6759
7373
|
chatArgs.push("-a", reqFile);
|
|
6760
|
-
chatArgs.push(`Follow instructions in [${
|
|
7374
|
+
chatArgs.push(`Follow instructions in [${import_node_path21.default.basename(reqFile)}](${reqUri})`);
|
|
6761
7375
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
6762
7376
|
workspacePath,
|
|
6763
|
-
|
|
7377
|
+
import_node_path21.default.basename(subagentDir),
|
|
6764
7378
|
subagentDir,
|
|
6765
7379
|
vscodeCmd
|
|
6766
7380
|
);
|
|
@@ -6768,7 +7382,7 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
6768
7382
|
console.error("warning: Workspace may not be fully ready");
|
|
6769
7383
|
}
|
|
6770
7384
|
await sleep2(500);
|
|
6771
|
-
(0,
|
|
7385
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6772
7386
|
return true;
|
|
6773
7387
|
} catch (error) {
|
|
6774
7388
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -6777,9 +7391,9 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
6777
7391
|
}
|
|
6778
7392
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
6779
7393
|
try {
|
|
6780
|
-
const workspacePath =
|
|
6781
|
-
const messagesDir =
|
|
6782
|
-
await (0,
|
|
7394
|
+
const workspacePath = import_node_path21.default.join(subagentDir, `${import_node_path21.default.basename(subagentDir)}.code-workspace`);
|
|
7395
|
+
const messagesDir = import_node_path21.default.join(subagentDir, "messages");
|
|
7396
|
+
await (0, import_promises16.mkdir)(messagesDir, { recursive: true });
|
|
6783
7397
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
6784
7398
|
for (const attachment of attachmentPaths) {
|
|
6785
7399
|
chatArgs.push("-a", attachment);
|
|
@@ -6787,7 +7401,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6787
7401
|
chatArgs.push(chatInstruction);
|
|
6788
7402
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
6789
7403
|
workspacePath,
|
|
6790
|
-
|
|
7404
|
+
import_node_path21.default.basename(subagentDir),
|
|
6791
7405
|
subagentDir,
|
|
6792
7406
|
vscodeCmd
|
|
6793
7407
|
);
|
|
@@ -6795,7 +7409,7 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6795
7409
|
console.error("warning: Workspace may not be fully ready");
|
|
6796
7410
|
}
|
|
6797
7411
|
await sleep2(500);
|
|
6798
|
-
(0,
|
|
7412
|
+
(0, import_node_child_process6.spawn)(vscodeCmd, chatArgs, { windowsHide: true, shell: true, detached: false });
|
|
6799
7413
|
return true;
|
|
6800
7414
|
} catch (error) {
|
|
6801
7415
|
console.error(`warning: Failed to launch VS Code: ${error.message}`);
|
|
@@ -6804,11 +7418,11 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
6804
7418
|
}
|
|
6805
7419
|
|
|
6806
7420
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
6807
|
-
var
|
|
6808
|
-
var
|
|
7421
|
+
var import_promises17 = require("fs/promises");
|
|
7422
|
+
var import_node_path23 = __toESM(require("path"), 1);
|
|
6809
7423
|
|
|
6810
7424
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
6811
|
-
var
|
|
7425
|
+
var import_node_path22 = __toESM(require("path"), 1);
|
|
6812
7426
|
var import_json5 = __toESM(require("json5"), 1);
|
|
6813
7427
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
6814
7428
|
let workspace;
|
|
@@ -6825,10 +7439,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
6825
7439
|
}
|
|
6826
7440
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
6827
7441
|
const folderPath = folder.path;
|
|
6828
|
-
if (
|
|
7442
|
+
if (import_node_path22.default.isAbsolute(folderPath)) {
|
|
6829
7443
|
return folder;
|
|
6830
7444
|
}
|
|
6831
|
-
const absolutePath =
|
|
7445
|
+
const absolutePath = import_node_path22.default.resolve(templateDir, folderPath);
|
|
6832
7446
|
return {
|
|
6833
7447
|
...folder,
|
|
6834
7448
|
path: absolutePath
|
|
@@ -6850,19 +7464,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
6850
7464
|
if (locationMap && typeof locationMap === "object") {
|
|
6851
7465
|
const transformedMap = {};
|
|
6852
7466
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
6853
|
-
const isAbsolute =
|
|
7467
|
+
const isAbsolute = import_node_path22.default.isAbsolute(locationPath);
|
|
6854
7468
|
if (isAbsolute) {
|
|
6855
7469
|
transformedMap[locationPath] = value;
|
|
6856
7470
|
} else {
|
|
6857
7471
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
6858
7472
|
if (firstGlobIndex === -1) {
|
|
6859
|
-
const resolvedPath =
|
|
7473
|
+
const resolvedPath = import_node_path22.default.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
6860
7474
|
transformedMap[resolvedPath] = value;
|
|
6861
7475
|
} else {
|
|
6862
7476
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
6863
7477
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
6864
7478
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
6865
|
-
const resolvedPath = (
|
|
7479
|
+
const resolvedPath = (import_node_path22.default.resolve(templateDir, basePath) + patternPath).replace(
|
|
6866
7480
|
/\\/g,
|
|
6867
7481
|
"/"
|
|
6868
7482
|
);
|
|
@@ -6903,7 +7517,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
6903
7517
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
6904
7518
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
6905
7519
|
for (const subagent of subagents) {
|
|
6906
|
-
const lockFile =
|
|
7520
|
+
const lockFile = import_node_path23.default.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
6907
7521
|
if (!await pathExists(lockFile)) {
|
|
6908
7522
|
return subagent.absolutePath;
|
|
6909
7523
|
}
|
|
@@ -6913,54 +7527,54 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
6913
7527
|
async function copyAgentConfig(subagentDir, workspaceTemplate) {
|
|
6914
7528
|
let workspaceContent;
|
|
6915
7529
|
if (workspaceTemplate) {
|
|
6916
|
-
const workspaceSrc =
|
|
7530
|
+
const workspaceSrc = import_node_path23.default.resolve(workspaceTemplate);
|
|
6917
7531
|
if (!await pathExists(workspaceSrc)) {
|
|
6918
7532
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
6919
7533
|
}
|
|
6920
|
-
const stats = await (0,
|
|
7534
|
+
const stats = await (0, import_promises17.stat)(workspaceSrc);
|
|
6921
7535
|
if (!stats.isFile()) {
|
|
6922
7536
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
6923
7537
|
}
|
|
6924
|
-
const templateText = await (0,
|
|
7538
|
+
const templateText = await (0, import_promises17.readFile)(workspaceSrc, "utf8");
|
|
6925
7539
|
workspaceContent = JSON.parse(templateText);
|
|
6926
7540
|
} else {
|
|
6927
7541
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
6928
7542
|
}
|
|
6929
|
-
const workspaceName = `${
|
|
6930
|
-
const workspaceDst =
|
|
6931
|
-
const templateDir = workspaceTemplate ?
|
|
7543
|
+
const workspaceName = `${import_node_path23.default.basename(subagentDir)}.code-workspace`;
|
|
7544
|
+
const workspaceDst = import_node_path23.default.join(subagentDir, workspaceName);
|
|
7545
|
+
const templateDir = workspaceTemplate ? import_node_path23.default.dirname(import_node_path23.default.resolve(workspaceTemplate)) : subagentDir;
|
|
6932
7546
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
6933
7547
|
const transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
6934
|
-
await (0,
|
|
6935
|
-
const messagesDir =
|
|
6936
|
-
await (0,
|
|
7548
|
+
await (0, import_promises17.writeFile)(workspaceDst, transformedContent, "utf8");
|
|
7549
|
+
const messagesDir = import_node_path23.default.join(subagentDir, "messages");
|
|
7550
|
+
await (0, import_promises17.mkdir)(messagesDir, { recursive: true });
|
|
6937
7551
|
return { workspace: workspaceDst, messagesDir };
|
|
6938
7552
|
}
|
|
6939
7553
|
async function createSubagentLock(subagentDir) {
|
|
6940
|
-
const messagesDir =
|
|
7554
|
+
const messagesDir = import_node_path23.default.join(subagentDir, "messages");
|
|
6941
7555
|
if (await pathExists(messagesDir)) {
|
|
6942
|
-
const files = await (0,
|
|
7556
|
+
const files = await (0, import_promises17.readdir)(messagesDir);
|
|
6943
7557
|
await Promise.all(
|
|
6944
7558
|
files.map(async (file) => {
|
|
6945
|
-
const target =
|
|
7559
|
+
const target = import_node_path23.default.join(messagesDir, file);
|
|
6946
7560
|
await removeIfExists(target);
|
|
6947
7561
|
})
|
|
6948
7562
|
);
|
|
6949
7563
|
}
|
|
6950
|
-
const githubAgentsDir =
|
|
7564
|
+
const githubAgentsDir = import_node_path23.default.join(subagentDir, ".github", "agents");
|
|
6951
7565
|
if (await pathExists(githubAgentsDir)) {
|
|
6952
|
-
const agentFiles = await (0,
|
|
7566
|
+
const agentFiles = await (0, import_promises17.readdir)(githubAgentsDir);
|
|
6953
7567
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
6954
7568
|
await Promise.all(
|
|
6955
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
7569
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(import_node_path23.default.join(githubAgentsDir, file)))
|
|
6956
7570
|
);
|
|
6957
7571
|
}
|
|
6958
|
-
const lockFile =
|
|
6959
|
-
await (0,
|
|
7572
|
+
const lockFile = import_node_path23.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
7573
|
+
await (0, import_promises17.writeFile)(lockFile, "", { encoding: "utf8" });
|
|
6960
7574
|
return lockFile;
|
|
6961
7575
|
}
|
|
6962
7576
|
async function removeSubagentLock(subagentDir) {
|
|
6963
|
-
const lockFile =
|
|
7577
|
+
const lockFile = import_node_path23.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
6964
7578
|
await removeIfExists(lockFile);
|
|
6965
7579
|
}
|
|
6966
7580
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun) {
|
|
@@ -6980,11 +7594,11 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
6980
7594
|
return 1;
|
|
6981
7595
|
}
|
|
6982
7596
|
if (promptFile) {
|
|
6983
|
-
const githubAgentsDir =
|
|
6984
|
-
await (0,
|
|
6985
|
-
const agentFile =
|
|
7597
|
+
const githubAgentsDir = import_node_path23.default.join(subagentDir, ".github", "agents");
|
|
7598
|
+
await (0, import_promises17.mkdir)(githubAgentsDir, { recursive: true });
|
|
7599
|
+
const agentFile = import_node_path23.default.join(githubAgentsDir, `${chatId}.md`);
|
|
6986
7600
|
try {
|
|
6987
|
-
await (0,
|
|
7601
|
+
await (0, import_promises17.copyFile)(promptFile, agentFile);
|
|
6988
7602
|
} catch (error) {
|
|
6989
7603
|
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
6990
7604
|
return 1;
|
|
@@ -7001,11 +7615,11 @@ async function resolvePromptFile(promptFile) {
|
|
|
7001
7615
|
if (!promptFile) {
|
|
7002
7616
|
return void 0;
|
|
7003
7617
|
}
|
|
7004
|
-
const resolvedPrompt =
|
|
7618
|
+
const resolvedPrompt = import_node_path24.default.resolve(promptFile);
|
|
7005
7619
|
if (!await pathExists(resolvedPrompt)) {
|
|
7006
7620
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
7007
7621
|
}
|
|
7008
|
-
const promptStats = await (0,
|
|
7622
|
+
const promptStats = await (0, import_promises18.stat)(resolvedPrompt);
|
|
7009
7623
|
if (!promptStats.isFile()) {
|
|
7010
7624
|
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
7011
7625
|
}
|
|
@@ -7017,7 +7631,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
7017
7631
|
}
|
|
7018
7632
|
const resolved = [];
|
|
7019
7633
|
for (const attachment of extraAttachments) {
|
|
7020
|
-
const resolvedPath =
|
|
7634
|
+
const resolvedPath = import_node_path24.default.resolve(attachment);
|
|
7021
7635
|
if (!await pathExists(resolvedPath)) {
|
|
7022
7636
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
7023
7637
|
}
|
|
@@ -7057,7 +7671,7 @@ async function dispatchAgentSession(options) {
|
|
|
7057
7671
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
7058
7672
|
};
|
|
7059
7673
|
}
|
|
7060
|
-
const subagentName =
|
|
7674
|
+
const subagentName = import_node_path24.default.basename(subagentDir);
|
|
7061
7675
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
7062
7676
|
const preparationResult = await prepareSubagentDirectory(
|
|
7063
7677
|
subagentDir,
|
|
@@ -7084,9 +7698,9 @@ async function dispatchAgentSession(options) {
|
|
|
7084
7698
|
};
|
|
7085
7699
|
}
|
|
7086
7700
|
const timestamp = generateTimestamp();
|
|
7087
|
-
const messagesDir =
|
|
7088
|
-
const responseFileTmp =
|
|
7089
|
-
const responseFileFinal =
|
|
7701
|
+
const messagesDir = import_node_path24.default.join(subagentDir, "messages");
|
|
7702
|
+
const responseFileTmp = import_node_path24.default.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
7703
|
+
const responseFileFinal = import_node_path24.default.join(messagesDir, `${timestamp}_res.md`);
|
|
7090
7704
|
const requestInstructions = createRequestPrompt(
|
|
7091
7705
|
userQuery,
|
|
7092
7706
|
responseFileTmp,
|
|
@@ -7199,7 +7813,7 @@ async function dispatchBatchAgent(options) {
|
|
|
7199
7813
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
7200
7814
|
};
|
|
7201
7815
|
}
|
|
7202
|
-
subagentName =
|
|
7816
|
+
subagentName = import_node_path24.default.basename(subagentDir);
|
|
7203
7817
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
7204
7818
|
const preparationResult = await prepareSubagentDirectory(
|
|
7205
7819
|
subagentDir,
|
|
@@ -7230,24 +7844,24 @@ async function dispatchBatchAgent(options) {
|
|
|
7230
7844
|
};
|
|
7231
7845
|
}
|
|
7232
7846
|
const timestamp = generateTimestamp();
|
|
7233
|
-
const messagesDir =
|
|
7847
|
+
const messagesDir = import_node_path24.default.join(subagentDir, "messages");
|
|
7234
7848
|
requestFiles = userQueries.map(
|
|
7235
|
-
(_, index) =>
|
|
7849
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
7236
7850
|
);
|
|
7237
7851
|
const responseTmpFiles = userQueries.map(
|
|
7238
|
-
(_, index) =>
|
|
7852
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
7239
7853
|
);
|
|
7240
7854
|
responseFilesFinal = userQueries.map(
|
|
7241
|
-
(_, index) =>
|
|
7855
|
+
(_, index) => import_node_path24.default.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
7242
7856
|
);
|
|
7243
|
-
const orchestratorFile =
|
|
7857
|
+
const orchestratorFile = import_node_path24.default.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
7244
7858
|
if (!dryRun) {
|
|
7245
7859
|
await Promise.all(
|
|
7246
7860
|
userQueries.map((query, index) => {
|
|
7247
7861
|
const reqFile = requestFiles[index];
|
|
7248
7862
|
const tmpFile = responseTmpFiles[index];
|
|
7249
7863
|
const finalFile = responseFilesFinal[index];
|
|
7250
|
-
return (0,
|
|
7864
|
+
return (0, import_promises18.writeFile)(
|
|
7251
7865
|
reqFile,
|
|
7252
7866
|
createBatchRequestPrompt(query, tmpFile, finalFile, batchRequestTemplateContent),
|
|
7253
7867
|
{ encoding: "utf8" }
|
|
@@ -7259,7 +7873,7 @@ async function dispatchBatchAgent(options) {
|
|
|
7259
7873
|
responseFilesFinal,
|
|
7260
7874
|
orchestratorTemplateContent
|
|
7261
7875
|
);
|
|
7262
|
-
await (0,
|
|
7876
|
+
await (0, import_promises18.writeFile)(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
7263
7877
|
}
|
|
7264
7878
|
const chatAttachments = [orchestratorFile, ...attachments];
|
|
7265
7879
|
const orchestratorUri = pathToFileUri2(orchestratorFile);
|
|
@@ -7329,8 +7943,8 @@ async function dispatchBatchAgent(options) {
|
|
|
7329
7943
|
}
|
|
7330
7944
|
|
|
7331
7945
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
7332
|
-
var
|
|
7333
|
-
var
|
|
7946
|
+
var import_promises19 = require("fs/promises");
|
|
7947
|
+
var import_node_path25 = __toESM(require("path"), 1);
|
|
7334
7948
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
7335
7949
|
folders: [
|
|
7336
7950
|
{
|
|
@@ -7361,7 +7975,7 @@ async function provisionSubagents(options) {
|
|
|
7361
7975
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
7362
7976
|
throw new Error("subagents must be a positive integer");
|
|
7363
7977
|
}
|
|
7364
|
-
const targetPath =
|
|
7978
|
+
const targetPath = import_node_path25.default.resolve(targetRoot);
|
|
7365
7979
|
if (!dryRun) {
|
|
7366
7980
|
await ensureDir(targetPath);
|
|
7367
7981
|
}
|
|
@@ -7381,7 +7995,7 @@ async function provisionSubagents(options) {
|
|
|
7381
7995
|
continue;
|
|
7382
7996
|
}
|
|
7383
7997
|
highestNumber = Math.max(highestNumber, parsed);
|
|
7384
|
-
const lockFile =
|
|
7998
|
+
const lockFile = import_node_path25.default.join(entry.absolutePath, lockName);
|
|
7385
7999
|
const locked = await pathExists(lockFile);
|
|
7386
8000
|
if (locked) {
|
|
7387
8001
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -7398,10 +8012,10 @@ async function provisionSubagents(options) {
|
|
|
7398
8012
|
break;
|
|
7399
8013
|
}
|
|
7400
8014
|
const subagentDir = subagent.absolutePath;
|
|
7401
|
-
const githubAgentsDir =
|
|
7402
|
-
const lockFile =
|
|
7403
|
-
const workspaceDst =
|
|
7404
|
-
const wakeupDst =
|
|
8015
|
+
const githubAgentsDir = import_node_path25.default.join(subagentDir, ".github", "agents");
|
|
8016
|
+
const lockFile = import_node_path25.default.join(subagentDir, lockName);
|
|
8017
|
+
const workspaceDst = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
8018
|
+
const wakeupDst = import_node_path25.default.join(githubAgentsDir, "wakeup.md");
|
|
7405
8019
|
const isLocked = await pathExists(lockFile);
|
|
7406
8020
|
if (isLocked && !force) {
|
|
7407
8021
|
continue;
|
|
@@ -7410,8 +8024,8 @@ async function provisionSubagents(options) {
|
|
|
7410
8024
|
if (!dryRun) {
|
|
7411
8025
|
await removeIfExists(lockFile);
|
|
7412
8026
|
await ensureDir(githubAgentsDir);
|
|
7413
|
-
await (0,
|
|
7414
|
-
await (0,
|
|
8027
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8028
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7415
8029
|
}
|
|
7416
8030
|
created.push(subagentDir);
|
|
7417
8031
|
lockedSubagents.delete(subagentDir);
|
|
@@ -7421,8 +8035,8 @@ async function provisionSubagents(options) {
|
|
|
7421
8035
|
if (!isLocked && force) {
|
|
7422
8036
|
if (!dryRun) {
|
|
7423
8037
|
await ensureDir(githubAgentsDir);
|
|
7424
|
-
await (0,
|
|
7425
|
-
await (0,
|
|
8038
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8039
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7426
8040
|
}
|
|
7427
8041
|
created.push(subagentDir);
|
|
7428
8042
|
subagentsProvisioned += 1;
|
|
@@ -7430,8 +8044,8 @@ async function provisionSubagents(options) {
|
|
|
7430
8044
|
}
|
|
7431
8045
|
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
7432
8046
|
await ensureDir(githubAgentsDir);
|
|
7433
|
-
await (0,
|
|
7434
|
-
await (0,
|
|
8047
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8048
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7435
8049
|
}
|
|
7436
8050
|
skippedExisting.push(subagentDir);
|
|
7437
8051
|
subagentsProvisioned += 1;
|
|
@@ -7439,15 +8053,15 @@ async function provisionSubagents(options) {
|
|
|
7439
8053
|
let nextIndex = highestNumber;
|
|
7440
8054
|
while (subagentsProvisioned < subagents) {
|
|
7441
8055
|
nextIndex += 1;
|
|
7442
|
-
const subagentDir =
|
|
7443
|
-
const githubAgentsDir =
|
|
7444
|
-
const workspaceDst =
|
|
7445
|
-
const wakeupDst =
|
|
8056
|
+
const subagentDir = import_node_path25.default.join(targetPath, `subagent-${nextIndex}`);
|
|
8057
|
+
const githubAgentsDir = import_node_path25.default.join(subagentDir, ".github", "agents");
|
|
8058
|
+
const workspaceDst = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
8059
|
+
const wakeupDst = import_node_path25.default.join(githubAgentsDir, "wakeup.md");
|
|
7446
8060
|
if (!dryRun) {
|
|
7447
8061
|
await ensureDir(subagentDir);
|
|
7448
8062
|
await ensureDir(githubAgentsDir);
|
|
7449
|
-
await (0,
|
|
7450
|
-
await (0,
|
|
8063
|
+
await (0, import_promises19.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
8064
|
+
await (0, import_promises19.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
7451
8065
|
}
|
|
7452
8066
|
created.push(subagentDir);
|
|
7453
8067
|
subagentsProvisioned += 1;
|
|
@@ -7626,7 +8240,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
7626
8240
|
return "";
|
|
7627
8241
|
}
|
|
7628
8242
|
const buildList = (files) => files.map((absolutePath) => {
|
|
7629
|
-
const fileName =
|
|
8243
|
+
const fileName = import_node_path26.default.basename(absolutePath);
|
|
7630
8244
|
const fileUri = pathToFileUri3(absolutePath);
|
|
7631
8245
|
return `* [${fileName}](${fileUri})`;
|
|
7632
8246
|
});
|
|
@@ -7651,8 +8265,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
7651
8265
|
}
|
|
7652
8266
|
const unique = /* @__PURE__ */ new Map();
|
|
7653
8267
|
for (const attachment of attachments) {
|
|
7654
|
-
const absolutePath =
|
|
7655
|
-
const normalized = absolutePath.split(
|
|
8268
|
+
const absolutePath = import_node_path26.default.resolve(attachment);
|
|
8269
|
+
const normalized = absolutePath.split(import_node_path26.default.sep).join("/");
|
|
7656
8270
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
7657
8271
|
if (!unique.has(absolutePath)) {
|
|
7658
8272
|
unique.set(absolutePath, absolutePath);
|
|
@@ -7667,7 +8281,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
7667
8281
|
}
|
|
7668
8282
|
const unique = /* @__PURE__ */ new Map();
|
|
7669
8283
|
for (const attachment of attachments) {
|
|
7670
|
-
const absolutePath =
|
|
8284
|
+
const absolutePath = import_node_path26.default.resolve(attachment);
|
|
7671
8285
|
if (!unique.has(absolutePath)) {
|
|
7672
8286
|
unique.set(absolutePath, absolutePath);
|
|
7673
8287
|
}
|
|
@@ -7675,7 +8289,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
7675
8289
|
return Array.from(unique.values());
|
|
7676
8290
|
}
|
|
7677
8291
|
function pathToFileUri3(filePath) {
|
|
7678
|
-
const absolutePath =
|
|
8292
|
+
const absolutePath = import_node_path26.default.isAbsolute(filePath) ? filePath : import_node_path26.default.resolve(filePath);
|
|
7679
8293
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
7680
8294
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
7681
8295
|
return `file:///${normalizedPath}`;
|
|
@@ -7688,7 +8302,7 @@ function normalizeAttachments(attachments) {
|
|
|
7688
8302
|
}
|
|
7689
8303
|
const deduped = /* @__PURE__ */ new Set();
|
|
7690
8304
|
for (const attachment of attachments) {
|
|
7691
|
-
deduped.add(
|
|
8305
|
+
deduped.add(import_node_path26.default.resolve(attachment));
|
|
7692
8306
|
}
|
|
7693
8307
|
return Array.from(deduped);
|
|
7694
8308
|
}
|
|
@@ -7697,7 +8311,7 @@ function mergeAttachments(all) {
|
|
|
7697
8311
|
for (const list of all) {
|
|
7698
8312
|
if (!list) continue;
|
|
7699
8313
|
for (const inputFile of list) {
|
|
7700
|
-
deduped.add(
|
|
8314
|
+
deduped.add(import_node_path26.default.resolve(inputFile));
|
|
7701
8315
|
}
|
|
7702
8316
|
}
|
|
7703
8317
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -7744,9 +8358,9 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
7744
8358
|
}
|
|
7745
8359
|
|
|
7746
8360
|
// src/evaluation/providers/targets-file.ts
|
|
7747
|
-
var
|
|
7748
|
-
var
|
|
7749
|
-
var
|
|
8361
|
+
var import_node_fs8 = require("fs");
|
|
8362
|
+
var import_promises20 = require("fs/promises");
|
|
8363
|
+
var import_node_path27 = __toESM(require("path"), 1);
|
|
7750
8364
|
var import_yaml4 = require("yaml");
|
|
7751
8365
|
function isRecord(value) {
|
|
7752
8366
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -7776,18 +8390,18 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
7776
8390
|
}
|
|
7777
8391
|
async function fileExists3(filePath) {
|
|
7778
8392
|
try {
|
|
7779
|
-
await (0,
|
|
8393
|
+
await (0, import_promises20.access)(filePath, import_node_fs8.constants.F_OK);
|
|
7780
8394
|
return true;
|
|
7781
8395
|
} catch {
|
|
7782
8396
|
return false;
|
|
7783
8397
|
}
|
|
7784
8398
|
}
|
|
7785
8399
|
async function readTargetDefinitions(filePath) {
|
|
7786
|
-
const absolutePath =
|
|
8400
|
+
const absolutePath = import_node_path27.default.resolve(filePath);
|
|
7787
8401
|
if (!await fileExists3(absolutePath)) {
|
|
7788
8402
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
7789
8403
|
}
|
|
7790
|
-
const raw = await (0,
|
|
8404
|
+
const raw = await (0, import_promises20.readFile)(absolutePath, "utf8");
|
|
7791
8405
|
const parsed = (0, import_yaml4.parse)(raw);
|
|
7792
8406
|
if (!isRecord(parsed)) {
|
|
7793
8407
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -7815,6 +8429,8 @@ function createProvider(target) {
|
|
|
7815
8429
|
return new CliProvider(target.name, target.config);
|
|
7816
8430
|
case "codex":
|
|
7817
8431
|
return new CodexProvider(target.name, target.config);
|
|
8432
|
+
case "copilot-cli":
|
|
8433
|
+
return new CopilotCliProvider(target.name, target.config);
|
|
7818
8434
|
case "pi-coding-agent":
|
|
7819
8435
|
return new PiCodingAgentProvider(target.name, target.config);
|
|
7820
8436
|
case "pi-agent-sdk":
|
|
@@ -7951,10 +8567,10 @@ async function execFileWithStdinBun(argv, stdinPayload, options) {
|
|
|
7951
8567
|
}
|
|
7952
8568
|
}
|
|
7953
8569
|
async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
7954
|
-
const { spawn:
|
|
8570
|
+
const { spawn: spawn6 } = await import("child_process");
|
|
7955
8571
|
return new Promise((resolve, reject) => {
|
|
7956
8572
|
const [cmd, ...args] = argv;
|
|
7957
|
-
const child =
|
|
8573
|
+
const child = spawn6(cmd, args, {
|
|
7958
8574
|
cwd: options.cwd,
|
|
7959
8575
|
stdio: ["pipe", "pipe", "pipe"],
|
|
7960
8576
|
// Merge additional env vars with process.env
|
|
@@ -7994,21 +8610,21 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
7994
8610
|
});
|
|
7995
8611
|
}
|
|
7996
8612
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
7997
|
-
const { mkdir:
|
|
7998
|
-
const { tmpdir:
|
|
7999
|
-
const
|
|
8000
|
-
const { randomUUID:
|
|
8001
|
-
const dir =
|
|
8002
|
-
await
|
|
8003
|
-
const stdinPath =
|
|
8004
|
-
const stdoutPath =
|
|
8005
|
-
const stderrPath =
|
|
8006
|
-
await
|
|
8613
|
+
const { mkdir: mkdir9, readFile: readFile11, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
|
|
8614
|
+
const { tmpdir: tmpdir5 } = await import("os");
|
|
8615
|
+
const path29 = await import("path");
|
|
8616
|
+
const { randomUUID: randomUUID5 } = await import("crypto");
|
|
8617
|
+
const dir = path29.join(tmpdir5(), `agentv-exec-${randomUUID5()}`);
|
|
8618
|
+
await mkdir9(dir, { recursive: true });
|
|
8619
|
+
const stdinPath = path29.join(dir, "stdin.txt");
|
|
8620
|
+
const stdoutPath = path29.join(dir, "stdout.txt");
|
|
8621
|
+
const stderrPath = path29.join(dir, "stderr.txt");
|
|
8622
|
+
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
8007
8623
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
8008
|
-
const { spawn:
|
|
8624
|
+
const { spawn: spawn6 } = await import("child_process");
|
|
8009
8625
|
try {
|
|
8010
8626
|
const exitCode = await new Promise((resolve, reject) => {
|
|
8011
|
-
const child =
|
|
8627
|
+
const child = spawn6(wrappedCommand, {
|
|
8012
8628
|
shell: true,
|
|
8013
8629
|
cwd: options.cwd,
|
|
8014
8630
|
stdio: ["ignore", "ignore", "ignore"],
|
|
@@ -8036,17 +8652,17 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
8036
8652
|
const stderr = (await readFile11(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
8037
8653
|
return { stdout, stderr, exitCode };
|
|
8038
8654
|
} finally {
|
|
8039
|
-
await
|
|
8655
|
+
await rm6(dir, { recursive: true, force: true });
|
|
8040
8656
|
}
|
|
8041
8657
|
}
|
|
8042
8658
|
|
|
8043
8659
|
// src/runtime/target-proxy.ts
|
|
8044
|
-
var
|
|
8660
|
+
var import_node_crypto5 = require("crypto");
|
|
8045
8661
|
var import_node_http = require("http");
|
|
8046
8662
|
var DEFAULT_MAX_CALLS = 50;
|
|
8047
8663
|
async function createTargetProxy(options) {
|
|
8048
8664
|
const { defaultProvider, targetResolver, availableTargets, maxCalls } = options;
|
|
8049
|
-
const token = (0,
|
|
8665
|
+
const token = (0, import_node_crypto5.randomBytes)(32).toString("hex");
|
|
8050
8666
|
let callCount = 0;
|
|
8051
8667
|
let isShutdown = false;
|
|
8052
8668
|
const targetsList = availableTargets ?? [defaultProvider.targetName];
|
|
@@ -8305,7 +8921,7 @@ var CodeEvaluator = class {
|
|
|
8305
8921
|
outputMessages: context.outputMessages ?? null,
|
|
8306
8922
|
guidelineFiles: context.evalCase.guideline_paths,
|
|
8307
8923
|
inputFiles: context.evalCase.file_paths.filter(
|
|
8308
|
-
(
|
|
8924
|
+
(path29) => !context.evalCase.guideline_paths.includes(path29)
|
|
8309
8925
|
),
|
|
8310
8926
|
inputMessages: context.evalCase.input_messages,
|
|
8311
8927
|
traceSummary: context.traceSummary ?? null,
|
|
@@ -8421,6 +9037,7 @@ var import_ai3 = require("ai");
|
|
|
8421
9037
|
// src/evaluation/providers/types.ts
|
|
8422
9038
|
var AGENT_PROVIDER_KINDS = [
|
|
8423
9039
|
"codex",
|
|
9040
|
+
"copilot-cli",
|
|
8424
9041
|
"pi-coding-agent",
|
|
8425
9042
|
"claude-code",
|
|
8426
9043
|
"vscode",
|
|
@@ -9252,115 +9869,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
9252
9869
|
* Evaluate a single field against the expected value.
|
|
9253
9870
|
*/
|
|
9254
9871
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
9255
|
-
const { path:
|
|
9256
|
-
const candidateValue = resolvePath(candidateData,
|
|
9257
|
-
const expectedValue = resolvePath(expectedData,
|
|
9872
|
+
const { path: path29, match, required = true, weight = 1 } = fieldConfig;
|
|
9873
|
+
const candidateValue = resolvePath(candidateData, path29);
|
|
9874
|
+
const expectedValue = resolvePath(expectedData, path29);
|
|
9258
9875
|
if (expectedValue === void 0) {
|
|
9259
9876
|
return {
|
|
9260
|
-
path:
|
|
9877
|
+
path: path29,
|
|
9261
9878
|
score: 1,
|
|
9262
9879
|
// No expected value means no comparison needed
|
|
9263
9880
|
weight,
|
|
9264
9881
|
hit: true,
|
|
9265
|
-
message: `${
|
|
9882
|
+
message: `${path29}: no expected value`
|
|
9266
9883
|
};
|
|
9267
9884
|
}
|
|
9268
9885
|
if (candidateValue === void 0) {
|
|
9269
9886
|
if (required) {
|
|
9270
9887
|
return {
|
|
9271
|
-
path:
|
|
9888
|
+
path: path29,
|
|
9272
9889
|
score: 0,
|
|
9273
9890
|
weight,
|
|
9274
9891
|
hit: false,
|
|
9275
|
-
message: `${
|
|
9892
|
+
message: `${path29} (required, missing)`
|
|
9276
9893
|
};
|
|
9277
9894
|
}
|
|
9278
9895
|
return {
|
|
9279
|
-
path:
|
|
9896
|
+
path: path29,
|
|
9280
9897
|
score: 1,
|
|
9281
9898
|
// Don't penalize missing optional fields
|
|
9282
9899
|
weight: 0,
|
|
9283
9900
|
// Zero weight means it won't affect the score
|
|
9284
9901
|
hit: true,
|
|
9285
|
-
message: `${
|
|
9902
|
+
message: `${path29}: optional field missing`
|
|
9286
9903
|
};
|
|
9287
9904
|
}
|
|
9288
9905
|
switch (match) {
|
|
9289
9906
|
case "exact":
|
|
9290
|
-
return this.compareExact(
|
|
9907
|
+
return this.compareExact(path29, candidateValue, expectedValue, weight);
|
|
9291
9908
|
case "numeric_tolerance":
|
|
9292
9909
|
return this.compareNumericTolerance(
|
|
9293
|
-
|
|
9910
|
+
path29,
|
|
9294
9911
|
candidateValue,
|
|
9295
9912
|
expectedValue,
|
|
9296
9913
|
fieldConfig,
|
|
9297
9914
|
weight
|
|
9298
9915
|
);
|
|
9299
9916
|
case "date":
|
|
9300
|
-
return this.compareDate(
|
|
9917
|
+
return this.compareDate(path29, candidateValue, expectedValue, fieldConfig, weight);
|
|
9301
9918
|
default:
|
|
9302
9919
|
return {
|
|
9303
|
-
path:
|
|
9920
|
+
path: path29,
|
|
9304
9921
|
score: 0,
|
|
9305
9922
|
weight,
|
|
9306
9923
|
hit: false,
|
|
9307
|
-
message: `${
|
|
9924
|
+
message: `${path29}: unknown match type "${match}"`
|
|
9308
9925
|
};
|
|
9309
9926
|
}
|
|
9310
9927
|
}
|
|
9311
9928
|
/**
|
|
9312
9929
|
* Exact equality comparison.
|
|
9313
9930
|
*/
|
|
9314
|
-
compareExact(
|
|
9931
|
+
compareExact(path29, candidateValue, expectedValue, weight) {
|
|
9315
9932
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
9316
9933
|
return {
|
|
9317
|
-
path:
|
|
9934
|
+
path: path29,
|
|
9318
9935
|
score: 1,
|
|
9319
9936
|
weight,
|
|
9320
9937
|
hit: true,
|
|
9321
|
-
message:
|
|
9938
|
+
message: path29
|
|
9322
9939
|
};
|
|
9323
9940
|
}
|
|
9324
9941
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
9325
9942
|
return {
|
|
9326
|
-
path:
|
|
9943
|
+
path: path29,
|
|
9327
9944
|
score: 0,
|
|
9328
9945
|
weight,
|
|
9329
9946
|
hit: false,
|
|
9330
|
-
message: `${
|
|
9947
|
+
message: `${path29} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
9331
9948
|
};
|
|
9332
9949
|
}
|
|
9333
9950
|
return {
|
|
9334
|
-
path:
|
|
9951
|
+
path: path29,
|
|
9335
9952
|
score: 0,
|
|
9336
9953
|
weight,
|
|
9337
9954
|
hit: false,
|
|
9338
|
-
message: `${
|
|
9955
|
+
message: `${path29} (value mismatch)`
|
|
9339
9956
|
};
|
|
9340
9957
|
}
|
|
9341
9958
|
/**
|
|
9342
9959
|
* Numeric comparison with absolute or relative tolerance.
|
|
9343
9960
|
*/
|
|
9344
|
-
compareNumericTolerance(
|
|
9961
|
+
compareNumericTolerance(path29, candidateValue, expectedValue, fieldConfig, weight) {
|
|
9345
9962
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
9346
9963
|
const candidateNum = toNumber(candidateValue);
|
|
9347
9964
|
const expectedNum = toNumber(expectedValue);
|
|
9348
9965
|
if (candidateNum === null || expectedNum === null) {
|
|
9349
9966
|
return {
|
|
9350
|
-
path:
|
|
9967
|
+
path: path29,
|
|
9351
9968
|
score: 0,
|
|
9352
9969
|
weight,
|
|
9353
9970
|
hit: false,
|
|
9354
|
-
message: `${
|
|
9971
|
+
message: `${path29} (non-numeric value)`
|
|
9355
9972
|
};
|
|
9356
9973
|
}
|
|
9357
9974
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
9358
9975
|
return {
|
|
9359
|
-
path:
|
|
9976
|
+
path: path29,
|
|
9360
9977
|
score: 0,
|
|
9361
9978
|
weight,
|
|
9362
9979
|
hit: false,
|
|
9363
|
-
message: `${
|
|
9980
|
+
message: `${path29} (invalid numeric value)`
|
|
9364
9981
|
};
|
|
9365
9982
|
}
|
|
9366
9983
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -9373,61 +9990,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
9373
9990
|
}
|
|
9374
9991
|
if (withinTolerance) {
|
|
9375
9992
|
return {
|
|
9376
|
-
path:
|
|
9993
|
+
path: path29,
|
|
9377
9994
|
score: 1,
|
|
9378
9995
|
weight,
|
|
9379
9996
|
hit: true,
|
|
9380
|
-
message: `${
|
|
9997
|
+
message: `${path29} (within tolerance: diff=${diff.toFixed(2)})`
|
|
9381
9998
|
};
|
|
9382
9999
|
}
|
|
9383
10000
|
return {
|
|
9384
|
-
path:
|
|
10001
|
+
path: path29,
|
|
9385
10002
|
score: 0,
|
|
9386
10003
|
weight,
|
|
9387
10004
|
hit: false,
|
|
9388
|
-
message: `${
|
|
10005
|
+
message: `${path29} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
9389
10006
|
};
|
|
9390
10007
|
}
|
|
9391
10008
|
/**
|
|
9392
10009
|
* Date comparison with format normalization.
|
|
9393
10010
|
*/
|
|
9394
|
-
compareDate(
|
|
10011
|
+
compareDate(path29, candidateValue, expectedValue, fieldConfig, weight) {
|
|
9395
10012
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
9396
10013
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
9397
10014
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
9398
10015
|
if (candidateDate === null) {
|
|
9399
10016
|
return {
|
|
9400
|
-
path:
|
|
10017
|
+
path: path29,
|
|
9401
10018
|
score: 0,
|
|
9402
10019
|
weight,
|
|
9403
10020
|
hit: false,
|
|
9404
|
-
message: `${
|
|
10021
|
+
message: `${path29} (unparseable candidate date)`
|
|
9405
10022
|
};
|
|
9406
10023
|
}
|
|
9407
10024
|
if (expectedDate === null) {
|
|
9408
10025
|
return {
|
|
9409
|
-
path:
|
|
10026
|
+
path: path29,
|
|
9410
10027
|
score: 0,
|
|
9411
10028
|
weight,
|
|
9412
10029
|
hit: false,
|
|
9413
|
-
message: `${
|
|
10030
|
+
message: `${path29} (unparseable expected date)`
|
|
9414
10031
|
};
|
|
9415
10032
|
}
|
|
9416
10033
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
9417
10034
|
return {
|
|
9418
|
-
path:
|
|
10035
|
+
path: path29,
|
|
9419
10036
|
score: 1,
|
|
9420
10037
|
weight,
|
|
9421
10038
|
hit: true,
|
|
9422
|
-
message:
|
|
10039
|
+
message: path29
|
|
9423
10040
|
};
|
|
9424
10041
|
}
|
|
9425
10042
|
return {
|
|
9426
|
-
path:
|
|
10043
|
+
path: path29,
|
|
9427
10044
|
score: 0,
|
|
9428
10045
|
weight,
|
|
9429
10046
|
hit: false,
|
|
9430
|
-
message: `${
|
|
10047
|
+
message: `${path29} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
9431
10048
|
};
|
|
9432
10049
|
}
|
|
9433
10050
|
/**
|
|
@@ -9467,11 +10084,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
9467
10084
|
};
|
|
9468
10085
|
}
|
|
9469
10086
|
};
|
|
9470
|
-
function resolvePath(obj,
|
|
9471
|
-
if (!
|
|
10087
|
+
function resolvePath(obj, path29) {
|
|
10088
|
+
if (!path29 || !obj) {
|
|
9472
10089
|
return void 0;
|
|
9473
10090
|
}
|
|
9474
|
-
const parts =
|
|
10091
|
+
const parts = path29.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
9475
10092
|
let current = obj;
|
|
9476
10093
|
for (const part of parts) {
|
|
9477
10094
|
if (current === null || current === void 0) {
|
|
@@ -9990,8 +10607,8 @@ var ToolTrajectoryEvaluator = class {
|
|
|
9990
10607
|
};
|
|
9991
10608
|
|
|
9992
10609
|
// src/evaluation/orchestrator.ts
|
|
9993
|
-
var
|
|
9994
|
-
var
|
|
10610
|
+
var import_node_crypto6 = require("crypto");
|
|
10611
|
+
var import_node_path28 = __toESM(require("path"), 1);
|
|
9995
10612
|
var import_micromatch4 = __toESM(require("micromatch"), 1);
|
|
9996
10613
|
|
|
9997
10614
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -10794,7 +11411,7 @@ async function runEvaluatorList(options) {
|
|
|
10794
11411
|
});
|
|
10795
11412
|
}
|
|
10796
11413
|
if (evaluator.type === "composite") {
|
|
10797
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
11414
|
+
const evalFileDir = evalCase.guideline_paths[0] ? import_node_path28.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
10798
11415
|
const createEvaluator = (memberConfig) => {
|
|
10799
11416
|
switch (memberConfig.type) {
|
|
10800
11417
|
case "llm_judge":
|
|
@@ -11150,7 +11767,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
|
|
|
11150
11767
|
};
|
|
11151
11768
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
11152
11769
|
const scriptPath = script[script.length - 1];
|
|
11153
|
-
const cwd =
|
|
11770
|
+
const cwd = import_node_path28.default.dirname(scriptPath);
|
|
11154
11771
|
try {
|
|
11155
11772
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
11156
11773
|
const prompt = stdout.trim();
|
|
@@ -11264,7 +11881,7 @@ function extractProviderError(response) {
|
|
|
11264
11881
|
return trimmed.length > 0 ? trimmed : void 0;
|
|
11265
11882
|
}
|
|
11266
11883
|
function createCacheKey(provider, target, evalCase, promptInputs) {
|
|
11267
|
-
const hash = (0,
|
|
11884
|
+
const hash = (0, import_node_crypto6.createHash)("sha256");
|
|
11268
11885
|
hash.update(provider.id);
|
|
11269
11886
|
hash.update(target.name);
|
|
11270
11887
|
hash.update(evalCase.id);
|
|
@@ -11425,6 +12042,7 @@ function createAgentKernel() {
|
|
|
11425
12042
|
computeTraceSummary,
|
|
11426
12043
|
consumeClaudeCodeLogEntries,
|
|
11427
12044
|
consumeCodexLogEntries,
|
|
12045
|
+
consumeCopilotCliLogEntries,
|
|
11428
12046
|
consumePiLogEntries,
|
|
11429
12047
|
createAgentKernel,
|
|
11430
12048
|
createProvider,
|
|
@@ -11464,6 +12082,7 @@ function createAgentKernel() {
|
|
|
11464
12082
|
scoreToVerdict,
|
|
11465
12083
|
subscribeToClaudeCodeLogEntries,
|
|
11466
12084
|
subscribeToCodexLogEntries,
|
|
12085
|
+
subscribeToCopilotCliLogEntries,
|
|
11467
12086
|
subscribeToPiLogEntries,
|
|
11468
12087
|
tokensPerTool
|
|
11469
12088
|
});
|