@agentv/core 2.14.2 → 2.14.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +390 -229
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +10 -2
- package/dist/index.d.ts +10 -2
- package/dist/index.js +373 -217
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1244,12 +1244,12 @@ function serializeAttributeValue(value) {
|
|
|
1244
1244
|
if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
|
|
1245
1245
|
return { stringValue: String(value) };
|
|
1246
1246
|
}
|
|
1247
|
-
var import_promises31,
|
|
1247
|
+
var import_promises31, import_node_path44, OtlpJsonFileExporter;
|
|
1248
1248
|
var init_otlp_json_file_exporter = __esm({
|
|
1249
1249
|
"src/observability/otlp-json-file-exporter.ts"() {
|
|
1250
1250
|
"use strict";
|
|
1251
1251
|
import_promises31 = require("fs/promises");
|
|
1252
|
-
|
|
1252
|
+
import_node_path44 = require("path");
|
|
1253
1253
|
OtlpJsonFileExporter = class {
|
|
1254
1254
|
// biome-ignore lint/suspicious/noExplicitAny: serialized span data
|
|
1255
1255
|
spans = [];
|
|
@@ -1288,7 +1288,7 @@ var init_otlp_json_file_exporter = __esm({
|
|
|
1288
1288
|
}
|
|
1289
1289
|
async flush() {
|
|
1290
1290
|
if (this.spans.length === 0) return;
|
|
1291
|
-
await (0, import_promises31.mkdir)((0,
|
|
1291
|
+
await (0, import_promises31.mkdir)((0, import_node_path44.dirname)(this.filePath), { recursive: true });
|
|
1292
1292
|
const otlpJson = {
|
|
1293
1293
|
resourceSpans: [
|
|
1294
1294
|
{
|
|
@@ -1319,13 +1319,13 @@ function hrTimeDiffMs(start, end) {
|
|
|
1319
1319
|
const diffNano = end[1] - start[1];
|
|
1320
1320
|
return Math.round(diffSec * 1e3 + diffNano / 1e6);
|
|
1321
1321
|
}
|
|
1322
|
-
var import_node_fs13, import_promises32,
|
|
1322
|
+
var import_node_fs13, import_promises32, import_node_path45, SimpleTraceFileExporter;
|
|
1323
1323
|
var init_simple_trace_file_exporter = __esm({
|
|
1324
1324
|
"src/observability/simple-trace-file-exporter.ts"() {
|
|
1325
1325
|
"use strict";
|
|
1326
1326
|
import_node_fs13 = require("fs");
|
|
1327
1327
|
import_promises32 = require("fs/promises");
|
|
1328
|
-
|
|
1328
|
+
import_node_path45 = require("path");
|
|
1329
1329
|
SimpleTraceFileExporter = class {
|
|
1330
1330
|
stream = null;
|
|
1331
1331
|
filePath;
|
|
@@ -1338,7 +1338,7 @@ var init_simple_trace_file_exporter = __esm({
|
|
|
1338
1338
|
async ensureStream() {
|
|
1339
1339
|
if (!this.streamReady) {
|
|
1340
1340
|
this.streamReady = (async () => {
|
|
1341
|
-
await (0, import_promises32.mkdir)((0,
|
|
1341
|
+
await (0, import_promises32.mkdir)((0, import_node_path45.dirname)(this.filePath), { recursive: true });
|
|
1342
1342
|
this.stream = (0, import_node_fs13.createWriteStream)(this.filePath, { flags: "w" });
|
|
1343
1343
|
return this.stream;
|
|
1344
1344
|
})();
|
|
@@ -1502,8 +1502,13 @@ __export(index_exports, {
|
|
|
1502
1502
|
findGitRoot: () => findGitRoot,
|
|
1503
1503
|
freeformEvaluationSchema: () => freeformEvaluationSchema,
|
|
1504
1504
|
generateRubrics: () => generateRubrics,
|
|
1505
|
+
getAgentvHome: () => getAgentvHome,
|
|
1506
|
+
getGitCacheRoot: () => getGitCacheRoot,
|
|
1505
1507
|
getHitCount: () => getHitCount,
|
|
1508
|
+
getSubagentsRoot: () => getSubagentsRoot,
|
|
1509
|
+
getTraceStateRoot: () => getTraceStateRoot,
|
|
1506
1510
|
getWorkspacePath: () => getWorkspacePath,
|
|
1511
|
+
getWorkspacesRoot: () => getWorkspacesRoot,
|
|
1507
1512
|
initializeBaseline: () => initializeBaseline,
|
|
1508
1513
|
isEvaluatorKind: () => isEvaluatorKind,
|
|
1509
1514
|
isGuidelineFile: () => isGuidelineFile,
|
|
@@ -1853,7 +1858,7 @@ async function resolveFileReference(ref, evalFileDir) {
|
|
|
1853
1858
|
const rawPath = extractFilePath(ref);
|
|
1854
1859
|
const absolutePattern = import_node_path.default.resolve(evalFileDir, rawPath);
|
|
1855
1860
|
if (isGlobPattern(rawPath)) {
|
|
1856
|
-
const matches = await (0, import_fast_glob.default)(absolutePattern, {
|
|
1861
|
+
const matches = await (0, import_fast_glob.default)(absolutePattern.replaceAll("\\", "/"), {
|
|
1857
1862
|
onlyFiles: true,
|
|
1858
1863
|
absolute: true
|
|
1859
1864
|
});
|
|
@@ -6784,7 +6789,7 @@ var import_node_path14 = __toESM(require("path"), 1);
|
|
|
6784
6789
|
var import_node_url2 = require("url");
|
|
6785
6790
|
var import_meta = {};
|
|
6786
6791
|
function resolvePlatformCliPath() {
|
|
6787
|
-
const
|
|
6792
|
+
const os3 = (0, import_node_os2.platform)();
|
|
6788
6793
|
const cpu = (0, import_node_os2.arch)();
|
|
6789
6794
|
const platformMap = {
|
|
6790
6795
|
linux: "linux",
|
|
@@ -6795,13 +6800,13 @@ function resolvePlatformCliPath() {
|
|
|
6795
6800
|
x64: "x64",
|
|
6796
6801
|
arm64: "arm64"
|
|
6797
6802
|
};
|
|
6798
|
-
const osPart = platformMap[
|
|
6803
|
+
const osPart = platformMap[os3];
|
|
6799
6804
|
const archPart = archMap[cpu];
|
|
6800
6805
|
if (!osPart || !archPart) {
|
|
6801
6806
|
return void 0;
|
|
6802
6807
|
}
|
|
6803
6808
|
const packageName = `@github/copilot-${osPart}-${archPart}`;
|
|
6804
|
-
const binaryName =
|
|
6809
|
+
const binaryName = os3 === "win32" ? "copilot.exe" : "copilot";
|
|
6805
6810
|
try {
|
|
6806
6811
|
const resolved = import_meta.resolve(`${packageName}/package.json`);
|
|
6807
6812
|
const packageJsonPath = resolved.startsWith("file:") ? (0, import_node_url2.fileURLToPath)(resolved) : resolved;
|
|
@@ -9488,8 +9493,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
9488
9493
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
9489
9494
|
if (!parseResult.success) {
|
|
9490
9495
|
const firstError = parseResult.error.errors[0];
|
|
9491
|
-
const
|
|
9492
|
-
const prefix =
|
|
9496
|
+
const path43 = firstError?.path.join(".") || "";
|
|
9497
|
+
const prefix = path43 ? `${target.name} ${path43}: ` : `${target.name}: `;
|
|
9493
9498
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
9494
9499
|
}
|
|
9495
9500
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -9699,12 +9704,12 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
9699
9704
|
// src/evaluation/providers/vscode-provider.ts
|
|
9700
9705
|
var import_node_child_process5 = require("child_process");
|
|
9701
9706
|
var import_promises22 = require("fs/promises");
|
|
9702
|
-
var
|
|
9707
|
+
var import_node_path30 = __toESM(require("path"), 1);
|
|
9703
9708
|
var import_node_util3 = require("util");
|
|
9704
9709
|
|
|
9705
9710
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
9706
9711
|
var import_promises20 = require("fs/promises");
|
|
9707
|
-
var
|
|
9712
|
+
var import_node_path28 = __toESM(require("path"), 1);
|
|
9708
9713
|
|
|
9709
9714
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
9710
9715
|
var import_node_fs7 = require("fs");
|
|
@@ -9972,17 +9977,46 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
9972
9977
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
9973
9978
|
var import_node_child_process4 = require("child_process");
|
|
9974
9979
|
var import_promises18 = require("fs/promises");
|
|
9975
|
-
var
|
|
9980
|
+
var import_node_path25 = __toESM(require("path"), 1);
|
|
9976
9981
|
var import_node_util2 = require("util");
|
|
9977
9982
|
|
|
9978
9983
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
9984
|
+
var import_node_path24 = __toESM(require("path"), 1);
|
|
9985
|
+
|
|
9986
|
+
// src/paths.ts
|
|
9979
9987
|
var import_node_os4 = __toESM(require("os"), 1);
|
|
9980
9988
|
var import_node_path23 = __toESM(require("path"), 1);
|
|
9989
|
+
var logged = false;
|
|
9990
|
+
function getAgentvHome() {
|
|
9991
|
+
const envHome = process.env.AGENTV_HOME;
|
|
9992
|
+
if (envHome) {
|
|
9993
|
+
if (!logged) {
|
|
9994
|
+
logged = true;
|
|
9995
|
+
console.warn(`Using AGENTV_HOME: ${envHome}`);
|
|
9996
|
+
}
|
|
9997
|
+
return envHome;
|
|
9998
|
+
}
|
|
9999
|
+
return import_node_path23.default.join(import_node_os4.default.homedir(), ".agentv");
|
|
10000
|
+
}
|
|
10001
|
+
function getWorkspacesRoot() {
|
|
10002
|
+
return import_node_path23.default.join(getAgentvHome(), "workspaces");
|
|
10003
|
+
}
|
|
10004
|
+
function getGitCacheRoot() {
|
|
10005
|
+
return import_node_path23.default.join(getAgentvHome(), "git-cache");
|
|
10006
|
+
}
|
|
10007
|
+
function getSubagentsRoot() {
|
|
10008
|
+
return import_node_path23.default.join(getAgentvHome(), "subagents");
|
|
10009
|
+
}
|
|
10010
|
+
function getTraceStateRoot() {
|
|
10011
|
+
return import_node_path23.default.join(getAgentvHome(), "trace-state");
|
|
10012
|
+
}
|
|
10013
|
+
|
|
10014
|
+
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
9981
10015
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
9982
10016
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
9983
10017
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
9984
10018
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
9985
|
-
return
|
|
10019
|
+
return import_node_path24.default.join(getSubagentsRoot(), folder);
|
|
9986
10020
|
}
|
|
9987
10021
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
9988
10022
|
|
|
@@ -9996,12 +10030,19 @@ description: 'Wake-up Signal'
|
|
|
9996
10030
|
model: Grok Code Fast 1 (copilot)
|
|
9997
10031
|
---`;
|
|
9998
10032
|
function spawnVsCode(vscodeCmd, args, options) {
|
|
9999
|
-
const
|
|
10033
|
+
const useShell = options?.shell ?? true;
|
|
10034
|
+
const command = useShell ? shellQuote(vscodeCmd) : vscodeCmd;
|
|
10035
|
+
const child = (0, import_node_child_process4.spawn)(command, args, {
|
|
10000
10036
|
windowsHide: true,
|
|
10001
|
-
shell:
|
|
10037
|
+
shell: useShell,
|
|
10002
10038
|
detached: false
|
|
10003
10039
|
});
|
|
10004
|
-
child.on("error", () => {
|
|
10040
|
+
child.on("error", (error) => {
|
|
10041
|
+
const label = options?.label ?? "spawn";
|
|
10042
|
+
const renderedArgs = args.map((value) => JSON.stringify(value)).join(" ");
|
|
10043
|
+
console.error(
|
|
10044
|
+
`[vscode] ${label} failed: command=${JSON.stringify(vscodeCmd)} args=${renderedArgs} error=${error.message}`
|
|
10045
|
+
);
|
|
10005
10046
|
});
|
|
10006
10047
|
return child;
|
|
10007
10048
|
}
|
|
@@ -10038,16 +10079,20 @@ async function checkWorkspaceOpened(workspaceName, vscodeCmd) {
|
|
|
10038
10079
|
async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir, vscodeCmd, pollInterval = 1, timeout = 60) {
|
|
10039
10080
|
const alreadyOpen = await checkWorkspaceOpened(workspaceName, vscodeCmd);
|
|
10040
10081
|
if (alreadyOpen) {
|
|
10041
|
-
spawnVsCode(
|
|
10082
|
+
const child = spawnVsCode(vscodeCmd, [workspacePath], { label: "focus-existing-workspace" });
|
|
10083
|
+
await raceSpawnError(child);
|
|
10042
10084
|
return true;
|
|
10043
10085
|
}
|
|
10044
|
-
const aliveFile =
|
|
10086
|
+
const aliveFile = import_node_path25.default.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
10045
10087
|
await removeIfExists(aliveFile);
|
|
10046
|
-
const githubAgentsDir =
|
|
10088
|
+
const githubAgentsDir = import_node_path25.default.join(subagentDir, ".github", "agents");
|
|
10047
10089
|
await (0, import_promises18.mkdir)(githubAgentsDir, { recursive: true });
|
|
10048
|
-
const wakeupDst =
|
|
10090
|
+
const wakeupDst = import_node_path25.default.join(githubAgentsDir, "wakeup.md");
|
|
10049
10091
|
await (0, import_promises18.writeFile)(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
10050
|
-
spawnVsCode(
|
|
10092
|
+
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
10093
|
+
label: "open-workspace"
|
|
10094
|
+
});
|
|
10095
|
+
await raceSpawnError(workspaceChild);
|
|
10051
10096
|
await sleep2(100);
|
|
10052
10097
|
const wakeupChatId = "wakeup";
|
|
10053
10098
|
const chatArgs = [
|
|
@@ -10055,9 +10100,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
10055
10100
|
"chat",
|
|
10056
10101
|
"-m",
|
|
10057
10102
|
wakeupChatId,
|
|
10058
|
-
`create a file named .alive in the ${
|
|
10103
|
+
`create a file named .alive in the ${import_node_path25.default.basename(subagentDir)} folder`
|
|
10059
10104
|
];
|
|
10060
|
-
spawnVsCode(
|
|
10105
|
+
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
10106
|
+
await raceSpawnError(wakeupChild);
|
|
10061
10107
|
const start = Date.now();
|
|
10062
10108
|
while (!await pathExists(aliveFile)) {
|
|
10063
10109
|
if (Date.now() - start > timeout * 1e3) {
|
|
@@ -10069,10 +10115,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
10069
10115
|
return true;
|
|
10070
10116
|
}
|
|
10071
10117
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
10072
|
-
const workspacePath =
|
|
10073
|
-
const messagesDir =
|
|
10118
|
+
const workspacePath = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
10119
|
+
const messagesDir = import_node_path25.default.join(subagentDir, "messages");
|
|
10074
10120
|
await (0, import_promises18.mkdir)(messagesDir, { recursive: true });
|
|
10075
|
-
const reqFile =
|
|
10121
|
+
const reqFile = import_node_path25.default.join(messagesDir, `${timestamp}_req.md`);
|
|
10076
10122
|
await (0, import_promises18.writeFile)(reqFile, requestInstructions, { encoding: "utf8" });
|
|
10077
10123
|
const reqUri = pathToFileUri2(reqFile);
|
|
10078
10124
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
@@ -10080,25 +10126,25 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
10080
10126
|
chatArgs.push("-a", attachment);
|
|
10081
10127
|
}
|
|
10082
10128
|
chatArgs.push("-a", reqFile);
|
|
10083
|
-
chatArgs.push(`Follow instructions in [${
|
|
10129
|
+
chatArgs.push(`Follow instructions in [${import_node_path25.default.basename(reqFile)}](${reqUri})`);
|
|
10084
10130
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
10085
10131
|
workspacePath,
|
|
10086
|
-
|
|
10132
|
+
import_node_path25.default.basename(subagentDir),
|
|
10087
10133
|
subagentDir,
|
|
10088
10134
|
vscodeCmd
|
|
10089
10135
|
);
|
|
10090
10136
|
if (!workspaceReady) {
|
|
10091
10137
|
throw new Error(
|
|
10092
|
-
`VS Code workspace '${
|
|
10138
|
+
`VS Code workspace '${import_node_path25.default.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
10093
10139
|
);
|
|
10094
10140
|
}
|
|
10095
10141
|
await sleep2(500);
|
|
10096
|
-
const child = spawnVsCode(
|
|
10142
|
+
const child = spawnVsCode(vscodeCmd, chatArgs, { label: "send-chat" });
|
|
10097
10143
|
await raceSpawnError(child);
|
|
10098
10144
|
}
|
|
10099
10145
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
10100
|
-
const workspacePath =
|
|
10101
|
-
const messagesDir =
|
|
10146
|
+
const workspacePath = import_node_path25.default.join(subagentDir, `${import_node_path25.default.basename(subagentDir)}.code-workspace`);
|
|
10147
|
+
const messagesDir = import_node_path25.default.join(subagentDir, "messages");
|
|
10102
10148
|
await (0, import_promises18.mkdir)(messagesDir, { recursive: true });
|
|
10103
10149
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
10104
10150
|
for (const attachment of attachmentPaths) {
|
|
@@ -10107,26 +10153,26 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
10107
10153
|
chatArgs.push(chatInstruction);
|
|
10108
10154
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
10109
10155
|
workspacePath,
|
|
10110
|
-
|
|
10156
|
+
import_node_path25.default.basename(subagentDir),
|
|
10111
10157
|
subagentDir,
|
|
10112
10158
|
vscodeCmd
|
|
10113
10159
|
);
|
|
10114
10160
|
if (!workspaceReady) {
|
|
10115
10161
|
throw new Error(
|
|
10116
|
-
`VS Code workspace '${
|
|
10162
|
+
`VS Code workspace '${import_node_path25.default.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
10117
10163
|
);
|
|
10118
10164
|
}
|
|
10119
10165
|
await sleep2(500);
|
|
10120
|
-
const child = spawnVsCode(
|
|
10166
|
+
const child = spawnVsCode(vscodeCmd, chatArgs, { label: "send-batch-chat" });
|
|
10121
10167
|
await raceSpawnError(child);
|
|
10122
10168
|
}
|
|
10123
10169
|
|
|
10124
10170
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
10125
10171
|
var import_promises19 = require("fs/promises");
|
|
10126
|
-
var
|
|
10172
|
+
var import_node_path27 = __toESM(require("path"), 1);
|
|
10127
10173
|
|
|
10128
10174
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
10129
|
-
var
|
|
10175
|
+
var import_node_path26 = __toESM(require("path"), 1);
|
|
10130
10176
|
var import_json5 = __toESM(require("json5"), 1);
|
|
10131
10177
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
10132
10178
|
let workspace;
|
|
@@ -10143,10 +10189,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
10143
10189
|
}
|
|
10144
10190
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
10145
10191
|
const folderPath = folder.path;
|
|
10146
|
-
if (
|
|
10192
|
+
if (import_node_path26.default.isAbsolute(folderPath)) {
|
|
10147
10193
|
return folder;
|
|
10148
10194
|
}
|
|
10149
|
-
const absolutePath =
|
|
10195
|
+
const absolutePath = import_node_path26.default.resolve(templateDir, folderPath);
|
|
10150
10196
|
return {
|
|
10151
10197
|
...folder,
|
|
10152
10198
|
path: absolutePath
|
|
@@ -10168,19 +10214,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
10168
10214
|
if (locationMap && typeof locationMap === "object") {
|
|
10169
10215
|
const transformedMap = {};
|
|
10170
10216
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
10171
|
-
const isAbsolute =
|
|
10217
|
+
const isAbsolute = import_node_path26.default.isAbsolute(locationPath);
|
|
10172
10218
|
if (isAbsolute) {
|
|
10173
10219
|
transformedMap[locationPath] = value;
|
|
10174
10220
|
} else {
|
|
10175
10221
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
10176
10222
|
if (firstGlobIndex === -1) {
|
|
10177
|
-
const resolvedPath =
|
|
10223
|
+
const resolvedPath = import_node_path26.default.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
10178
10224
|
transformedMap[resolvedPath] = value;
|
|
10179
10225
|
} else {
|
|
10180
10226
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
10181
10227
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
10182
10228
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
10183
|
-
const resolvedPath = (
|
|
10229
|
+
const resolvedPath = (import_node_path26.default.resolve(templateDir, basePath) + patternPath).replace(
|
|
10184
10230
|
/\\/g,
|
|
10185
10231
|
"/"
|
|
10186
10232
|
);
|
|
@@ -10221,7 +10267,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
10221
10267
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
10222
10268
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
10223
10269
|
for (const subagent of subagents) {
|
|
10224
|
-
const lockFile =
|
|
10270
|
+
const lockFile = import_node_path27.default.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
10225
10271
|
if (!await pathExists(lockFile)) {
|
|
10226
10272
|
return subagent.absolutePath;
|
|
10227
10273
|
}
|
|
@@ -10231,7 +10277,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
10231
10277
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
10232
10278
|
let workspaceContent;
|
|
10233
10279
|
if (workspaceTemplate) {
|
|
10234
|
-
const workspaceSrc =
|
|
10280
|
+
const workspaceSrc = import_node_path27.default.resolve(workspaceTemplate);
|
|
10235
10281
|
if (!await pathExists(workspaceSrc)) {
|
|
10236
10282
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
10237
10283
|
}
|
|
@@ -10244,13 +10290,13 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
10244
10290
|
} else {
|
|
10245
10291
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
10246
10292
|
}
|
|
10247
|
-
const workspaceName = `${
|
|
10248
|
-
const workspaceDst =
|
|
10249
|
-
const templateDir = workspaceTemplate ?
|
|
10293
|
+
const workspaceName = `${import_node_path27.default.basename(subagentDir)}.code-workspace`;
|
|
10294
|
+
const workspaceDst = import_node_path27.default.join(subagentDir, workspaceName);
|
|
10295
|
+
const templateDir = workspaceTemplate ? import_node_path27.default.dirname(import_node_path27.default.resolve(workspaceTemplate)) : subagentDir;
|
|
10250
10296
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
10251
10297
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
10252
10298
|
if (cwd) {
|
|
10253
|
-
const absCwd =
|
|
10299
|
+
const absCwd = import_node_path27.default.resolve(cwd);
|
|
10254
10300
|
const parsed = JSON.parse(transformedContent);
|
|
10255
10301
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
10256
10302
|
if (!alreadyPresent) {
|
|
@@ -10259,35 +10305,35 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
10259
10305
|
}
|
|
10260
10306
|
}
|
|
10261
10307
|
await (0, import_promises19.writeFile)(workspaceDst, transformedContent, "utf8");
|
|
10262
|
-
const messagesDir =
|
|
10308
|
+
const messagesDir = import_node_path27.default.join(subagentDir, "messages");
|
|
10263
10309
|
await (0, import_promises19.mkdir)(messagesDir, { recursive: true });
|
|
10264
10310
|
return { workspace: workspaceDst, messagesDir };
|
|
10265
10311
|
}
|
|
10266
10312
|
async function createSubagentLock(subagentDir) {
|
|
10267
|
-
const messagesDir =
|
|
10313
|
+
const messagesDir = import_node_path27.default.join(subagentDir, "messages");
|
|
10268
10314
|
if (await pathExists(messagesDir)) {
|
|
10269
10315
|
const files = await (0, import_promises19.readdir)(messagesDir);
|
|
10270
10316
|
await Promise.all(
|
|
10271
10317
|
files.map(async (file) => {
|
|
10272
|
-
const target =
|
|
10318
|
+
const target = import_node_path27.default.join(messagesDir, file);
|
|
10273
10319
|
await removeIfExists(target);
|
|
10274
10320
|
})
|
|
10275
10321
|
);
|
|
10276
10322
|
}
|
|
10277
|
-
const githubAgentsDir =
|
|
10323
|
+
const githubAgentsDir = import_node_path27.default.join(subagentDir, ".github", "agents");
|
|
10278
10324
|
if (await pathExists(githubAgentsDir)) {
|
|
10279
10325
|
const agentFiles = await (0, import_promises19.readdir)(githubAgentsDir);
|
|
10280
10326
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
10281
10327
|
await Promise.all(
|
|
10282
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
10328
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(import_node_path27.default.join(githubAgentsDir, file)))
|
|
10283
10329
|
);
|
|
10284
10330
|
}
|
|
10285
|
-
const lockFile =
|
|
10331
|
+
const lockFile = import_node_path27.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
10286
10332
|
await (0, import_promises19.writeFile)(lockFile, "", { encoding: "utf8" });
|
|
10287
10333
|
return lockFile;
|
|
10288
10334
|
}
|
|
10289
10335
|
async function removeSubagentLock(subagentDir) {
|
|
10290
|
-
const lockFile =
|
|
10336
|
+
const lockFile = import_node_path27.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
10291
10337
|
await removeIfExists(lockFile);
|
|
10292
10338
|
}
|
|
10293
10339
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -10307,9 +10353,9 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
10307
10353
|
return 1;
|
|
10308
10354
|
}
|
|
10309
10355
|
if (promptFile) {
|
|
10310
|
-
const githubAgentsDir =
|
|
10356
|
+
const githubAgentsDir = import_node_path27.default.join(subagentDir, ".github", "agents");
|
|
10311
10357
|
await (0, import_promises19.mkdir)(githubAgentsDir, { recursive: true });
|
|
10312
|
-
const agentFile =
|
|
10358
|
+
const agentFile = import_node_path27.default.join(githubAgentsDir, `${chatId}.md`);
|
|
10313
10359
|
try {
|
|
10314
10360
|
await (0, import_promises19.copyFile)(promptFile, agentFile);
|
|
10315
10361
|
} catch (error) {
|
|
@@ -10328,7 +10374,7 @@ async function resolvePromptFile(promptFile) {
|
|
|
10328
10374
|
if (!promptFile) {
|
|
10329
10375
|
return void 0;
|
|
10330
10376
|
}
|
|
10331
|
-
const resolvedPrompt =
|
|
10377
|
+
const resolvedPrompt = import_node_path28.default.resolve(promptFile);
|
|
10332
10378
|
if (!await pathExists(resolvedPrompt)) {
|
|
10333
10379
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
10334
10380
|
}
|
|
@@ -10344,7 +10390,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
10344
10390
|
}
|
|
10345
10391
|
const resolved = [];
|
|
10346
10392
|
for (const attachment of extraAttachments) {
|
|
10347
|
-
const resolvedPath =
|
|
10393
|
+
const resolvedPath = import_node_path28.default.resolve(attachment);
|
|
10348
10394
|
if (!await pathExists(resolvedPath)) {
|
|
10349
10395
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
10350
10396
|
}
|
|
@@ -10386,7 +10432,7 @@ async function dispatchAgentSession(options) {
|
|
|
10386
10432
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
10387
10433
|
};
|
|
10388
10434
|
}
|
|
10389
|
-
const subagentName =
|
|
10435
|
+
const subagentName = import_node_path28.default.basename(subagentDir);
|
|
10390
10436
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
10391
10437
|
const preparationResult = await prepareSubagentDirectory(
|
|
10392
10438
|
subagentDir,
|
|
@@ -10414,9 +10460,9 @@ async function dispatchAgentSession(options) {
|
|
|
10414
10460
|
};
|
|
10415
10461
|
}
|
|
10416
10462
|
const timestamp = generateTimestamp();
|
|
10417
|
-
const messagesDir =
|
|
10418
|
-
const responseFileTmp =
|
|
10419
|
-
const responseFileFinal =
|
|
10463
|
+
const messagesDir = import_node_path28.default.join(subagentDir, "messages");
|
|
10464
|
+
const responseFileTmp = import_node_path28.default.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
10465
|
+
const responseFileFinal = import_node_path28.default.join(messagesDir, `${timestamp}_res.md`);
|
|
10420
10466
|
const requestInstructions = createRequestPrompt(
|
|
10421
10467
|
userQuery,
|
|
10422
10468
|
responseFileTmp,
|
|
@@ -10521,7 +10567,7 @@ async function dispatchBatchAgent(options) {
|
|
|
10521
10567
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
10522
10568
|
};
|
|
10523
10569
|
}
|
|
10524
|
-
subagentName =
|
|
10570
|
+
subagentName = import_node_path28.default.basename(subagentDir);
|
|
10525
10571
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
10526
10572
|
const preparationResult = await prepareSubagentDirectory(
|
|
10527
10573
|
subagentDir,
|
|
@@ -10552,17 +10598,17 @@ async function dispatchBatchAgent(options) {
|
|
|
10552
10598
|
};
|
|
10553
10599
|
}
|
|
10554
10600
|
const timestamp = generateTimestamp();
|
|
10555
|
-
const messagesDir =
|
|
10601
|
+
const messagesDir = import_node_path28.default.join(subagentDir, "messages");
|
|
10556
10602
|
requestFiles = userQueries.map(
|
|
10557
|
-
(_, index) =>
|
|
10603
|
+
(_, index) => import_node_path28.default.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
10558
10604
|
);
|
|
10559
10605
|
const responseTmpFiles = userQueries.map(
|
|
10560
|
-
(_, index) =>
|
|
10606
|
+
(_, index) => import_node_path28.default.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
10561
10607
|
);
|
|
10562
10608
|
responseFilesFinal = userQueries.map(
|
|
10563
|
-
(_, index) =>
|
|
10609
|
+
(_, index) => import_node_path28.default.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
10564
10610
|
);
|
|
10565
|
-
const orchestratorFile =
|
|
10611
|
+
const orchestratorFile = import_node_path28.default.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
10566
10612
|
if (!dryRun) {
|
|
10567
10613
|
await Promise.all(
|
|
10568
10614
|
userQueries.map((query, index) => {
|
|
@@ -10648,7 +10694,7 @@ async function dispatchBatchAgent(options) {
|
|
|
10648
10694
|
|
|
10649
10695
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
10650
10696
|
var import_promises21 = require("fs/promises");
|
|
10651
|
-
var
|
|
10697
|
+
var import_node_path29 = __toESM(require("path"), 1);
|
|
10652
10698
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
10653
10699
|
folders: [
|
|
10654
10700
|
{
|
|
@@ -10679,7 +10725,7 @@ async function provisionSubagents(options) {
|
|
|
10679
10725
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
10680
10726
|
throw new Error("subagents must be a positive integer");
|
|
10681
10727
|
}
|
|
10682
|
-
const targetPath =
|
|
10728
|
+
const targetPath = import_node_path29.default.resolve(targetRoot);
|
|
10683
10729
|
if (!dryRun) {
|
|
10684
10730
|
await ensureDir(targetPath);
|
|
10685
10731
|
}
|
|
@@ -10699,7 +10745,7 @@ async function provisionSubagents(options) {
|
|
|
10699
10745
|
continue;
|
|
10700
10746
|
}
|
|
10701
10747
|
highestNumber = Math.max(highestNumber, parsed);
|
|
10702
|
-
const lockFile =
|
|
10748
|
+
const lockFile = import_node_path29.default.join(entry.absolutePath, lockName);
|
|
10703
10749
|
const locked = await pathExists(lockFile);
|
|
10704
10750
|
if (locked) {
|
|
10705
10751
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -10716,10 +10762,10 @@ async function provisionSubagents(options) {
|
|
|
10716
10762
|
break;
|
|
10717
10763
|
}
|
|
10718
10764
|
const subagentDir = subagent.absolutePath;
|
|
10719
|
-
const githubAgentsDir =
|
|
10720
|
-
const lockFile =
|
|
10721
|
-
const workspaceDst =
|
|
10722
|
-
const wakeupDst =
|
|
10765
|
+
const githubAgentsDir = import_node_path29.default.join(subagentDir, ".github", "agents");
|
|
10766
|
+
const lockFile = import_node_path29.default.join(subagentDir, lockName);
|
|
10767
|
+
const workspaceDst = import_node_path29.default.join(subagentDir, `${import_node_path29.default.basename(subagentDir)}.code-workspace`);
|
|
10768
|
+
const wakeupDst = import_node_path29.default.join(githubAgentsDir, "wakeup.md");
|
|
10723
10769
|
const isLocked = await pathExists(lockFile);
|
|
10724
10770
|
if (isLocked && !force) {
|
|
10725
10771
|
continue;
|
|
@@ -10757,10 +10803,10 @@ async function provisionSubagents(options) {
|
|
|
10757
10803
|
let nextIndex = highestNumber;
|
|
10758
10804
|
while (subagentsProvisioned < subagents) {
|
|
10759
10805
|
nextIndex += 1;
|
|
10760
|
-
const subagentDir =
|
|
10761
|
-
const githubAgentsDir =
|
|
10762
|
-
const workspaceDst =
|
|
10763
|
-
const wakeupDst =
|
|
10806
|
+
const subagentDir = import_node_path29.default.join(targetPath, `subagent-${nextIndex}`);
|
|
10807
|
+
const githubAgentsDir = import_node_path29.default.join(subagentDir, ".github", "agents");
|
|
10808
|
+
const workspaceDst = import_node_path29.default.join(subagentDir, `${import_node_path29.default.basename(subagentDir)}.code-workspace`);
|
|
10809
|
+
const wakeupDst = import_node_path29.default.join(githubAgentsDir, "wakeup.md");
|
|
10764
10810
|
if (!dryRun) {
|
|
10765
10811
|
await ensureDir(subagentDir);
|
|
10766
10812
|
await ensureDir(githubAgentsDir);
|
|
@@ -10954,7 +11000,7 @@ var VSCodeProvider = class {
|
|
|
10954
11000
|
async function locateVSCodeExecutable(candidate) {
|
|
10955
11001
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
10956
11002
|
if (includesPathSeparator) {
|
|
10957
|
-
const resolved =
|
|
11003
|
+
const resolved = import_node_path30.default.isAbsolute(candidate) ? candidate : import_node_path30.default.resolve(candidate);
|
|
10958
11004
|
try {
|
|
10959
11005
|
await (0, import_promises22.access)(resolved, import_promises22.constants.F_OK);
|
|
10960
11006
|
return resolved;
|
|
@@ -10983,7 +11029,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
10983
11029
|
return void 0;
|
|
10984
11030
|
}
|
|
10985
11031
|
try {
|
|
10986
|
-
const stats = await (0, import_promises22.stat)(
|
|
11032
|
+
const stats = await (0, import_promises22.stat)(import_node_path30.default.resolve(template));
|
|
10987
11033
|
return stats.isFile() ? template : void 0;
|
|
10988
11034
|
} catch {
|
|
10989
11035
|
return template;
|
|
@@ -11009,7 +11055,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
11009
11055
|
return "";
|
|
11010
11056
|
}
|
|
11011
11057
|
const buildList = (files) => files.map((absolutePath) => {
|
|
11012
|
-
const fileName =
|
|
11058
|
+
const fileName = import_node_path30.default.basename(absolutePath);
|
|
11013
11059
|
const fileUri = pathToFileUri3(absolutePath);
|
|
11014
11060
|
return `* [${fileName}](${fileUri})`;
|
|
11015
11061
|
});
|
|
@@ -11034,8 +11080,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
11034
11080
|
}
|
|
11035
11081
|
const unique = /* @__PURE__ */ new Map();
|
|
11036
11082
|
for (const attachment of attachments) {
|
|
11037
|
-
const absolutePath =
|
|
11038
|
-
const normalized = absolutePath.split(
|
|
11083
|
+
const absolutePath = import_node_path30.default.resolve(attachment);
|
|
11084
|
+
const normalized = absolutePath.split(import_node_path30.default.sep).join("/");
|
|
11039
11085
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
11040
11086
|
if (!unique.has(absolutePath)) {
|
|
11041
11087
|
unique.set(absolutePath, absolutePath);
|
|
@@ -11050,7 +11096,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
11050
11096
|
}
|
|
11051
11097
|
const unique = /* @__PURE__ */ new Map();
|
|
11052
11098
|
for (const attachment of attachments) {
|
|
11053
|
-
const absolutePath =
|
|
11099
|
+
const absolutePath = import_node_path30.default.resolve(attachment);
|
|
11054
11100
|
if (!unique.has(absolutePath)) {
|
|
11055
11101
|
unique.set(absolutePath, absolutePath);
|
|
11056
11102
|
}
|
|
@@ -11058,7 +11104,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
11058
11104
|
return Array.from(unique.values());
|
|
11059
11105
|
}
|
|
11060
11106
|
function pathToFileUri3(filePath) {
|
|
11061
|
-
const absolutePath =
|
|
11107
|
+
const absolutePath = import_node_path30.default.isAbsolute(filePath) ? filePath : import_node_path30.default.resolve(filePath);
|
|
11062
11108
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
11063
11109
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
11064
11110
|
return `file:///${normalizedPath}`;
|
|
@@ -11071,7 +11117,7 @@ function normalizeAttachments(attachments) {
|
|
|
11071
11117
|
}
|
|
11072
11118
|
const deduped = /* @__PURE__ */ new Set();
|
|
11073
11119
|
for (const attachment of attachments) {
|
|
11074
|
-
deduped.add(
|
|
11120
|
+
deduped.add(import_node_path30.default.resolve(attachment));
|
|
11075
11121
|
}
|
|
11076
11122
|
return Array.from(deduped);
|
|
11077
11123
|
}
|
|
@@ -11080,7 +11126,7 @@ function mergeAttachments(all) {
|
|
|
11080
11126
|
for (const list of all) {
|
|
11081
11127
|
if (!list) continue;
|
|
11082
11128
|
for (const inputFile of list) {
|
|
11083
|
-
deduped.add(
|
|
11129
|
+
deduped.add(import_node_path30.default.resolve(inputFile));
|
|
11084
11130
|
}
|
|
11085
11131
|
}
|
|
11086
11132
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -11129,7 +11175,7 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
11129
11175
|
// src/evaluation/providers/targets-file.ts
|
|
11130
11176
|
var import_node_fs8 = require("fs");
|
|
11131
11177
|
var import_promises23 = require("fs/promises");
|
|
11132
|
-
var
|
|
11178
|
+
var import_node_path31 = __toESM(require("path"), 1);
|
|
11133
11179
|
var import_yaml5 = require("yaml");
|
|
11134
11180
|
function isRecord(value) {
|
|
11135
11181
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -11166,7 +11212,7 @@ async function fileExists3(filePath) {
|
|
|
11166
11212
|
}
|
|
11167
11213
|
}
|
|
11168
11214
|
async function readTargetDefinitions(filePath) {
|
|
11169
|
-
const absolutePath =
|
|
11215
|
+
const absolutePath = import_node_path31.default.resolve(filePath);
|
|
11170
11216
|
if (!await fileExists3(absolutePath)) {
|
|
11171
11217
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
11172
11218
|
}
|
|
@@ -11186,16 +11232,16 @@ function listTargetNames(definitions) {
|
|
|
11186
11232
|
}
|
|
11187
11233
|
|
|
11188
11234
|
// src/evaluation/providers/provider-discovery.ts
|
|
11189
|
-
var
|
|
11235
|
+
var import_node_path32 = __toESM(require("path"), 1);
|
|
11190
11236
|
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
11191
11237
|
async function discoverProviders(registry, baseDir) {
|
|
11192
11238
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
11193
11239
|
const candidateDirs = [];
|
|
11194
|
-
let dir =
|
|
11195
|
-
const root =
|
|
11240
|
+
let dir = import_node_path32.default.resolve(baseDir);
|
|
11241
|
+
const root = import_node_path32.default.parse(dir).root;
|
|
11196
11242
|
while (dir !== root) {
|
|
11197
|
-
candidateDirs.push(
|
|
11198
|
-
dir =
|
|
11243
|
+
candidateDirs.push(import_node_path32.default.join(dir, ".agentv", "providers"));
|
|
11244
|
+
dir = import_node_path32.default.dirname(dir);
|
|
11199
11245
|
}
|
|
11200
11246
|
let files = [];
|
|
11201
11247
|
for (const providersDir of candidateDirs) {
|
|
@@ -11211,7 +11257,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
11211
11257
|
}
|
|
11212
11258
|
const discoveredKinds = [];
|
|
11213
11259
|
for (const filePath of files) {
|
|
11214
|
-
const basename =
|
|
11260
|
+
const basename = import_node_path32.default.basename(filePath);
|
|
11215
11261
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
11216
11262
|
if (registry.has(kindName)) {
|
|
11217
11263
|
continue;
|
|
@@ -11318,7 +11364,7 @@ function negateScore(score) {
|
|
|
11318
11364
|
// src/evaluation/evaluators/code-evaluator.ts
|
|
11319
11365
|
var import_promises24 = require("fs/promises");
|
|
11320
11366
|
var import_node_os5 = require("os");
|
|
11321
|
-
var
|
|
11367
|
+
var import_node_path33 = require("path");
|
|
11322
11368
|
|
|
11323
11369
|
// src/runtime/exec.ts
|
|
11324
11370
|
function shellEscapePath(value) {
|
|
@@ -11420,13 +11466,13 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
11420
11466
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
11421
11467
|
const { mkdir: mkdir16, readFile: readFile13, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
|
|
11422
11468
|
const { tmpdir: tmpdir3 } = await import("os");
|
|
11423
|
-
const
|
|
11469
|
+
const path43 = await import("path");
|
|
11424
11470
|
const { randomUUID: randomUUID8 } = await import("crypto");
|
|
11425
|
-
const dir =
|
|
11471
|
+
const dir = path43.join(tmpdir3(), `agentv-exec-${randomUUID8()}`);
|
|
11426
11472
|
await mkdir16(dir, { recursive: true });
|
|
11427
|
-
const stdinPath =
|
|
11428
|
-
const stdoutPath =
|
|
11429
|
-
const stderrPath =
|
|
11473
|
+
const stdinPath = path43.join(dir, "stdin.txt");
|
|
11474
|
+
const stdoutPath = path43.join(dir, "stdout.txt");
|
|
11475
|
+
const stderrPath = path43.join(dir, "stderr.txt");
|
|
11430
11476
|
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
11431
11477
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
11432
11478
|
const { spawn: spawn4 } = await import("child_process");
|
|
@@ -11762,8 +11808,8 @@ var CodeEvaluator = class {
|
|
|
11762
11808
|
if (outputForPayload) {
|
|
11763
11809
|
const serialized = JSON.stringify(outputForPayload);
|
|
11764
11810
|
if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) {
|
|
11765
|
-
const tmpDir = await (0, import_promises24.mkdtemp)((0,
|
|
11766
|
-
outputPath = (0,
|
|
11811
|
+
const tmpDir = await (0, import_promises24.mkdtemp)((0, import_node_path33.join)((0, import_node_os5.tmpdir)(), "agentv-judge-"));
|
|
11812
|
+
outputPath = (0, import_node_path33.join)(tmpDir, "output.json");
|
|
11767
11813
|
await (0, import_promises24.writeFile)(outputPath, serialized);
|
|
11768
11814
|
outputForPayload = null;
|
|
11769
11815
|
}
|
|
@@ -11778,7 +11824,7 @@ var CodeEvaluator = class {
|
|
|
11778
11824
|
outputPath,
|
|
11779
11825
|
guidelineFiles: context2.evalCase.guideline_paths,
|
|
11780
11826
|
inputFiles: context2.evalCase.file_paths.filter(
|
|
11781
|
-
(
|
|
11827
|
+
(path43) => !context2.evalCase.guideline_paths.includes(path43)
|
|
11782
11828
|
),
|
|
11783
11829
|
input: context2.evalCase.input,
|
|
11784
11830
|
trace: context2.trace ?? null,
|
|
@@ -11876,7 +11922,7 @@ var CodeEvaluator = class {
|
|
|
11876
11922
|
await proxyShutdown();
|
|
11877
11923
|
}
|
|
11878
11924
|
if (outputPath) {
|
|
11879
|
-
await (0, import_promises24.rm)((0,
|
|
11925
|
+
await (0, import_promises24.rm)((0, import_node_path33.dirname)(outputPath), { recursive: true, force: true }).catch(() => {
|
|
11880
11926
|
});
|
|
11881
11927
|
}
|
|
11882
11928
|
}
|
|
@@ -13005,115 +13051,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
13005
13051
|
* Evaluate a single field against the expected value.
|
|
13006
13052
|
*/
|
|
13007
13053
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
13008
|
-
const { path:
|
|
13009
|
-
const candidateValue = resolvePath(candidateData,
|
|
13010
|
-
const expectedValue = resolvePath(expectedData,
|
|
13054
|
+
const { path: path43, match, required = true, weight = 1 } = fieldConfig;
|
|
13055
|
+
const candidateValue = resolvePath(candidateData, path43);
|
|
13056
|
+
const expectedValue = resolvePath(expectedData, path43);
|
|
13011
13057
|
if (expectedValue === void 0) {
|
|
13012
13058
|
return {
|
|
13013
|
-
path:
|
|
13059
|
+
path: path43,
|
|
13014
13060
|
score: 1,
|
|
13015
13061
|
// No expected value means no comparison needed
|
|
13016
13062
|
weight,
|
|
13017
13063
|
hit: true,
|
|
13018
|
-
message: `${
|
|
13064
|
+
message: `${path43}: no expected value`
|
|
13019
13065
|
};
|
|
13020
13066
|
}
|
|
13021
13067
|
if (candidateValue === void 0) {
|
|
13022
13068
|
if (required) {
|
|
13023
13069
|
return {
|
|
13024
|
-
path:
|
|
13070
|
+
path: path43,
|
|
13025
13071
|
score: 0,
|
|
13026
13072
|
weight,
|
|
13027
13073
|
hit: false,
|
|
13028
|
-
message: `${
|
|
13074
|
+
message: `${path43} (required, missing)`
|
|
13029
13075
|
};
|
|
13030
13076
|
}
|
|
13031
13077
|
return {
|
|
13032
|
-
path:
|
|
13078
|
+
path: path43,
|
|
13033
13079
|
score: 1,
|
|
13034
13080
|
// Don't penalize missing optional fields
|
|
13035
13081
|
weight: 0,
|
|
13036
13082
|
// Zero weight means it won't affect the score
|
|
13037
13083
|
hit: true,
|
|
13038
|
-
message: `${
|
|
13084
|
+
message: `${path43}: optional field missing`
|
|
13039
13085
|
};
|
|
13040
13086
|
}
|
|
13041
13087
|
switch (match) {
|
|
13042
13088
|
case "exact":
|
|
13043
|
-
return this.compareExact(
|
|
13089
|
+
return this.compareExact(path43, candidateValue, expectedValue, weight);
|
|
13044
13090
|
case "numeric_tolerance":
|
|
13045
13091
|
return this.compareNumericTolerance(
|
|
13046
|
-
|
|
13092
|
+
path43,
|
|
13047
13093
|
candidateValue,
|
|
13048
13094
|
expectedValue,
|
|
13049
13095
|
fieldConfig,
|
|
13050
13096
|
weight
|
|
13051
13097
|
);
|
|
13052
13098
|
case "date":
|
|
13053
|
-
return this.compareDate(
|
|
13099
|
+
return this.compareDate(path43, candidateValue, expectedValue, fieldConfig, weight);
|
|
13054
13100
|
default:
|
|
13055
13101
|
return {
|
|
13056
|
-
path:
|
|
13102
|
+
path: path43,
|
|
13057
13103
|
score: 0,
|
|
13058
13104
|
weight,
|
|
13059
13105
|
hit: false,
|
|
13060
|
-
message: `${
|
|
13106
|
+
message: `${path43}: unknown match type "${match}"`
|
|
13061
13107
|
};
|
|
13062
13108
|
}
|
|
13063
13109
|
}
|
|
13064
13110
|
/**
|
|
13065
13111
|
* Exact equality comparison.
|
|
13066
13112
|
*/
|
|
13067
|
-
compareExact(
|
|
13113
|
+
compareExact(path43, candidateValue, expectedValue, weight) {
|
|
13068
13114
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
13069
13115
|
return {
|
|
13070
|
-
path:
|
|
13116
|
+
path: path43,
|
|
13071
13117
|
score: 1,
|
|
13072
13118
|
weight,
|
|
13073
13119
|
hit: true,
|
|
13074
|
-
message:
|
|
13120
|
+
message: path43
|
|
13075
13121
|
};
|
|
13076
13122
|
}
|
|
13077
13123
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
13078
13124
|
return {
|
|
13079
|
-
path:
|
|
13125
|
+
path: path43,
|
|
13080
13126
|
score: 0,
|
|
13081
13127
|
weight,
|
|
13082
13128
|
hit: false,
|
|
13083
|
-
message: `${
|
|
13129
|
+
message: `${path43} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
13084
13130
|
};
|
|
13085
13131
|
}
|
|
13086
13132
|
return {
|
|
13087
|
-
path:
|
|
13133
|
+
path: path43,
|
|
13088
13134
|
score: 0,
|
|
13089
13135
|
weight,
|
|
13090
13136
|
hit: false,
|
|
13091
|
-
message: `${
|
|
13137
|
+
message: `${path43} (value mismatch)`
|
|
13092
13138
|
};
|
|
13093
13139
|
}
|
|
13094
13140
|
/**
|
|
13095
13141
|
* Numeric comparison with absolute or relative tolerance.
|
|
13096
13142
|
*/
|
|
13097
|
-
compareNumericTolerance(
|
|
13143
|
+
compareNumericTolerance(path43, candidateValue, expectedValue, fieldConfig, weight) {
|
|
13098
13144
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
13099
13145
|
const candidateNum = toNumber2(candidateValue);
|
|
13100
13146
|
const expectedNum = toNumber2(expectedValue);
|
|
13101
13147
|
if (candidateNum === null || expectedNum === null) {
|
|
13102
13148
|
return {
|
|
13103
|
-
path:
|
|
13149
|
+
path: path43,
|
|
13104
13150
|
score: 0,
|
|
13105
13151
|
weight,
|
|
13106
13152
|
hit: false,
|
|
13107
|
-
message: `${
|
|
13153
|
+
message: `${path43} (non-numeric value)`
|
|
13108
13154
|
};
|
|
13109
13155
|
}
|
|
13110
13156
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
13111
13157
|
return {
|
|
13112
|
-
path:
|
|
13158
|
+
path: path43,
|
|
13113
13159
|
score: 0,
|
|
13114
13160
|
weight,
|
|
13115
13161
|
hit: false,
|
|
13116
|
-
message: `${
|
|
13162
|
+
message: `${path43} (invalid numeric value)`
|
|
13117
13163
|
};
|
|
13118
13164
|
}
|
|
13119
13165
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -13126,61 +13172,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
13126
13172
|
}
|
|
13127
13173
|
if (withinTolerance) {
|
|
13128
13174
|
return {
|
|
13129
|
-
path:
|
|
13175
|
+
path: path43,
|
|
13130
13176
|
score: 1,
|
|
13131
13177
|
weight,
|
|
13132
13178
|
hit: true,
|
|
13133
|
-
message: `${
|
|
13179
|
+
message: `${path43} (within tolerance: diff=${diff.toFixed(2)})`
|
|
13134
13180
|
};
|
|
13135
13181
|
}
|
|
13136
13182
|
return {
|
|
13137
|
-
path:
|
|
13183
|
+
path: path43,
|
|
13138
13184
|
score: 0,
|
|
13139
13185
|
weight,
|
|
13140
13186
|
hit: false,
|
|
13141
|
-
message: `${
|
|
13187
|
+
message: `${path43} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
13142
13188
|
};
|
|
13143
13189
|
}
|
|
13144
13190
|
/**
|
|
13145
13191
|
* Date comparison with format normalization.
|
|
13146
13192
|
*/
|
|
13147
|
-
compareDate(
|
|
13193
|
+
compareDate(path43, candidateValue, expectedValue, fieldConfig, weight) {
|
|
13148
13194
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
13149
13195
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
13150
13196
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
13151
13197
|
if (candidateDate === null) {
|
|
13152
13198
|
return {
|
|
13153
|
-
path:
|
|
13199
|
+
path: path43,
|
|
13154
13200
|
score: 0,
|
|
13155
13201
|
weight,
|
|
13156
13202
|
hit: false,
|
|
13157
|
-
message: `${
|
|
13203
|
+
message: `${path43} (unparseable candidate date)`
|
|
13158
13204
|
};
|
|
13159
13205
|
}
|
|
13160
13206
|
if (expectedDate === null) {
|
|
13161
13207
|
return {
|
|
13162
|
-
path:
|
|
13208
|
+
path: path43,
|
|
13163
13209
|
score: 0,
|
|
13164
13210
|
weight,
|
|
13165
13211
|
hit: false,
|
|
13166
|
-
message: `${
|
|
13212
|
+
message: `${path43} (unparseable expected date)`
|
|
13167
13213
|
};
|
|
13168
13214
|
}
|
|
13169
13215
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
13170
13216
|
return {
|
|
13171
|
-
path:
|
|
13217
|
+
path: path43,
|
|
13172
13218
|
score: 1,
|
|
13173
13219
|
weight,
|
|
13174
13220
|
hit: true,
|
|
13175
|
-
message:
|
|
13221
|
+
message: path43
|
|
13176
13222
|
};
|
|
13177
13223
|
}
|
|
13178
13224
|
return {
|
|
13179
|
-
path:
|
|
13225
|
+
path: path43,
|
|
13180
13226
|
score: 0,
|
|
13181
13227
|
weight,
|
|
13182
13228
|
hit: false,
|
|
13183
|
-
message: `${
|
|
13229
|
+
message: `${path43} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
13184
13230
|
};
|
|
13185
13231
|
}
|
|
13186
13232
|
/**
|
|
@@ -13221,11 +13267,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
13221
13267
|
};
|
|
13222
13268
|
}
|
|
13223
13269
|
};
|
|
13224
|
-
function resolvePath(obj,
|
|
13225
|
-
if (!
|
|
13270
|
+
function resolvePath(obj, path43) {
|
|
13271
|
+
if (!path43 || !obj) {
|
|
13226
13272
|
return void 0;
|
|
13227
13273
|
}
|
|
13228
|
-
const parts =
|
|
13274
|
+
const parts = path43.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
13229
13275
|
let current = obj;
|
|
13230
13276
|
for (const part of parts) {
|
|
13231
13277
|
if (current === null || current === void 0) {
|
|
@@ -13357,7 +13403,7 @@ var LatencyEvaluator = class {
|
|
|
13357
13403
|
|
|
13358
13404
|
// src/evaluation/evaluators/agent-judge.ts
|
|
13359
13405
|
var import_promises25 = __toESM(require("fs/promises"), 1);
|
|
13360
|
-
var
|
|
13406
|
+
var import_node_path34 = __toESM(require("path"), 1);
|
|
13361
13407
|
var import_ai4 = require("ai");
|
|
13362
13408
|
var import_zod5 = require("zod");
|
|
13363
13409
|
var DEFAULT_MAX_STEPS = 10;
|
|
@@ -13706,8 +13752,8 @@ ${outputSchema}`;
|
|
|
13706
13752
|
}
|
|
13707
13753
|
};
|
|
13708
13754
|
function resolveSandboxed(basePath, relativePath) {
|
|
13709
|
-
const resolved =
|
|
13710
|
-
if (!resolved.startsWith(basePath +
|
|
13755
|
+
const resolved = import_node_path34.default.resolve(basePath, relativePath);
|
|
13756
|
+
if (!resolved.startsWith(basePath + import_node_path34.default.sep) && resolved !== basePath) {
|
|
13711
13757
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
13712
13758
|
}
|
|
13713
13759
|
return resolved;
|
|
@@ -13790,11 +13836,11 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
13790
13836
|
for (const entry of entries) {
|
|
13791
13837
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
13792
13838
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
13793
|
-
const fullPath =
|
|
13839
|
+
const fullPath = import_node_path34.default.join(dirPath, entry.name);
|
|
13794
13840
|
if (entry.isDirectory()) {
|
|
13795
13841
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
13796
13842
|
} else if (entry.isFile()) {
|
|
13797
|
-
const ext =
|
|
13843
|
+
const ext = import_node_path34.default.extname(entry.name).toLowerCase();
|
|
13798
13844
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
13799
13845
|
try {
|
|
13800
13846
|
const stat8 = await import_promises25.default.stat(fullPath);
|
|
@@ -13806,7 +13852,7 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
13806
13852
|
regex.lastIndex = 0;
|
|
13807
13853
|
if (regex.test(lines[i])) {
|
|
13808
13854
|
matches.push({
|
|
13809
|
-
file:
|
|
13855
|
+
file: import_node_path34.default.relative(workspacePath, fullPath),
|
|
13810
13856
|
line: i + 1,
|
|
13811
13857
|
text: lines[i].substring(0, 200)
|
|
13812
13858
|
});
|
|
@@ -14043,8 +14089,8 @@ var TokenUsageEvaluator = class {
|
|
|
14043
14089
|
};
|
|
14044
14090
|
|
|
14045
14091
|
// src/evaluation/evaluators/tool-trajectory.ts
|
|
14046
|
-
function getNestedValue(obj,
|
|
14047
|
-
const parts =
|
|
14092
|
+
function getNestedValue(obj, path43) {
|
|
14093
|
+
const parts = path43.split(".");
|
|
14048
14094
|
let current = obj;
|
|
14049
14095
|
for (const part of parts) {
|
|
14050
14096
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -14607,7 +14653,7 @@ function runEqualsAssertion(output, value) {
|
|
|
14607
14653
|
// src/evaluation/orchestrator.ts
|
|
14608
14654
|
var import_node_crypto9 = require("crypto");
|
|
14609
14655
|
var import_promises29 = require("fs/promises");
|
|
14610
|
-
var
|
|
14656
|
+
var import_node_path41 = __toESM(require("path"), 1);
|
|
14611
14657
|
var import_micromatch4 = __toESM(require("micromatch"), 1);
|
|
14612
14658
|
|
|
14613
14659
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -14798,7 +14844,7 @@ var DeterministicAssertionEvaluator = class {
|
|
|
14798
14844
|
var import_node_fs9 = require("fs");
|
|
14799
14845
|
|
|
14800
14846
|
// src/evaluation/evaluators/prompt-resolution.ts
|
|
14801
|
-
var
|
|
14847
|
+
var import_node_path35 = __toESM(require("path"), 1);
|
|
14802
14848
|
async function resolveCustomPrompt(promptConfig, context2, timeoutMs) {
|
|
14803
14849
|
if (promptConfig.resolvedPromptScript && promptConfig.resolvedPromptScript.length > 0) {
|
|
14804
14850
|
if (!context2) {
|
|
@@ -14847,7 +14893,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
|
14847
14893
|
};
|
|
14848
14894
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
14849
14895
|
const scriptPath = script[script.length - 1];
|
|
14850
|
-
const cwd =
|
|
14896
|
+
const cwd = import_node_path35.default.dirname(scriptPath);
|
|
14851
14897
|
try {
|
|
14852
14898
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
14853
14899
|
const prompt = stdout.trim();
|
|
@@ -15140,16 +15186,16 @@ function createBuiltinRegistry() {
|
|
|
15140
15186
|
}
|
|
15141
15187
|
|
|
15142
15188
|
// src/evaluation/registry/assertion-discovery.ts
|
|
15143
|
-
var
|
|
15189
|
+
var import_node_path36 = __toESM(require("path"), 1);
|
|
15144
15190
|
var import_fast_glob3 = __toESM(require("fast-glob"), 1);
|
|
15145
15191
|
async function discoverAssertions(registry, baseDir) {
|
|
15146
15192
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
15147
15193
|
const candidateDirs = [];
|
|
15148
|
-
let dir =
|
|
15149
|
-
const root =
|
|
15194
|
+
let dir = import_node_path36.default.resolve(baseDir);
|
|
15195
|
+
const root = import_node_path36.default.parse(dir).root;
|
|
15150
15196
|
while (dir !== root) {
|
|
15151
|
-
candidateDirs.push(
|
|
15152
|
-
dir =
|
|
15197
|
+
candidateDirs.push(import_node_path36.default.join(dir, ".agentv", "assertions"));
|
|
15198
|
+
dir = import_node_path36.default.dirname(dir);
|
|
15153
15199
|
}
|
|
15154
15200
|
let files = [];
|
|
15155
15201
|
for (const assertionsDir of candidateDirs) {
|
|
@@ -15165,7 +15211,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
15165
15211
|
}
|
|
15166
15212
|
const discoveredTypes = [];
|
|
15167
15213
|
for (const filePath of files) {
|
|
15168
|
-
const basename =
|
|
15214
|
+
const basename = import_node_path36.default.basename(filePath);
|
|
15169
15215
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
15170
15216
|
if (registry.has(typeName)) {
|
|
15171
15217
|
continue;
|
|
@@ -15325,7 +15371,7 @@ function getTCritical(df) {
|
|
|
15325
15371
|
// src/evaluation/workspace/file-changes.ts
|
|
15326
15372
|
var import_node_child_process6 = require("child_process");
|
|
15327
15373
|
var import_node_fs10 = require("fs");
|
|
15328
|
-
var
|
|
15374
|
+
var import_node_path37 = __toESM(require("path"), 1);
|
|
15329
15375
|
var import_node_util4 = require("util");
|
|
15330
15376
|
var execAsync4 = (0, import_node_util4.promisify)(import_node_child_process6.exec);
|
|
15331
15377
|
function gitExecOpts(workspacePath) {
|
|
@@ -15359,10 +15405,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
15359
15405
|
}
|
|
15360
15406
|
for (const entry of entries) {
|
|
15361
15407
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
15362
|
-
const childPath =
|
|
15408
|
+
const childPath = import_node_path37.default.join(workspacePath, entry);
|
|
15363
15409
|
try {
|
|
15364
15410
|
if (!(0, import_node_fs10.statSync)(childPath).isDirectory()) continue;
|
|
15365
|
-
if (!(0, import_node_fs10.statSync)(
|
|
15411
|
+
if (!(0, import_node_fs10.statSync)(import_node_path37.default.join(childPath, ".git")).isDirectory()) continue;
|
|
15366
15412
|
} catch {
|
|
15367
15413
|
continue;
|
|
15368
15414
|
}
|
|
@@ -15373,9 +15419,7 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
15373
15419
|
|
|
15374
15420
|
// src/evaluation/workspace/manager.ts
|
|
15375
15421
|
var import_promises26 = require("fs/promises");
|
|
15376
|
-
var
|
|
15377
|
-
var import_node_path37 = __toESM(require("path"), 1);
|
|
15378
|
-
var DEFAULT_WORKSPACE_ROOT = import_node_path37.default.join(import_node_os6.default.homedir(), ".agentv", "workspaces");
|
|
15422
|
+
var import_node_path38 = __toESM(require("path"), 1);
|
|
15379
15423
|
var TemplateNotFoundError = class extends Error {
|
|
15380
15424
|
constructor(templatePath) {
|
|
15381
15425
|
super(`Workspace template not found: ${templatePath}`);
|
|
@@ -15404,15 +15448,15 @@ async function isDirectory(filePath) {
|
|
|
15404
15448
|
}
|
|
15405
15449
|
}
|
|
15406
15450
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
15407
|
-
const root = workspaceRoot ??
|
|
15408
|
-
return
|
|
15451
|
+
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
15452
|
+
return import_node_path38.default.join(root, evalRunId, caseId);
|
|
15409
15453
|
}
|
|
15410
15454
|
async function copyDirectoryRecursive(src, dest) {
|
|
15411
15455
|
await (0, import_promises26.mkdir)(dest, { recursive: true });
|
|
15412
15456
|
const entries = await (0, import_promises26.readdir)(src, { withFileTypes: true });
|
|
15413
15457
|
for (const entry of entries) {
|
|
15414
|
-
const srcPath =
|
|
15415
|
-
const destPath =
|
|
15458
|
+
const srcPath = import_node_path38.default.join(src, entry.name);
|
|
15459
|
+
const destPath = import_node_path38.default.join(dest, entry.name);
|
|
15416
15460
|
if (entry.name === ".git") {
|
|
15417
15461
|
continue;
|
|
15418
15462
|
}
|
|
@@ -15424,7 +15468,7 @@ async function copyDirectoryRecursive(src, dest) {
|
|
|
15424
15468
|
}
|
|
15425
15469
|
}
|
|
15426
15470
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
15427
|
-
const resolvedTemplatePath =
|
|
15471
|
+
const resolvedTemplatePath = import_node_path38.default.resolve(templatePath);
|
|
15428
15472
|
if (!await fileExists2(resolvedTemplatePath)) {
|
|
15429
15473
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
15430
15474
|
}
|
|
@@ -15472,8 +15516,8 @@ async function cleanupWorkspace(workspacePath) {
|
|
|
15472
15516
|
}
|
|
15473
15517
|
}
|
|
15474
15518
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
15475
|
-
const root = workspaceRoot ??
|
|
15476
|
-
const evalDir =
|
|
15519
|
+
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
15520
|
+
const evalDir = import_node_path38.default.join(root, evalRunId);
|
|
15477
15521
|
if (await fileExists2(evalDir)) {
|
|
15478
15522
|
await (0, import_promises26.rm)(evalDir, { recursive: true, force: true });
|
|
15479
15523
|
}
|
|
@@ -15484,11 +15528,9 @@ var import_node_child_process7 = require("child_process");
|
|
|
15484
15528
|
var import_node_crypto8 = require("crypto");
|
|
15485
15529
|
var import_node_fs11 = require("fs");
|
|
15486
15530
|
var import_promises27 = require("fs/promises");
|
|
15487
|
-
var
|
|
15488
|
-
var import_node_path38 = __toESM(require("path"), 1);
|
|
15531
|
+
var import_node_path39 = __toESM(require("path"), 1);
|
|
15489
15532
|
var import_node_util5 = require("util");
|
|
15490
15533
|
var execFileAsync = (0, import_node_util5.promisify)(import_node_child_process7.execFile);
|
|
15491
|
-
var DEFAULT_CACHE_DIR = import_node_path38.default.join(import_node_os7.default.homedir(), ".agentv", "git-cache");
|
|
15492
15534
|
var DEFAULT_TIMEOUT_MS2 = 3e5;
|
|
15493
15535
|
var LOCK_TIMEOUT_MS = 6e4;
|
|
15494
15536
|
function gitEnv() {
|
|
@@ -15546,8 +15588,35 @@ async function releaseLock(lockPath) {
|
|
|
15546
15588
|
}
|
|
15547
15589
|
var RepoManager = class {
|
|
15548
15590
|
cacheDir;
|
|
15549
|
-
|
|
15550
|
-
|
|
15591
|
+
verbose;
|
|
15592
|
+
constructor(cacheDir, verbose = false) {
|
|
15593
|
+
this.cacheDir = cacheDir ?? getGitCacheRoot();
|
|
15594
|
+
this.verbose = verbose;
|
|
15595
|
+
}
|
|
15596
|
+
async runGit(args, opts) {
|
|
15597
|
+
const startedAt = Date.now();
|
|
15598
|
+
if (this.verbose) {
|
|
15599
|
+
console.log(
|
|
15600
|
+
`[repo] git start cwd=${opts?.cwd ?? process.cwd()} args=${args.join(" ")}`
|
|
15601
|
+
);
|
|
15602
|
+
}
|
|
15603
|
+
try {
|
|
15604
|
+
const output = await git(args, opts);
|
|
15605
|
+
if (this.verbose) {
|
|
15606
|
+
console.log(
|
|
15607
|
+
`[repo] git ok durationMs=${Date.now() - startedAt} args=${args.join(" ")}`
|
|
15608
|
+
);
|
|
15609
|
+
}
|
|
15610
|
+
return output;
|
|
15611
|
+
} catch (error) {
|
|
15612
|
+
if (this.verbose) {
|
|
15613
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
15614
|
+
console.log(
|
|
15615
|
+
`[repo] git fail durationMs=${Date.now() - startedAt} args=${args.join(" ")} error=${message}`
|
|
15616
|
+
);
|
|
15617
|
+
}
|
|
15618
|
+
throw error;
|
|
15619
|
+
}
|
|
15551
15620
|
}
|
|
15552
15621
|
/**
|
|
15553
15622
|
* Ensure a bare mirror cache exists for the given source.
|
|
@@ -15556,11 +15625,19 @@ var RepoManager = class {
|
|
|
15556
15625
|
*/
|
|
15557
15626
|
async ensureCache(source, depth, resolve) {
|
|
15558
15627
|
const key = cacheKey(source);
|
|
15559
|
-
const cachePath =
|
|
15628
|
+
const cachePath = import_node_path39.default.join(this.cacheDir, key);
|
|
15560
15629
|
const lockPath = `${cachePath}.lock`;
|
|
15561
|
-
const cacheExists = (0, import_node_fs11.existsSync)(
|
|
15630
|
+
const cacheExists = (0, import_node_fs11.existsSync)(import_node_path39.default.join(cachePath, "HEAD"));
|
|
15631
|
+
if (this.verbose) {
|
|
15632
|
+
console.log(
|
|
15633
|
+
`[repo] ensureCache source=${getSourceUrl(source)} resolve=${resolve ?? "remote"} cache=${cachePath}`
|
|
15634
|
+
);
|
|
15635
|
+
}
|
|
15562
15636
|
if (resolve === "local") {
|
|
15563
15637
|
if (cacheExists) {
|
|
15638
|
+
if (this.verbose) {
|
|
15639
|
+
console.log(`[repo] using existing local cache ${cachePath}`);
|
|
15640
|
+
}
|
|
15564
15641
|
return cachePath;
|
|
15565
15642
|
}
|
|
15566
15643
|
const url = getSourceUrl(source);
|
|
@@ -15569,15 +15646,27 @@ var RepoManager = class {
|
|
|
15569
15646
|
);
|
|
15570
15647
|
}
|
|
15571
15648
|
await (0, import_promises27.mkdir)(this.cacheDir, { recursive: true });
|
|
15649
|
+
const lockStartedAt = Date.now();
|
|
15572
15650
|
await acquireLock(lockPath);
|
|
15651
|
+
if (this.verbose) {
|
|
15652
|
+
console.log(
|
|
15653
|
+
`[repo] lock acquired path=${lockPath} waitedMs=${Date.now() - lockStartedAt}`
|
|
15654
|
+
);
|
|
15655
|
+
}
|
|
15573
15656
|
try {
|
|
15574
15657
|
if (cacheExists) {
|
|
15658
|
+
if (this.verbose) {
|
|
15659
|
+
console.log(`[repo] refreshing existing cache ${cachePath}`);
|
|
15660
|
+
}
|
|
15575
15661
|
const fetchArgs = ["fetch", "--prune"];
|
|
15576
15662
|
if (depth) {
|
|
15577
15663
|
fetchArgs.push("--depth", String(depth));
|
|
15578
15664
|
}
|
|
15579
|
-
await
|
|
15665
|
+
await this.runGit(fetchArgs, { cwd: cachePath });
|
|
15580
15666
|
} else {
|
|
15667
|
+
if (this.verbose) {
|
|
15668
|
+
console.log(`[repo] creating new cache ${cachePath}`);
|
|
15669
|
+
}
|
|
15581
15670
|
const cloneArgs = ["clone", "--mirror", "--bare"];
|
|
15582
15671
|
if (depth) {
|
|
15583
15672
|
cloneArgs.push("--depth", String(depth));
|
|
@@ -15585,10 +15674,13 @@ var RepoManager = class {
|
|
|
15585
15674
|
const sourceUrl = getSourceUrl(source);
|
|
15586
15675
|
const cloneUrl = depth && source.type === "local" ? `file://${sourceUrl}` : sourceUrl;
|
|
15587
15676
|
cloneArgs.push(cloneUrl, cachePath);
|
|
15588
|
-
await
|
|
15677
|
+
await this.runGit(cloneArgs);
|
|
15589
15678
|
}
|
|
15590
15679
|
} finally {
|
|
15591
15680
|
await releaseLock(lockPath);
|
|
15681
|
+
if (this.verbose) {
|
|
15682
|
+
console.log(`[repo] lock released path=${lockPath}`);
|
|
15683
|
+
}
|
|
15592
15684
|
}
|
|
15593
15685
|
return cachePath;
|
|
15594
15686
|
}
|
|
@@ -15597,7 +15689,13 @@ var RepoManager = class {
|
|
|
15597
15689
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
15598
15690
|
*/
|
|
15599
15691
|
async materialize(repo, workspacePath) {
|
|
15600
|
-
const targetDir =
|
|
15692
|
+
const targetDir = import_node_path39.default.join(workspacePath, repo.path);
|
|
15693
|
+
const startedAt = Date.now();
|
|
15694
|
+
if (this.verbose) {
|
|
15695
|
+
console.log(
|
|
15696
|
+
`[repo] materialize start path=${repo.path} source=${getSourceUrl(repo.source)} workspace=${workspacePath}`
|
|
15697
|
+
);
|
|
15698
|
+
}
|
|
15601
15699
|
const cachePath = await this.ensureCache(
|
|
15602
15700
|
repo.source,
|
|
15603
15701
|
repo.clone?.depth,
|
|
@@ -15613,10 +15711,10 @@ var RepoManager = class {
|
|
|
15613
15711
|
cloneArgs.push("--no-checkout");
|
|
15614
15712
|
const cloneUrl = repo.clone?.depth || repo.clone?.filter ? `file://${cachePath}` : cachePath;
|
|
15615
15713
|
cloneArgs.push(cloneUrl, targetDir);
|
|
15616
|
-
await
|
|
15714
|
+
await this.runGit(cloneArgs);
|
|
15617
15715
|
if (repo.clone?.sparse?.length) {
|
|
15618
|
-
await
|
|
15619
|
-
await
|
|
15716
|
+
await this.runGit(["sparse-checkout", "init", "--cone"], { cwd: targetDir });
|
|
15717
|
+
await this.runGit(["sparse-checkout", "set", ...repo.clone.sparse], { cwd: targetDir });
|
|
15620
15718
|
}
|
|
15621
15719
|
const ref = repo.checkout?.ref ?? "HEAD";
|
|
15622
15720
|
const resolve = repo.checkout?.resolve ?? "remote";
|
|
@@ -15624,7 +15722,7 @@ var RepoManager = class {
|
|
|
15624
15722
|
if (resolve === "remote" && repo.source.type === "git") {
|
|
15625
15723
|
const url = getSourceUrl(repo.source);
|
|
15626
15724
|
try {
|
|
15627
|
-
const lsOutput = await
|
|
15725
|
+
const lsOutput = await this.runGit(["ls-remote", url, ref]);
|
|
15628
15726
|
const match = lsOutput.split(" ")[0];
|
|
15629
15727
|
if (!match) {
|
|
15630
15728
|
throw new Error(`Ref '${ref}' not found on remote ${url}`);
|
|
@@ -15637,17 +15735,26 @@ var RepoManager = class {
|
|
|
15637
15735
|
} else {
|
|
15638
15736
|
resolvedSha = ref;
|
|
15639
15737
|
}
|
|
15640
|
-
|
|
15738
|
+
if (this.verbose) {
|
|
15739
|
+
console.log(
|
|
15740
|
+
`[repo] checkout path=${repo.path} ref=${ref} resolved=${resolvedSha} resolve=${resolve}`
|
|
15741
|
+
);
|
|
15742
|
+
}
|
|
15743
|
+
await this.runGit(["checkout", resolvedSha], { cwd: targetDir });
|
|
15641
15744
|
const ancestor = repo.checkout?.ancestor ?? 0;
|
|
15642
15745
|
if (ancestor > 0) {
|
|
15643
15746
|
try {
|
|
15644
|
-
const ancestorSha = await
|
|
15645
|
-
|
|
15747
|
+
const ancestorSha = await this.runGit(["rev-parse", `HEAD~${ancestor}`], {
|
|
15748
|
+
cwd: targetDir
|
|
15749
|
+
});
|
|
15750
|
+
await this.runGit(["checkout", ancestorSha], { cwd: targetDir });
|
|
15646
15751
|
} catch {
|
|
15647
15752
|
if (repo.clone?.depth) {
|
|
15648
|
-
await
|
|
15649
|
-
const ancestorSha = await
|
|
15650
|
-
|
|
15753
|
+
await this.runGit(["fetch", "--deepen", String(ancestor)], { cwd: targetDir });
|
|
15754
|
+
const ancestorSha = await this.runGit(["rev-parse", `HEAD~${ancestor}`], {
|
|
15755
|
+
cwd: targetDir
|
|
15756
|
+
});
|
|
15757
|
+
await this.runGit(["checkout", ancestorSha], { cwd: targetDir });
|
|
15651
15758
|
} else {
|
|
15652
15759
|
throw new Error(
|
|
15653
15760
|
`Cannot resolve ancestor ${ancestor} of ref '${ref}'. If using shallow clone, increase clone.depth to at least ${ancestor + 1}.`
|
|
@@ -15655,27 +15762,38 @@ var RepoManager = class {
|
|
|
15655
15762
|
}
|
|
15656
15763
|
}
|
|
15657
15764
|
}
|
|
15765
|
+
if (this.verbose) {
|
|
15766
|
+
console.log(
|
|
15767
|
+
`[repo] materialize done path=${repo.path} target=${targetDir} durationMs=${Date.now() - startedAt}`
|
|
15768
|
+
);
|
|
15769
|
+
}
|
|
15658
15770
|
}
|
|
15659
15771
|
/** Materialize all repos into the workspace. */
|
|
15660
15772
|
async materializeAll(repos, workspacePath) {
|
|
15773
|
+
if (this.verbose) {
|
|
15774
|
+
console.log(`[repo] materializeAll count=${repos.length} workspace=${workspacePath}`);
|
|
15775
|
+
}
|
|
15661
15776
|
for (const repo of repos) {
|
|
15662
15777
|
await this.materialize(repo, workspacePath);
|
|
15663
15778
|
}
|
|
15779
|
+
if (this.verbose) {
|
|
15780
|
+
console.log("[repo] materializeAll complete");
|
|
15781
|
+
}
|
|
15664
15782
|
}
|
|
15665
15783
|
/** Reset repos in workspace to their checkout state. */
|
|
15666
15784
|
async reset(repos, workspacePath, strategy) {
|
|
15667
15785
|
if (strategy === "recreate") {
|
|
15668
15786
|
for (const repo of repos) {
|
|
15669
|
-
const targetDir =
|
|
15787
|
+
const targetDir = import_node_path39.default.join(workspacePath, repo.path);
|
|
15670
15788
|
await (0, import_promises27.rm)(targetDir, { recursive: true, force: true });
|
|
15671
15789
|
}
|
|
15672
15790
|
await this.materializeAll(repos, workspacePath);
|
|
15673
15791
|
return;
|
|
15674
15792
|
}
|
|
15675
15793
|
for (const repo of repos) {
|
|
15676
|
-
const targetDir =
|
|
15677
|
-
await
|
|
15678
|
-
await
|
|
15794
|
+
const targetDir = import_node_path39.default.join(workspacePath, repo.path);
|
|
15795
|
+
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
15796
|
+
await this.runGit(["clean", "-fd"], { cwd: targetDir });
|
|
15679
15797
|
}
|
|
15680
15798
|
}
|
|
15681
15799
|
/**
|
|
@@ -15685,12 +15803,12 @@ var RepoManager = class {
|
|
|
15685
15803
|
async seedCache(localPath, remoteUrl, opts) {
|
|
15686
15804
|
const source = { type: "git", url: remoteUrl };
|
|
15687
15805
|
const key = cacheKey(source);
|
|
15688
|
-
const cachePath =
|
|
15806
|
+
const cachePath = import_node_path39.default.join(this.cacheDir, key);
|
|
15689
15807
|
const lockPath = `${cachePath}.lock`;
|
|
15690
15808
|
await (0, import_promises27.mkdir)(this.cacheDir, { recursive: true });
|
|
15691
15809
|
await acquireLock(lockPath);
|
|
15692
15810
|
try {
|
|
15693
|
-
if ((0, import_node_fs11.existsSync)(
|
|
15811
|
+
if ((0, import_node_fs11.existsSync)(import_node_path39.default.join(cachePath, "HEAD"))) {
|
|
15694
15812
|
if (!opts?.force) {
|
|
15695
15813
|
throw new Error(
|
|
15696
15814
|
`Cache already exists for ${remoteUrl} at ${cachePath}. Use force to overwrite.`
|
|
@@ -15713,16 +15831,16 @@ var RepoManager = class {
|
|
|
15713
15831
|
|
|
15714
15832
|
// src/evaluation/workspace/resolve.ts
|
|
15715
15833
|
var import_promises28 = require("fs/promises");
|
|
15716
|
-
var
|
|
15834
|
+
var import_node_path40 = __toESM(require("path"), 1);
|
|
15717
15835
|
async function resolveWorkspaceTemplate(templatePath) {
|
|
15718
15836
|
if (!templatePath) {
|
|
15719
15837
|
return void 0;
|
|
15720
15838
|
}
|
|
15721
|
-
const resolved =
|
|
15839
|
+
const resolved = import_node_path40.default.resolve(templatePath);
|
|
15722
15840
|
const stats = await (0, import_promises28.stat)(resolved);
|
|
15723
15841
|
if (stats.isFile()) {
|
|
15724
15842
|
return {
|
|
15725
|
-
dir:
|
|
15843
|
+
dir: import_node_path40.default.dirname(resolved),
|
|
15726
15844
|
workspaceFile: resolved
|
|
15727
15845
|
};
|
|
15728
15846
|
}
|
|
@@ -15734,14 +15852,14 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
15734
15852
|
if (workspaceFiles.length === 1) {
|
|
15735
15853
|
return {
|
|
15736
15854
|
dir: resolved,
|
|
15737
|
-
workspaceFile:
|
|
15855
|
+
workspaceFile: import_node_path40.default.join(resolved, workspaceFiles[0])
|
|
15738
15856
|
};
|
|
15739
15857
|
}
|
|
15740
15858
|
if (workspaceFiles.length > 1) {
|
|
15741
15859
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
15742
15860
|
return {
|
|
15743
15861
|
dir: resolved,
|
|
15744
|
-
workspaceFile: conventionFile ?
|
|
15862
|
+
workspaceFile: conventionFile ? import_node_path40.default.join(resolved, conventionFile) : void 0
|
|
15745
15863
|
};
|
|
15746
15864
|
}
|
|
15747
15865
|
return { dir: resolved };
|
|
@@ -15897,7 +16015,7 @@ async function runEvaluation(options) {
|
|
|
15897
16015
|
];
|
|
15898
16016
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveJudgeProvider);
|
|
15899
16017
|
const typeRegistry = createBuiltinRegistry();
|
|
15900
|
-
const discoveryBaseDir = evalFilePath ?
|
|
16018
|
+
const discoveryBaseDir = evalFilePath ? import_node_path41.default.dirname(import_node_path41.default.resolve(evalFilePath)) : process.cwd();
|
|
15901
16019
|
const evalDir = discoveryBaseDir;
|
|
15902
16020
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
15903
16021
|
const providerRegistry = createBuiltinProviderRegistry();
|
|
@@ -15953,10 +16071,18 @@ async function runEvaluation(options) {
|
|
|
15953
16071
|
const resolvedTemplate = await resolveWorkspaceTemplate(rawTemplate);
|
|
15954
16072
|
const workspaceTemplate = resolvedTemplate?.dir;
|
|
15955
16073
|
let suiteWorkspaceFile = resolvedTemplate?.workspaceFile;
|
|
16074
|
+
const setupLog = (message) => {
|
|
16075
|
+
if (verbose) {
|
|
16076
|
+
console.log(`[setup] ${message}`);
|
|
16077
|
+
}
|
|
16078
|
+
};
|
|
15956
16079
|
const isPerTestIsolation = suiteWorkspace?.isolation === "per_test";
|
|
15957
16080
|
const hasSharedWorkspace = !!(workspaceTemplate || suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation);
|
|
15958
16081
|
const requestedWorkers = options.maxConcurrency ?? target.workers ?? 1;
|
|
15959
16082
|
const workers = hasSharedWorkspace ? 1 : requestedWorkers;
|
|
16083
|
+
setupLog(
|
|
16084
|
+
`sharedWorkspace=${hasSharedWorkspace} perTestIsolation=${isPerTestIsolation} requestedWorkers=${requestedWorkers} effectiveWorkers=${workers}`
|
|
16085
|
+
);
|
|
15960
16086
|
if (hasSharedWorkspace && requestedWorkers > 1) {
|
|
15961
16087
|
console.warn(
|
|
15962
16088
|
`Warning: Shared workspace requires sequential execution. Overriding workers from ${requestedWorkers} to 1.`
|
|
@@ -15967,14 +16093,16 @@ async function runEvaluation(options) {
|
|
|
15967
16093
|
let sharedBaselineCommit;
|
|
15968
16094
|
let beforeAllOutput;
|
|
15969
16095
|
if (workspaceTemplate) {
|
|
16096
|
+
setupLog(`creating shared workspace from template: ${workspaceTemplate}`);
|
|
15970
16097
|
try {
|
|
15971
16098
|
sharedWorkspacePath = await createTempWorkspace(workspaceTemplate, evalRunId, "shared");
|
|
16099
|
+
setupLog(`shared workspace created at: ${sharedWorkspacePath}`);
|
|
15972
16100
|
} catch (error) {
|
|
15973
16101
|
const message = error instanceof Error ? error.message : String(error);
|
|
15974
16102
|
throw new Error(`Failed to create shared workspace: ${message}`);
|
|
15975
16103
|
}
|
|
15976
16104
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
15977
|
-
const copiedWorkspaceFile =
|
|
16105
|
+
const copiedWorkspaceFile = import_node_path41.default.join(sharedWorkspacePath, import_node_path41.default.basename(suiteWorkspaceFile));
|
|
15978
16106
|
try {
|
|
15979
16107
|
await (0, import_promises29.stat)(copiedWorkspaceFile);
|
|
15980
16108
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
@@ -15984,11 +16112,14 @@ async function runEvaluation(options) {
|
|
|
15984
16112
|
} else if (suiteWorkspace?.before_all || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
|
|
15985
16113
|
sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
|
|
15986
16114
|
await (0, import_promises29.mkdir)(sharedWorkspacePath, { recursive: true });
|
|
16115
|
+
setupLog(`created empty shared workspace at: ${sharedWorkspacePath}`);
|
|
15987
16116
|
}
|
|
15988
|
-
const repoManager = suiteWorkspace?.repos?.length ? new RepoManager() : void 0;
|
|
16117
|
+
const repoManager = suiteWorkspace?.repos?.length ? new RepoManager(void 0, verbose) : void 0;
|
|
15989
16118
|
if (repoManager && sharedWorkspacePath && suiteWorkspace?.repos && !isPerTestIsolation) {
|
|
16119
|
+
setupLog(`materializing ${suiteWorkspace.repos.length} shared repo(s) into ${sharedWorkspacePath}`);
|
|
15990
16120
|
try {
|
|
15991
16121
|
await repoManager.materializeAll(suiteWorkspace.repos, sharedWorkspacePath);
|
|
16122
|
+
setupLog("shared repo materialization complete");
|
|
15992
16123
|
} catch (error) {
|
|
15993
16124
|
const message = error instanceof Error ? error.message : String(error);
|
|
15994
16125
|
if (sharedWorkspacePath) {
|
|
@@ -15999,6 +16130,10 @@ async function runEvaluation(options) {
|
|
|
15999
16130
|
}
|
|
16000
16131
|
}
|
|
16001
16132
|
if (sharedWorkspacePath && suiteWorkspace?.before_all) {
|
|
16133
|
+
const beforeAllCommand = (suiteWorkspace.before_all.command ?? suiteWorkspace.before_all.script ?? []).join(" ");
|
|
16134
|
+
setupLog(
|
|
16135
|
+
`running shared before_all in cwd=${suiteWorkspace.before_all.cwd ?? evalDir} command=${beforeAllCommand}`
|
|
16136
|
+
);
|
|
16002
16137
|
const scriptContext = {
|
|
16003
16138
|
workspacePath: sharedWorkspacePath,
|
|
16004
16139
|
testId: "__before_all__",
|
|
@@ -16007,6 +16142,7 @@ async function runEvaluation(options) {
|
|
|
16007
16142
|
};
|
|
16008
16143
|
try {
|
|
16009
16144
|
beforeAllOutput = await executeWorkspaceScript(suiteWorkspace.before_all, scriptContext);
|
|
16145
|
+
setupLog("shared before_all completed");
|
|
16010
16146
|
} catch (error) {
|
|
16011
16147
|
const message = error instanceof Error ? error.message : String(error);
|
|
16012
16148
|
if (sharedWorkspacePath) {
|
|
@@ -16019,7 +16155,9 @@ async function runEvaluation(options) {
|
|
|
16019
16155
|
if (sharedWorkspacePath) {
|
|
16020
16156
|
try {
|
|
16021
16157
|
sharedBaselineCommit = await initializeBaseline(sharedWorkspacePath);
|
|
16158
|
+
setupLog(`shared baseline initialized: ${sharedBaselineCommit}`);
|
|
16022
16159
|
} catch {
|
|
16160
|
+
setupLog("shared baseline initialization skipped (non-fatal)");
|
|
16023
16161
|
}
|
|
16024
16162
|
}
|
|
16025
16163
|
let nextWorkerId = 1;
|
|
@@ -16423,6 +16561,7 @@ async function runEvalCase(options) {
|
|
|
16423
16561
|
repoManager,
|
|
16424
16562
|
evalDir
|
|
16425
16563
|
} = options;
|
|
16564
|
+
const setupDebug = process.env.AGENTV_SETUP_DEBUG === "1";
|
|
16426
16565
|
const formattingMode = usesFileReferencePrompt(provider) ? "agent" : "lm";
|
|
16427
16566
|
const promptInputs = await buildPromptInputs(evalCase, formattingMode);
|
|
16428
16567
|
const typeRegistry = providedTypeRegistry ?? createBuiltinRegistry();
|
|
@@ -16460,7 +16599,7 @@ async function runEvalCase(options) {
|
|
|
16460
16599
|
);
|
|
16461
16600
|
}
|
|
16462
16601
|
if (caseWorkspaceFile && workspacePath) {
|
|
16463
|
-
const copiedFile =
|
|
16602
|
+
const copiedFile = import_node_path41.default.join(workspacePath, import_node_path41.default.basename(caseWorkspaceFile));
|
|
16464
16603
|
try {
|
|
16465
16604
|
await (0, import_promises29.stat)(copiedFile);
|
|
16466
16605
|
caseWorkspaceFile = copiedFile;
|
|
@@ -16473,9 +16612,17 @@ async function runEvalCase(options) {
|
|
|
16473
16612
|
await (0, import_promises29.mkdir)(workspacePath, { recursive: true });
|
|
16474
16613
|
}
|
|
16475
16614
|
if (evalCase.workspace?.repos?.length && workspacePath) {
|
|
16476
|
-
const perCaseRepoManager = new RepoManager();
|
|
16615
|
+
const perCaseRepoManager = new RepoManager(void 0, setupDebug);
|
|
16477
16616
|
try {
|
|
16617
|
+
if (setupDebug) {
|
|
16618
|
+
console.log(
|
|
16619
|
+
`[setup] test=${evalCase.id} materializing ${evalCase.workspace.repos.length} per-test repo(s) into ${workspacePath}`
|
|
16620
|
+
);
|
|
16621
|
+
}
|
|
16478
16622
|
await perCaseRepoManager.materializeAll(evalCase.workspace.repos, workspacePath);
|
|
16623
|
+
if (setupDebug) {
|
|
16624
|
+
console.log(`[setup] test=${evalCase.id} per-test repo materialization complete`);
|
|
16625
|
+
}
|
|
16479
16626
|
} catch (error) {
|
|
16480
16627
|
const message = error instanceof Error ? error.message : String(error);
|
|
16481
16628
|
return buildErrorResult(
|
|
@@ -16491,6 +16638,12 @@ async function runEvalCase(options) {
|
|
|
16491
16638
|
}
|
|
16492
16639
|
}
|
|
16493
16640
|
if (workspacePath && evalCase.workspace?.before_all) {
|
|
16641
|
+
const beforeAllCommand = (evalCase.workspace.before_all.command ?? evalCase.workspace.before_all.script ?? []).join(" ");
|
|
16642
|
+
if (setupDebug) {
|
|
16643
|
+
console.log(
|
|
16644
|
+
`[setup] test=${evalCase.id} running before_all in cwd=${evalCase.workspace.before_all.cwd ?? evalDir} command=${beforeAllCommand}`
|
|
16645
|
+
);
|
|
16646
|
+
}
|
|
16494
16647
|
const scriptContext = {
|
|
16495
16648
|
workspacePath,
|
|
16496
16649
|
testId: evalCase.id,
|
|
@@ -16504,6 +16657,9 @@ async function runEvalCase(options) {
|
|
|
16504
16657
|
evalCase.workspace.before_all,
|
|
16505
16658
|
scriptContext
|
|
16506
16659
|
);
|
|
16660
|
+
if (setupDebug) {
|
|
16661
|
+
console.log(`[setup] test=${evalCase.id} before_all completed`);
|
|
16662
|
+
}
|
|
16507
16663
|
} catch (error) {
|
|
16508
16664
|
const message = error instanceof Error ? error.message : String(error);
|
|
16509
16665
|
if (forceCleanup && workspacePath) {
|
|
@@ -17053,7 +17209,7 @@ async function runEvaluatorList(options) {
|
|
|
17053
17209
|
fileChanges,
|
|
17054
17210
|
workspacePath
|
|
17055
17211
|
};
|
|
17056
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
17212
|
+
const evalFileDir = evalCase.guideline_paths[0] ? import_node_path41.default.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
17057
17213
|
const dispatchContext = {
|
|
17058
17214
|
judgeProvider,
|
|
17059
17215
|
targetResolver,
|
|
@@ -17356,7 +17512,7 @@ function computeWeightedMean(entries) {
|
|
|
17356
17512
|
|
|
17357
17513
|
// src/evaluation/evaluate.ts
|
|
17358
17514
|
var import_node_fs12 = require("fs");
|
|
17359
|
-
var
|
|
17515
|
+
var import_node_path42 = __toESM(require("path"), 1);
|
|
17360
17516
|
async function evaluate(config) {
|
|
17361
17517
|
const startTime = Date.now();
|
|
17362
17518
|
if (config.tests && config.specFile) {
|
|
@@ -17378,13 +17534,13 @@ async function evaluate(config) {
|
|
|
17378
17534
|
let evalCases;
|
|
17379
17535
|
let testFilePath;
|
|
17380
17536
|
if (config.specFile) {
|
|
17381
|
-
testFilePath =
|
|
17537
|
+
testFilePath = import_node_path42.default.resolve(config.specFile);
|
|
17382
17538
|
evalCases = await loadTests(testFilePath, repoRoot, {
|
|
17383
17539
|
verbose: config.verbose,
|
|
17384
17540
|
filter: config.filter
|
|
17385
17541
|
});
|
|
17386
17542
|
} else {
|
|
17387
|
-
testFilePath =
|
|
17543
|
+
testFilePath = import_node_path42.default.join(process.cwd(), "__programmatic__.yaml");
|
|
17388
17544
|
evalCases = (config.tests ?? []).map((test) => {
|
|
17389
17545
|
const input = typeof test.input === "string" ? [{ role: "user", content: test.input }] : test.input;
|
|
17390
17546
|
const question = typeof test.input === "string" ? test.input : test.input.find((m) => m.role === "user")?.content ?? "";
|
|
@@ -17470,10 +17626,10 @@ function computeSummary(results, durationMs) {
|
|
|
17470
17626
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
17471
17627
|
async function discoverDefaultTarget(repoRoot) {
|
|
17472
17628
|
const cwd = process.cwd();
|
|
17473
|
-
const chain = buildDirectoryChain2(
|
|
17629
|
+
const chain = buildDirectoryChain2(import_node_path42.default.join(cwd, "_placeholder"), repoRoot);
|
|
17474
17630
|
for (const dir of chain) {
|
|
17475
17631
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
17476
|
-
const targetsPath =
|
|
17632
|
+
const targetsPath = import_node_path42.default.join(dir, candidate);
|
|
17477
17633
|
if (!(0, import_node_fs12.existsSync)(targetsPath)) continue;
|
|
17478
17634
|
try {
|
|
17479
17635
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -17488,10 +17644,10 @@ async function discoverDefaultTarget(repoRoot) {
|
|
|
17488
17644
|
async function loadEnvHierarchy(repoRoot) {
|
|
17489
17645
|
const { readFileSync: readFileSync2 } = await import("fs");
|
|
17490
17646
|
const cwd = process.cwd();
|
|
17491
|
-
const chain = buildDirectoryChain2(
|
|
17647
|
+
const chain = buildDirectoryChain2(import_node_path42.default.join(cwd, "_placeholder"), repoRoot);
|
|
17492
17648
|
const envFiles = [];
|
|
17493
17649
|
for (const dir of chain) {
|
|
17494
|
-
const envPath =
|
|
17650
|
+
const envPath = import_node_path42.default.join(dir, ".env");
|
|
17495
17651
|
if ((0, import_node_fs12.existsSync)(envPath)) envFiles.push(envPath);
|
|
17496
17652
|
}
|
|
17497
17653
|
for (let i = envFiles.length - 1; i >= 0; i--) {
|
|
@@ -17673,7 +17829,7 @@ function buildPrompt(criteria, question, referenceAnswer) {
|
|
|
17673
17829
|
|
|
17674
17830
|
// src/evaluation/cache/response-cache.ts
|
|
17675
17831
|
var import_promises30 = require("fs/promises");
|
|
17676
|
-
var
|
|
17832
|
+
var import_node_path43 = __toESM(require("path"), 1);
|
|
17677
17833
|
var DEFAULT_CACHE_PATH = ".agentv/cache";
|
|
17678
17834
|
var ResponseCache = class {
|
|
17679
17835
|
cachePath;
|
|
@@ -17691,13 +17847,13 @@ var ResponseCache = class {
|
|
|
17691
17847
|
}
|
|
17692
17848
|
async set(key, value) {
|
|
17693
17849
|
const filePath = this.keyToPath(key);
|
|
17694
|
-
const dir =
|
|
17850
|
+
const dir = import_node_path43.default.dirname(filePath);
|
|
17695
17851
|
await (0, import_promises30.mkdir)(dir, { recursive: true });
|
|
17696
17852
|
await (0, import_promises30.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
17697
17853
|
}
|
|
17698
17854
|
keyToPath(key) {
|
|
17699
17855
|
const prefix = key.slice(0, 2);
|
|
17700
|
-
return
|
|
17856
|
+
return import_node_path43.default.join(this.cachePath, prefix, `${key}.json`);
|
|
17701
17857
|
}
|
|
17702
17858
|
};
|
|
17703
17859
|
function shouldEnableCache(params) {
|
|
@@ -18229,8 +18385,13 @@ function createAgentKernel() {
|
|
|
18229
18385
|
findGitRoot,
|
|
18230
18386
|
freeformEvaluationSchema,
|
|
18231
18387
|
generateRubrics,
|
|
18388
|
+
getAgentvHome,
|
|
18389
|
+
getGitCacheRoot,
|
|
18232
18390
|
getHitCount,
|
|
18391
|
+
getSubagentsRoot,
|
|
18392
|
+
getTraceStateRoot,
|
|
18233
18393
|
getWorkspacePath,
|
|
18394
|
+
getWorkspacesRoot,
|
|
18234
18395
|
initializeBaseline,
|
|
18235
18396
|
isEvaluatorKind,
|
|
18236
18397
|
isGuidelineFile,
|