@agentv/core 3.13.3 → 3.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-ZB3AUPES.js → chunk-HP5PFOVK.js} +48 -1
- package/dist/{chunk-ZB3AUPES.js.map → chunk-HP5PFOVK.js.map} +1 -1
- package/dist/evaluation/validation/index.cjs +1 -0
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +1 -1
- package/dist/index.cjs +877 -434
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +62 -2
- package/dist/index.d.ts +62 -2
- package/dist/index.js +728 -333
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1315,12 +1315,12 @@ function serializeAttributeValue(value) {
|
|
|
1315
1315
|
if (Array.isArray(value)) return { arrayValue: { values: value.map(serializeAttributeValue) } };
|
|
1316
1316
|
return { stringValue: String(value) };
|
|
1317
1317
|
}
|
|
1318
|
-
var
|
|
1318
|
+
var import_promises35, import_node_path51, OtlpJsonFileExporter;
|
|
1319
1319
|
var init_otlp_json_file_exporter = __esm({
|
|
1320
1320
|
"src/observability/otlp-json-file-exporter.ts"() {
|
|
1321
1321
|
"use strict";
|
|
1322
|
-
|
|
1323
|
-
|
|
1322
|
+
import_promises35 = require("fs/promises");
|
|
1323
|
+
import_node_path51 = require("path");
|
|
1324
1324
|
OtlpJsonFileExporter = class {
|
|
1325
1325
|
// biome-ignore lint/suspicious/noExplicitAny: serialized span data
|
|
1326
1326
|
spans = [];
|
|
@@ -1359,7 +1359,7 @@ var init_otlp_json_file_exporter = __esm({
|
|
|
1359
1359
|
}
|
|
1360
1360
|
async flush() {
|
|
1361
1361
|
if (this.spans.length === 0) return;
|
|
1362
|
-
await (0,
|
|
1362
|
+
await (0, import_promises35.mkdir)((0, import_node_path51.dirname)(this.filePath), { recursive: true });
|
|
1363
1363
|
const otlpJson = {
|
|
1364
1364
|
resourceSpans: [
|
|
1365
1365
|
{
|
|
@@ -1441,6 +1441,7 @@ __export(index_exports, {
|
|
|
1441
1441
|
defineConfig: () => defineConfig,
|
|
1442
1442
|
detectFormat: () => detectFormat,
|
|
1443
1443
|
discoverAssertions: () => discoverAssertions,
|
|
1444
|
+
discoverCopilotSessions: () => discoverCopilotSessions,
|
|
1444
1445
|
discoverGraders: () => discoverGraders,
|
|
1445
1446
|
discoverJudges: () => discoverGraders,
|
|
1446
1447
|
discoverProviders: () => discoverProviders,
|
|
@@ -4675,22 +4676,31 @@ async function loadTestById(evalFilePath, repoRoot, evalId) {
|
|
|
4675
4676
|
return match;
|
|
4676
4677
|
}
|
|
4677
4678
|
var loadEvalCaseById = loadTestById;
|
|
4679
|
+
function parseCommandArray(source) {
|
|
4680
|
+
if (typeof source === "string") {
|
|
4681
|
+
const parts = source.trim().split(/\s+/);
|
|
4682
|
+
return parts.length > 0 && parts[0] !== "" ? parts : void 0;
|
|
4683
|
+
}
|
|
4684
|
+
if (Array.isArray(source)) {
|
|
4685
|
+
const arr = source.filter((s) => typeof s === "string");
|
|
4686
|
+
return arr.length > 0 ? arr : void 0;
|
|
4687
|
+
}
|
|
4688
|
+
return void 0;
|
|
4689
|
+
}
|
|
4678
4690
|
function parseWorkspaceScriptConfig(raw, evalFileDir) {
|
|
4679
4691
|
if (!isJsonObject(raw)) return void 0;
|
|
4680
4692
|
const obj = raw;
|
|
4681
4693
|
if (obj.script !== void 0 && obj.command === void 0) {
|
|
4682
4694
|
logWarning5("'script' is deprecated. Use 'command' instead.");
|
|
4683
4695
|
}
|
|
4684
|
-
const
|
|
4685
|
-
if (!
|
|
4686
|
-
const commandArr = commandSource.filter((s) => typeof s === "string");
|
|
4687
|
-
if (commandArr.length === 0) return void 0;
|
|
4696
|
+
const command = parseCommandArray(obj.command ?? obj.script);
|
|
4697
|
+
if (!command) return void 0;
|
|
4688
4698
|
const timeoutMs = typeof obj.timeout_ms === "number" ? obj.timeout_ms : void 0;
|
|
4689
4699
|
let cwd = typeof obj.cwd === "string" ? obj.cwd : void 0;
|
|
4690
4700
|
if (cwd && !import_node_path8.default.isAbsolute(cwd)) {
|
|
4691
4701
|
cwd = import_node_path8.default.resolve(evalFileDir, cwd);
|
|
4692
4702
|
}
|
|
4693
|
-
const config = { command
|
|
4703
|
+
const config = { command };
|
|
4694
4704
|
if (timeoutMs !== void 0) {
|
|
4695
4705
|
return { ...config, timeout_ms: timeoutMs, ...cwd !== void 0 && { cwd } };
|
|
4696
4706
|
}
|
|
@@ -8246,10 +8256,268 @@ function summarizeAcpEvent(eventType, data) {
|
|
|
8246
8256
|
}
|
|
8247
8257
|
}
|
|
8248
8258
|
|
|
8249
|
-
// src/evaluation/providers/copilot-
|
|
8250
|
-
var
|
|
8259
|
+
// src/evaluation/providers/copilot-log.ts
|
|
8260
|
+
var import_promises16 = require("fs/promises");
|
|
8261
|
+
var import_node_os4 = require("os");
|
|
8262
|
+
var import_node_path19 = __toESM(require("path"), 1);
|
|
8263
|
+
|
|
8264
|
+
// src/evaluation/providers/copilot-log-parser.ts
|
|
8265
|
+
function parseCopilotEvents(eventsJsonl) {
|
|
8266
|
+
const messages = [];
|
|
8267
|
+
const meta = { sessionId: "", model: "", cwd: "" };
|
|
8268
|
+
let totalInputTokens = 0;
|
|
8269
|
+
let totalOutputTokens = 0;
|
|
8270
|
+
let hasUsage = false;
|
|
8271
|
+
let startTimestamp;
|
|
8272
|
+
let endTimestamp;
|
|
8273
|
+
const toolCallsInProgress = /* @__PURE__ */ new Map();
|
|
8274
|
+
const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
|
|
8275
|
+
for (const line of lines) {
|
|
8276
|
+
let event;
|
|
8277
|
+
try {
|
|
8278
|
+
event = JSON.parse(line);
|
|
8279
|
+
} catch {
|
|
8280
|
+
continue;
|
|
8281
|
+
}
|
|
8282
|
+
const eventType = event.type;
|
|
8283
|
+
if (!eventType) continue;
|
|
8284
|
+
const data = event.data ?? {};
|
|
8285
|
+
switch (eventType) {
|
|
8286
|
+
case "session.start": {
|
|
8287
|
+
meta.sessionId = String(data.sessionId ?? "");
|
|
8288
|
+
const ctx = data.context;
|
|
8289
|
+
meta.cwd = String(ctx?.cwd ?? "");
|
|
8290
|
+
meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
|
|
8291
|
+
meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
|
|
8292
|
+
const ts = event.timestamp ?? data.startTime;
|
|
8293
|
+
meta.startedAt = ts ? String(ts) : void 0;
|
|
8294
|
+
startTimestamp = ts ? String(ts) : void 0;
|
|
8295
|
+
break;
|
|
8296
|
+
}
|
|
8297
|
+
case "user.message": {
|
|
8298
|
+
messages.push({
|
|
8299
|
+
role: "user",
|
|
8300
|
+
content: data.content != null ? String(data.content) : ""
|
|
8301
|
+
});
|
|
8302
|
+
break;
|
|
8303
|
+
}
|
|
8304
|
+
case "assistant.message": {
|
|
8305
|
+
const toolRequests = data.toolRequests;
|
|
8306
|
+
const toolCalls = (toolRequests ?? []).map((req) => ({
|
|
8307
|
+
tool: String(req.name ?? req.toolName ?? ""),
|
|
8308
|
+
input: req.arguments,
|
|
8309
|
+
id: req.toolCallId ? String(req.toolCallId) : void 0
|
|
8310
|
+
}));
|
|
8311
|
+
messages.push({
|
|
8312
|
+
role: "assistant",
|
|
8313
|
+
content: data.content != null ? String(data.content) : void 0,
|
|
8314
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : void 0
|
|
8315
|
+
});
|
|
8316
|
+
break;
|
|
8317
|
+
}
|
|
8318
|
+
case "skill.invoked": {
|
|
8319
|
+
const skillName = String(data.name ?? "");
|
|
8320
|
+
messages.push({
|
|
8321
|
+
role: "assistant",
|
|
8322
|
+
toolCalls: [
|
|
8323
|
+
{
|
|
8324
|
+
tool: "Skill",
|
|
8325
|
+
input: { skill: skillName }
|
|
8326
|
+
}
|
|
8327
|
+
]
|
|
8328
|
+
});
|
|
8329
|
+
break;
|
|
8330
|
+
}
|
|
8331
|
+
case "tool.execution_start": {
|
|
8332
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
8333
|
+
if (toolCallId) {
|
|
8334
|
+
toolCallsInProgress.set(toolCallId, {
|
|
8335
|
+
toolName: String(data.toolName ?? ""),
|
|
8336
|
+
input: data.arguments,
|
|
8337
|
+
toolCallId
|
|
8338
|
+
});
|
|
8339
|
+
}
|
|
8340
|
+
break;
|
|
8341
|
+
}
|
|
8342
|
+
case "tool.execution_complete": {
|
|
8343
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
8344
|
+
const started = toolCallsInProgress.get(toolCallId);
|
|
8345
|
+
if (started) {
|
|
8346
|
+
toolCallsInProgress.delete(toolCallId);
|
|
8347
|
+
messages.push({
|
|
8348
|
+
role: "assistant",
|
|
8349
|
+
toolCalls: [
|
|
8350
|
+
{
|
|
8351
|
+
tool: started.toolName,
|
|
8352
|
+
input: started.input,
|
|
8353
|
+
output: data.result,
|
|
8354
|
+
id: toolCallId
|
|
8355
|
+
}
|
|
8356
|
+
]
|
|
8357
|
+
});
|
|
8358
|
+
}
|
|
8359
|
+
break;
|
|
8360
|
+
}
|
|
8361
|
+
case "session.shutdown": {
|
|
8362
|
+
endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
|
|
8363
|
+
const modelMetrics = data.modelMetrics;
|
|
8364
|
+
if (modelMetrics) {
|
|
8365
|
+
for (const metrics of Object.values(modelMetrics)) {
|
|
8366
|
+
if (metrics.usage) {
|
|
8367
|
+
hasUsage = true;
|
|
8368
|
+
totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
|
|
8369
|
+
totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
|
|
8370
|
+
}
|
|
8371
|
+
}
|
|
8372
|
+
}
|
|
8373
|
+
const currentModel = data.currentModel;
|
|
8374
|
+
if (currentModel && !meta.model) {
|
|
8375
|
+
meta.model = String(currentModel);
|
|
8376
|
+
}
|
|
8377
|
+
break;
|
|
8378
|
+
}
|
|
8379
|
+
}
|
|
8380
|
+
}
|
|
8381
|
+
let durationMs;
|
|
8382
|
+
if (startTimestamp && endTimestamp) {
|
|
8383
|
+
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
8384
|
+
}
|
|
8385
|
+
return {
|
|
8386
|
+
messages,
|
|
8387
|
+
meta,
|
|
8388
|
+
tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
|
|
8389
|
+
durationMs
|
|
8390
|
+
};
|
|
8391
|
+
}
|
|
8392
|
+
|
|
8393
|
+
// src/evaluation/providers/copilot-session-discovery.ts
|
|
8251
8394
|
var import_promises15 = require("fs/promises");
|
|
8395
|
+
var import_node_os3 = require("os");
|
|
8252
8396
|
var import_node_path18 = __toESM(require("path"), 1);
|
|
8397
|
+
var import_yaml6 = require("yaml");
|
|
8398
|
+
var DEFAULT_SESSION_STATE_DIR = () => import_node_path18.default.join((0, import_node_os3.homedir)(), ".copilot", "session-state");
|
|
8399
|
+
async function discoverCopilotSessions(opts) {
|
|
8400
|
+
const sessionStateDir = opts?.sessionStateDir ?? DEFAULT_SESSION_STATE_DIR();
|
|
8401
|
+
const limit = opts?.limit ?? 10;
|
|
8402
|
+
let entries;
|
|
8403
|
+
try {
|
|
8404
|
+
entries = await (0, import_promises15.readdir)(sessionStateDir);
|
|
8405
|
+
} catch {
|
|
8406
|
+
return [];
|
|
8407
|
+
}
|
|
8408
|
+
const sessions = [];
|
|
8409
|
+
for (const entry of entries) {
|
|
8410
|
+
const sessionDir = import_node_path18.default.join(sessionStateDir, entry);
|
|
8411
|
+
const workspacePath = import_node_path18.default.join(sessionDir, "workspace.yaml");
|
|
8412
|
+
const eventsPath = import_node_path18.default.join(sessionDir, "events.jsonl");
|
|
8413
|
+
try {
|
|
8414
|
+
const workspaceContent = await (0, import_promises15.readFile)(workspacePath, "utf8");
|
|
8415
|
+
const workspace = (0, import_yaml6.parse)(workspaceContent) ?? {};
|
|
8416
|
+
const cwd = String(workspace.cwd ?? "");
|
|
8417
|
+
let updatedAt;
|
|
8418
|
+
try {
|
|
8419
|
+
const eventsStat = await (0, import_promises15.stat)(eventsPath);
|
|
8420
|
+
updatedAt = eventsStat.mtime;
|
|
8421
|
+
} catch {
|
|
8422
|
+
updatedAt = /* @__PURE__ */ new Date(0);
|
|
8423
|
+
}
|
|
8424
|
+
let isActive = true;
|
|
8425
|
+
try {
|
|
8426
|
+
const fd = await import("fs/promises").then((fs3) => fs3.open(eventsPath, "r"));
|
|
8427
|
+
try {
|
|
8428
|
+
const fstat = await fd.stat();
|
|
8429
|
+
const tailSize = Math.min(fstat.size, 4096);
|
|
8430
|
+
const buf = Buffer.alloc(tailSize);
|
|
8431
|
+
await fd.read(buf, 0, tailSize, Math.max(0, fstat.size - tailSize));
|
|
8432
|
+
isActive = !buf.toString("utf8").includes('"session.shutdown"');
|
|
8433
|
+
} finally {
|
|
8434
|
+
await fd.close();
|
|
8435
|
+
}
|
|
8436
|
+
} catch {
|
|
8437
|
+
}
|
|
8438
|
+
sessions.push({
|
|
8439
|
+
sessionId: entry,
|
|
8440
|
+
sessionDir,
|
|
8441
|
+
cwd,
|
|
8442
|
+
repository: workspace.repository ? String(workspace.repository) : void 0,
|
|
8443
|
+
updatedAt,
|
|
8444
|
+
isActive
|
|
8445
|
+
});
|
|
8446
|
+
} catch {
|
|
8447
|
+
}
|
|
8448
|
+
}
|
|
8449
|
+
let filtered = sessions;
|
|
8450
|
+
if (opts?.cwd) {
|
|
8451
|
+
filtered = filtered.filter((s) => s.cwd === opts.cwd);
|
|
8452
|
+
}
|
|
8453
|
+
if (opts?.repository) {
|
|
8454
|
+
filtered = filtered.filter((s) => s.repository === opts.repository);
|
|
8455
|
+
}
|
|
8456
|
+
filtered.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
8457
|
+
return filtered.slice(0, limit);
|
|
8458
|
+
}
|
|
8459
|
+
|
|
8460
|
+
// src/evaluation/providers/copilot-log.ts
|
|
8461
|
+
var CopilotLogProvider = class {
|
|
8462
|
+
id;
|
|
8463
|
+
kind = "copilot-log";
|
|
8464
|
+
targetName;
|
|
8465
|
+
config;
|
|
8466
|
+
constructor(targetName, config) {
|
|
8467
|
+
this.targetName = targetName;
|
|
8468
|
+
this.id = `copilot-log:${targetName}`;
|
|
8469
|
+
this.config = config;
|
|
8470
|
+
}
|
|
8471
|
+
async invoke(_request) {
|
|
8472
|
+
const sessionDir = await this.resolveSessionDir();
|
|
8473
|
+
const eventsPath = import_node_path19.default.join(sessionDir, "events.jsonl");
|
|
8474
|
+
let eventsContent;
|
|
8475
|
+
try {
|
|
8476
|
+
eventsContent = await (0, import_promises16.readFile)(eventsPath, "utf8");
|
|
8477
|
+
} catch (err) {
|
|
8478
|
+
throw new Error(
|
|
8479
|
+
`Failed to read Copilot session transcript at ${eventsPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
8480
|
+
);
|
|
8481
|
+
}
|
|
8482
|
+
const parsed = parseCopilotEvents(eventsContent);
|
|
8483
|
+
return {
|
|
8484
|
+
output: parsed.messages,
|
|
8485
|
+
tokenUsage: parsed.tokenUsage,
|
|
8486
|
+
durationMs: parsed.durationMs,
|
|
8487
|
+
startTime: parsed.meta.startedAt
|
|
8488
|
+
};
|
|
8489
|
+
}
|
|
8490
|
+
async resolveSessionDir() {
|
|
8491
|
+
if (this.config.sessionDir) {
|
|
8492
|
+
return this.config.sessionDir;
|
|
8493
|
+
}
|
|
8494
|
+
if (this.config.sessionId) {
|
|
8495
|
+
const stateDir = this.config.sessionStateDir ?? import_node_path19.default.join((0, import_node_os4.homedir)(), ".copilot", "session-state");
|
|
8496
|
+
return import_node_path19.default.join(stateDir, this.config.sessionId);
|
|
8497
|
+
}
|
|
8498
|
+
if (this.config.discover === "latest") {
|
|
8499
|
+
const sessions = await discoverCopilotSessions({
|
|
8500
|
+
sessionStateDir: this.config.sessionStateDir,
|
|
8501
|
+
cwd: this.config.cwd,
|
|
8502
|
+
limit: 1
|
|
8503
|
+
});
|
|
8504
|
+
if (sessions.length === 0) {
|
|
8505
|
+
throw new Error(
|
|
8506
|
+
`No Copilot CLI sessions found${this.config.cwd ? ` for cwd=${this.config.cwd}` : ""}. Check that sessions exist in ${this.config.sessionStateDir ?? "~/.copilot/session-state/"}`
|
|
8507
|
+
);
|
|
8508
|
+
}
|
|
8509
|
+
return sessions[0].sessionDir;
|
|
8510
|
+
}
|
|
8511
|
+
throw new Error(
|
|
8512
|
+
'CopilotLogProvider requires one of: sessionDir, sessionId, or discover="latest"'
|
|
8513
|
+
);
|
|
8514
|
+
}
|
|
8515
|
+
};
|
|
8516
|
+
|
|
8517
|
+
// src/evaluation/providers/copilot-sdk.ts
|
|
8518
|
+
var import_node_crypto6 = require("crypto");
|
|
8519
|
+
var import_promises17 = require("fs/promises");
|
|
8520
|
+
var import_node_path20 = __toESM(require("path"), 1);
|
|
8253
8521
|
|
|
8254
8522
|
// src/evaluation/providers/copilot-sdk-log-tracker.ts
|
|
8255
8523
|
var GLOBAL_LOGS_KEY4 = Symbol.for("agentv.copilotSdkLogs");
|
|
@@ -8539,10 +8807,10 @@ var CopilotSdkProvider = class {
|
|
|
8539
8807
|
}
|
|
8540
8808
|
resolveCwd(cwdOverride) {
|
|
8541
8809
|
if (cwdOverride) {
|
|
8542
|
-
return
|
|
8810
|
+
return import_node_path20.default.resolve(cwdOverride);
|
|
8543
8811
|
}
|
|
8544
8812
|
if (this.config.cwd) {
|
|
8545
|
-
return
|
|
8813
|
+
return import_node_path20.default.resolve(this.config.cwd);
|
|
8546
8814
|
}
|
|
8547
8815
|
return void 0;
|
|
8548
8816
|
}
|
|
@@ -8551,9 +8819,9 @@ var CopilotSdkProvider = class {
|
|
|
8551
8819
|
return void 0;
|
|
8552
8820
|
}
|
|
8553
8821
|
if (this.config.logDir) {
|
|
8554
|
-
return
|
|
8822
|
+
return import_node_path20.default.resolve(this.config.logDir);
|
|
8555
8823
|
}
|
|
8556
|
-
return
|
|
8824
|
+
return import_node_path20.default.join(process.cwd(), ".agentv", "logs", "copilot-sdk");
|
|
8557
8825
|
}
|
|
8558
8826
|
async createStreamLogger(request) {
|
|
8559
8827
|
const logDir = this.resolveLogDirectory();
|
|
@@ -8561,13 +8829,13 @@ var CopilotSdkProvider = class {
|
|
|
8561
8829
|
return void 0;
|
|
8562
8830
|
}
|
|
8563
8831
|
try {
|
|
8564
|
-
await (0,
|
|
8832
|
+
await (0, import_promises17.mkdir)(logDir, { recursive: true });
|
|
8565
8833
|
} catch (error) {
|
|
8566
8834
|
const message = error instanceof Error ? error.message : String(error);
|
|
8567
8835
|
console.warn(`Skipping Copilot SDK stream logging (could not create ${logDir}): ${message}`);
|
|
8568
8836
|
return void 0;
|
|
8569
8837
|
}
|
|
8570
|
-
const filePath =
|
|
8838
|
+
const filePath = import_node_path20.default.join(logDir, buildLogFilename4(request, this.targetName, "copilot-sdk"));
|
|
8571
8839
|
try {
|
|
8572
8840
|
const logger = await CopilotStreamLogger.create(
|
|
8573
8841
|
{
|
|
@@ -8662,9 +8930,9 @@ var MockProvider = class {
|
|
|
8662
8930
|
var import_node_child_process4 = require("child_process");
|
|
8663
8931
|
var import_node_crypto7 = require("crypto");
|
|
8664
8932
|
var import_node_fs8 = require("fs");
|
|
8665
|
-
var
|
|
8666
|
-
var
|
|
8667
|
-
var
|
|
8933
|
+
var import_promises18 = require("fs/promises");
|
|
8934
|
+
var import_node_os5 = require("os");
|
|
8935
|
+
var import_node_path21 = __toESM(require("path"), 1);
|
|
8668
8936
|
|
|
8669
8937
|
// src/evaluation/providers/pi-log-tracker.ts
|
|
8670
8938
|
var GLOBAL_LOGS_KEY5 = Symbol.for("agentv.piLogs");
|
|
@@ -8767,13 +9035,14 @@ var PiCliProvider = class {
|
|
|
8767
9035
|
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
8768
9036
|
const startTime = (/* @__PURE__ */ new Date()).toISOString();
|
|
8769
9037
|
const startMs = Date.now();
|
|
8770
|
-
const
|
|
9038
|
+
const hasExternalCwd = !!(request.cwd || this.config.cwd);
|
|
9039
|
+
const workspaceRoot = hasExternalCwd ? void 0 : await this.createWorkspace();
|
|
9040
|
+
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
8771
9041
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
8772
9042
|
try {
|
|
8773
|
-
const promptFile =
|
|
8774
|
-
await (0,
|
|
9043
|
+
const promptFile = import_node_path21.default.join(cwd, PROMPT_FILENAME);
|
|
9044
|
+
await (0, import_promises18.writeFile)(promptFile, request.question, "utf8");
|
|
8775
9045
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
8776
|
-
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
8777
9046
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
8778
9047
|
if (result.timedOut) {
|
|
8779
9048
|
throw new Error(
|
|
@@ -8815,7 +9084,7 @@ var PiCliProvider = class {
|
|
|
8815
9084
|
args,
|
|
8816
9085
|
executable: this.config.executable,
|
|
8817
9086
|
promptFile,
|
|
8818
|
-
workspace: workspaceRoot,
|
|
9087
|
+
workspace: workspaceRoot ?? cwd,
|
|
8819
9088
|
inputFiles,
|
|
8820
9089
|
logFile: logger?.filePath
|
|
8821
9090
|
},
|
|
@@ -8827,17 +9096,22 @@ var PiCliProvider = class {
|
|
|
8827
9096
|
};
|
|
8828
9097
|
} finally {
|
|
8829
9098
|
await logger?.close();
|
|
8830
|
-
|
|
9099
|
+
if (workspaceRoot) {
|
|
9100
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
9101
|
+
}
|
|
8831
9102
|
}
|
|
8832
9103
|
}
|
|
8833
9104
|
resolveCwd(workspaceRoot, cwdOverride) {
|
|
8834
9105
|
if (cwdOverride) {
|
|
8835
|
-
return
|
|
9106
|
+
return import_node_path21.default.resolve(cwdOverride);
|
|
9107
|
+
}
|
|
9108
|
+
if (this.config.cwd) {
|
|
9109
|
+
return import_node_path21.default.resolve(this.config.cwd);
|
|
8836
9110
|
}
|
|
8837
|
-
if (
|
|
9111
|
+
if (workspaceRoot) {
|
|
8838
9112
|
return workspaceRoot;
|
|
8839
9113
|
}
|
|
8840
|
-
return
|
|
9114
|
+
return process.cwd();
|
|
8841
9115
|
}
|
|
8842
9116
|
buildPiArgs(prompt, inputFiles) {
|
|
8843
9117
|
const args = [];
|
|
@@ -8915,22 +9189,42 @@ ${prompt}` : prompt;
|
|
|
8915
9189
|
env[envKey] = this.config.apiKey;
|
|
8916
9190
|
}
|
|
8917
9191
|
}
|
|
9192
|
+
if (this.config.subprovider) {
|
|
9193
|
+
const provider = this.config.subprovider.toLowerCase();
|
|
9194
|
+
const PROVIDER_OWN_PREFIXES = {
|
|
9195
|
+
openrouter: ["OPENROUTER_"],
|
|
9196
|
+
anthropic: ["ANTHROPIC_"],
|
|
9197
|
+
openai: ["OPENAI_"],
|
|
9198
|
+
azure: ["AZURE_OPENAI_"],
|
|
9199
|
+
google: ["GEMINI_", "GOOGLE_GENERATIVE_AI_"],
|
|
9200
|
+
gemini: ["GEMINI_", "GOOGLE_GENERATIVE_AI_"],
|
|
9201
|
+
groq: ["GROQ_"],
|
|
9202
|
+
xai: ["XAI_"]
|
|
9203
|
+
};
|
|
9204
|
+
const ownPrefixes = PROVIDER_OWN_PREFIXES[provider] ?? [];
|
|
9205
|
+
const allOtherPrefixes = Object.entries(PROVIDER_OWN_PREFIXES).filter(([key]) => key !== provider).flatMap(([, prefixes]) => prefixes);
|
|
9206
|
+
for (const key of Object.keys(env)) {
|
|
9207
|
+
if (allOtherPrefixes.some((prefix) => key.startsWith(prefix)) && !ownPrefixes.some((prefix) => key.startsWith(prefix))) {
|
|
9208
|
+
delete env[key];
|
|
9209
|
+
}
|
|
9210
|
+
}
|
|
9211
|
+
}
|
|
8918
9212
|
return env;
|
|
8919
9213
|
}
|
|
8920
9214
|
async createWorkspace() {
|
|
8921
|
-
return await (0,
|
|
9215
|
+
return await (0, import_promises18.mkdtemp)(import_node_path21.default.join((0, import_node_os5.tmpdir)(), WORKSPACE_PREFIX));
|
|
8922
9216
|
}
|
|
8923
9217
|
async cleanupWorkspace(workspaceRoot) {
|
|
8924
9218
|
try {
|
|
8925
|
-
await (0,
|
|
9219
|
+
await (0, import_promises18.rm)(workspaceRoot, { recursive: true, force: true });
|
|
8926
9220
|
} catch {
|
|
8927
9221
|
}
|
|
8928
9222
|
}
|
|
8929
9223
|
resolveLogDirectory() {
|
|
8930
9224
|
if (this.config.logDir) {
|
|
8931
|
-
return
|
|
9225
|
+
return import_node_path21.default.resolve(this.config.logDir);
|
|
8932
9226
|
}
|
|
8933
|
-
return
|
|
9227
|
+
return import_node_path21.default.join(process.cwd(), ".agentv", "logs", "pi-cli");
|
|
8934
9228
|
}
|
|
8935
9229
|
async createStreamLogger(request) {
|
|
8936
9230
|
const logDir = this.resolveLogDirectory();
|
|
@@ -8938,13 +9232,13 @@ ${prompt}` : prompt;
|
|
|
8938
9232
|
return void 0;
|
|
8939
9233
|
}
|
|
8940
9234
|
try {
|
|
8941
|
-
await (0,
|
|
9235
|
+
await (0, import_promises18.mkdir)(logDir, { recursive: true });
|
|
8942
9236
|
} catch (error) {
|
|
8943
9237
|
const message = error instanceof Error ? error.message : String(error);
|
|
8944
9238
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
8945
9239
|
return void 0;
|
|
8946
9240
|
}
|
|
8947
|
-
const filePath =
|
|
9241
|
+
const filePath = import_node_path21.default.join(logDir, buildLogFilename5(request, this.targetName));
|
|
8948
9242
|
try {
|
|
8949
9243
|
const logger = await PiStreamLogger.create({
|
|
8950
9244
|
filePath,
|
|
@@ -9142,6 +9436,10 @@ function summarizePiEvent(event) {
|
|
|
9142
9436
|
}
|
|
9143
9437
|
return `message_update: ${eventType}`;
|
|
9144
9438
|
}
|
|
9439
|
+
case "tool_execution_start":
|
|
9440
|
+
return `tool_start: ${record.toolName}`;
|
|
9441
|
+
case "tool_execution_end":
|
|
9442
|
+
return `tool_end: ${record.toolName}`;
|
|
9145
9443
|
default:
|
|
9146
9444
|
return type;
|
|
9147
9445
|
}
|
|
@@ -9172,25 +9470,89 @@ function parsePiJsonl(output) {
|
|
|
9172
9470
|
return parsed;
|
|
9173
9471
|
}
|
|
9174
9472
|
function extractMessages(events) {
|
|
9473
|
+
let messages;
|
|
9175
9474
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
9176
9475
|
const event = events[i];
|
|
9177
9476
|
if (!event || typeof event !== "object") continue;
|
|
9178
9477
|
const record = event;
|
|
9179
9478
|
if (record.type !== "agent_end") continue;
|
|
9180
|
-
const
|
|
9181
|
-
if (!Array.isArray(
|
|
9182
|
-
|
|
9479
|
+
const msgs = record.messages;
|
|
9480
|
+
if (!Array.isArray(msgs)) continue;
|
|
9481
|
+
messages = msgs.map(convertPiMessage).filter((m) => m !== void 0);
|
|
9482
|
+
break;
|
|
9483
|
+
}
|
|
9484
|
+
if (!messages) {
|
|
9485
|
+
messages = [];
|
|
9486
|
+
for (const event of events) {
|
|
9487
|
+
if (!event || typeof event !== "object") continue;
|
|
9488
|
+
const record = event;
|
|
9489
|
+
if (record.type === "turn_end") {
|
|
9490
|
+
const converted = convertPiMessage(record.message);
|
|
9491
|
+
if (converted) messages.push(converted);
|
|
9492
|
+
}
|
|
9493
|
+
}
|
|
9494
|
+
}
|
|
9495
|
+
const eventToolCalls = extractToolCallsFromEvents(events);
|
|
9496
|
+
if (eventToolCalls.length > 0) {
|
|
9497
|
+
injectEventToolCalls(messages, eventToolCalls);
|
|
9183
9498
|
}
|
|
9184
|
-
|
|
9499
|
+
return messages;
|
|
9500
|
+
}
|
|
9501
|
+
function extractToolCallsFromEvents(events) {
|
|
9502
|
+
const starts = /* @__PURE__ */ new Map();
|
|
9503
|
+
const results = /* @__PURE__ */ new Map();
|
|
9185
9504
|
for (const event of events) {
|
|
9186
9505
|
if (!event || typeof event !== "object") continue;
|
|
9187
|
-
const
|
|
9188
|
-
|
|
9189
|
-
|
|
9190
|
-
|
|
9506
|
+
const r = event;
|
|
9507
|
+
const type = r.type;
|
|
9508
|
+
if (type === "tool_execution_start" && typeof r.toolName === "string") {
|
|
9509
|
+
const id = typeof r.toolCallId === "string" ? r.toolCallId : void 0;
|
|
9510
|
+
starts.set(id ?? `anon-${starts.size}`, { tool: r.toolName, input: r.args });
|
|
9511
|
+
} else if (type === "tool_execution_end") {
|
|
9512
|
+
const id = typeof r.toolCallId === "string" ? r.toolCallId : void 0;
|
|
9513
|
+
if (id) results.set(id, r.result);
|
|
9191
9514
|
}
|
|
9192
9515
|
}
|
|
9193
|
-
|
|
9516
|
+
const toolCalls = [];
|
|
9517
|
+
for (const [id, { tool: tool2, input }] of starts) {
|
|
9518
|
+
toolCalls.push({
|
|
9519
|
+
tool: tool2,
|
|
9520
|
+
input,
|
|
9521
|
+
id: id.startsWith("anon-") ? void 0 : id,
|
|
9522
|
+
output: results.get(id)
|
|
9523
|
+
});
|
|
9524
|
+
}
|
|
9525
|
+
return toolCalls;
|
|
9526
|
+
}
|
|
9527
|
+
function injectEventToolCalls(messages, eventToolCalls) {
|
|
9528
|
+
const existingIds = /* @__PURE__ */ new Set();
|
|
9529
|
+
const existingTools = /* @__PURE__ */ new Set();
|
|
9530
|
+
for (const msg of messages) {
|
|
9531
|
+
if (!msg.toolCalls) continue;
|
|
9532
|
+
for (const tc of msg.toolCalls) {
|
|
9533
|
+
if (tc.id) existingIds.add(tc.id);
|
|
9534
|
+
existingTools.add(`${tc.tool}:${JSON.stringify(tc.input)}`);
|
|
9535
|
+
}
|
|
9536
|
+
}
|
|
9537
|
+
const missing = eventToolCalls.filter((tc) => {
|
|
9538
|
+
if (tc.id && existingIds.has(tc.id)) return false;
|
|
9539
|
+
if (existingTools.has(`${tc.tool}:${JSON.stringify(tc.input)}`)) return false;
|
|
9540
|
+
return true;
|
|
9541
|
+
});
|
|
9542
|
+
if (missing.length === 0) return;
|
|
9543
|
+
let targetIdx = -1;
|
|
9544
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
9545
|
+
if (messages[i].role === "assistant") {
|
|
9546
|
+
targetIdx = i;
|
|
9547
|
+
break;
|
|
9548
|
+
}
|
|
9549
|
+
}
|
|
9550
|
+
if (targetIdx >= 0) {
|
|
9551
|
+
const target = messages[targetIdx];
|
|
9552
|
+
messages[targetIdx] = { ...target, toolCalls: [...target.toolCalls ?? [], ...missing] };
|
|
9553
|
+
} else {
|
|
9554
|
+
messages.push({ role: "assistant", content: "", toolCalls: missing });
|
|
9555
|
+
}
|
|
9194
9556
|
}
|
|
9195
9557
|
function extractTokenUsage(events) {
|
|
9196
9558
|
for (let i = events.length - 1; i >= 0; i--) {
|
|
@@ -9285,15 +9647,13 @@ function extractToolCalls3(content) {
|
|
|
9285
9647
|
input: p.input,
|
|
9286
9648
|
id: typeof p.id === "string" ? p.id : void 0
|
|
9287
9649
|
});
|
|
9288
|
-
}
|
|
9289
|
-
if (p.type === "toolCall" && typeof p.name === "string") {
|
|
9650
|
+
} else if ((p.type === "toolCall" || p.type === "tool_call") && typeof p.name === "string") {
|
|
9290
9651
|
toolCalls.push({
|
|
9291
9652
|
tool: p.name,
|
|
9292
|
-
input: p.arguments,
|
|
9653
|
+
input: p.arguments ?? p.input,
|
|
9293
9654
|
id: typeof p.id === "string" ? p.id : void 0
|
|
9294
9655
|
});
|
|
9295
|
-
}
|
|
9296
|
-
if (p.type === "tool_result" && typeof p.tool_use_id === "string") {
|
|
9656
|
+
} else if (p.type === "tool_result" && typeof p.tool_use_id === "string") {
|
|
9297
9657
|
const existing = toolCalls.find((tc) => tc.id === p.tool_use_id);
|
|
9298
9658
|
if (existing) {
|
|
9299
9659
|
const idx = toolCalls.indexOf(existing);
|
|
@@ -9384,14 +9744,17 @@ async function defaultPiRunner(options) {
|
|
|
9384
9744
|
var import_node_child_process5 = require("child_process");
|
|
9385
9745
|
var import_node_crypto8 = require("crypto");
|
|
9386
9746
|
var import_node_fs9 = require("fs");
|
|
9387
|
-
var
|
|
9388
|
-
var
|
|
9747
|
+
var import_promises19 = require("fs/promises");
|
|
9748
|
+
var import_node_path22 = __toESM(require("path"), 1);
|
|
9389
9749
|
var import_node_readline = require("readline");
|
|
9750
|
+
var import_node_url3 = require("url");
|
|
9751
|
+
var import_meta2 = {};
|
|
9390
9752
|
var piCodingAgentModule = null;
|
|
9391
9753
|
var piAiModule = null;
|
|
9754
|
+
var loadingPromise = null;
|
|
9392
9755
|
async function promptInstall() {
|
|
9393
9756
|
if (!process.stdout.isTTY) return false;
|
|
9394
|
-
const rl = (0, import_node_readline.createInterface)({ input: process.stdin, output: process.
|
|
9757
|
+
const rl = (0, import_node_readline.createInterface)({ input: process.stdin, output: process.stdout });
|
|
9395
9758
|
try {
|
|
9396
9759
|
return await new Promise((resolve) => {
|
|
9397
9760
|
rl.question(
|
|
@@ -9403,43 +9766,74 @@ async function promptInstall() {
|
|
|
9403
9766
|
rl.close();
|
|
9404
9767
|
}
|
|
9405
9768
|
}
|
|
9406
|
-
|
|
9407
|
-
|
|
9769
|
+
function findAgentvRoot() {
|
|
9770
|
+
const thisFile = (0, import_node_url3.fileURLToPath)(import_meta2.url);
|
|
9771
|
+
let dir = import_node_path22.default.dirname(thisFile);
|
|
9772
|
+
for (let i = 0; i < 10; i++) {
|
|
9408
9773
|
try {
|
|
9774
|
+
const pkg = import_node_path22.default.join(dir, "package.json");
|
|
9775
|
+
(0, import_node_fs9.accessSync)(pkg);
|
|
9776
|
+
return dir;
|
|
9777
|
+
} catch {
|
|
9778
|
+
const parent = import_node_path22.default.dirname(dir);
|
|
9779
|
+
if (parent === dir) break;
|
|
9780
|
+
dir = parent;
|
|
9781
|
+
}
|
|
9782
|
+
}
|
|
9783
|
+
return import_node_path22.default.dirname(thisFile);
|
|
9784
|
+
}
|
|
9785
|
+
async function doLoadSdkModules() {
|
|
9786
|
+
try {
|
|
9787
|
+
[piCodingAgentModule, piAiModule] = await Promise.all([
|
|
9788
|
+
import("@mariozechner/pi-coding-agent"),
|
|
9789
|
+
import("@mariozechner/pi-ai")
|
|
9790
|
+
]);
|
|
9791
|
+
} catch {
|
|
9792
|
+
if (await promptInstall()) {
|
|
9793
|
+
const installDir = findAgentvRoot();
|
|
9794
|
+
console.error(`Installing @mariozechner/pi-coding-agent into ${installDir}...`);
|
|
9795
|
+
(0, import_node_child_process5.execSync)("bun add @mariozechner/pi-coding-agent", {
|
|
9796
|
+
cwd: installDir,
|
|
9797
|
+
stdio: "inherit"
|
|
9798
|
+
});
|
|
9409
9799
|
[piCodingAgentModule, piAiModule] = await Promise.all([
|
|
9410
9800
|
import("@mariozechner/pi-coding-agent"),
|
|
9411
9801
|
import("@mariozechner/pi-ai")
|
|
9412
9802
|
]);
|
|
9413
|
-
}
|
|
9414
|
-
|
|
9415
|
-
|
|
9416
|
-
|
|
9417
|
-
|
|
9418
|
-
|
|
9419
|
-
|
|
9420
|
-
|
|
9421
|
-
|
|
9422
|
-
|
|
9423
|
-
|
|
9424
|
-
|
|
9425
|
-
|
|
9803
|
+
} else {
|
|
9804
|
+
throw new Error(
|
|
9805
|
+
"pi-coding-agent SDK is not installed. Install it with:\n bun add @mariozechner/pi-coding-agent"
|
|
9806
|
+
);
|
|
9807
|
+
}
|
|
9808
|
+
}
|
|
9809
|
+
}
|
|
9810
|
+
async function loadSdkModules() {
|
|
9811
|
+
if (!piCodingAgentModule || !piAiModule) {
|
|
9812
|
+
if (!loadingPromise) {
|
|
9813
|
+
loadingPromise = doLoadSdkModules().catch((err) => {
|
|
9814
|
+
loadingPromise = null;
|
|
9815
|
+
throw err;
|
|
9816
|
+
});
|
|
9426
9817
|
}
|
|
9818
|
+
await loadingPromise;
|
|
9427
9819
|
}
|
|
9820
|
+
const piSdk = piCodingAgentModule;
|
|
9821
|
+
const piAi = piAiModule;
|
|
9428
9822
|
const toolMap = {
|
|
9429
|
-
read:
|
|
9430
|
-
bash:
|
|
9431
|
-
edit:
|
|
9432
|
-
write:
|
|
9433
|
-
grep:
|
|
9434
|
-
find:
|
|
9435
|
-
ls:
|
|
9823
|
+
read: piSdk.readTool,
|
|
9824
|
+
bash: piSdk.bashTool,
|
|
9825
|
+
edit: piSdk.editTool,
|
|
9826
|
+
write: piSdk.writeTool,
|
|
9827
|
+
grep: piSdk.grepTool,
|
|
9828
|
+
find: piSdk.findTool,
|
|
9829
|
+
ls: piSdk.lsTool
|
|
9436
9830
|
};
|
|
9437
9831
|
return {
|
|
9438
|
-
createAgentSession:
|
|
9439
|
-
codingTools:
|
|
9832
|
+
createAgentSession: piSdk.createAgentSession,
|
|
9833
|
+
codingTools: piSdk.codingTools,
|
|
9440
9834
|
toolMap,
|
|
9441
|
-
SessionManager:
|
|
9442
|
-
getModel:
|
|
9835
|
+
SessionManager: piSdk.SessionManager,
|
|
9836
|
+
getModel: piAi.getModel
|
|
9443
9837
|
};
|
|
9444
9838
|
}
|
|
9445
9839
|
var PiCodingAgentProvider = class {
|
|
@@ -9628,10 +10022,10 @@ ${fileList}`;
|
|
|
9628
10022
|
}
|
|
9629
10023
|
resolveCwd(cwdOverride) {
|
|
9630
10024
|
if (cwdOverride) {
|
|
9631
|
-
return
|
|
10025
|
+
return import_node_path22.default.resolve(cwdOverride);
|
|
9632
10026
|
}
|
|
9633
10027
|
if (this.config.cwd) {
|
|
9634
|
-
return
|
|
10028
|
+
return import_node_path22.default.resolve(this.config.cwd);
|
|
9635
10029
|
}
|
|
9636
10030
|
return process.cwd();
|
|
9637
10031
|
}
|
|
@@ -9650,9 +10044,9 @@ ${fileList}`;
|
|
|
9650
10044
|
}
|
|
9651
10045
|
resolveLogDirectory() {
|
|
9652
10046
|
if (this.config.logDir) {
|
|
9653
|
-
return
|
|
10047
|
+
return import_node_path22.default.resolve(this.config.logDir);
|
|
9654
10048
|
}
|
|
9655
|
-
return
|
|
10049
|
+
return import_node_path22.default.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
9656
10050
|
}
|
|
9657
10051
|
async createStreamLogger(request) {
|
|
9658
10052
|
const logDir = this.resolveLogDirectory();
|
|
@@ -9660,13 +10054,13 @@ ${fileList}`;
|
|
|
9660
10054
|
return void 0;
|
|
9661
10055
|
}
|
|
9662
10056
|
try {
|
|
9663
|
-
await (0,
|
|
10057
|
+
await (0, import_promises19.mkdir)(logDir, { recursive: true });
|
|
9664
10058
|
} catch (error) {
|
|
9665
10059
|
const message = error instanceof Error ? error.message : String(error);
|
|
9666
10060
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
9667
10061
|
return void 0;
|
|
9668
10062
|
}
|
|
9669
|
-
const filePath =
|
|
10063
|
+
const filePath = import_node_path22.default.join(logDir, buildLogFilename6(request, this.targetName));
|
|
9670
10064
|
try {
|
|
9671
10065
|
const logger = await PiStreamLogger2.create({
|
|
9672
10066
|
filePath,
|
|
@@ -9878,7 +10272,7 @@ var ProviderRegistry = class {
|
|
|
9878
10272
|
};
|
|
9879
10273
|
|
|
9880
10274
|
// src/evaluation/providers/targets.ts
|
|
9881
|
-
var
|
|
10275
|
+
var import_node_path23 = __toESM(require("path"), 1);
|
|
9882
10276
|
var import_zod3 = require("zod");
|
|
9883
10277
|
var CliHealthcheckHttpInputSchema = import_zod3.z.object({
|
|
9884
10278
|
url: import_zod3.z.string().min(1, "healthcheck URL is required"),
|
|
@@ -9975,11 +10369,11 @@ function normalizeCliHealthcheck(input, env, targetName, evalFilePath) {
|
|
|
9975
10369
|
allowLiteral: true,
|
|
9976
10370
|
optionalEnv: true
|
|
9977
10371
|
});
|
|
9978
|
-
if (cwd && evalFilePath && !
|
|
9979
|
-
cwd =
|
|
10372
|
+
if (cwd && evalFilePath && !import_node_path23.default.isAbsolute(cwd)) {
|
|
10373
|
+
cwd = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), cwd);
|
|
9980
10374
|
}
|
|
9981
10375
|
if (!cwd && evalFilePath) {
|
|
9982
|
-
cwd =
|
|
10376
|
+
cwd = import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath));
|
|
9983
10377
|
}
|
|
9984
10378
|
return {
|
|
9985
10379
|
command,
|
|
@@ -10002,15 +10396,15 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
|
|
|
10002
10396
|
optionalEnv: true
|
|
10003
10397
|
}
|
|
10004
10398
|
);
|
|
10005
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10006
|
-
workspaceTemplate =
|
|
10399
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
10400
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10007
10401
|
}
|
|
10008
10402
|
let cwd = resolveOptionalString(input.cwd, env, `${targetName} working directory`, {
|
|
10009
10403
|
allowLiteral: true,
|
|
10010
10404
|
optionalEnv: true
|
|
10011
10405
|
});
|
|
10012
|
-
if (cwd && evalFilePath && !
|
|
10013
|
-
cwd =
|
|
10406
|
+
if (cwd && evalFilePath && !import_node_path23.default.isAbsolute(cwd)) {
|
|
10407
|
+
cwd = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), cwd);
|
|
10014
10408
|
}
|
|
10015
10409
|
if (cwd && workspaceTemplate) {
|
|
10016
10410
|
throw new Error(
|
|
@@ -10018,7 +10412,7 @@ function normalizeCliTargetInput(input, env, evalFilePath) {
|
|
|
10018
10412
|
);
|
|
10019
10413
|
}
|
|
10020
10414
|
if (!cwd && !workspaceTemplate && evalFilePath) {
|
|
10021
|
-
cwd =
|
|
10415
|
+
cwd = import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath));
|
|
10022
10416
|
}
|
|
10023
10417
|
const timeoutSeconds = input.timeout_seconds ?? input.timeoutSeconds;
|
|
10024
10418
|
const timeoutMs = timeoutSeconds !== void 0 ? Math.floor(timeoutSeconds * 1e3) : void 0;
|
|
@@ -10196,6 +10590,15 @@ function resolveTargetDefinition(definition, env = process.env, evalFilePath) {
|
|
|
10196
10590
|
providerBatching,
|
|
10197
10591
|
config: resolveCopilotCliConfig(parsed, env, evalFilePath)
|
|
10198
10592
|
};
|
|
10593
|
+
case "copilot-log":
|
|
10594
|
+
return {
|
|
10595
|
+
kind: "copilot-log",
|
|
10596
|
+
name: parsed.name,
|
|
10597
|
+
graderTarget: parsed.grader_target ?? parsed.judge_target,
|
|
10598
|
+
workers: parsed.workers,
|
|
10599
|
+
providerBatching,
|
|
10600
|
+
config: resolveCopilotLogConfig(parsed, env)
|
|
10601
|
+
};
|
|
10199
10602
|
case "pi":
|
|
10200
10603
|
case "pi-coding-agent":
|
|
10201
10604
|
return {
|
|
@@ -10440,8 +10843,8 @@ function resolveCodexConfig(target, env, evalFilePath) {
|
|
|
10440
10843
|
optionalEnv: true
|
|
10441
10844
|
}
|
|
10442
10845
|
);
|
|
10443
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10444
|
-
workspaceTemplate =
|
|
10846
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
10847
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10445
10848
|
}
|
|
10446
10849
|
if (cwd && workspaceTemplate) {
|
|
10447
10850
|
throw new Error(
|
|
@@ -10525,8 +10928,8 @@ function resolveCopilotSdkConfig(target, env, evalFilePath) {
|
|
|
10525
10928
|
optionalEnv: true
|
|
10526
10929
|
}
|
|
10527
10930
|
);
|
|
10528
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10529
|
-
workspaceTemplate =
|
|
10931
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
10932
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10530
10933
|
}
|
|
10531
10934
|
if (cwd && workspaceTemplate) {
|
|
10532
10935
|
throw new Error(
|
|
@@ -10590,8 +10993,8 @@ function resolveCopilotCliConfig(target, env, evalFilePath) {
|
|
|
10590
10993
|
optionalEnv: true
|
|
10591
10994
|
}
|
|
10592
10995
|
);
|
|
10593
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10594
|
-
workspaceTemplate =
|
|
10996
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
10997
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10595
10998
|
}
|
|
10596
10999
|
if (cwd && workspaceTemplate) {
|
|
10597
11000
|
throw new Error(
|
|
@@ -10679,8 +11082,8 @@ function resolvePiCodingAgentConfig(target, env, evalFilePath) {
|
|
|
10679
11082
|
optionalEnv: true
|
|
10680
11083
|
}
|
|
10681
11084
|
);
|
|
10682
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10683
|
-
workspaceTemplate =
|
|
11085
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
11086
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10684
11087
|
}
|
|
10685
11088
|
if (cwd && workspaceTemplate) {
|
|
10686
11089
|
throw new Error(
|
|
@@ -10759,8 +11162,8 @@ function resolvePiCliConfig(target, env, evalFilePath) {
|
|
|
10759
11162
|
`${target.name} pi-cli workspace template`,
|
|
10760
11163
|
{ allowLiteral: true, optionalEnv: true }
|
|
10761
11164
|
);
|
|
10762
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10763
|
-
workspaceTemplate =
|
|
11165
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
11166
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10764
11167
|
}
|
|
10765
11168
|
if (cwd && workspaceTemplate) {
|
|
10766
11169
|
throw new Error(`${target.name}: 'cwd' and 'workspace_template' are mutually exclusive.`);
|
|
@@ -10813,8 +11216,8 @@ function resolveClaudeConfig(target, env, evalFilePath) {
|
|
|
10813
11216
|
optionalEnv: true
|
|
10814
11217
|
}
|
|
10815
11218
|
);
|
|
10816
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10817
|
-
workspaceTemplate =
|
|
11219
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
11220
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10818
11221
|
}
|
|
10819
11222
|
if (cwd && workspaceTemplate) {
|
|
10820
11223
|
throw new Error(
|
|
@@ -10872,8 +11275,8 @@ function resolveVSCodeConfig(target, env, insiders, evalFilePath) {
|
|
|
10872
11275
|
optionalEnv: true
|
|
10873
11276
|
}
|
|
10874
11277
|
) : void 0;
|
|
10875
|
-
if (workspaceTemplate && evalFilePath && !
|
|
10876
|
-
workspaceTemplate =
|
|
11278
|
+
if (workspaceTemplate && evalFilePath && !import_node_path23.default.isAbsolute(workspaceTemplate)) {
|
|
11279
|
+
workspaceTemplate = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), workspaceTemplate);
|
|
10877
11280
|
}
|
|
10878
11281
|
const executableSource = target.executable;
|
|
10879
11282
|
const waitSource = target.wait;
|
|
@@ -10914,8 +11317,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
10914
11317
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
10915
11318
|
if (!parseResult.success) {
|
|
10916
11319
|
const firstError = parseResult.error.errors[0];
|
|
10917
|
-
const
|
|
10918
|
-
const prefix =
|
|
11320
|
+
const path50 = firstError?.path.join(".") || "";
|
|
11321
|
+
const prefix = path50 ? `${target.name} ${path50}: ` : `${target.name}: `;
|
|
10919
11322
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
10920
11323
|
}
|
|
10921
11324
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -10936,11 +11339,11 @@ function resolveDiscoveredProviderConfig(target, providerKind, env, evalFilePath
|
|
|
10936
11339
|
allowLiteral: true,
|
|
10937
11340
|
optionalEnv: true
|
|
10938
11341
|
});
|
|
10939
|
-
if (cwd && evalFilePath && !
|
|
10940
|
-
cwd =
|
|
11342
|
+
if (cwd && evalFilePath && !import_node_path23.default.isAbsolute(cwd)) {
|
|
11343
|
+
cwd = import_node_path23.default.resolve(import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath)), cwd);
|
|
10941
11344
|
}
|
|
10942
11345
|
if (!cwd && evalFilePath) {
|
|
10943
|
-
cwd =
|
|
11346
|
+
cwd = import_node_path23.default.dirname(import_node_path23.default.resolve(evalFilePath));
|
|
10944
11347
|
}
|
|
10945
11348
|
return {
|
|
10946
11349
|
command,
|
|
@@ -10988,6 +11391,43 @@ function resolveString(source, env, description, allowLiteral = false) {
|
|
|
10988
11391
|
}
|
|
10989
11392
|
return value;
|
|
10990
11393
|
}
|
|
11394
|
+
function resolveDiscover(value, targetName) {
|
|
11395
|
+
if (value === void 0 || value === null) return void 0;
|
|
11396
|
+
if (value === "latest") return "latest";
|
|
11397
|
+
throw new Error(`Target "${targetName}": discover must be "latest" (got "${String(value)}")`);
|
|
11398
|
+
}
|
|
11399
|
+
function resolveCopilotLogConfig(target, env) {
|
|
11400
|
+
const sessionDirSource = target.session_dir ?? target.sessionDir;
|
|
11401
|
+
const sessionIdSource = target.session_id ?? target.sessionId;
|
|
11402
|
+
const discoverSource = target.discover;
|
|
11403
|
+
const sessionStateDirSource = target.session_state_dir ?? target.sessionStateDir;
|
|
11404
|
+
const cwdSource = target.cwd;
|
|
11405
|
+
return {
|
|
11406
|
+
sessionDir: resolveOptionalString(
|
|
11407
|
+
sessionDirSource,
|
|
11408
|
+
env,
|
|
11409
|
+
`${target.name} copilot-log session_dir`,
|
|
11410
|
+
{ allowLiteral: true, optionalEnv: true }
|
|
11411
|
+
),
|
|
11412
|
+
sessionId: resolveOptionalString(
|
|
11413
|
+
sessionIdSource,
|
|
11414
|
+
env,
|
|
11415
|
+
`${target.name} copilot-log session_id`,
|
|
11416
|
+
{ allowLiteral: true, optionalEnv: true }
|
|
11417
|
+
),
|
|
11418
|
+
discover: resolveDiscover(discoverSource, target.name),
|
|
11419
|
+
sessionStateDir: resolveOptionalString(
|
|
11420
|
+
sessionStateDirSource,
|
|
11421
|
+
env,
|
|
11422
|
+
`${target.name} copilot-log session_state_dir`,
|
|
11423
|
+
{ allowLiteral: true, optionalEnv: true }
|
|
11424
|
+
),
|
|
11425
|
+
cwd: resolveOptionalString(cwdSource, env, `${target.name} copilot-log cwd`, {
|
|
11426
|
+
allowLiteral: true,
|
|
11427
|
+
optionalEnv: true
|
|
11428
|
+
})
|
|
11429
|
+
};
|
|
11430
|
+
}
|
|
10991
11431
|
function resolveOptionalString(source, env, description, options) {
|
|
10992
11432
|
if (source === void 0 || source === null) {
|
|
10993
11433
|
return void 0;
|
|
@@ -11124,40 +11564,40 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
11124
11564
|
|
|
11125
11565
|
// src/evaluation/providers/vscode-provider.ts
|
|
11126
11566
|
var import_node_child_process7 = require("child_process");
|
|
11127
|
-
var
|
|
11128
|
-
var
|
|
11567
|
+
var import_promises26 = require("fs/promises");
|
|
11568
|
+
var import_node_path35 = __toESM(require("path"), 1);
|
|
11129
11569
|
var import_node_util3 = require("util");
|
|
11130
11570
|
|
|
11131
11571
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
11132
|
-
var
|
|
11133
|
-
var
|
|
11572
|
+
var import_promises24 = require("fs/promises");
|
|
11573
|
+
var import_node_path33 = __toESM(require("path"), 1);
|
|
11134
11574
|
|
|
11135
11575
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
11136
11576
|
var import_node_fs10 = require("fs");
|
|
11137
|
-
var
|
|
11138
|
-
var
|
|
11577
|
+
var import_promises20 = require("fs/promises");
|
|
11578
|
+
var import_node_path24 = __toESM(require("path"), 1);
|
|
11139
11579
|
async function pathExists(target) {
|
|
11140
11580
|
try {
|
|
11141
|
-
await (0,
|
|
11581
|
+
await (0, import_promises20.access)(target, import_node_fs10.constants.F_OK);
|
|
11142
11582
|
return true;
|
|
11143
11583
|
} catch {
|
|
11144
11584
|
return false;
|
|
11145
11585
|
}
|
|
11146
11586
|
}
|
|
11147
11587
|
async function ensureDir(target) {
|
|
11148
|
-
await (0,
|
|
11588
|
+
await (0, import_promises20.mkdir)(target, { recursive: true });
|
|
11149
11589
|
}
|
|
11150
11590
|
async function readDirEntries(target) {
|
|
11151
|
-
const entries = await (0,
|
|
11591
|
+
const entries = await (0, import_promises20.readdir)(target, { withFileTypes: true });
|
|
11152
11592
|
return entries.map((entry) => ({
|
|
11153
11593
|
name: entry.name,
|
|
11154
|
-
absolutePath:
|
|
11594
|
+
absolutePath: import_node_path24.default.join(target, entry.name),
|
|
11155
11595
|
isDirectory: entry.isDirectory()
|
|
11156
11596
|
}));
|
|
11157
11597
|
}
|
|
11158
11598
|
async function removeIfExists(target) {
|
|
11159
11599
|
try {
|
|
11160
|
-
await (0,
|
|
11600
|
+
await (0, import_promises20.rm)(target, { force: true, recursive: false });
|
|
11161
11601
|
} catch (error) {
|
|
11162
11602
|
if (error.code !== "ENOENT") {
|
|
11163
11603
|
throw error;
|
|
@@ -11166,9 +11606,9 @@ async function removeIfExists(target) {
|
|
|
11166
11606
|
}
|
|
11167
11607
|
|
|
11168
11608
|
// src/evaluation/providers/vscode/utils/path.ts
|
|
11169
|
-
var
|
|
11609
|
+
var import_node_path25 = __toESM(require("path"), 1);
|
|
11170
11610
|
function pathToFileUri2(filePath) {
|
|
11171
|
-
const absolutePath =
|
|
11611
|
+
const absolutePath = import_node_path25.default.isAbsolute(filePath) ? filePath : import_node_path25.default.resolve(filePath);
|
|
11172
11612
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
11173
11613
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
11174
11614
|
return `file:///${normalizedPath}`;
|
|
@@ -11177,7 +11617,7 @@ function pathToFileUri2(filePath) {
|
|
|
11177
11617
|
}
|
|
11178
11618
|
|
|
11179
11619
|
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
11180
|
-
var
|
|
11620
|
+
var import_node_path26 = __toESM(require("path"), 1);
|
|
11181
11621
|
|
|
11182
11622
|
// src/evaluation/providers/vscode/utils/template.ts
|
|
11183
11623
|
function renderTemplate2(content, variables) {
|
|
@@ -11269,8 +11709,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
11269
11709
|
});
|
|
11270
11710
|
}
|
|
11271
11711
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
11272
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
11273
|
-
const responseList = responseFiles.map((file) => `"${
|
|
11712
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${import_node_path26.default.basename(file)}`).join("\n");
|
|
11713
|
+
const responseList = responseFiles.map((file) => `"${import_node_path26.default.basename(file)}"`).join(", ");
|
|
11274
11714
|
return renderTemplate2(templateContent, {
|
|
11275
11715
|
requestFiles: requestLines,
|
|
11276
11716
|
responseList
|
|
@@ -11278,8 +11718,8 @@ function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateCont
|
|
|
11278
11718
|
}
|
|
11279
11719
|
|
|
11280
11720
|
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
11281
|
-
var
|
|
11282
|
-
var
|
|
11721
|
+
var import_promises21 = require("fs/promises");
|
|
11722
|
+
var import_node_path27 = __toESM(require("path"), 1);
|
|
11283
11723
|
|
|
11284
11724
|
// src/evaluation/providers/vscode/utils/time.ts
|
|
11285
11725
|
function sleep2(ms) {
|
|
@@ -11317,7 +11757,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
11317
11757
|
const maxAttempts = 10;
|
|
11318
11758
|
while (attempts < maxAttempts) {
|
|
11319
11759
|
try {
|
|
11320
|
-
const content = await (0,
|
|
11760
|
+
const content = await (0, import_promises21.readFile)(responseFileFinal, { encoding: "utf8" });
|
|
11321
11761
|
if (!silent) {
|
|
11322
11762
|
process.stdout.write(`${content}
|
|
11323
11763
|
`);
|
|
@@ -11338,7 +11778,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
11338
11778
|
}
|
|
11339
11779
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
11340
11780
|
if (!silent) {
|
|
11341
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
11781
|
+
const fileList = responseFilesFinal.map((file) => import_node_path27.default.basename(file)).join(", ");
|
|
11342
11782
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
11343
11783
|
}
|
|
11344
11784
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -11347,7 +11787,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
11347
11787
|
while (pending.size > 0) {
|
|
11348
11788
|
if (Date.now() >= deadline) {
|
|
11349
11789
|
if (!silent) {
|
|
11350
|
-
const remaining = [...pending].map((f) =>
|
|
11790
|
+
const remaining = [...pending].map((f) => import_node_path27.default.basename(f)).join(", ");
|
|
11351
11791
|
console.error(
|
|
11352
11792
|
`error: timed out after ${Math.round(timeoutMs / 1e3)}s waiting for batch responses. Still pending: ${remaining}`
|
|
11353
11793
|
);
|
|
@@ -11374,7 +11814,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
11374
11814
|
const maxAttempts = 10;
|
|
11375
11815
|
while (attempts < maxAttempts) {
|
|
11376
11816
|
try {
|
|
11377
|
-
const content = await (0,
|
|
11817
|
+
const content = await (0, import_promises21.readFile)(file, { encoding: "utf8" });
|
|
11378
11818
|
if (!silent) {
|
|
11379
11819
|
process.stdout.write(`${content}
|
|
11380
11820
|
`);
|
|
@@ -11397,16 +11837,16 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
11397
11837
|
|
|
11398
11838
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
11399
11839
|
var import_node_child_process6 = require("child_process");
|
|
11400
|
-
var
|
|
11401
|
-
var
|
|
11840
|
+
var import_promises22 = require("fs/promises");
|
|
11841
|
+
var import_node_path30 = __toESM(require("path"), 1);
|
|
11402
11842
|
var import_node_util2 = require("util");
|
|
11403
11843
|
|
|
11404
11844
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
11405
|
-
var
|
|
11845
|
+
var import_node_path29 = __toESM(require("path"), 1);
|
|
11406
11846
|
|
|
11407
11847
|
// src/paths.ts
|
|
11408
|
-
var
|
|
11409
|
-
var
|
|
11848
|
+
var import_node_os6 = __toESM(require("os"), 1);
|
|
11849
|
+
var import_node_path28 = __toESM(require("path"), 1);
|
|
11410
11850
|
var logged = false;
|
|
11411
11851
|
function getAgentvHome() {
|
|
11412
11852
|
const envHome = process.env.AGENTV_HOME;
|
|
@@ -11417,19 +11857,19 @@ function getAgentvHome() {
|
|
|
11417
11857
|
}
|
|
11418
11858
|
return envHome;
|
|
11419
11859
|
}
|
|
11420
|
-
return
|
|
11860
|
+
return import_node_path28.default.join(import_node_os6.default.homedir(), ".agentv");
|
|
11421
11861
|
}
|
|
11422
11862
|
function getWorkspacesRoot() {
|
|
11423
|
-
return
|
|
11863
|
+
return import_node_path28.default.join(getAgentvHome(), "workspaces");
|
|
11424
11864
|
}
|
|
11425
11865
|
function getSubagentsRoot() {
|
|
11426
|
-
return
|
|
11866
|
+
return import_node_path28.default.join(getAgentvHome(), "subagents");
|
|
11427
11867
|
}
|
|
11428
11868
|
function getTraceStateRoot() {
|
|
11429
|
-
return
|
|
11869
|
+
return import_node_path28.default.join(getAgentvHome(), "trace-state");
|
|
11430
11870
|
}
|
|
11431
11871
|
function getWorkspacePoolRoot() {
|
|
11432
|
-
return
|
|
11872
|
+
return import_node_path28.default.join(getAgentvHome(), "workspace-pool");
|
|
11433
11873
|
}
|
|
11434
11874
|
|
|
11435
11875
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
@@ -11437,7 +11877,7 @@ var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
|
11437
11877
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
11438
11878
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
11439
11879
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
11440
|
-
return
|
|
11880
|
+
return import_node_path29.default.join(getSubagentsRoot(), folder);
|
|
11441
11881
|
}
|
|
11442
11882
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
11443
11883
|
|
|
@@ -11504,12 +11944,12 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
11504
11944
|
await raceSpawnError(child);
|
|
11505
11945
|
return true;
|
|
11506
11946
|
}
|
|
11507
|
-
const aliveFile =
|
|
11947
|
+
const aliveFile = import_node_path30.default.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
11508
11948
|
await removeIfExists(aliveFile);
|
|
11509
|
-
const githubAgentsDir =
|
|
11510
|
-
await (0,
|
|
11511
|
-
const wakeupDst =
|
|
11512
|
-
await (0,
|
|
11949
|
+
const githubAgentsDir = import_node_path30.default.join(subagentDir, ".github", "agents");
|
|
11950
|
+
await (0, import_promises22.mkdir)(githubAgentsDir, { recursive: true });
|
|
11951
|
+
const wakeupDst = import_node_path30.default.join(githubAgentsDir, "wakeup.md");
|
|
11952
|
+
await (0, import_promises22.writeFile)(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
11513
11953
|
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
11514
11954
|
label: "open-workspace"
|
|
11515
11955
|
});
|
|
@@ -11521,7 +11961,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
11521
11961
|
"chat",
|
|
11522
11962
|
"-m",
|
|
11523
11963
|
wakeupChatId,
|
|
11524
|
-
`create a file named .alive in the ${
|
|
11964
|
+
`create a file named .alive in the ${import_node_path30.default.basename(subagentDir)} folder`
|
|
11525
11965
|
];
|
|
11526
11966
|
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
11527
11967
|
await raceSpawnError(wakeupChild);
|
|
@@ -11536,27 +11976,27 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
11536
11976
|
return true;
|
|
11537
11977
|
}
|
|
11538
11978
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
11539
|
-
const workspacePath =
|
|
11540
|
-
const messagesDir =
|
|
11541
|
-
await (0,
|
|
11542
|
-
const reqFile =
|
|
11543
|
-
await (0,
|
|
11979
|
+
const workspacePath = import_node_path30.default.join(subagentDir, `${import_node_path30.default.basename(subagentDir)}.code-workspace`);
|
|
11980
|
+
const messagesDir = import_node_path30.default.join(subagentDir, "messages");
|
|
11981
|
+
await (0, import_promises22.mkdir)(messagesDir, { recursive: true });
|
|
11982
|
+
const reqFile = import_node_path30.default.join(messagesDir, `${timestamp}_req.md`);
|
|
11983
|
+
await (0, import_promises22.writeFile)(reqFile, requestInstructions, { encoding: "utf8" });
|
|
11544
11984
|
const reqUri = pathToFileUri2(reqFile);
|
|
11545
11985
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
11546
11986
|
for (const attachment of attachmentPaths) {
|
|
11547
11987
|
chatArgs.push("-a", attachment);
|
|
11548
11988
|
}
|
|
11549
11989
|
chatArgs.push("-a", reqFile);
|
|
11550
|
-
chatArgs.push(`Follow instructions in [${
|
|
11990
|
+
chatArgs.push(`Follow instructions in [${import_node_path30.default.basename(reqFile)}](${reqUri})`);
|
|
11551
11991
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
11552
11992
|
workspacePath,
|
|
11553
|
-
|
|
11993
|
+
import_node_path30.default.basename(subagentDir),
|
|
11554
11994
|
subagentDir,
|
|
11555
11995
|
vscodeCmd
|
|
11556
11996
|
);
|
|
11557
11997
|
if (!workspaceReady) {
|
|
11558
11998
|
throw new Error(
|
|
11559
|
-
`VS Code workspace '${
|
|
11999
|
+
`VS Code workspace '${import_node_path30.default.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
11560
12000
|
);
|
|
11561
12001
|
}
|
|
11562
12002
|
await sleep2(500);
|
|
@@ -11564,9 +12004,9 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
11564
12004
|
await raceSpawnError(child);
|
|
11565
12005
|
}
|
|
11566
12006
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
11567
|
-
const workspacePath =
|
|
11568
|
-
const messagesDir =
|
|
11569
|
-
await (0,
|
|
12007
|
+
const workspacePath = import_node_path30.default.join(subagentDir, `${import_node_path30.default.basename(subagentDir)}.code-workspace`);
|
|
12008
|
+
const messagesDir = import_node_path30.default.join(subagentDir, "messages");
|
|
12009
|
+
await (0, import_promises22.mkdir)(messagesDir, { recursive: true });
|
|
11570
12010
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
11571
12011
|
for (const attachment of attachmentPaths) {
|
|
11572
12012
|
chatArgs.push("-a", attachment);
|
|
@@ -11574,13 +12014,13 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
11574
12014
|
chatArgs.push(chatInstruction);
|
|
11575
12015
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
11576
12016
|
workspacePath,
|
|
11577
|
-
|
|
12017
|
+
import_node_path30.default.basename(subagentDir),
|
|
11578
12018
|
subagentDir,
|
|
11579
12019
|
vscodeCmd
|
|
11580
12020
|
);
|
|
11581
12021
|
if (!workspaceReady) {
|
|
11582
12022
|
throw new Error(
|
|
11583
|
-
`VS Code workspace '${
|
|
12023
|
+
`VS Code workspace '${import_node_path30.default.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
11584
12024
|
);
|
|
11585
12025
|
}
|
|
11586
12026
|
await sleep2(500);
|
|
@@ -11589,11 +12029,11 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
11589
12029
|
}
|
|
11590
12030
|
|
|
11591
12031
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
11592
|
-
var
|
|
11593
|
-
var
|
|
12032
|
+
var import_promises23 = require("fs/promises");
|
|
12033
|
+
var import_node_path32 = __toESM(require("path"), 1);
|
|
11594
12034
|
|
|
11595
12035
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
11596
|
-
var
|
|
12036
|
+
var import_node_path31 = __toESM(require("path"), 1);
|
|
11597
12037
|
var import_json5 = __toESM(require("json5"), 1);
|
|
11598
12038
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
11599
12039
|
let workspace;
|
|
@@ -11610,10 +12050,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
11610
12050
|
}
|
|
11611
12051
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
11612
12052
|
const folderPath = folder.path;
|
|
11613
|
-
if (
|
|
12053
|
+
if (import_node_path31.default.isAbsolute(folderPath)) {
|
|
11614
12054
|
return folder;
|
|
11615
12055
|
}
|
|
11616
|
-
const absolutePath =
|
|
12056
|
+
const absolutePath = import_node_path31.default.resolve(templateDir, folderPath);
|
|
11617
12057
|
return {
|
|
11618
12058
|
...folder,
|
|
11619
12059
|
path: absolutePath
|
|
@@ -11635,19 +12075,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
11635
12075
|
if (locationMap && typeof locationMap === "object") {
|
|
11636
12076
|
const transformedMap = {};
|
|
11637
12077
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
11638
|
-
const isAbsolute =
|
|
12078
|
+
const isAbsolute = import_node_path31.default.isAbsolute(locationPath);
|
|
11639
12079
|
if (isAbsolute) {
|
|
11640
12080
|
transformedMap[locationPath] = value;
|
|
11641
12081
|
} else {
|
|
11642
12082
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
11643
12083
|
if (firstGlobIndex === -1) {
|
|
11644
|
-
const resolvedPath =
|
|
12084
|
+
const resolvedPath = import_node_path31.default.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
11645
12085
|
transformedMap[resolvedPath] = value;
|
|
11646
12086
|
} else {
|
|
11647
12087
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
11648
12088
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
11649
12089
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
11650
|
-
const resolvedPath = (
|
|
12090
|
+
const resolvedPath = (import_node_path31.default.resolve(templateDir, basePath) + patternPath).replace(
|
|
11651
12091
|
/\\/g,
|
|
11652
12092
|
"/"
|
|
11653
12093
|
);
|
|
@@ -11688,7 +12128,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
11688
12128
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
11689
12129
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
11690
12130
|
for (const subagent of subagents) {
|
|
11691
|
-
const lockFile =
|
|
12131
|
+
const lockFile = import_node_path32.default.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
11692
12132
|
if (!await pathExists(lockFile)) {
|
|
11693
12133
|
return subagent.absolutePath;
|
|
11694
12134
|
}
|
|
@@ -11698,26 +12138,26 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
11698
12138
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
11699
12139
|
let workspaceContent;
|
|
11700
12140
|
if (workspaceTemplate) {
|
|
11701
|
-
const workspaceSrc =
|
|
12141
|
+
const workspaceSrc = import_node_path32.default.resolve(workspaceTemplate);
|
|
11702
12142
|
if (!await pathExists(workspaceSrc)) {
|
|
11703
12143
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
11704
12144
|
}
|
|
11705
|
-
const stats = await (0,
|
|
12145
|
+
const stats = await (0, import_promises23.stat)(workspaceSrc);
|
|
11706
12146
|
if (!stats.isFile()) {
|
|
11707
12147
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
11708
12148
|
}
|
|
11709
|
-
const templateText = await (0,
|
|
12149
|
+
const templateText = await (0, import_promises23.readFile)(workspaceSrc, "utf8");
|
|
11710
12150
|
workspaceContent = JSON.parse(templateText);
|
|
11711
12151
|
} else {
|
|
11712
12152
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
11713
12153
|
}
|
|
11714
|
-
const workspaceName = `${
|
|
11715
|
-
const workspaceDst =
|
|
11716
|
-
const templateDir = workspaceTemplate ?
|
|
12154
|
+
const workspaceName = `${import_node_path32.default.basename(subagentDir)}.code-workspace`;
|
|
12155
|
+
const workspaceDst = import_node_path32.default.join(subagentDir, workspaceName);
|
|
12156
|
+
const templateDir = workspaceTemplate ? import_node_path32.default.dirname(import_node_path32.default.resolve(workspaceTemplate)) : subagentDir;
|
|
11717
12157
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
11718
12158
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
11719
12159
|
if (cwd) {
|
|
11720
|
-
const absCwd =
|
|
12160
|
+
const absCwd = import_node_path32.default.resolve(cwd);
|
|
11721
12161
|
const parsed = JSON.parse(transformedContent);
|
|
11722
12162
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
11723
12163
|
if (!alreadyPresent) {
|
|
@@ -11725,36 +12165,36 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
11725
12165
|
transformedContent = JSON.stringify(parsed, null, 2);
|
|
11726
12166
|
}
|
|
11727
12167
|
}
|
|
11728
|
-
await (0,
|
|
11729
|
-
const messagesDir =
|
|
11730
|
-
await (0,
|
|
12168
|
+
await (0, import_promises23.writeFile)(workspaceDst, transformedContent, "utf8");
|
|
12169
|
+
const messagesDir = import_node_path32.default.join(subagentDir, "messages");
|
|
12170
|
+
await (0, import_promises23.mkdir)(messagesDir, { recursive: true });
|
|
11731
12171
|
return { workspace: workspaceDst, messagesDir };
|
|
11732
12172
|
}
|
|
11733
12173
|
async function createSubagentLock(subagentDir) {
|
|
11734
|
-
const messagesDir =
|
|
12174
|
+
const messagesDir = import_node_path32.default.join(subagentDir, "messages");
|
|
11735
12175
|
if (await pathExists(messagesDir)) {
|
|
11736
|
-
const files = await (0,
|
|
12176
|
+
const files = await (0, import_promises23.readdir)(messagesDir);
|
|
11737
12177
|
await Promise.all(
|
|
11738
12178
|
files.map(async (file) => {
|
|
11739
|
-
const target =
|
|
12179
|
+
const target = import_node_path32.default.join(messagesDir, file);
|
|
11740
12180
|
await removeIfExists(target);
|
|
11741
12181
|
})
|
|
11742
12182
|
);
|
|
11743
12183
|
}
|
|
11744
|
-
const githubAgentsDir =
|
|
12184
|
+
const githubAgentsDir = import_node_path32.default.join(subagentDir, ".github", "agents");
|
|
11745
12185
|
if (await pathExists(githubAgentsDir)) {
|
|
11746
|
-
const agentFiles = await (0,
|
|
12186
|
+
const agentFiles = await (0, import_promises23.readdir)(githubAgentsDir);
|
|
11747
12187
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
11748
12188
|
await Promise.all(
|
|
11749
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
12189
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(import_node_path32.default.join(githubAgentsDir, file)))
|
|
11750
12190
|
);
|
|
11751
12191
|
}
|
|
11752
|
-
const lockFile =
|
|
11753
|
-
await (0,
|
|
12192
|
+
const lockFile = import_node_path32.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
12193
|
+
await (0, import_promises23.writeFile)(lockFile, "", { encoding: "utf8" });
|
|
11754
12194
|
return lockFile;
|
|
11755
12195
|
}
|
|
11756
12196
|
async function removeSubagentLock(subagentDir) {
|
|
11757
|
-
const lockFile =
|
|
12197
|
+
const lockFile = import_node_path32.default.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
11758
12198
|
await removeIfExists(lockFile);
|
|
11759
12199
|
}
|
|
11760
12200
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -11774,11 +12214,11 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
11774
12214
|
return 1;
|
|
11775
12215
|
}
|
|
11776
12216
|
if (promptFile) {
|
|
11777
|
-
const githubAgentsDir =
|
|
11778
|
-
await (0,
|
|
11779
|
-
const agentFile =
|
|
12217
|
+
const githubAgentsDir = import_node_path32.default.join(subagentDir, ".github", "agents");
|
|
12218
|
+
await (0, import_promises23.mkdir)(githubAgentsDir, { recursive: true });
|
|
12219
|
+
const agentFile = import_node_path32.default.join(githubAgentsDir, `${chatId}.md`);
|
|
11780
12220
|
try {
|
|
11781
|
-
await (0,
|
|
12221
|
+
await (0, import_promises23.copyFile)(promptFile, agentFile);
|
|
11782
12222
|
} catch (error) {
|
|
11783
12223
|
console.error(`error: Failed to copy prompt file to agent mode: ${error.message}`);
|
|
11784
12224
|
return 1;
|
|
@@ -11795,11 +12235,11 @@ async function resolvePromptFile(promptFile) {
|
|
|
11795
12235
|
if (!promptFile) {
|
|
11796
12236
|
return void 0;
|
|
11797
12237
|
}
|
|
11798
|
-
const resolvedPrompt =
|
|
12238
|
+
const resolvedPrompt = import_node_path33.default.resolve(promptFile);
|
|
11799
12239
|
if (!await pathExists(resolvedPrompt)) {
|
|
11800
12240
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
11801
12241
|
}
|
|
11802
|
-
const promptStats = await (0,
|
|
12242
|
+
const promptStats = await (0, import_promises24.stat)(resolvedPrompt);
|
|
11803
12243
|
if (!promptStats.isFile()) {
|
|
11804
12244
|
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
11805
12245
|
}
|
|
@@ -11811,7 +12251,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
11811
12251
|
}
|
|
11812
12252
|
const resolved = [];
|
|
11813
12253
|
for (const attachment of extraAttachments) {
|
|
11814
|
-
const resolvedPath =
|
|
12254
|
+
const resolvedPath = import_node_path33.default.resolve(attachment);
|
|
11815
12255
|
if (!await pathExists(resolvedPath)) {
|
|
11816
12256
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
11817
12257
|
}
|
|
@@ -11853,7 +12293,7 @@ async function dispatchAgentSession(options) {
|
|
|
11853
12293
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
11854
12294
|
};
|
|
11855
12295
|
}
|
|
11856
|
-
const subagentName =
|
|
12296
|
+
const subagentName = import_node_path33.default.basename(subagentDir);
|
|
11857
12297
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
11858
12298
|
const preparationResult = await prepareSubagentDirectory(
|
|
11859
12299
|
subagentDir,
|
|
@@ -11881,9 +12321,9 @@ async function dispatchAgentSession(options) {
|
|
|
11881
12321
|
};
|
|
11882
12322
|
}
|
|
11883
12323
|
const timestamp = generateTimestamp();
|
|
11884
|
-
const messagesDir =
|
|
11885
|
-
const responseFileTmp =
|
|
11886
|
-
const responseFileFinal =
|
|
12324
|
+
const messagesDir = import_node_path33.default.join(subagentDir, "messages");
|
|
12325
|
+
const responseFileTmp = import_node_path33.default.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
12326
|
+
const responseFileFinal = import_node_path33.default.join(messagesDir, `${timestamp}_res.md`);
|
|
11887
12327
|
const requestInstructions = createRequestPrompt(
|
|
11888
12328
|
userQuery,
|
|
11889
12329
|
responseFileTmp,
|
|
@@ -11988,7 +12428,7 @@ async function dispatchBatchAgent(options) {
|
|
|
11988
12428
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
11989
12429
|
};
|
|
11990
12430
|
}
|
|
11991
|
-
subagentName =
|
|
12431
|
+
subagentName = import_node_path33.default.basename(subagentDir);
|
|
11992
12432
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
11993
12433
|
const preparationResult = await prepareSubagentDirectory(
|
|
11994
12434
|
subagentDir,
|
|
@@ -12019,24 +12459,24 @@ async function dispatchBatchAgent(options) {
|
|
|
12019
12459
|
};
|
|
12020
12460
|
}
|
|
12021
12461
|
const timestamp = generateTimestamp();
|
|
12022
|
-
const messagesDir =
|
|
12462
|
+
const messagesDir = import_node_path33.default.join(subagentDir, "messages");
|
|
12023
12463
|
requestFiles = userQueries.map(
|
|
12024
|
-
(_, index) =>
|
|
12464
|
+
(_, index) => import_node_path33.default.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
12025
12465
|
);
|
|
12026
12466
|
const responseTmpFiles = userQueries.map(
|
|
12027
|
-
(_, index) =>
|
|
12467
|
+
(_, index) => import_node_path33.default.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
12028
12468
|
);
|
|
12029
12469
|
responseFilesFinal = userQueries.map(
|
|
12030
|
-
(_, index) =>
|
|
12470
|
+
(_, index) => import_node_path33.default.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
12031
12471
|
);
|
|
12032
|
-
const orchestratorFile =
|
|
12472
|
+
const orchestratorFile = import_node_path33.default.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
12033
12473
|
if (!dryRun) {
|
|
12034
12474
|
await Promise.all(
|
|
12035
12475
|
userQueries.map((query, index) => {
|
|
12036
12476
|
const reqFile = requestFiles[index];
|
|
12037
12477
|
const tmpFile = responseTmpFiles[index];
|
|
12038
12478
|
const finalFile = responseFilesFinal[index];
|
|
12039
|
-
return (0,
|
|
12479
|
+
return (0, import_promises24.writeFile)(
|
|
12040
12480
|
reqFile,
|
|
12041
12481
|
createBatchRequestPrompt(query, tmpFile, finalFile, batchRequestTemplateContent),
|
|
12042
12482
|
{ encoding: "utf8" }
|
|
@@ -12048,7 +12488,7 @@ async function dispatchBatchAgent(options) {
|
|
|
12048
12488
|
responseFilesFinal,
|
|
12049
12489
|
orchestratorTemplateContent
|
|
12050
12490
|
);
|
|
12051
|
-
await (0,
|
|
12491
|
+
await (0, import_promises24.writeFile)(orchestratorFile, orchestratorContent, { encoding: "utf8" });
|
|
12052
12492
|
}
|
|
12053
12493
|
const chatAttachments = [orchestratorFile, ...attachments];
|
|
12054
12494
|
const orchestratorUri = pathToFileUri2(orchestratorFile);
|
|
@@ -12114,8 +12554,8 @@ async function dispatchBatchAgent(options) {
|
|
|
12114
12554
|
}
|
|
12115
12555
|
|
|
12116
12556
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
12117
|
-
var
|
|
12118
|
-
var
|
|
12557
|
+
var import_promises25 = require("fs/promises");
|
|
12558
|
+
var import_node_path34 = __toESM(require("path"), 1);
|
|
12119
12559
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
12120
12560
|
folders: [
|
|
12121
12561
|
{
|
|
@@ -12146,7 +12586,7 @@ async function provisionSubagents(options) {
|
|
|
12146
12586
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
12147
12587
|
throw new Error("subagents must be a positive integer");
|
|
12148
12588
|
}
|
|
12149
|
-
const targetPath =
|
|
12589
|
+
const targetPath = import_node_path34.default.resolve(targetRoot);
|
|
12150
12590
|
if (!dryRun) {
|
|
12151
12591
|
await ensureDir(targetPath);
|
|
12152
12592
|
}
|
|
@@ -12166,7 +12606,7 @@ async function provisionSubagents(options) {
|
|
|
12166
12606
|
continue;
|
|
12167
12607
|
}
|
|
12168
12608
|
highestNumber = Math.max(highestNumber, parsed);
|
|
12169
|
-
const lockFile =
|
|
12609
|
+
const lockFile = import_node_path34.default.join(entry.absolutePath, lockName);
|
|
12170
12610
|
const locked = await pathExists(lockFile);
|
|
12171
12611
|
if (locked) {
|
|
12172
12612
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -12183,10 +12623,10 @@ async function provisionSubagents(options) {
|
|
|
12183
12623
|
break;
|
|
12184
12624
|
}
|
|
12185
12625
|
const subagentDir = subagent.absolutePath;
|
|
12186
|
-
const githubAgentsDir =
|
|
12187
|
-
const lockFile =
|
|
12188
|
-
const workspaceDst =
|
|
12189
|
-
const wakeupDst =
|
|
12626
|
+
const githubAgentsDir = import_node_path34.default.join(subagentDir, ".github", "agents");
|
|
12627
|
+
const lockFile = import_node_path34.default.join(subagentDir, lockName);
|
|
12628
|
+
const workspaceDst = import_node_path34.default.join(subagentDir, `${import_node_path34.default.basename(subagentDir)}.code-workspace`);
|
|
12629
|
+
const wakeupDst = import_node_path34.default.join(githubAgentsDir, "wakeup.md");
|
|
12190
12630
|
const isLocked = await pathExists(lockFile);
|
|
12191
12631
|
if (isLocked && !force) {
|
|
12192
12632
|
continue;
|
|
@@ -12195,8 +12635,8 @@ async function provisionSubagents(options) {
|
|
|
12195
12635
|
if (!dryRun) {
|
|
12196
12636
|
await removeIfExists(lockFile);
|
|
12197
12637
|
await ensureDir(githubAgentsDir);
|
|
12198
|
-
await (0,
|
|
12199
|
-
await (0,
|
|
12638
|
+
await (0, import_promises25.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
12639
|
+
await (0, import_promises25.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
12200
12640
|
}
|
|
12201
12641
|
created.push(subagentDir);
|
|
12202
12642
|
lockedSubagents.delete(subagentDir);
|
|
@@ -12206,8 +12646,8 @@ async function provisionSubagents(options) {
|
|
|
12206
12646
|
if (!isLocked && force) {
|
|
12207
12647
|
if (!dryRun) {
|
|
12208
12648
|
await ensureDir(githubAgentsDir);
|
|
12209
|
-
await (0,
|
|
12210
|
-
await (0,
|
|
12649
|
+
await (0, import_promises25.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
12650
|
+
await (0, import_promises25.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
12211
12651
|
}
|
|
12212
12652
|
created.push(subagentDir);
|
|
12213
12653
|
subagentsProvisioned += 1;
|
|
@@ -12215,8 +12655,8 @@ async function provisionSubagents(options) {
|
|
|
12215
12655
|
}
|
|
12216
12656
|
if (!dryRun && !await pathExists(workspaceDst)) {
|
|
12217
12657
|
await ensureDir(githubAgentsDir);
|
|
12218
|
-
await (0,
|
|
12219
|
-
await (0,
|
|
12658
|
+
await (0, import_promises25.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
12659
|
+
await (0, import_promises25.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
12220
12660
|
}
|
|
12221
12661
|
skippedExisting.push(subagentDir);
|
|
12222
12662
|
subagentsProvisioned += 1;
|
|
@@ -12224,15 +12664,15 @@ async function provisionSubagents(options) {
|
|
|
12224
12664
|
let nextIndex = highestNumber;
|
|
12225
12665
|
while (subagentsProvisioned < subagents) {
|
|
12226
12666
|
nextIndex += 1;
|
|
12227
|
-
const subagentDir =
|
|
12228
|
-
const githubAgentsDir =
|
|
12229
|
-
const workspaceDst =
|
|
12230
|
-
const wakeupDst =
|
|
12667
|
+
const subagentDir = import_node_path34.default.join(targetPath, `subagent-${nextIndex}`);
|
|
12668
|
+
const githubAgentsDir = import_node_path34.default.join(subagentDir, ".github", "agents");
|
|
12669
|
+
const workspaceDst = import_node_path34.default.join(subagentDir, `${import_node_path34.default.basename(subagentDir)}.code-workspace`);
|
|
12670
|
+
const wakeupDst = import_node_path34.default.join(githubAgentsDir, "wakeup.md");
|
|
12231
12671
|
if (!dryRun) {
|
|
12232
12672
|
await ensureDir(subagentDir);
|
|
12233
12673
|
await ensureDir(githubAgentsDir);
|
|
12234
|
-
await (0,
|
|
12235
|
-
await (0,
|
|
12674
|
+
await (0, import_promises25.writeFile)(workspaceDst, JSON.stringify(workspaceTemplate, null, 2), "utf8");
|
|
12675
|
+
await (0, import_promises25.writeFile)(wakeupDst, wakeupContent, "utf8");
|
|
12236
12676
|
}
|
|
12237
12677
|
created.push(subagentDir);
|
|
12238
12678
|
subagentsProvisioned += 1;
|
|
@@ -12417,9 +12857,9 @@ var VSCodeProvider = class {
|
|
|
12417
12857
|
async function locateVSCodeExecutable(candidate) {
|
|
12418
12858
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
12419
12859
|
if (includesPathSeparator) {
|
|
12420
|
-
const resolved =
|
|
12860
|
+
const resolved = import_node_path35.default.isAbsolute(candidate) ? candidate : import_node_path35.default.resolve(candidate);
|
|
12421
12861
|
try {
|
|
12422
|
-
await (0,
|
|
12862
|
+
await (0, import_promises26.access)(resolved, import_promises26.constants.F_OK);
|
|
12423
12863
|
return resolved;
|
|
12424
12864
|
} catch {
|
|
12425
12865
|
throw new Error(
|
|
@@ -12432,7 +12872,7 @@ async function locateVSCodeExecutable(candidate) {
|
|
|
12432
12872
|
const { stdout } = await execAsync3(`${locator} ${candidate}`);
|
|
12433
12873
|
const lines = stdout.split(/\r?\n/).map((line) => line.trim()).filter((line) => line.length > 0);
|
|
12434
12874
|
if (lines.length > 0 && lines[0]) {
|
|
12435
|
-
await (0,
|
|
12875
|
+
await (0, import_promises26.access)(lines[0], import_promises26.constants.F_OK);
|
|
12436
12876
|
return lines[0];
|
|
12437
12877
|
}
|
|
12438
12878
|
} catch {
|
|
@@ -12446,7 +12886,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
12446
12886
|
return void 0;
|
|
12447
12887
|
}
|
|
12448
12888
|
try {
|
|
12449
|
-
const stats = await (0,
|
|
12889
|
+
const stats = await (0, import_promises26.stat)(import_node_path35.default.resolve(template));
|
|
12450
12890
|
return stats.isFile() ? template : void 0;
|
|
12451
12891
|
} catch {
|
|
12452
12892
|
return template;
|
|
@@ -12470,7 +12910,7 @@ function buildMandatoryPrereadBlock2(attachmentFiles) {
|
|
|
12470
12910
|
return "";
|
|
12471
12911
|
}
|
|
12472
12912
|
const buildList = (files) => files.map((absolutePath) => {
|
|
12473
|
-
const fileName =
|
|
12913
|
+
const fileName = import_node_path35.default.basename(absolutePath);
|
|
12474
12914
|
const fileUri = pathToFileUri3(absolutePath);
|
|
12475
12915
|
return `* [${fileName}](${fileUri})`;
|
|
12476
12916
|
});
|
|
@@ -12491,7 +12931,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
12491
12931
|
}
|
|
12492
12932
|
const unique = /* @__PURE__ */ new Map();
|
|
12493
12933
|
for (const attachment of attachments) {
|
|
12494
|
-
const absolutePath =
|
|
12934
|
+
const absolutePath = import_node_path35.default.resolve(attachment);
|
|
12495
12935
|
if (!unique.has(absolutePath)) {
|
|
12496
12936
|
unique.set(absolutePath, absolutePath);
|
|
12497
12937
|
}
|
|
@@ -12499,7 +12939,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
12499
12939
|
return Array.from(unique.values());
|
|
12500
12940
|
}
|
|
12501
12941
|
function pathToFileUri3(filePath) {
|
|
12502
|
-
const absolutePath =
|
|
12942
|
+
const absolutePath = import_node_path35.default.isAbsolute(filePath) ? filePath : import_node_path35.default.resolve(filePath);
|
|
12503
12943
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
12504
12944
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
12505
12945
|
return `file:///${normalizedPath}`;
|
|
@@ -12512,7 +12952,7 @@ function normalizeAttachments(attachments) {
|
|
|
12512
12952
|
}
|
|
12513
12953
|
const deduped = /* @__PURE__ */ new Set();
|
|
12514
12954
|
for (const attachment of attachments) {
|
|
12515
|
-
deduped.add(
|
|
12955
|
+
deduped.add(import_node_path35.default.resolve(attachment));
|
|
12516
12956
|
}
|
|
12517
12957
|
return Array.from(deduped);
|
|
12518
12958
|
}
|
|
@@ -12521,7 +12961,7 @@ function mergeAttachments(all) {
|
|
|
12521
12961
|
for (const list of all) {
|
|
12522
12962
|
if (!list) continue;
|
|
12523
12963
|
for (const inputFile of list) {
|
|
12524
|
-
deduped.add(
|
|
12964
|
+
deduped.add(import_node_path35.default.resolve(inputFile));
|
|
12525
12965
|
}
|
|
12526
12966
|
}
|
|
12527
12967
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -12569,9 +13009,9 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
12569
13009
|
|
|
12570
13010
|
// src/evaluation/providers/targets-file.ts
|
|
12571
13011
|
var import_node_fs11 = require("fs");
|
|
12572
|
-
var
|
|
12573
|
-
var
|
|
12574
|
-
var
|
|
13012
|
+
var import_promises27 = require("fs/promises");
|
|
13013
|
+
var import_node_path36 = __toESM(require("path"), 1);
|
|
13014
|
+
var import_yaml7 = require("yaml");
|
|
12575
13015
|
function isRecord(value) {
|
|
12576
13016
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
12577
13017
|
}
|
|
@@ -12600,19 +13040,19 @@ function assertTargetDefinition(value, index, filePath) {
|
|
|
12600
13040
|
}
|
|
12601
13041
|
async function fileExists3(filePath) {
|
|
12602
13042
|
try {
|
|
12603
|
-
await (0,
|
|
13043
|
+
await (0, import_promises27.access)(filePath, import_node_fs11.constants.F_OK);
|
|
12604
13044
|
return true;
|
|
12605
13045
|
} catch {
|
|
12606
13046
|
return false;
|
|
12607
13047
|
}
|
|
12608
13048
|
}
|
|
12609
13049
|
async function readTargetDefinitions(filePath) {
|
|
12610
|
-
const absolutePath =
|
|
13050
|
+
const absolutePath = import_node_path36.default.resolve(filePath);
|
|
12611
13051
|
if (!await fileExists3(absolutePath)) {
|
|
12612
13052
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
12613
13053
|
}
|
|
12614
|
-
const raw = await (0,
|
|
12615
|
-
const parsed = (0,
|
|
13054
|
+
const raw = await (0, import_promises27.readFile)(absolutePath, "utf8");
|
|
13055
|
+
const parsed = (0, import_yaml7.parse)(raw);
|
|
12616
13056
|
if (!isRecord(parsed)) {
|
|
12617
13057
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
12618
13058
|
}
|
|
@@ -12627,16 +13067,16 @@ function listTargetNames(definitions) {
|
|
|
12627
13067
|
}
|
|
12628
13068
|
|
|
12629
13069
|
// src/evaluation/providers/provider-discovery.ts
|
|
12630
|
-
var
|
|
13070
|
+
var import_node_path37 = __toESM(require("path"), 1);
|
|
12631
13071
|
var import_fast_glob2 = __toESM(require("fast-glob"), 1);
|
|
12632
13072
|
async function discoverProviders(registry, baseDir) {
|
|
12633
13073
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
12634
13074
|
const candidateDirs = [];
|
|
12635
|
-
let dir =
|
|
12636
|
-
const root =
|
|
13075
|
+
let dir = import_node_path37.default.resolve(baseDir);
|
|
13076
|
+
const root = import_node_path37.default.parse(dir).root;
|
|
12637
13077
|
while (dir !== root) {
|
|
12638
|
-
candidateDirs.push(
|
|
12639
|
-
dir =
|
|
13078
|
+
candidateDirs.push(import_node_path37.default.join(dir, ".agentv", "providers"));
|
|
13079
|
+
dir = import_node_path37.default.dirname(dir);
|
|
12640
13080
|
}
|
|
12641
13081
|
let files = [];
|
|
12642
13082
|
for (const providersDir of candidateDirs) {
|
|
@@ -12652,7 +13092,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
12652
13092
|
}
|
|
12653
13093
|
const discoveredKinds = [];
|
|
12654
13094
|
for (const filePath of files) {
|
|
12655
|
-
const basename =
|
|
13095
|
+
const basename = import_node_path37.default.basename(filePath);
|
|
12656
13096
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
12657
13097
|
if (registry.has(kindName)) {
|
|
12658
13098
|
continue;
|
|
@@ -12670,7 +13110,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
12670
13110
|
// src/evaluation/providers/index.ts
|
|
12671
13111
|
function createBuiltinProviderRegistry() {
|
|
12672
13112
|
const registry = new ProviderRegistry();
|
|
12673
|
-
registry.register("openai", (t) => new OpenAIProvider(t.name, t.config)).register("openrouter", (t) => new OpenRouterProvider(t.name, t.config)).register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-cli", (t) => new PiCliProvider(t.name, t.config)).register("claude-cli", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude-sdk", (t) => new ClaudeSdkProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("agentv", (t) => new AgentvProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
|
|
13113
|
+
registry.register("openai", (t) => new OpenAIProvider(t.name, t.config)).register("openrouter", (t) => new OpenRouterProvider(t.name, t.config)).register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("copilot-log", (t) => new CopilotLogProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-cli", (t) => new PiCliProvider(t.name, t.config)).register("claude-cli", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude-sdk", (t) => new ClaudeSdkProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("agentv", (t) => new AgentvProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
|
|
12674
13114
|
"vscode-insiders",
|
|
12675
13115
|
(t) => new VSCodeProvider(t.name, t.config, "vscode-insiders")
|
|
12676
13116
|
);
|
|
@@ -12775,9 +13215,9 @@ function negateScore(score) {
|
|
|
12775
13215
|
}
|
|
12776
13216
|
|
|
12777
13217
|
// src/evaluation/evaluators/code-evaluator.ts
|
|
12778
|
-
var
|
|
12779
|
-
var
|
|
12780
|
-
var
|
|
13218
|
+
var import_promises28 = require("fs/promises");
|
|
13219
|
+
var import_node_os7 = require("os");
|
|
13220
|
+
var import_node_path38 = require("path");
|
|
12781
13221
|
|
|
12782
13222
|
// src/runtime/exec.ts
|
|
12783
13223
|
function shellEscapePath(value) {
|
|
@@ -12877,15 +13317,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
12877
13317
|
});
|
|
12878
13318
|
}
|
|
12879
13319
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
12880
|
-
const { mkdir: mkdir17, readFile:
|
|
13320
|
+
const { mkdir: mkdir17, readFile: readFile16, rm: rm6, writeFile: writeFile9 } = await import("fs/promises");
|
|
12881
13321
|
const { tmpdir: tmpdir3 } = await import("os");
|
|
12882
|
-
const
|
|
13322
|
+
const path50 = await import("path");
|
|
12883
13323
|
const { randomUUID: randomUUID10 } = await import("crypto");
|
|
12884
|
-
const dir =
|
|
13324
|
+
const dir = path50.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
|
|
12885
13325
|
await mkdir17(dir, { recursive: true });
|
|
12886
|
-
const stdinPath =
|
|
12887
|
-
const stdoutPath =
|
|
12888
|
-
const stderrPath =
|
|
13326
|
+
const stdinPath = path50.join(dir, "stdin.txt");
|
|
13327
|
+
const stdoutPath = path50.join(dir, "stdout.txt");
|
|
13328
|
+
const stderrPath = path50.join(dir, "stderr.txt");
|
|
12889
13329
|
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
12890
13330
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
12891
13331
|
const { spawn: spawn5 } = await import("child_process");
|
|
@@ -12915,8 +13355,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
12915
13355
|
resolve(code ?? 0);
|
|
12916
13356
|
});
|
|
12917
13357
|
});
|
|
12918
|
-
const stdout = (await
|
|
12919
|
-
const stderr = (await
|
|
13358
|
+
const stdout = (await readFile16(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
13359
|
+
const stderr = (await readFile16(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
12920
13360
|
return { stdout, stderr, exitCode };
|
|
12921
13361
|
} finally {
|
|
12922
13362
|
await rm6(dir, { recursive: true, force: true });
|
|
@@ -13221,9 +13661,9 @@ var CodeEvaluator = class {
|
|
|
13221
13661
|
if (outputForPayload) {
|
|
13222
13662
|
const serialized = JSON.stringify(outputForPayload);
|
|
13223
13663
|
if (serialized.length > FILE_BACKED_OUTPUT_THRESHOLD) {
|
|
13224
|
-
const tmpDir = await (0,
|
|
13225
|
-
outputPath = (0,
|
|
13226
|
-
await (0,
|
|
13664
|
+
const tmpDir = await (0, import_promises28.mkdtemp)((0, import_node_path38.join)((0, import_node_os7.tmpdir)(), "agentv-grader-"));
|
|
13665
|
+
outputPath = (0, import_node_path38.join)(tmpDir, "output.json");
|
|
13666
|
+
await (0, import_promises28.writeFile)(outputPath, serialized);
|
|
13227
13667
|
outputForPayload = null;
|
|
13228
13668
|
}
|
|
13229
13669
|
}
|
|
@@ -13332,7 +13772,7 @@ var CodeEvaluator = class {
|
|
|
13332
13772
|
await proxyShutdown();
|
|
13333
13773
|
}
|
|
13334
13774
|
if (outputPath) {
|
|
13335
|
-
await (0,
|
|
13775
|
+
await (0, import_promises28.rm)((0, import_node_path38.dirname)(outputPath), { recursive: true, force: true }).catch(() => {
|
|
13336
13776
|
});
|
|
13337
13777
|
}
|
|
13338
13778
|
}
|
|
@@ -13395,8 +13835,8 @@ function isAgentProvider(provider) {
|
|
|
13395
13835
|
}
|
|
13396
13836
|
|
|
13397
13837
|
// src/evaluation/evaluators/llm-grader.ts
|
|
13398
|
-
var
|
|
13399
|
-
var
|
|
13838
|
+
var import_promises29 = __toESM(require("fs/promises"), 1);
|
|
13839
|
+
var import_node_path39 = __toESM(require("path"), 1);
|
|
13400
13840
|
var import_ai2 = require("ai");
|
|
13401
13841
|
var import_zod4 = require("zod");
|
|
13402
13842
|
var DEFAULT_MAX_STEPS = 10;
|
|
@@ -14251,8 +14691,8 @@ function calculateScoreRangeResult(result, rubrics) {
|
|
|
14251
14691
|
};
|
|
14252
14692
|
}
|
|
14253
14693
|
function resolveSandboxed(basePath, relativePath) {
|
|
14254
|
-
const resolved =
|
|
14255
|
-
if (!resolved.startsWith(basePath +
|
|
14694
|
+
const resolved = import_node_path39.default.resolve(basePath, relativePath);
|
|
14695
|
+
if (!resolved.startsWith(basePath + import_node_path39.default.sep) && resolved !== basePath) {
|
|
14256
14696
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
14257
14697
|
}
|
|
14258
14698
|
return resolved;
|
|
@@ -14267,7 +14707,7 @@ function createFilesystemTools(workspacePath) {
|
|
|
14267
14707
|
execute: async (input) => {
|
|
14268
14708
|
try {
|
|
14269
14709
|
const resolved = resolveSandboxed(workspacePath, input.path);
|
|
14270
|
-
const entries = await
|
|
14710
|
+
const entries = await import_promises29.default.readdir(resolved, { withFileTypes: true });
|
|
14271
14711
|
return entries.map((e) => ({
|
|
14272
14712
|
name: e.name,
|
|
14273
14713
|
type: e.isDirectory() ? "directory" : "file"
|
|
@@ -14285,20 +14725,20 @@ function createFilesystemTools(workspacePath) {
|
|
|
14285
14725
|
execute: async (input) => {
|
|
14286
14726
|
try {
|
|
14287
14727
|
const resolved = resolveSandboxed(workspacePath, input.path);
|
|
14288
|
-
const
|
|
14289
|
-
if (
|
|
14728
|
+
const stat9 = await import_promises29.default.stat(resolved);
|
|
14729
|
+
if (stat9.isDirectory()) {
|
|
14290
14730
|
return { error: `'${input.path}' is a directory, not a file` };
|
|
14291
14731
|
}
|
|
14292
|
-
const buffer = Buffer.alloc(Math.min(
|
|
14293
|
-
const fd = await
|
|
14732
|
+
const buffer = Buffer.alloc(Math.min(stat9.size, MAX_FILE_SIZE));
|
|
14733
|
+
const fd = await import_promises29.default.open(resolved, "r");
|
|
14294
14734
|
try {
|
|
14295
14735
|
await fd.read(buffer, 0, buffer.length, 0);
|
|
14296
14736
|
} finally {
|
|
14297
14737
|
await fd.close();
|
|
14298
14738
|
}
|
|
14299
14739
|
const content = buffer.toString("utf-8");
|
|
14300
|
-
const truncated =
|
|
14301
|
-
return { content, truncated, size:
|
|
14740
|
+
const truncated = stat9.size > MAX_FILE_SIZE;
|
|
14741
|
+
return { content, truncated, size: stat9.size };
|
|
14302
14742
|
} catch (error) {
|
|
14303
14743
|
return { error: error instanceof Error ? error.message : String(error) };
|
|
14304
14744
|
}
|
|
@@ -14335,30 +14775,30 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
14335
14775
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
14336
14776
|
let entries;
|
|
14337
14777
|
try {
|
|
14338
|
-
entries = await
|
|
14778
|
+
entries = await import_promises29.default.readdir(dirPath, { withFileTypes: true });
|
|
14339
14779
|
} catch {
|
|
14340
14780
|
return;
|
|
14341
14781
|
}
|
|
14342
14782
|
for (const entry of entries) {
|
|
14343
14783
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
14344
14784
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
14345
|
-
const fullPath =
|
|
14785
|
+
const fullPath = import_node_path39.default.join(dirPath, entry.name);
|
|
14346
14786
|
if (entry.isDirectory()) {
|
|
14347
14787
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
14348
14788
|
} else if (entry.isFile()) {
|
|
14349
|
-
const ext =
|
|
14789
|
+
const ext = import_node_path39.default.extname(entry.name).toLowerCase();
|
|
14350
14790
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
14351
14791
|
try {
|
|
14352
|
-
const
|
|
14353
|
-
if (
|
|
14354
|
-
const content = await
|
|
14792
|
+
const stat9 = await import_promises29.default.stat(fullPath);
|
|
14793
|
+
if (stat9.size > MAX_FILE_SIZE) continue;
|
|
14794
|
+
const content = await import_promises29.default.readFile(fullPath, "utf-8");
|
|
14355
14795
|
const lines = content.split("\n");
|
|
14356
14796
|
for (let i = 0; i < lines.length; i++) {
|
|
14357
14797
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
14358
14798
|
regex.lastIndex = 0;
|
|
14359
14799
|
if (regex.test(lines[i])) {
|
|
14360
14800
|
matches.push({
|
|
14361
|
-
file:
|
|
14801
|
+
file: import_node_path39.default.relative(workspacePath, fullPath),
|
|
14362
14802
|
line: i + 1,
|
|
14363
14803
|
text: lines[i].substring(0, 200)
|
|
14364
14804
|
});
|
|
@@ -14991,115 +15431,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
14991
15431
|
* Evaluate a single field against the expected value.
|
|
14992
15432
|
*/
|
|
14993
15433
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
14994
|
-
const { path:
|
|
14995
|
-
const candidateValue = resolvePath(candidateData,
|
|
14996
|
-
const expectedValue = resolvePath(expectedData,
|
|
15434
|
+
const { path: path50, match, required = true, weight = 1 } = fieldConfig;
|
|
15435
|
+
const candidateValue = resolvePath(candidateData, path50);
|
|
15436
|
+
const expectedValue = resolvePath(expectedData, path50);
|
|
14997
15437
|
if (expectedValue === void 0) {
|
|
14998
15438
|
return {
|
|
14999
|
-
path:
|
|
15439
|
+
path: path50,
|
|
15000
15440
|
score: 1,
|
|
15001
15441
|
// No expected value means no comparison needed
|
|
15002
15442
|
weight,
|
|
15003
15443
|
hit: true,
|
|
15004
|
-
message: `${
|
|
15444
|
+
message: `${path50}: no expected value`
|
|
15005
15445
|
};
|
|
15006
15446
|
}
|
|
15007
15447
|
if (candidateValue === void 0) {
|
|
15008
15448
|
if (required) {
|
|
15009
15449
|
return {
|
|
15010
|
-
path:
|
|
15450
|
+
path: path50,
|
|
15011
15451
|
score: 0,
|
|
15012
15452
|
weight,
|
|
15013
15453
|
hit: false,
|
|
15014
|
-
message: `${
|
|
15454
|
+
message: `${path50} (required, missing)`
|
|
15015
15455
|
};
|
|
15016
15456
|
}
|
|
15017
15457
|
return {
|
|
15018
|
-
path:
|
|
15458
|
+
path: path50,
|
|
15019
15459
|
score: 1,
|
|
15020
15460
|
// Don't penalize missing optional fields
|
|
15021
15461
|
weight: 0,
|
|
15022
15462
|
// Zero weight means it won't affect the score
|
|
15023
15463
|
hit: true,
|
|
15024
|
-
message: `${
|
|
15464
|
+
message: `${path50}: optional field missing`
|
|
15025
15465
|
};
|
|
15026
15466
|
}
|
|
15027
15467
|
switch (match) {
|
|
15028
15468
|
case "exact":
|
|
15029
|
-
return this.compareExact(
|
|
15469
|
+
return this.compareExact(path50, candidateValue, expectedValue, weight);
|
|
15030
15470
|
case "numeric_tolerance":
|
|
15031
15471
|
return this.compareNumericTolerance(
|
|
15032
|
-
|
|
15472
|
+
path50,
|
|
15033
15473
|
candidateValue,
|
|
15034
15474
|
expectedValue,
|
|
15035
15475
|
fieldConfig,
|
|
15036
15476
|
weight
|
|
15037
15477
|
);
|
|
15038
15478
|
case "date":
|
|
15039
|
-
return this.compareDate(
|
|
15479
|
+
return this.compareDate(path50, candidateValue, expectedValue, fieldConfig, weight);
|
|
15040
15480
|
default:
|
|
15041
15481
|
return {
|
|
15042
|
-
path:
|
|
15482
|
+
path: path50,
|
|
15043
15483
|
score: 0,
|
|
15044
15484
|
weight,
|
|
15045
15485
|
hit: false,
|
|
15046
|
-
message: `${
|
|
15486
|
+
message: `${path50}: unknown match type "${match}"`
|
|
15047
15487
|
};
|
|
15048
15488
|
}
|
|
15049
15489
|
}
|
|
15050
15490
|
/**
|
|
15051
15491
|
* Exact equality comparison.
|
|
15052
15492
|
*/
|
|
15053
|
-
compareExact(
|
|
15493
|
+
compareExact(path50, candidateValue, expectedValue, weight) {
|
|
15054
15494
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
15055
15495
|
return {
|
|
15056
|
-
path:
|
|
15496
|
+
path: path50,
|
|
15057
15497
|
score: 1,
|
|
15058
15498
|
weight,
|
|
15059
15499
|
hit: true,
|
|
15060
|
-
message:
|
|
15500
|
+
message: path50
|
|
15061
15501
|
};
|
|
15062
15502
|
}
|
|
15063
15503
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
15064
15504
|
return {
|
|
15065
|
-
path:
|
|
15505
|
+
path: path50,
|
|
15066
15506
|
score: 0,
|
|
15067
15507
|
weight,
|
|
15068
15508
|
hit: false,
|
|
15069
|
-
message: `${
|
|
15509
|
+
message: `${path50} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
15070
15510
|
};
|
|
15071
15511
|
}
|
|
15072
15512
|
return {
|
|
15073
|
-
path:
|
|
15513
|
+
path: path50,
|
|
15074
15514
|
score: 0,
|
|
15075
15515
|
weight,
|
|
15076
15516
|
hit: false,
|
|
15077
|
-
message: `${
|
|
15517
|
+
message: `${path50} (value mismatch)`
|
|
15078
15518
|
};
|
|
15079
15519
|
}
|
|
15080
15520
|
/**
|
|
15081
15521
|
* Numeric comparison with absolute or relative tolerance.
|
|
15082
15522
|
*/
|
|
15083
|
-
compareNumericTolerance(
|
|
15523
|
+
compareNumericTolerance(path50, candidateValue, expectedValue, fieldConfig, weight) {
|
|
15084
15524
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
15085
15525
|
const candidateNum = toNumber(candidateValue);
|
|
15086
15526
|
const expectedNum = toNumber(expectedValue);
|
|
15087
15527
|
if (candidateNum === null || expectedNum === null) {
|
|
15088
15528
|
return {
|
|
15089
|
-
path:
|
|
15529
|
+
path: path50,
|
|
15090
15530
|
score: 0,
|
|
15091
15531
|
weight,
|
|
15092
15532
|
hit: false,
|
|
15093
|
-
message: `${
|
|
15533
|
+
message: `${path50} (non-numeric value)`
|
|
15094
15534
|
};
|
|
15095
15535
|
}
|
|
15096
15536
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
15097
15537
|
return {
|
|
15098
|
-
path:
|
|
15538
|
+
path: path50,
|
|
15099
15539
|
score: 0,
|
|
15100
15540
|
weight,
|
|
15101
15541
|
hit: false,
|
|
15102
|
-
message: `${
|
|
15542
|
+
message: `${path50} (invalid numeric value)`
|
|
15103
15543
|
};
|
|
15104
15544
|
}
|
|
15105
15545
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -15112,61 +15552,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
15112
15552
|
}
|
|
15113
15553
|
if (withinTolerance) {
|
|
15114
15554
|
return {
|
|
15115
|
-
path:
|
|
15555
|
+
path: path50,
|
|
15116
15556
|
score: 1,
|
|
15117
15557
|
weight,
|
|
15118
15558
|
hit: true,
|
|
15119
|
-
message: `${
|
|
15559
|
+
message: `${path50} (within tolerance: diff=${diff.toFixed(2)})`
|
|
15120
15560
|
};
|
|
15121
15561
|
}
|
|
15122
15562
|
return {
|
|
15123
|
-
path:
|
|
15563
|
+
path: path50,
|
|
15124
15564
|
score: 0,
|
|
15125
15565
|
weight,
|
|
15126
15566
|
hit: false,
|
|
15127
|
-
message: `${
|
|
15567
|
+
message: `${path50} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
15128
15568
|
};
|
|
15129
15569
|
}
|
|
15130
15570
|
/**
|
|
15131
15571
|
* Date comparison with format normalization.
|
|
15132
15572
|
*/
|
|
15133
|
-
compareDate(
|
|
15573
|
+
compareDate(path50, candidateValue, expectedValue, fieldConfig, weight) {
|
|
15134
15574
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
15135
15575
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
15136
15576
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
15137
15577
|
if (candidateDate === null) {
|
|
15138
15578
|
return {
|
|
15139
|
-
path:
|
|
15579
|
+
path: path50,
|
|
15140
15580
|
score: 0,
|
|
15141
15581
|
weight,
|
|
15142
15582
|
hit: false,
|
|
15143
|
-
message: `${
|
|
15583
|
+
message: `${path50} (unparseable candidate date)`
|
|
15144
15584
|
};
|
|
15145
15585
|
}
|
|
15146
15586
|
if (expectedDate === null) {
|
|
15147
15587
|
return {
|
|
15148
|
-
path:
|
|
15588
|
+
path: path50,
|
|
15149
15589
|
score: 0,
|
|
15150
15590
|
weight,
|
|
15151
15591
|
hit: false,
|
|
15152
|
-
message: `${
|
|
15592
|
+
message: `${path50} (unparseable expected date)`
|
|
15153
15593
|
};
|
|
15154
15594
|
}
|
|
15155
15595
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
15156
15596
|
return {
|
|
15157
|
-
path:
|
|
15597
|
+
path: path50,
|
|
15158
15598
|
score: 1,
|
|
15159
15599
|
weight,
|
|
15160
15600
|
hit: true,
|
|
15161
|
-
message:
|
|
15601
|
+
message: path50
|
|
15162
15602
|
};
|
|
15163
15603
|
}
|
|
15164
15604
|
return {
|
|
15165
|
-
path:
|
|
15605
|
+
path: path50,
|
|
15166
15606
|
score: 0,
|
|
15167
15607
|
weight,
|
|
15168
15608
|
hit: false,
|
|
15169
|
-
message: `${
|
|
15609
|
+
message: `${path50} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
15170
15610
|
};
|
|
15171
15611
|
}
|
|
15172
15612
|
/**
|
|
@@ -15199,11 +15639,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
15199
15639
|
};
|
|
15200
15640
|
}
|
|
15201
15641
|
};
|
|
15202
|
-
function resolvePath(obj,
|
|
15203
|
-
if (!
|
|
15642
|
+
function resolvePath(obj, path50) {
|
|
15643
|
+
if (!path50 || !obj) {
|
|
15204
15644
|
return void 0;
|
|
15205
15645
|
}
|
|
15206
|
-
const parts =
|
|
15646
|
+
const parts = path50.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
15207
15647
|
let current = obj;
|
|
15208
15648
|
for (const part of parts) {
|
|
15209
15649
|
if (current === null || current === void 0) {
|
|
@@ -15371,6 +15811,7 @@ var PROVIDER_TOOL_SEMANTICS = {
|
|
|
15371
15811
|
"pi-coding-agent": PI_CODING_AGENT_MATCHER,
|
|
15372
15812
|
"pi-cli": PI_CODING_AGENT_MATCHER,
|
|
15373
15813
|
"copilot-cli": COPILOT_MATCHER,
|
|
15814
|
+
"copilot-log": COPILOT_MATCHER,
|
|
15374
15815
|
"copilot-sdk": COPILOT_MATCHER,
|
|
15375
15816
|
vscode: COPILOT_MATCHER,
|
|
15376
15817
|
"vscode-insiders": COPILOT_MATCHER
|
|
@@ -15397,8 +15838,9 @@ var SkillTriggerEvaluator = class {
|
|
|
15397
15838
|
let triggered = false;
|
|
15398
15839
|
let evidence = "";
|
|
15399
15840
|
for (const toolCall of allToolCalls) {
|
|
15841
|
+
const toolName = toolCall.tool ?? "";
|
|
15400
15842
|
const input = toolCall.input ?? {};
|
|
15401
|
-
if (matcher.skillTools.includes(
|
|
15843
|
+
if (matcher.skillTools.includes(toolName)) {
|
|
15402
15844
|
const skillArg = String(input[matcher.skillInputField] ?? "");
|
|
15403
15845
|
if (skillArg.includes(skillName)) {
|
|
15404
15846
|
triggered = true;
|
|
@@ -15406,12 +15848,12 @@ var SkillTriggerEvaluator = class {
|
|
|
15406
15848
|
break;
|
|
15407
15849
|
}
|
|
15408
15850
|
} else if (matcher.skillToolPrefixes?.some(
|
|
15409
|
-
(prefix) =>
|
|
15851
|
+
(prefix) => toolName.startsWith(prefix) && toolName.includes(skillName)
|
|
15410
15852
|
)) {
|
|
15411
15853
|
triggered = true;
|
|
15412
|
-
evidence = `Skill tool invoked via tool name "${
|
|
15854
|
+
evidence = `Skill tool invoked via tool name "${toolName}"`;
|
|
15413
15855
|
break;
|
|
15414
|
-
} else if (matcher.readTools.includes(
|
|
15856
|
+
} else if (matcher.readTools.includes(toolName)) {
|
|
15415
15857
|
const filePath = this.readPathFromInput(input, matcher);
|
|
15416
15858
|
if (filePath.includes(skillName)) {
|
|
15417
15859
|
triggered = true;
|
|
@@ -15419,10 +15861,10 @@ var SkillTriggerEvaluator = class {
|
|
|
15419
15861
|
break;
|
|
15420
15862
|
}
|
|
15421
15863
|
} else if (matcher.readToolPrefixes?.some(
|
|
15422
|
-
(prefix) =>
|
|
15864
|
+
(prefix) => toolName.startsWith(prefix) && toolName.includes(skillName)
|
|
15423
15865
|
)) {
|
|
15424
15866
|
triggered = true;
|
|
15425
|
-
evidence = `Read tool loaded skill file via tool name "${
|
|
15867
|
+
evidence = `Read tool loaded skill file via tool name "${toolName}"`;
|
|
15426
15868
|
break;
|
|
15427
15869
|
}
|
|
15428
15870
|
}
|
|
@@ -15684,8 +16126,8 @@ var TokenUsageEvaluator = class {
|
|
|
15684
16126
|
};
|
|
15685
16127
|
|
|
15686
16128
|
// src/evaluation/evaluators/tool-trajectory.ts
|
|
15687
|
-
function getNestedValue(obj,
|
|
15688
|
-
const parts =
|
|
16129
|
+
function getNestedValue(obj, path50) {
|
|
16130
|
+
const parts = path50.split(".");
|
|
15689
16131
|
let current = obj;
|
|
15690
16132
|
for (const part of parts) {
|
|
15691
16133
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -16306,8 +16748,8 @@ function runEqualsAssertion(output, value) {
|
|
|
16306
16748
|
|
|
16307
16749
|
// src/evaluation/orchestrator.ts
|
|
16308
16750
|
var import_node_crypto11 = require("crypto");
|
|
16309
|
-
var
|
|
16310
|
-
var
|
|
16751
|
+
var import_promises33 = require("fs/promises");
|
|
16752
|
+
var import_node_path48 = __toESM(require("path"), 1);
|
|
16311
16753
|
var import_micromatch3 = __toESM(require("micromatch"), 1);
|
|
16312
16754
|
|
|
16313
16755
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -16521,7 +16963,7 @@ var InlineAssertEvaluator = class {
|
|
|
16521
16963
|
};
|
|
16522
16964
|
|
|
16523
16965
|
// src/evaluation/evaluators/prompt-resolution.ts
|
|
16524
|
-
var
|
|
16966
|
+
var import_node_path40 = __toESM(require("path"), 1);
|
|
16525
16967
|
async function resolveCustomPrompt(promptConfig, context2, timeoutMs) {
|
|
16526
16968
|
if (promptConfig.resolvedPromptScript && promptConfig.resolvedPromptScript.length > 0) {
|
|
16527
16969
|
if (!context2) {
|
|
@@ -16567,7 +17009,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
|
16567
17009
|
};
|
|
16568
17010
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
16569
17011
|
const scriptPath = script[script.length - 1];
|
|
16570
|
-
const cwd =
|
|
17012
|
+
const cwd = import_node_path40.default.dirname(scriptPath);
|
|
16571
17013
|
try {
|
|
16572
17014
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
16573
17015
|
const prompt = stdout.trim();
|
|
@@ -16839,16 +17281,16 @@ function createBuiltinRegistry() {
|
|
|
16839
17281
|
}
|
|
16840
17282
|
|
|
16841
17283
|
// src/evaluation/registry/assertion-discovery.ts
|
|
16842
|
-
var
|
|
17284
|
+
var import_node_path41 = __toESM(require("path"), 1);
|
|
16843
17285
|
var import_fast_glob3 = __toESM(require("fast-glob"), 1);
|
|
16844
17286
|
async function discoverAssertions(registry, baseDir) {
|
|
16845
17287
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
16846
17288
|
const candidateDirs = [];
|
|
16847
|
-
let dir =
|
|
16848
|
-
const root =
|
|
17289
|
+
let dir = import_node_path41.default.resolve(baseDir);
|
|
17290
|
+
const root = import_node_path41.default.parse(dir).root;
|
|
16849
17291
|
while (dir !== root) {
|
|
16850
|
-
candidateDirs.push(
|
|
16851
|
-
dir =
|
|
17292
|
+
candidateDirs.push(import_node_path41.default.join(dir, ".agentv", "assertions"));
|
|
17293
|
+
dir = import_node_path41.default.dirname(dir);
|
|
16852
17294
|
}
|
|
16853
17295
|
let files = [];
|
|
16854
17296
|
for (const assertionsDir of candidateDirs) {
|
|
@@ -16864,7 +17306,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
16864
17306
|
}
|
|
16865
17307
|
const discoveredTypes = [];
|
|
16866
17308
|
for (const filePath of files) {
|
|
16867
|
-
const basename =
|
|
17309
|
+
const basename = import_node_path41.default.basename(filePath);
|
|
16868
17310
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
16869
17311
|
if (registry.has(typeName)) {
|
|
16870
17312
|
continue;
|
|
@@ -16882,17 +17324,17 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
16882
17324
|
}
|
|
16883
17325
|
|
|
16884
17326
|
// src/evaluation/registry/grader-discovery.ts
|
|
16885
|
-
var
|
|
17327
|
+
var import_node_path42 = __toESM(require("path"), 1);
|
|
16886
17328
|
var import_fast_glob4 = __toESM(require("fast-glob"), 1);
|
|
16887
17329
|
async function discoverGraders(registry, baseDir) {
|
|
16888
17330
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
16889
17331
|
const candidateDirs = [];
|
|
16890
|
-
let dir =
|
|
16891
|
-
const root =
|
|
17332
|
+
let dir = import_node_path42.default.resolve(baseDir);
|
|
17333
|
+
const root = import_node_path42.default.parse(dir).root;
|
|
16892
17334
|
while (dir !== root) {
|
|
16893
|
-
candidateDirs.push(
|
|
16894
|
-
candidateDirs.push(
|
|
16895
|
-
dir =
|
|
17335
|
+
candidateDirs.push(import_node_path42.default.join(dir, ".agentv", "graders"));
|
|
17336
|
+
candidateDirs.push(import_node_path42.default.join(dir, ".agentv", "judges"));
|
|
17337
|
+
dir = import_node_path42.default.dirname(dir);
|
|
16896
17338
|
}
|
|
16897
17339
|
let files = [];
|
|
16898
17340
|
for (const gradersDir of candidateDirs) {
|
|
@@ -16908,7 +17350,7 @@ async function discoverGraders(registry, baseDir) {
|
|
|
16908
17350
|
}
|
|
16909
17351
|
const discoveredTypes = [];
|
|
16910
17352
|
for (const filePath of files) {
|
|
16911
|
-
const basename =
|
|
17353
|
+
const basename = import_node_path42.default.basename(filePath);
|
|
16912
17354
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
16913
17355
|
if (registry.has(typeName)) {
|
|
16914
17356
|
continue;
|
|
@@ -17068,7 +17510,7 @@ function getTCritical(df) {
|
|
|
17068
17510
|
// src/evaluation/workspace/file-changes.ts
|
|
17069
17511
|
var import_node_child_process8 = require("child_process");
|
|
17070
17512
|
var import_node_fs12 = require("fs");
|
|
17071
|
-
var
|
|
17513
|
+
var import_node_path43 = __toESM(require("path"), 1);
|
|
17072
17514
|
var import_node_util4 = require("util");
|
|
17073
17515
|
var execAsync4 = (0, import_node_util4.promisify)(import_node_child_process8.exec);
|
|
17074
17516
|
function gitExecOpts(workspacePath) {
|
|
@@ -17102,10 +17544,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
17102
17544
|
}
|
|
17103
17545
|
for (const entry of entries) {
|
|
17104
17546
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
17105
|
-
const childPath =
|
|
17547
|
+
const childPath = import_node_path43.default.join(workspacePath, entry);
|
|
17106
17548
|
try {
|
|
17107
17549
|
if (!(0, import_node_fs12.statSync)(childPath).isDirectory()) continue;
|
|
17108
|
-
if (!(0, import_node_fs12.statSync)(
|
|
17550
|
+
if (!(0, import_node_fs12.statSync)(import_node_path43.default.join(childPath, ".git")).isDirectory()) continue;
|
|
17109
17551
|
} catch {
|
|
17110
17552
|
continue;
|
|
17111
17553
|
}
|
|
@@ -17115,8 +17557,8 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
17115
17557
|
}
|
|
17116
17558
|
|
|
17117
17559
|
// src/evaluation/workspace/manager.ts
|
|
17118
|
-
var
|
|
17119
|
-
var
|
|
17560
|
+
var import_promises30 = require("fs/promises");
|
|
17561
|
+
var import_node_path44 = __toESM(require("path"), 1);
|
|
17120
17562
|
var TemplateNotFoundError = class extends Error {
|
|
17121
17563
|
constructor(templatePath) {
|
|
17122
17564
|
super(`Workspace template not found: ${templatePath}`);
|
|
@@ -17138,7 +17580,7 @@ var WorkspaceCreationError = class extends Error {
|
|
|
17138
17580
|
};
|
|
17139
17581
|
async function isDirectory(filePath) {
|
|
17140
17582
|
try {
|
|
17141
|
-
const stats = await (0,
|
|
17583
|
+
const stats = await (0, import_promises30.stat)(filePath);
|
|
17142
17584
|
return stats.isDirectory();
|
|
17143
17585
|
} catch {
|
|
17144
17586
|
return false;
|
|
@@ -17146,26 +17588,26 @@ async function isDirectory(filePath) {
|
|
|
17146
17588
|
}
|
|
17147
17589
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
17148
17590
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
17149
|
-
return
|
|
17591
|
+
return import_node_path44.default.join(root, evalRunId, caseId);
|
|
17150
17592
|
}
|
|
17151
17593
|
async function copyDirectoryRecursive(src, dest) {
|
|
17152
|
-
await (0,
|
|
17153
|
-
const entries = await (0,
|
|
17594
|
+
await (0, import_promises30.mkdir)(dest, { recursive: true });
|
|
17595
|
+
const entries = await (0, import_promises30.readdir)(src, { withFileTypes: true });
|
|
17154
17596
|
for (const entry of entries) {
|
|
17155
|
-
const srcPath =
|
|
17156
|
-
const destPath =
|
|
17597
|
+
const srcPath = import_node_path44.default.join(src, entry.name);
|
|
17598
|
+
const destPath = import_node_path44.default.join(dest, entry.name);
|
|
17157
17599
|
if (entry.name === ".git") {
|
|
17158
17600
|
continue;
|
|
17159
17601
|
}
|
|
17160
17602
|
if (entry.isDirectory()) {
|
|
17161
17603
|
await copyDirectoryRecursive(srcPath, destPath);
|
|
17162
17604
|
} else {
|
|
17163
|
-
await (0,
|
|
17605
|
+
await (0, import_promises30.cp)(srcPath, destPath, { preserveTimestamps: true });
|
|
17164
17606
|
}
|
|
17165
17607
|
}
|
|
17166
17608
|
}
|
|
17167
17609
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
17168
|
-
const resolvedTemplatePath =
|
|
17610
|
+
const resolvedTemplatePath = import_node_path44.default.resolve(templatePath);
|
|
17169
17611
|
if (!await fileExists2(resolvedTemplatePath)) {
|
|
17170
17612
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
17171
17613
|
}
|
|
@@ -17175,7 +17617,7 @@ async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoo
|
|
|
17175
17617
|
const workspacePath = getWorkspacePath(evalRunId, caseId, workspaceRoot);
|
|
17176
17618
|
try {
|
|
17177
17619
|
if (await fileExists2(workspacePath)) {
|
|
17178
|
-
await (0,
|
|
17620
|
+
await (0, import_promises30.rm)(workspacePath, { recursive: true, force: true });
|
|
17179
17621
|
}
|
|
17180
17622
|
await copyDirectoryRecursive(resolvedTemplatePath, workspacePath);
|
|
17181
17623
|
return workspacePath;
|
|
@@ -17209,14 +17651,14 @@ async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoo
|
|
|
17209
17651
|
}
|
|
17210
17652
|
async function cleanupWorkspace(workspacePath) {
|
|
17211
17653
|
if (await fileExists2(workspacePath)) {
|
|
17212
|
-
await (0,
|
|
17654
|
+
await (0, import_promises30.rm)(workspacePath, { recursive: true, force: true });
|
|
17213
17655
|
}
|
|
17214
17656
|
}
|
|
17215
17657
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
17216
17658
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
17217
|
-
const evalDir =
|
|
17659
|
+
const evalDir = import_node_path44.default.join(root, evalRunId);
|
|
17218
17660
|
if (await fileExists2(evalDir)) {
|
|
17219
|
-
await (0,
|
|
17661
|
+
await (0, import_promises30.rm)(evalDir, { recursive: true, force: true });
|
|
17220
17662
|
}
|
|
17221
17663
|
}
|
|
17222
17664
|
|
|
@@ -17224,8 +17666,8 @@ async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
|
17224
17666
|
var import_node_child_process9 = require("child_process");
|
|
17225
17667
|
var import_node_crypto10 = require("crypto");
|
|
17226
17668
|
var import_node_fs13 = require("fs");
|
|
17227
|
-
var
|
|
17228
|
-
var
|
|
17669
|
+
var import_promises31 = require("fs/promises");
|
|
17670
|
+
var import_node_path45 = __toESM(require("path"), 1);
|
|
17229
17671
|
var import_node_util5 = require("util");
|
|
17230
17672
|
var execFileAsync = (0, import_node_util5.promisify)(import_node_child_process9.execFile);
|
|
17231
17673
|
function gitEnv() {
|
|
@@ -17276,11 +17718,11 @@ function computeWorkspaceFingerprint(repos) {
|
|
|
17276
17718
|
return (0, import_node_crypto10.createHash)("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
17277
17719
|
}
|
|
17278
17720
|
async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
17279
|
-
await (0,
|
|
17280
|
-
const entries = await (0,
|
|
17721
|
+
await (0, import_promises31.mkdir)(dest, { recursive: true });
|
|
17722
|
+
const entries = await (0, import_promises31.readdir)(src, { withFileTypes: true });
|
|
17281
17723
|
for (const entry of entries) {
|
|
17282
|
-
const srcPath =
|
|
17283
|
-
const destPath =
|
|
17724
|
+
const srcPath = import_node_path45.default.join(src, entry.name);
|
|
17725
|
+
const destPath = import_node_path45.default.join(dest, entry.name);
|
|
17284
17726
|
if (entry.name === ".git") {
|
|
17285
17727
|
continue;
|
|
17286
17728
|
}
|
|
@@ -17290,7 +17732,7 @@ async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
|
17290
17732
|
}
|
|
17291
17733
|
await copyDirectoryRecursive2(srcPath, destPath, skipDirs);
|
|
17292
17734
|
} else {
|
|
17293
|
-
await (0,
|
|
17735
|
+
await (0, import_promises31.cp)(srcPath, destPath, { preserveTimestamps: true, force: true });
|
|
17294
17736
|
}
|
|
17295
17737
|
}
|
|
17296
17738
|
}
|
|
@@ -17313,8 +17755,8 @@ var WorkspacePoolManager = class {
|
|
|
17313
17755
|
async acquireWorkspace(options) {
|
|
17314
17756
|
const { templatePath, repos, maxSlots, repoManager, poolReset } = options;
|
|
17315
17757
|
const fingerprint = computeWorkspaceFingerprint(repos);
|
|
17316
|
-
const poolDir =
|
|
17317
|
-
await (0,
|
|
17758
|
+
const poolDir = import_node_path45.default.join(this.poolRoot, fingerprint);
|
|
17759
|
+
await (0, import_promises31.mkdir)(poolDir, { recursive: true });
|
|
17318
17760
|
const drifted = await this.checkDrift(poolDir, fingerprint);
|
|
17319
17761
|
if (drifted) {
|
|
17320
17762
|
console.warn(
|
|
@@ -17323,7 +17765,7 @@ var WorkspacePoolManager = class {
|
|
|
17323
17765
|
await this.removeAllSlots(poolDir);
|
|
17324
17766
|
}
|
|
17325
17767
|
for (let i = 0; i < maxSlots; i++) {
|
|
17326
|
-
const slotPath =
|
|
17768
|
+
const slotPath = import_node_path45.default.join(poolDir, `slot-${i}`);
|
|
17327
17769
|
const lockPath = `${slotPath}.lock`;
|
|
17328
17770
|
const locked = await this.tryLock(lockPath);
|
|
17329
17771
|
if (!locked) {
|
|
@@ -17341,7 +17783,7 @@ var WorkspacePoolManager = class {
|
|
|
17341
17783
|
poolDir
|
|
17342
17784
|
};
|
|
17343
17785
|
}
|
|
17344
|
-
await (0,
|
|
17786
|
+
await (0, import_promises31.mkdir)(slotPath, { recursive: true });
|
|
17345
17787
|
if (templatePath) {
|
|
17346
17788
|
await copyDirectoryRecursive2(templatePath, slotPath);
|
|
17347
17789
|
}
|
|
@@ -17365,7 +17807,7 @@ var WorkspacePoolManager = class {
|
|
|
17365
17807
|
/** Remove lock file to release a slot. */
|
|
17366
17808
|
async releaseSlot(slot) {
|
|
17367
17809
|
try {
|
|
17368
|
-
await (0,
|
|
17810
|
+
await (0, import_promises31.unlink)(slot.lockPath);
|
|
17369
17811
|
} catch {
|
|
17370
17812
|
}
|
|
17371
17813
|
}
|
|
@@ -17378,21 +17820,21 @@ var WorkspacePoolManager = class {
|
|
|
17378
17820
|
async tryLock(lockPath) {
|
|
17379
17821
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
17380
17822
|
try {
|
|
17381
|
-
await (0,
|
|
17823
|
+
await (0, import_promises31.writeFile)(lockPath, String(process.pid), { flag: "wx" });
|
|
17382
17824
|
return true;
|
|
17383
17825
|
} catch (err) {
|
|
17384
17826
|
if (err.code !== "EEXIST") {
|
|
17385
17827
|
throw err;
|
|
17386
17828
|
}
|
|
17387
17829
|
try {
|
|
17388
|
-
const pidStr = await (0,
|
|
17830
|
+
const pidStr = await (0, import_promises31.readFile)(lockPath, "utf-8");
|
|
17389
17831
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
17390
17832
|
if (!Number.isNaN(pid)) {
|
|
17391
17833
|
try {
|
|
17392
17834
|
process.kill(pid, 0);
|
|
17393
17835
|
return false;
|
|
17394
17836
|
} catch {
|
|
17395
|
-
await (0,
|
|
17837
|
+
await (0, import_promises31.unlink)(lockPath).catch(() => {
|
|
17396
17838
|
});
|
|
17397
17839
|
continue;
|
|
17398
17840
|
}
|
|
@@ -17410,9 +17852,9 @@ var WorkspacePoolManager = class {
|
|
|
17410
17852
|
* Returns false (no drift) if metadata.json doesn't exist (first use).
|
|
17411
17853
|
*/
|
|
17412
17854
|
async checkDrift(poolDir, fingerprint) {
|
|
17413
|
-
const metadataPath =
|
|
17855
|
+
const metadataPath = import_node_path45.default.join(poolDir, "metadata.json");
|
|
17414
17856
|
try {
|
|
17415
|
-
const raw = await (0,
|
|
17857
|
+
const raw = await (0, import_promises31.readFile)(metadataPath, "utf-8");
|
|
17416
17858
|
const metadata = JSON.parse(raw);
|
|
17417
17859
|
return metadata.fingerprint !== fingerprint;
|
|
17418
17860
|
} catch {
|
|
@@ -17427,17 +17869,17 @@ var WorkspacePoolManager = class {
|
|
|
17427
17869
|
repos,
|
|
17428
17870
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17429
17871
|
};
|
|
17430
|
-
await (0,
|
|
17872
|
+
await (0, import_promises31.writeFile)(import_node_path45.default.join(poolDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
17431
17873
|
}
|
|
17432
17874
|
/** Remove all slot directories and their lock files from a pool directory. */
|
|
17433
17875
|
async removeAllSlots(poolDir) {
|
|
17434
|
-
const entries = await (0,
|
|
17876
|
+
const entries = await (0, import_promises31.readdir)(poolDir);
|
|
17435
17877
|
for (const entry of entries) {
|
|
17436
17878
|
if (entry.startsWith("slot-") && !entry.endsWith(".lock")) {
|
|
17437
|
-
const lockPath =
|
|
17879
|
+
const lockPath = import_node_path45.default.join(poolDir, `${entry}.lock`);
|
|
17438
17880
|
if ((0, import_node_fs13.existsSync)(lockPath)) {
|
|
17439
17881
|
try {
|
|
17440
|
-
const pidStr = await (0,
|
|
17882
|
+
const pidStr = await (0, import_promises31.readFile)(lockPath, "utf-8");
|
|
17441
17883
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
17442
17884
|
if (!Number.isNaN(pid)) {
|
|
17443
17885
|
try {
|
|
@@ -17450,12 +17892,12 @@ var WorkspacePoolManager = class {
|
|
|
17450
17892
|
} catch {
|
|
17451
17893
|
}
|
|
17452
17894
|
}
|
|
17453
|
-
await (0,
|
|
17454
|
-
await (0,
|
|
17895
|
+
await (0, import_promises31.rm)(import_node_path45.default.join(poolDir, entry), { recursive: true, force: true });
|
|
17896
|
+
await (0, import_promises31.rm)(lockPath, { force: true }).catch(() => {
|
|
17455
17897
|
});
|
|
17456
17898
|
}
|
|
17457
17899
|
}
|
|
17458
|
-
await (0,
|
|
17900
|
+
await (0, import_promises31.rm)(import_node_path45.default.join(poolDir, "metadata.json"), { force: true }).catch(() => {
|
|
17459
17901
|
});
|
|
17460
17902
|
}
|
|
17461
17903
|
/**
|
|
@@ -17465,7 +17907,7 @@ var WorkspacePoolManager = class {
|
|
|
17465
17907
|
*/
|
|
17466
17908
|
async resetSlot(slotPath, templatePath, repos, poolReset = "fast") {
|
|
17467
17909
|
for (const repo of repos) {
|
|
17468
|
-
const repoDir =
|
|
17910
|
+
const repoDir = import_node_path45.default.join(slotPath, repo.path);
|
|
17469
17911
|
if (!(0, import_node_fs13.existsSync)(repoDir)) {
|
|
17470
17912
|
continue;
|
|
17471
17913
|
}
|
|
@@ -17492,7 +17934,7 @@ var WorkspacePoolManager = class {
|
|
|
17492
17934
|
// src/evaluation/workspace/repo-manager.ts
|
|
17493
17935
|
var import_node_child_process10 = require("child_process");
|
|
17494
17936
|
var import_node_fs14 = require("fs");
|
|
17495
|
-
var
|
|
17937
|
+
var import_node_path46 = __toESM(require("path"), 1);
|
|
17496
17938
|
var import_node_util6 = require("util");
|
|
17497
17939
|
var execFileAsync2 = (0, import_node_util6.promisify)(import_node_child_process10.execFile);
|
|
17498
17940
|
var DEFAULT_TIMEOUT_MS2 = 3e5;
|
|
@@ -17592,7 +18034,7 @@ ${lines.join("\n")}`;
|
|
|
17592
18034
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
17593
18035
|
*/
|
|
17594
18036
|
async materialize(repo, workspacePath) {
|
|
17595
|
-
const targetDir =
|
|
18037
|
+
const targetDir = import_node_path46.default.join(workspacePath, repo.path);
|
|
17596
18038
|
const sourceUrl = getSourceUrl(repo.source);
|
|
17597
18039
|
const startedAt = Date.now();
|
|
17598
18040
|
if (this.verbose) {
|
|
@@ -17683,7 +18125,7 @@ ${lines.join("\n")}`;
|
|
|
17683
18125
|
async reset(repos, workspacePath, reset) {
|
|
17684
18126
|
const cleanFlag = reset === "strict" ? "-fdx" : "-fd";
|
|
17685
18127
|
for (const repo of repos) {
|
|
17686
|
-
const targetDir =
|
|
18128
|
+
const targetDir = import_node_path46.default.join(workspacePath, repo.path);
|
|
17687
18129
|
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
17688
18130
|
await this.runGit(["clean", cleanFlag], { cwd: targetDir });
|
|
17689
18131
|
}
|
|
@@ -17691,36 +18133,36 @@ ${lines.join("\n")}`;
|
|
|
17691
18133
|
};
|
|
17692
18134
|
|
|
17693
18135
|
// src/evaluation/workspace/resolve.ts
|
|
17694
|
-
var
|
|
17695
|
-
var
|
|
18136
|
+
var import_promises32 = require("fs/promises");
|
|
18137
|
+
var import_node_path47 = __toESM(require("path"), 1);
|
|
17696
18138
|
async function resolveWorkspaceTemplate(templatePath) {
|
|
17697
18139
|
if (!templatePath) {
|
|
17698
18140
|
return void 0;
|
|
17699
18141
|
}
|
|
17700
|
-
const resolved =
|
|
17701
|
-
const stats = await (0,
|
|
18142
|
+
const resolved = import_node_path47.default.resolve(templatePath);
|
|
18143
|
+
const stats = await (0, import_promises32.stat)(resolved);
|
|
17702
18144
|
if (stats.isFile()) {
|
|
17703
18145
|
return {
|
|
17704
|
-
dir:
|
|
18146
|
+
dir: import_node_path47.default.dirname(resolved),
|
|
17705
18147
|
workspaceFile: resolved
|
|
17706
18148
|
};
|
|
17707
18149
|
}
|
|
17708
18150
|
if (!stats.isDirectory()) {
|
|
17709
18151
|
throw new Error(`workspace template is neither a file nor a directory: ${resolved}`);
|
|
17710
18152
|
}
|
|
17711
|
-
const entries = await (0,
|
|
18153
|
+
const entries = await (0, import_promises32.readdir)(resolved);
|
|
17712
18154
|
const workspaceFiles = entries.filter((e) => e.endsWith(".code-workspace"));
|
|
17713
18155
|
if (workspaceFiles.length === 1) {
|
|
17714
18156
|
return {
|
|
17715
18157
|
dir: resolved,
|
|
17716
|
-
workspaceFile:
|
|
18158
|
+
workspaceFile: import_node_path47.default.join(resolved, workspaceFiles[0])
|
|
17717
18159
|
};
|
|
17718
18160
|
}
|
|
17719
18161
|
if (workspaceFiles.length > 1) {
|
|
17720
18162
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
17721
18163
|
return {
|
|
17722
18164
|
dir: resolved,
|
|
17723
|
-
workspaceFile: conventionFile ?
|
|
18165
|
+
workspaceFile: conventionFile ? import_node_path47.default.join(resolved, conventionFile) : void 0
|
|
17724
18166
|
};
|
|
17725
18167
|
}
|
|
17726
18168
|
return { dir: resolved };
|
|
@@ -17936,7 +18378,7 @@ async function runEvaluation(options) {
|
|
|
17936
18378
|
];
|
|
17937
18379
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveGraderProvider);
|
|
17938
18380
|
const typeRegistry = createBuiltinRegistry();
|
|
17939
|
-
const discoveryBaseDir = evalFilePath ?
|
|
18381
|
+
const discoveryBaseDir = evalFilePath ? import_node_path48.default.dirname(import_node_path48.default.resolve(evalFilePath)) : process.cwd();
|
|
17940
18382
|
const evalDir = discoveryBaseDir;
|
|
17941
18383
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
17942
18384
|
await discoverGraders(typeRegistry, discoveryBaseDir);
|
|
@@ -18076,14 +18518,14 @@ async function runEvaluation(options) {
|
|
|
18076
18518
|
let staticMaterialised = false;
|
|
18077
18519
|
if (useStaticWorkspace && configuredStaticPath) {
|
|
18078
18520
|
const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
|
|
18079
|
-
const dirExists = await (0,
|
|
18521
|
+
const dirExists = await (0, import_promises33.stat)(configuredStaticPath).then(
|
|
18080
18522
|
(s) => s.isDirectory(),
|
|
18081
18523
|
() => false
|
|
18082
18524
|
);
|
|
18083
|
-
const isEmpty = dirExists ? (await (0,
|
|
18525
|
+
const isEmpty = dirExists ? (await (0, import_promises33.readdir)(configuredStaticPath)).length === 0 : false;
|
|
18084
18526
|
if (isYamlConfiguredPath && (!dirExists || isEmpty)) {
|
|
18085
18527
|
if (!dirExists) {
|
|
18086
|
-
await (0,
|
|
18528
|
+
await (0, import_promises33.mkdir)(configuredStaticPath, { recursive: true });
|
|
18087
18529
|
}
|
|
18088
18530
|
if (workspaceTemplate) {
|
|
18089
18531
|
await copyDirectoryRecursive(workspaceTemplate, configuredStaticPath);
|
|
@@ -18128,14 +18570,14 @@ async function runEvaluation(options) {
|
|
|
18128
18570
|
}
|
|
18129
18571
|
} else if (suiteWorkspace?.hooks || suiteWorkspace?.repos?.length && !isPerTestIsolation) {
|
|
18130
18572
|
sharedWorkspacePath = getWorkspacePath(evalRunId, "shared");
|
|
18131
|
-
await (0,
|
|
18573
|
+
await (0, import_promises33.mkdir)(sharedWorkspacePath, { recursive: true });
|
|
18132
18574
|
setupLog(`created empty shared workspace at: ${sharedWorkspacePath}`);
|
|
18133
18575
|
}
|
|
18134
18576
|
try {
|
|
18135
18577
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
18136
|
-
const copiedWorkspaceFile =
|
|
18578
|
+
const copiedWorkspaceFile = import_node_path48.default.join(sharedWorkspacePath, import_node_path48.default.basename(suiteWorkspaceFile));
|
|
18137
18579
|
try {
|
|
18138
|
-
await (0,
|
|
18580
|
+
await (0, import_promises33.stat)(copiedWorkspaceFile);
|
|
18139
18581
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
18140
18582
|
} catch {
|
|
18141
18583
|
}
|
|
@@ -18715,9 +19157,9 @@ async function runEvalCase(options) {
|
|
|
18715
19157
|
);
|
|
18716
19158
|
}
|
|
18717
19159
|
if (caseWorkspaceFile && workspacePath) {
|
|
18718
|
-
const copiedFile =
|
|
19160
|
+
const copiedFile = import_node_path48.default.join(workspacePath, import_node_path48.default.basename(caseWorkspaceFile));
|
|
18719
19161
|
try {
|
|
18720
|
-
await (0,
|
|
19162
|
+
await (0, import_promises33.stat)(copiedFile);
|
|
18721
19163
|
caseWorkspaceFile = copiedFile;
|
|
18722
19164
|
} catch {
|
|
18723
19165
|
}
|
|
@@ -18725,7 +19167,7 @@ async function runEvalCase(options) {
|
|
|
18725
19167
|
}
|
|
18726
19168
|
if (!workspacePath && (evalCase.workspace?.hooks || evalCase.workspace?.repos?.length) && evalRunId) {
|
|
18727
19169
|
workspacePath = getWorkspacePath(evalRunId, evalCase.id);
|
|
18728
|
-
await (0,
|
|
19170
|
+
await (0, import_promises33.mkdir)(workspacePath, { recursive: true });
|
|
18729
19171
|
}
|
|
18730
19172
|
if (evalCase.workspace?.repos?.length && workspacePath) {
|
|
18731
19173
|
const localPathErrors = RepoManager.validateLocalPaths(evalCase.workspace.repos);
|
|
@@ -18777,11 +19219,11 @@ async function runEvalCase(options) {
|
|
|
18777
19219
|
const files = evalCase.metadata.agent_skills_files;
|
|
18778
19220
|
if (baseDir && files.length > 0) {
|
|
18779
19221
|
for (const relPath of files) {
|
|
18780
|
-
const srcPath =
|
|
18781
|
-
const destPath =
|
|
19222
|
+
const srcPath = import_node_path48.default.resolve(baseDir, relPath);
|
|
19223
|
+
const destPath = import_node_path48.default.resolve(workspacePath, relPath);
|
|
18782
19224
|
try {
|
|
18783
|
-
await (0,
|
|
18784
|
-
await (0,
|
|
19225
|
+
await (0, import_promises33.mkdir)(import_node_path48.default.dirname(destPath), { recursive: true });
|
|
19226
|
+
await (0, import_promises33.copyFile)(srcPath, destPath);
|
|
18785
19227
|
} catch (error) {
|
|
18786
19228
|
const message = error instanceof Error ? error.message : String(error);
|
|
18787
19229
|
return buildErrorResult(
|
|
@@ -19426,7 +19868,7 @@ async function runEvaluatorList(options) {
|
|
|
19426
19868
|
fileChanges,
|
|
19427
19869
|
workspacePath
|
|
19428
19870
|
};
|
|
19429
|
-
const evalFileDir = evalCase.file_paths[0] ?
|
|
19871
|
+
const evalFileDir = evalCase.file_paths[0] ? import_node_path48.default.dirname(evalCase.file_paths[0]) : process.cwd();
|
|
19430
19872
|
const dispatchContext = {
|
|
19431
19873
|
graderProvider,
|
|
19432
19874
|
targetResolver,
|
|
@@ -19760,7 +20202,7 @@ function computeWeightedMean(entries) {
|
|
|
19760
20202
|
|
|
19761
20203
|
// src/evaluation/evaluate.ts
|
|
19762
20204
|
var import_node_fs15 = require("fs");
|
|
19763
|
-
var
|
|
20205
|
+
var import_node_path49 = __toESM(require("path"), 1);
|
|
19764
20206
|
|
|
19765
20207
|
// src/evaluation/providers/function-provider.ts
|
|
19766
20208
|
function createFunctionProvider(taskFn) {
|
|
@@ -19797,7 +20239,7 @@ async function evaluate(config) {
|
|
|
19797
20239
|
}
|
|
19798
20240
|
const gitRoot = await findGitRoot(process.cwd());
|
|
19799
20241
|
const repoRoot = gitRoot ?? process.cwd();
|
|
19800
|
-
const testFilePath = config.specFile ?
|
|
20242
|
+
const testFilePath = config.specFile ? import_node_path49.default.resolve(config.specFile) : import_node_path49.default.join(process.cwd(), "__programmatic__.yaml");
|
|
19801
20243
|
await loadEnvHierarchy(repoRoot, testFilePath);
|
|
19802
20244
|
let resolvedTarget;
|
|
19803
20245
|
let taskProvider;
|
|
@@ -19918,10 +20360,10 @@ function computeSummary(results, durationMs) {
|
|
|
19918
20360
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
19919
20361
|
async function discoverDefaultTarget(repoRoot) {
|
|
19920
20362
|
const cwd = process.cwd();
|
|
19921
|
-
const chain = buildDirectoryChain2(
|
|
20363
|
+
const chain = buildDirectoryChain2(import_node_path49.default.join(cwd, "_placeholder"), repoRoot);
|
|
19922
20364
|
for (const dir of chain) {
|
|
19923
20365
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
19924
|
-
const targetsPath =
|
|
20366
|
+
const targetsPath = import_node_path49.default.join(dir, candidate);
|
|
19925
20367
|
if (!(0, import_node_fs15.existsSync)(targetsPath)) continue;
|
|
19926
20368
|
try {
|
|
19927
20369
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -19938,7 +20380,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
|
|
|
19938
20380
|
const chain = buildDirectoryChain2(startPath, repoRoot);
|
|
19939
20381
|
const envFiles = [];
|
|
19940
20382
|
for (const dir of chain) {
|
|
19941
|
-
const envPath =
|
|
20383
|
+
const envPath = import_node_path49.default.join(dir, ".env");
|
|
19942
20384
|
if ((0, import_node_fs15.existsSync)(envPath)) envFiles.push(envPath);
|
|
19943
20385
|
}
|
|
19944
20386
|
for (let i = 0; i < envFiles.length; i++) {
|
|
@@ -20117,8 +20559,8 @@ function buildPrompt(criteria, question, referenceAnswer) {
|
|
|
20117
20559
|
}
|
|
20118
20560
|
|
|
20119
20561
|
// src/evaluation/cache/response-cache.ts
|
|
20120
|
-
var
|
|
20121
|
-
var
|
|
20562
|
+
var import_promises34 = require("fs/promises");
|
|
20563
|
+
var import_node_path50 = __toESM(require("path"), 1);
|
|
20122
20564
|
var DEFAULT_CACHE_PATH = ".agentv/cache";
|
|
20123
20565
|
var ResponseCache = class {
|
|
20124
20566
|
cachePath;
|
|
@@ -20128,7 +20570,7 @@ var ResponseCache = class {
|
|
|
20128
20570
|
async get(key) {
|
|
20129
20571
|
const filePath = this.keyToPath(key);
|
|
20130
20572
|
try {
|
|
20131
|
-
const data = await (0,
|
|
20573
|
+
const data = await (0, import_promises34.readFile)(filePath, "utf8");
|
|
20132
20574
|
return JSON.parse(data);
|
|
20133
20575
|
} catch {
|
|
20134
20576
|
return void 0;
|
|
@@ -20136,13 +20578,13 @@ var ResponseCache = class {
|
|
|
20136
20578
|
}
|
|
20137
20579
|
async set(key, value) {
|
|
20138
20580
|
const filePath = this.keyToPath(key);
|
|
20139
|
-
const dir =
|
|
20140
|
-
await (0,
|
|
20141
|
-
await (0,
|
|
20581
|
+
const dir = import_node_path50.default.dirname(filePath);
|
|
20582
|
+
await (0, import_promises34.mkdir)(dir, { recursive: true });
|
|
20583
|
+
await (0, import_promises34.writeFile)(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
20142
20584
|
}
|
|
20143
20585
|
keyToPath(key) {
|
|
20144
20586
|
const prefix = key.slice(0, 2);
|
|
20145
|
-
return
|
|
20587
|
+
return import_node_path50.default.join(this.cachePath, prefix, `${key}.json`);
|
|
20146
20588
|
}
|
|
20147
20589
|
};
|
|
20148
20590
|
function shouldEnableCache(params) {
|
|
@@ -20767,6 +21209,7 @@ function createAgentKernel() {
|
|
|
20767
21209
|
defineConfig,
|
|
20768
21210
|
detectFormat,
|
|
20769
21211
|
discoverAssertions,
|
|
21212
|
+
discoverCopilotSessions,
|
|
20770
21213
|
discoverGraders,
|
|
20771
21214
|
discoverJudges,
|
|
20772
21215
|
discoverProviders,
|