@agentv/core 3.13.3 → 3.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-ZB3AUPES.js → chunk-HP5PFOVK.js} +48 -1
- package/dist/{chunk-ZB3AUPES.js.map → chunk-HP5PFOVK.js.map} +1 -1
- package/dist/evaluation/validation/index.cjs +1 -0
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +1 -1
- package/dist/index.cjs +763 -415
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +62 -2
- package/dist/index.d.ts +62 -2
- package/dist/index.js +614 -314
- package/dist/index.js.map +1 -1
- package/package.json +5 -2
package/dist/index.js
CHANGED
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
readTextFile,
|
|
20
20
|
resolveFileReference,
|
|
21
21
|
resolveTargetDefinition
|
|
22
|
-
} from "./chunk-
|
|
22
|
+
} from "./chunk-HP5PFOVK.js";
|
|
23
23
|
import {
|
|
24
24
|
AgentvProvider
|
|
25
25
|
} from "./chunk-W5YDZWT4.js";
|
|
@@ -6407,10 +6407,268 @@ function summarizeAcpEvent(eventType, data) {
|
|
|
6407
6407
|
}
|
|
6408
6408
|
}
|
|
6409
6409
|
|
|
6410
|
+
// src/evaluation/providers/copilot-log.ts
|
|
6411
|
+
import { readFile as readFile8 } from "node:fs/promises";
|
|
6412
|
+
import { homedir as homedir2 } from "node:os";
|
|
6413
|
+
import path17 from "node:path";
|
|
6414
|
+
|
|
6415
|
+
// src/evaluation/providers/copilot-log-parser.ts
|
|
6416
|
+
function parseCopilotEvents(eventsJsonl) {
|
|
6417
|
+
const messages = [];
|
|
6418
|
+
const meta = { sessionId: "", model: "", cwd: "" };
|
|
6419
|
+
let totalInputTokens = 0;
|
|
6420
|
+
let totalOutputTokens = 0;
|
|
6421
|
+
let hasUsage = false;
|
|
6422
|
+
let startTimestamp;
|
|
6423
|
+
let endTimestamp;
|
|
6424
|
+
const toolCallsInProgress = /* @__PURE__ */ new Map();
|
|
6425
|
+
const lines = eventsJsonl.split("\n").filter((l) => l.trim().length > 0);
|
|
6426
|
+
for (const line of lines) {
|
|
6427
|
+
let event;
|
|
6428
|
+
try {
|
|
6429
|
+
event = JSON.parse(line);
|
|
6430
|
+
} catch {
|
|
6431
|
+
continue;
|
|
6432
|
+
}
|
|
6433
|
+
const eventType = event.type;
|
|
6434
|
+
if (!eventType) continue;
|
|
6435
|
+
const data = event.data ?? {};
|
|
6436
|
+
switch (eventType) {
|
|
6437
|
+
case "session.start": {
|
|
6438
|
+
meta.sessionId = String(data.sessionId ?? "");
|
|
6439
|
+
const ctx = data.context;
|
|
6440
|
+
meta.cwd = String(ctx?.cwd ?? "");
|
|
6441
|
+
meta.repository = ctx?.repository ? String(ctx.repository) : void 0;
|
|
6442
|
+
meta.branch = ctx?.branch ? String(ctx.branch) : void 0;
|
|
6443
|
+
const ts = event.timestamp ?? data.startTime;
|
|
6444
|
+
meta.startedAt = ts ? String(ts) : void 0;
|
|
6445
|
+
startTimestamp = ts ? String(ts) : void 0;
|
|
6446
|
+
break;
|
|
6447
|
+
}
|
|
6448
|
+
case "user.message": {
|
|
6449
|
+
messages.push({
|
|
6450
|
+
role: "user",
|
|
6451
|
+
content: data.content != null ? String(data.content) : ""
|
|
6452
|
+
});
|
|
6453
|
+
break;
|
|
6454
|
+
}
|
|
6455
|
+
case "assistant.message": {
|
|
6456
|
+
const toolRequests = data.toolRequests;
|
|
6457
|
+
const toolCalls = (toolRequests ?? []).map((req) => ({
|
|
6458
|
+
tool: String(req.name ?? req.toolName ?? ""),
|
|
6459
|
+
input: req.arguments,
|
|
6460
|
+
id: req.toolCallId ? String(req.toolCallId) : void 0
|
|
6461
|
+
}));
|
|
6462
|
+
messages.push({
|
|
6463
|
+
role: "assistant",
|
|
6464
|
+
content: data.content != null ? String(data.content) : void 0,
|
|
6465
|
+
toolCalls: toolCalls.length > 0 ? toolCalls : void 0
|
|
6466
|
+
});
|
|
6467
|
+
break;
|
|
6468
|
+
}
|
|
6469
|
+
case "skill.invoked": {
|
|
6470
|
+
const skillName = String(data.name ?? "");
|
|
6471
|
+
messages.push({
|
|
6472
|
+
role: "assistant",
|
|
6473
|
+
toolCalls: [
|
|
6474
|
+
{
|
|
6475
|
+
tool: "Skill",
|
|
6476
|
+
input: { skill: skillName }
|
|
6477
|
+
}
|
|
6478
|
+
]
|
|
6479
|
+
});
|
|
6480
|
+
break;
|
|
6481
|
+
}
|
|
6482
|
+
case "tool.execution_start": {
|
|
6483
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6484
|
+
if (toolCallId) {
|
|
6485
|
+
toolCallsInProgress.set(toolCallId, {
|
|
6486
|
+
toolName: String(data.toolName ?? ""),
|
|
6487
|
+
input: data.arguments,
|
|
6488
|
+
toolCallId
|
|
6489
|
+
});
|
|
6490
|
+
}
|
|
6491
|
+
break;
|
|
6492
|
+
}
|
|
6493
|
+
case "tool.execution_complete": {
|
|
6494
|
+
const toolCallId = String(data.toolCallId ?? "");
|
|
6495
|
+
const started = toolCallsInProgress.get(toolCallId);
|
|
6496
|
+
if (started) {
|
|
6497
|
+
toolCallsInProgress.delete(toolCallId);
|
|
6498
|
+
messages.push({
|
|
6499
|
+
role: "assistant",
|
|
6500
|
+
toolCalls: [
|
|
6501
|
+
{
|
|
6502
|
+
tool: started.toolName,
|
|
6503
|
+
input: started.input,
|
|
6504
|
+
output: data.result,
|
|
6505
|
+
id: toolCallId
|
|
6506
|
+
}
|
|
6507
|
+
]
|
|
6508
|
+
});
|
|
6509
|
+
}
|
|
6510
|
+
break;
|
|
6511
|
+
}
|
|
6512
|
+
case "session.shutdown": {
|
|
6513
|
+
endTimestamp = event.timestamp ? String(event.timestamp) : void 0;
|
|
6514
|
+
const modelMetrics = data.modelMetrics;
|
|
6515
|
+
if (modelMetrics) {
|
|
6516
|
+
for (const metrics of Object.values(modelMetrics)) {
|
|
6517
|
+
if (metrics.usage) {
|
|
6518
|
+
hasUsage = true;
|
|
6519
|
+
totalInputTokens += Number(metrics.usage.inputTokens ?? 0);
|
|
6520
|
+
totalOutputTokens += Number(metrics.usage.outputTokens ?? 0);
|
|
6521
|
+
}
|
|
6522
|
+
}
|
|
6523
|
+
}
|
|
6524
|
+
const currentModel = data.currentModel;
|
|
6525
|
+
if (currentModel && !meta.model) {
|
|
6526
|
+
meta.model = String(currentModel);
|
|
6527
|
+
}
|
|
6528
|
+
break;
|
|
6529
|
+
}
|
|
6530
|
+
}
|
|
6531
|
+
}
|
|
6532
|
+
let durationMs;
|
|
6533
|
+
if (startTimestamp && endTimestamp) {
|
|
6534
|
+
durationMs = new Date(endTimestamp).getTime() - new Date(startTimestamp).getTime();
|
|
6535
|
+
}
|
|
6536
|
+
return {
|
|
6537
|
+
messages,
|
|
6538
|
+
meta,
|
|
6539
|
+
tokenUsage: hasUsage ? { input: totalInputTokens, output: totalOutputTokens } : void 0,
|
|
6540
|
+
durationMs
|
|
6541
|
+
};
|
|
6542
|
+
}
|
|
6543
|
+
|
|
6544
|
+
// src/evaluation/providers/copilot-session-discovery.ts
|
|
6545
|
+
import { readFile as readFile7, readdir, stat } from "node:fs/promises";
|
|
6546
|
+
import { homedir } from "node:os";
|
|
6547
|
+
import path16 from "node:path";
|
|
6548
|
+
import { parse as parseYaml2 } from "yaml";
|
|
6549
|
+
var DEFAULT_SESSION_STATE_DIR = () => path16.join(homedir(), ".copilot", "session-state");
|
|
6550
|
+
async function discoverCopilotSessions(opts) {
|
|
6551
|
+
const sessionStateDir = opts?.sessionStateDir ?? DEFAULT_SESSION_STATE_DIR();
|
|
6552
|
+
const limit = opts?.limit ?? 10;
|
|
6553
|
+
let entries;
|
|
6554
|
+
try {
|
|
6555
|
+
entries = await readdir(sessionStateDir);
|
|
6556
|
+
} catch {
|
|
6557
|
+
return [];
|
|
6558
|
+
}
|
|
6559
|
+
const sessions = [];
|
|
6560
|
+
for (const entry of entries) {
|
|
6561
|
+
const sessionDir = path16.join(sessionStateDir, entry);
|
|
6562
|
+
const workspacePath = path16.join(sessionDir, "workspace.yaml");
|
|
6563
|
+
const eventsPath = path16.join(sessionDir, "events.jsonl");
|
|
6564
|
+
try {
|
|
6565
|
+
const workspaceContent = await readFile7(workspacePath, "utf8");
|
|
6566
|
+
const workspace = parseYaml2(workspaceContent) ?? {};
|
|
6567
|
+
const cwd = String(workspace.cwd ?? "");
|
|
6568
|
+
let updatedAt;
|
|
6569
|
+
try {
|
|
6570
|
+
const eventsStat = await stat(eventsPath);
|
|
6571
|
+
updatedAt = eventsStat.mtime;
|
|
6572
|
+
} catch {
|
|
6573
|
+
updatedAt = /* @__PURE__ */ new Date(0);
|
|
6574
|
+
}
|
|
6575
|
+
let isActive = true;
|
|
6576
|
+
try {
|
|
6577
|
+
const fd = await import("node:fs/promises").then((fs3) => fs3.open(eventsPath, "r"));
|
|
6578
|
+
try {
|
|
6579
|
+
const fstat = await fd.stat();
|
|
6580
|
+
const tailSize = Math.min(fstat.size, 4096);
|
|
6581
|
+
const buf = Buffer.alloc(tailSize);
|
|
6582
|
+
await fd.read(buf, 0, tailSize, Math.max(0, fstat.size - tailSize));
|
|
6583
|
+
isActive = !buf.toString("utf8").includes('"session.shutdown"');
|
|
6584
|
+
} finally {
|
|
6585
|
+
await fd.close();
|
|
6586
|
+
}
|
|
6587
|
+
} catch {
|
|
6588
|
+
}
|
|
6589
|
+
sessions.push({
|
|
6590
|
+
sessionId: entry,
|
|
6591
|
+
sessionDir,
|
|
6592
|
+
cwd,
|
|
6593
|
+
repository: workspace.repository ? String(workspace.repository) : void 0,
|
|
6594
|
+
updatedAt,
|
|
6595
|
+
isActive
|
|
6596
|
+
});
|
|
6597
|
+
} catch {
|
|
6598
|
+
}
|
|
6599
|
+
}
|
|
6600
|
+
let filtered = sessions;
|
|
6601
|
+
if (opts?.cwd) {
|
|
6602
|
+
filtered = filtered.filter((s) => s.cwd === opts.cwd);
|
|
6603
|
+
}
|
|
6604
|
+
if (opts?.repository) {
|
|
6605
|
+
filtered = filtered.filter((s) => s.repository === opts.repository);
|
|
6606
|
+
}
|
|
6607
|
+
filtered.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
6608
|
+
return filtered.slice(0, limit);
|
|
6609
|
+
}
|
|
6610
|
+
|
|
6611
|
+
// src/evaluation/providers/copilot-log.ts
|
|
6612
|
+
var CopilotLogProvider = class {
|
|
6613
|
+
id;
|
|
6614
|
+
kind = "copilot-log";
|
|
6615
|
+
targetName;
|
|
6616
|
+
config;
|
|
6617
|
+
constructor(targetName, config) {
|
|
6618
|
+
this.targetName = targetName;
|
|
6619
|
+
this.id = `copilot-log:${targetName}`;
|
|
6620
|
+
this.config = config;
|
|
6621
|
+
}
|
|
6622
|
+
async invoke(_request) {
|
|
6623
|
+
const sessionDir = await this.resolveSessionDir();
|
|
6624
|
+
const eventsPath = path17.join(sessionDir, "events.jsonl");
|
|
6625
|
+
let eventsContent;
|
|
6626
|
+
try {
|
|
6627
|
+
eventsContent = await readFile8(eventsPath, "utf8");
|
|
6628
|
+
} catch (err) {
|
|
6629
|
+
throw new Error(
|
|
6630
|
+
`Failed to read Copilot session transcript at ${eventsPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
6631
|
+
);
|
|
6632
|
+
}
|
|
6633
|
+
const parsed = parseCopilotEvents(eventsContent);
|
|
6634
|
+
return {
|
|
6635
|
+
output: parsed.messages,
|
|
6636
|
+
tokenUsage: parsed.tokenUsage,
|
|
6637
|
+
durationMs: parsed.durationMs,
|
|
6638
|
+
startTime: parsed.meta.startedAt
|
|
6639
|
+
};
|
|
6640
|
+
}
|
|
6641
|
+
async resolveSessionDir() {
|
|
6642
|
+
if (this.config.sessionDir) {
|
|
6643
|
+
return this.config.sessionDir;
|
|
6644
|
+
}
|
|
6645
|
+
if (this.config.sessionId) {
|
|
6646
|
+
const stateDir = this.config.sessionStateDir ?? path17.join(homedir2(), ".copilot", "session-state");
|
|
6647
|
+
return path17.join(stateDir, this.config.sessionId);
|
|
6648
|
+
}
|
|
6649
|
+
if (this.config.discover === "latest") {
|
|
6650
|
+
const sessions = await discoverCopilotSessions({
|
|
6651
|
+
sessionStateDir: this.config.sessionStateDir,
|
|
6652
|
+
cwd: this.config.cwd,
|
|
6653
|
+
limit: 1
|
|
6654
|
+
});
|
|
6655
|
+
if (sessions.length === 0) {
|
|
6656
|
+
throw new Error(
|
|
6657
|
+
`No Copilot CLI sessions found${this.config.cwd ? ` for cwd=${this.config.cwd}` : ""}. Check that sessions exist in ${this.config.sessionStateDir ?? "~/.copilot/session-state/"}`
|
|
6658
|
+
);
|
|
6659
|
+
}
|
|
6660
|
+
return sessions[0].sessionDir;
|
|
6661
|
+
}
|
|
6662
|
+
throw new Error(
|
|
6663
|
+
'CopilotLogProvider requires one of: sessionDir, sessionId, or discover="latest"'
|
|
6664
|
+
);
|
|
6665
|
+
}
|
|
6666
|
+
};
|
|
6667
|
+
|
|
6410
6668
|
// src/evaluation/providers/copilot-sdk.ts
|
|
6411
6669
|
import { randomUUID as randomUUID6 } from "node:crypto";
|
|
6412
6670
|
import { mkdir as mkdir5 } from "node:fs/promises";
|
|
6413
|
-
import
|
|
6671
|
+
import path18 from "node:path";
|
|
6414
6672
|
|
|
6415
6673
|
// src/evaluation/providers/copilot-sdk-log-tracker.ts
|
|
6416
6674
|
var GLOBAL_LOGS_KEY4 = Symbol.for("agentv.copilotSdkLogs");
|
|
@@ -6700,10 +6958,10 @@ var CopilotSdkProvider = class {
|
|
|
6700
6958
|
}
|
|
6701
6959
|
resolveCwd(cwdOverride) {
|
|
6702
6960
|
if (cwdOverride) {
|
|
6703
|
-
return
|
|
6961
|
+
return path18.resolve(cwdOverride);
|
|
6704
6962
|
}
|
|
6705
6963
|
if (this.config.cwd) {
|
|
6706
|
-
return
|
|
6964
|
+
return path18.resolve(this.config.cwd);
|
|
6707
6965
|
}
|
|
6708
6966
|
return void 0;
|
|
6709
6967
|
}
|
|
@@ -6712,9 +6970,9 @@ var CopilotSdkProvider = class {
|
|
|
6712
6970
|
return void 0;
|
|
6713
6971
|
}
|
|
6714
6972
|
if (this.config.logDir) {
|
|
6715
|
-
return
|
|
6973
|
+
return path18.resolve(this.config.logDir);
|
|
6716
6974
|
}
|
|
6717
|
-
return
|
|
6975
|
+
return path18.join(process.cwd(), ".agentv", "logs", "copilot-sdk");
|
|
6718
6976
|
}
|
|
6719
6977
|
async createStreamLogger(request) {
|
|
6720
6978
|
const logDir = this.resolveLogDirectory();
|
|
@@ -6728,7 +6986,7 @@ var CopilotSdkProvider = class {
|
|
|
6728
6986
|
console.warn(`Skipping Copilot SDK stream logging (could not create ${logDir}): ${message}`);
|
|
6729
6987
|
return void 0;
|
|
6730
6988
|
}
|
|
6731
|
-
const filePath =
|
|
6989
|
+
const filePath = path18.join(logDir, buildLogFilename4(request, this.targetName, "copilot-sdk"));
|
|
6732
6990
|
try {
|
|
6733
6991
|
const logger = await CopilotStreamLogger.create(
|
|
6734
6992
|
{
|
|
@@ -6825,7 +7083,7 @@ import { randomUUID as randomUUID7 } from "node:crypto";
|
|
|
6825
7083
|
import { createWriteStream as createWriteStream5 } from "node:fs";
|
|
6826
7084
|
import { mkdir as mkdir6, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
6827
7085
|
import { tmpdir } from "node:os";
|
|
6828
|
-
import
|
|
7086
|
+
import path19 from "node:path";
|
|
6829
7087
|
|
|
6830
7088
|
// src/evaluation/providers/pi-log-tracker.ts
|
|
6831
7089
|
var GLOBAL_LOGS_KEY5 = Symbol.for("agentv.piLogs");
|
|
@@ -6928,13 +7186,14 @@ var PiCliProvider = class {
|
|
|
6928
7186
|
const inputFiles = normalizeInputFiles(request.inputFiles);
|
|
6929
7187
|
const startTime = (/* @__PURE__ */ new Date()).toISOString();
|
|
6930
7188
|
const startMs = Date.now();
|
|
6931
|
-
const
|
|
7189
|
+
const hasExternalCwd = !!(request.cwd || this.config.cwd);
|
|
7190
|
+
const workspaceRoot = hasExternalCwd ? void 0 : await this.createWorkspace();
|
|
7191
|
+
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
6932
7192
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
6933
7193
|
try {
|
|
6934
|
-
const promptFile =
|
|
7194
|
+
const promptFile = path19.join(cwd, PROMPT_FILENAME);
|
|
6935
7195
|
await writeFile(promptFile, request.question, "utf8");
|
|
6936
7196
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
6937
|
-
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
6938
7197
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
6939
7198
|
if (result.timedOut) {
|
|
6940
7199
|
throw new Error(
|
|
@@ -6976,7 +7235,7 @@ var PiCliProvider = class {
|
|
|
6976
7235
|
args,
|
|
6977
7236
|
executable: this.config.executable,
|
|
6978
7237
|
promptFile,
|
|
6979
|
-
workspace: workspaceRoot,
|
|
7238
|
+
workspace: workspaceRoot ?? cwd,
|
|
6980
7239
|
inputFiles,
|
|
6981
7240
|
logFile: logger?.filePath
|
|
6982
7241
|
},
|
|
@@ -6988,17 +7247,22 @@ var PiCliProvider = class {
|
|
|
6988
7247
|
};
|
|
6989
7248
|
} finally {
|
|
6990
7249
|
await logger?.close();
|
|
6991
|
-
|
|
7250
|
+
if (workspaceRoot) {
|
|
7251
|
+
await this.cleanupWorkspace(workspaceRoot);
|
|
7252
|
+
}
|
|
6992
7253
|
}
|
|
6993
7254
|
}
|
|
6994
7255
|
resolveCwd(workspaceRoot, cwdOverride) {
|
|
6995
7256
|
if (cwdOverride) {
|
|
6996
|
-
return
|
|
7257
|
+
return path19.resolve(cwdOverride);
|
|
6997
7258
|
}
|
|
6998
|
-
if (
|
|
7259
|
+
if (this.config.cwd) {
|
|
7260
|
+
return path19.resolve(this.config.cwd);
|
|
7261
|
+
}
|
|
7262
|
+
if (workspaceRoot) {
|
|
6999
7263
|
return workspaceRoot;
|
|
7000
7264
|
}
|
|
7001
|
-
return
|
|
7265
|
+
return process.cwd();
|
|
7002
7266
|
}
|
|
7003
7267
|
buildPiArgs(prompt, inputFiles) {
|
|
7004
7268
|
const args = [];
|
|
@@ -7079,7 +7343,7 @@ ${prompt}` : prompt;
|
|
|
7079
7343
|
return env;
|
|
7080
7344
|
}
|
|
7081
7345
|
async createWorkspace() {
|
|
7082
|
-
return await mkdtemp(
|
|
7346
|
+
return await mkdtemp(path19.join(tmpdir(), WORKSPACE_PREFIX));
|
|
7083
7347
|
}
|
|
7084
7348
|
async cleanupWorkspace(workspaceRoot) {
|
|
7085
7349
|
try {
|
|
@@ -7089,9 +7353,9 @@ ${prompt}` : prompt;
|
|
|
7089
7353
|
}
|
|
7090
7354
|
resolveLogDirectory() {
|
|
7091
7355
|
if (this.config.logDir) {
|
|
7092
|
-
return
|
|
7356
|
+
return path19.resolve(this.config.logDir);
|
|
7093
7357
|
}
|
|
7094
|
-
return
|
|
7358
|
+
return path19.join(process.cwd(), ".agentv", "logs", "pi-cli");
|
|
7095
7359
|
}
|
|
7096
7360
|
async createStreamLogger(request) {
|
|
7097
7361
|
const logDir = this.resolveLogDirectory();
|
|
@@ -7105,7 +7369,7 @@ ${prompt}` : prompt;
|
|
|
7105
7369
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
7106
7370
|
return void 0;
|
|
7107
7371
|
}
|
|
7108
|
-
const filePath =
|
|
7372
|
+
const filePath = path19.join(logDir, buildLogFilename5(request, this.targetName));
|
|
7109
7373
|
try {
|
|
7110
7374
|
const logger = await PiStreamLogger.create({
|
|
7111
7375
|
filePath,
|
|
@@ -7544,15 +7808,17 @@ async function defaultPiRunner(options) {
|
|
|
7544
7808
|
// src/evaluation/providers/pi-coding-agent.ts
|
|
7545
7809
|
import { execSync } from "node:child_process";
|
|
7546
7810
|
import { randomUUID as randomUUID8 } from "node:crypto";
|
|
7547
|
-
import { createWriteStream as createWriteStream6 } from "node:fs";
|
|
7811
|
+
import { accessSync, createWriteStream as createWriteStream6 } from "node:fs";
|
|
7548
7812
|
import { mkdir as mkdir7 } from "node:fs/promises";
|
|
7549
|
-
import
|
|
7813
|
+
import path20 from "node:path";
|
|
7550
7814
|
import { createInterface } from "node:readline";
|
|
7815
|
+
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
7551
7816
|
var piCodingAgentModule = null;
|
|
7552
7817
|
var piAiModule = null;
|
|
7818
|
+
var loadingPromise = null;
|
|
7553
7819
|
async function promptInstall() {
|
|
7554
7820
|
if (!process.stdout.isTTY) return false;
|
|
7555
|
-
const rl = createInterface({ input: process.stdin, output: process.
|
|
7821
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
7556
7822
|
try {
|
|
7557
7823
|
return await new Promise((resolve) => {
|
|
7558
7824
|
rl.question(
|
|
@@ -7564,43 +7830,74 @@ async function promptInstall() {
|
|
|
7564
7830
|
rl.close();
|
|
7565
7831
|
}
|
|
7566
7832
|
}
|
|
7567
|
-
|
|
7568
|
-
|
|
7833
|
+
function findAgentvRoot() {
|
|
7834
|
+
const thisFile = fileURLToPath3(import.meta.url);
|
|
7835
|
+
let dir = path20.dirname(thisFile);
|
|
7836
|
+
for (let i = 0; i < 10; i++) {
|
|
7569
7837
|
try {
|
|
7838
|
+
const pkg = path20.join(dir, "package.json");
|
|
7839
|
+
accessSync(pkg);
|
|
7840
|
+
return dir;
|
|
7841
|
+
} catch {
|
|
7842
|
+
const parent = path20.dirname(dir);
|
|
7843
|
+
if (parent === dir) break;
|
|
7844
|
+
dir = parent;
|
|
7845
|
+
}
|
|
7846
|
+
}
|
|
7847
|
+
return path20.dirname(thisFile);
|
|
7848
|
+
}
|
|
7849
|
+
async function doLoadSdkModules() {
|
|
7850
|
+
try {
|
|
7851
|
+
[piCodingAgentModule, piAiModule] = await Promise.all([
|
|
7852
|
+
import("@mariozechner/pi-coding-agent"),
|
|
7853
|
+
import("@mariozechner/pi-ai")
|
|
7854
|
+
]);
|
|
7855
|
+
} catch {
|
|
7856
|
+
if (await promptInstall()) {
|
|
7857
|
+
const installDir = findAgentvRoot();
|
|
7858
|
+
console.error(`Installing @mariozechner/pi-coding-agent into ${installDir}...`);
|
|
7859
|
+
execSync("bun add @mariozechner/pi-coding-agent", {
|
|
7860
|
+
cwd: installDir,
|
|
7861
|
+
stdio: "inherit"
|
|
7862
|
+
});
|
|
7570
7863
|
[piCodingAgentModule, piAiModule] = await Promise.all([
|
|
7571
7864
|
import("@mariozechner/pi-coding-agent"),
|
|
7572
7865
|
import("@mariozechner/pi-ai")
|
|
7573
7866
|
]);
|
|
7574
|
-
}
|
|
7575
|
-
|
|
7576
|
-
|
|
7577
|
-
|
|
7578
|
-
[piCodingAgentModule, piAiModule] = await Promise.all([
|
|
7579
|
-
import("@mariozechner/pi-coding-agent"),
|
|
7580
|
-
import("@mariozechner/pi-ai")
|
|
7581
|
-
]);
|
|
7582
|
-
} else {
|
|
7583
|
-
throw new Error(
|
|
7584
|
-
"pi-coding-agent SDK is not installed. Install it with:\n bun add @mariozechner/pi-coding-agent"
|
|
7585
|
-
);
|
|
7586
|
-
}
|
|
7867
|
+
} else {
|
|
7868
|
+
throw new Error(
|
|
7869
|
+
"pi-coding-agent SDK is not installed. Install it with:\n bun add @mariozechner/pi-coding-agent"
|
|
7870
|
+
);
|
|
7587
7871
|
}
|
|
7588
7872
|
}
|
|
7873
|
+
}
|
|
7874
|
+
async function loadSdkModules() {
|
|
7875
|
+
if (!piCodingAgentModule || !piAiModule) {
|
|
7876
|
+
if (!loadingPromise) {
|
|
7877
|
+
loadingPromise = doLoadSdkModules().catch((err) => {
|
|
7878
|
+
loadingPromise = null;
|
|
7879
|
+
throw err;
|
|
7880
|
+
});
|
|
7881
|
+
}
|
|
7882
|
+
await loadingPromise;
|
|
7883
|
+
}
|
|
7884
|
+
const piSdk = piCodingAgentModule;
|
|
7885
|
+
const piAi = piAiModule;
|
|
7589
7886
|
const toolMap = {
|
|
7590
|
-
read:
|
|
7591
|
-
bash:
|
|
7592
|
-
edit:
|
|
7593
|
-
write:
|
|
7594
|
-
grep:
|
|
7595
|
-
find:
|
|
7596
|
-
ls:
|
|
7887
|
+
read: piSdk.readTool,
|
|
7888
|
+
bash: piSdk.bashTool,
|
|
7889
|
+
edit: piSdk.editTool,
|
|
7890
|
+
write: piSdk.writeTool,
|
|
7891
|
+
grep: piSdk.grepTool,
|
|
7892
|
+
find: piSdk.findTool,
|
|
7893
|
+
ls: piSdk.lsTool
|
|
7597
7894
|
};
|
|
7598
7895
|
return {
|
|
7599
|
-
createAgentSession:
|
|
7600
|
-
codingTools:
|
|
7896
|
+
createAgentSession: piSdk.createAgentSession,
|
|
7897
|
+
codingTools: piSdk.codingTools,
|
|
7601
7898
|
toolMap,
|
|
7602
|
-
SessionManager:
|
|
7603
|
-
getModel:
|
|
7899
|
+
SessionManager: piSdk.SessionManager,
|
|
7900
|
+
getModel: piAi.getModel
|
|
7604
7901
|
};
|
|
7605
7902
|
}
|
|
7606
7903
|
var PiCodingAgentProvider = class {
|
|
@@ -7789,10 +8086,10 @@ ${fileList}`;
|
|
|
7789
8086
|
}
|
|
7790
8087
|
resolveCwd(cwdOverride) {
|
|
7791
8088
|
if (cwdOverride) {
|
|
7792
|
-
return
|
|
8089
|
+
return path20.resolve(cwdOverride);
|
|
7793
8090
|
}
|
|
7794
8091
|
if (this.config.cwd) {
|
|
7795
|
-
return
|
|
8092
|
+
return path20.resolve(this.config.cwd);
|
|
7796
8093
|
}
|
|
7797
8094
|
return process.cwd();
|
|
7798
8095
|
}
|
|
@@ -7811,9 +8108,9 @@ ${fileList}`;
|
|
|
7811
8108
|
}
|
|
7812
8109
|
resolveLogDirectory() {
|
|
7813
8110
|
if (this.config.logDir) {
|
|
7814
|
-
return
|
|
8111
|
+
return path20.resolve(this.config.logDir);
|
|
7815
8112
|
}
|
|
7816
|
-
return
|
|
8113
|
+
return path20.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
7817
8114
|
}
|
|
7818
8115
|
async createStreamLogger(request) {
|
|
7819
8116
|
const logDir = this.resolveLogDirectory();
|
|
@@ -7827,7 +8124,7 @@ ${fileList}`;
|
|
|
7827
8124
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
7828
8125
|
return void 0;
|
|
7829
8126
|
}
|
|
7830
|
-
const filePath =
|
|
8127
|
+
const filePath = path20.join(logDir, buildLogFilename6(request, this.targetName));
|
|
7831
8128
|
try {
|
|
7832
8129
|
const logger = await PiStreamLogger2.create({
|
|
7833
8130
|
filePath,
|
|
@@ -8040,18 +8337,18 @@ var ProviderRegistry = class {
|
|
|
8040
8337
|
|
|
8041
8338
|
// src/evaluation/providers/vscode-provider.ts
|
|
8042
8339
|
import { exec as exec2 } from "node:child_process";
|
|
8043
|
-
import { constants as constants3, access as access3, stat as
|
|
8044
|
-
import
|
|
8340
|
+
import { constants as constants3, access as access3, stat as stat5 } from "node:fs/promises";
|
|
8341
|
+
import path32 from "node:path";
|
|
8045
8342
|
import { promisify as promisify3 } from "node:util";
|
|
8046
8343
|
|
|
8047
8344
|
// src/evaluation/providers/vscode/dispatch/agentDispatch.ts
|
|
8048
|
-
import { stat as
|
|
8049
|
-
import
|
|
8345
|
+
import { stat as stat4, writeFile as writeFile4 } from "node:fs/promises";
|
|
8346
|
+
import path30 from "node:path";
|
|
8050
8347
|
|
|
8051
8348
|
// src/evaluation/providers/vscode/utils/fs.ts
|
|
8052
8349
|
import { constants as constants2 } from "node:fs";
|
|
8053
|
-
import { access as access2, mkdir as mkdir8, readdir, rm as rm2, stat } from "node:fs/promises";
|
|
8054
|
-
import
|
|
8350
|
+
import { access as access2, mkdir as mkdir8, readdir as readdir2, rm as rm2, stat as stat2 } from "node:fs/promises";
|
|
8351
|
+
import path21 from "node:path";
|
|
8055
8352
|
async function pathExists(target) {
|
|
8056
8353
|
try {
|
|
8057
8354
|
await access2(target, constants2.F_OK);
|
|
@@ -8064,10 +8361,10 @@ async function ensureDir(target) {
|
|
|
8064
8361
|
await mkdir8(target, { recursive: true });
|
|
8065
8362
|
}
|
|
8066
8363
|
async function readDirEntries(target) {
|
|
8067
|
-
const entries = await
|
|
8364
|
+
const entries = await readdir2(target, { withFileTypes: true });
|
|
8068
8365
|
return entries.map((entry) => ({
|
|
8069
8366
|
name: entry.name,
|
|
8070
|
-
absolutePath:
|
|
8367
|
+
absolutePath: path21.join(target, entry.name),
|
|
8071
8368
|
isDirectory: entry.isDirectory()
|
|
8072
8369
|
}));
|
|
8073
8370
|
}
|
|
@@ -8082,9 +8379,9 @@ async function removeIfExists(target) {
|
|
|
8082
8379
|
}
|
|
8083
8380
|
|
|
8084
8381
|
// src/evaluation/providers/vscode/utils/path.ts
|
|
8085
|
-
import
|
|
8382
|
+
import path22 from "node:path";
|
|
8086
8383
|
function pathToFileUri2(filePath) {
|
|
8087
|
-
const absolutePath =
|
|
8384
|
+
const absolutePath = path22.isAbsolute(filePath) ? filePath : path22.resolve(filePath);
|
|
8088
8385
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
8089
8386
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
8090
8387
|
return `file:///${normalizedPath}`;
|
|
@@ -8093,7 +8390,7 @@ function pathToFileUri2(filePath) {
|
|
|
8093
8390
|
}
|
|
8094
8391
|
|
|
8095
8392
|
// src/evaluation/providers/vscode/dispatch/promptBuilder.ts
|
|
8096
|
-
import
|
|
8393
|
+
import path23 from "node:path";
|
|
8097
8394
|
|
|
8098
8395
|
// src/evaluation/providers/vscode/utils/template.ts
|
|
8099
8396
|
function renderTemplate2(content, variables) {
|
|
@@ -8185,8 +8482,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
8185
8482
|
});
|
|
8186
8483
|
}
|
|
8187
8484
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
8188
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
8189
|
-
const responseList = responseFiles.map((file) => `"${
|
|
8485
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path23.basename(file)}`).join("\n");
|
|
8486
|
+
const responseList = responseFiles.map((file) => `"${path23.basename(file)}"`).join(", ");
|
|
8190
8487
|
return renderTemplate2(templateContent, {
|
|
8191
8488
|
requestFiles: requestLines,
|
|
8192
8489
|
responseList
|
|
@@ -8194,8 +8491,8 @@ function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateCont
|
|
|
8194
8491
|
}
|
|
8195
8492
|
|
|
8196
8493
|
// src/evaluation/providers/vscode/dispatch/responseWaiter.ts
|
|
8197
|
-
import { readFile as
|
|
8198
|
-
import
|
|
8494
|
+
import { readFile as readFile9 } from "node:fs/promises";
|
|
8495
|
+
import path24 from "node:path";
|
|
8199
8496
|
|
|
8200
8497
|
// src/evaluation/providers/vscode/utils/time.ts
|
|
8201
8498
|
function sleep2(ms) {
|
|
@@ -8233,7 +8530,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
8233
8530
|
const maxAttempts = 10;
|
|
8234
8531
|
while (attempts < maxAttempts) {
|
|
8235
8532
|
try {
|
|
8236
|
-
const content = await
|
|
8533
|
+
const content = await readFile9(responseFileFinal, { encoding: "utf8" });
|
|
8237
8534
|
if (!silent) {
|
|
8238
8535
|
process.stdout.write(`${content}
|
|
8239
8536
|
`);
|
|
@@ -8254,7 +8551,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
8254
8551
|
}
|
|
8255
8552
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
8256
8553
|
if (!silent) {
|
|
8257
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
8554
|
+
const fileList = responseFilesFinal.map((file) => path24.basename(file)).join(", ");
|
|
8258
8555
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
8259
8556
|
}
|
|
8260
8557
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -8263,7 +8560,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
8263
8560
|
while (pending.size > 0) {
|
|
8264
8561
|
if (Date.now() >= deadline) {
|
|
8265
8562
|
if (!silent) {
|
|
8266
|
-
const remaining = [...pending].map((f) =>
|
|
8563
|
+
const remaining = [...pending].map((f) => path24.basename(f)).join(", ");
|
|
8267
8564
|
console.error(
|
|
8268
8565
|
`error: timed out after ${Math.round(timeoutMs / 1e3)}s waiting for batch responses. Still pending: ${remaining}`
|
|
8269
8566
|
);
|
|
@@ -8290,7 +8587,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
8290
8587
|
const maxAttempts = 10;
|
|
8291
8588
|
while (attempts < maxAttempts) {
|
|
8292
8589
|
try {
|
|
8293
|
-
const content = await
|
|
8590
|
+
const content = await readFile9(file, { encoding: "utf8" });
|
|
8294
8591
|
if (!silent) {
|
|
8295
8592
|
process.stdout.write(`${content}
|
|
8296
8593
|
`);
|
|
@@ -8314,15 +8611,15 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
8314
8611
|
// src/evaluation/providers/vscode/dispatch/vscodeProcess.ts
|
|
8315
8612
|
import { exec, spawn as spawn4 } from "node:child_process";
|
|
8316
8613
|
import { mkdir as mkdir9, writeFile as writeFile2 } from "node:fs/promises";
|
|
8317
|
-
import
|
|
8614
|
+
import path27 from "node:path";
|
|
8318
8615
|
import { promisify as promisify2 } from "node:util";
|
|
8319
8616
|
|
|
8320
8617
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
8321
|
-
import
|
|
8618
|
+
import path26 from "node:path";
|
|
8322
8619
|
|
|
8323
8620
|
// src/paths.ts
|
|
8324
8621
|
import os2 from "node:os";
|
|
8325
|
-
import
|
|
8622
|
+
import path25 from "node:path";
|
|
8326
8623
|
var logged = false;
|
|
8327
8624
|
function getAgentvHome() {
|
|
8328
8625
|
const envHome = process.env.AGENTV_HOME;
|
|
@@ -8333,19 +8630,19 @@ function getAgentvHome() {
|
|
|
8333
8630
|
}
|
|
8334
8631
|
return envHome;
|
|
8335
8632
|
}
|
|
8336
|
-
return
|
|
8633
|
+
return path25.join(os2.homedir(), ".agentv");
|
|
8337
8634
|
}
|
|
8338
8635
|
function getWorkspacesRoot() {
|
|
8339
|
-
return
|
|
8636
|
+
return path25.join(getAgentvHome(), "workspaces");
|
|
8340
8637
|
}
|
|
8341
8638
|
function getSubagentsRoot() {
|
|
8342
|
-
return
|
|
8639
|
+
return path25.join(getAgentvHome(), "subagents");
|
|
8343
8640
|
}
|
|
8344
8641
|
function getTraceStateRoot() {
|
|
8345
|
-
return
|
|
8642
|
+
return path25.join(getAgentvHome(), "trace-state");
|
|
8346
8643
|
}
|
|
8347
8644
|
function getWorkspacePoolRoot() {
|
|
8348
|
-
return
|
|
8645
|
+
return path25.join(getAgentvHome(), "workspace-pool");
|
|
8349
8646
|
}
|
|
8350
8647
|
|
|
8351
8648
|
// src/evaluation/providers/vscode/dispatch/constants.ts
|
|
@@ -8353,7 +8650,7 @@ var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
|
8353
8650
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
8354
8651
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
8355
8652
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
8356
|
-
return
|
|
8653
|
+
return path26.join(getSubagentsRoot(), folder);
|
|
8357
8654
|
}
|
|
8358
8655
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
8359
8656
|
|
|
@@ -8420,11 +8717,11 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
8420
8717
|
await raceSpawnError(child);
|
|
8421
8718
|
return true;
|
|
8422
8719
|
}
|
|
8423
|
-
const aliveFile =
|
|
8720
|
+
const aliveFile = path27.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
8424
8721
|
await removeIfExists(aliveFile);
|
|
8425
|
-
const githubAgentsDir =
|
|
8722
|
+
const githubAgentsDir = path27.join(subagentDir, ".github", "agents");
|
|
8426
8723
|
await mkdir9(githubAgentsDir, { recursive: true });
|
|
8427
|
-
const wakeupDst =
|
|
8724
|
+
const wakeupDst = path27.join(githubAgentsDir, "wakeup.md");
|
|
8428
8725
|
await writeFile2(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
8429
8726
|
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
8430
8727
|
label: "open-workspace"
|
|
@@ -8437,7 +8734,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
8437
8734
|
"chat",
|
|
8438
8735
|
"-m",
|
|
8439
8736
|
wakeupChatId,
|
|
8440
|
-
`create a file named .alive in the ${
|
|
8737
|
+
`create a file named .alive in the ${path27.basename(subagentDir)} folder`
|
|
8441
8738
|
];
|
|
8442
8739
|
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
8443
8740
|
await raceSpawnError(wakeupChild);
|
|
@@ -8452,10 +8749,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
8452
8749
|
return true;
|
|
8453
8750
|
}
|
|
8454
8751
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
8455
|
-
const workspacePath =
|
|
8456
|
-
const messagesDir =
|
|
8752
|
+
const workspacePath = path27.join(subagentDir, `${path27.basename(subagentDir)}.code-workspace`);
|
|
8753
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
8457
8754
|
await mkdir9(messagesDir, { recursive: true });
|
|
8458
|
-
const reqFile =
|
|
8755
|
+
const reqFile = path27.join(messagesDir, `${timestamp}_req.md`);
|
|
8459
8756
|
await writeFile2(reqFile, requestInstructions, { encoding: "utf8" });
|
|
8460
8757
|
const reqUri = pathToFileUri2(reqFile);
|
|
8461
8758
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
@@ -8463,16 +8760,16 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
8463
8760
|
chatArgs.push("-a", attachment);
|
|
8464
8761
|
}
|
|
8465
8762
|
chatArgs.push("-a", reqFile);
|
|
8466
|
-
chatArgs.push(`Follow instructions in [${
|
|
8763
|
+
chatArgs.push(`Follow instructions in [${path27.basename(reqFile)}](${reqUri})`);
|
|
8467
8764
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
8468
8765
|
workspacePath,
|
|
8469
|
-
|
|
8766
|
+
path27.basename(subagentDir),
|
|
8470
8767
|
subagentDir,
|
|
8471
8768
|
vscodeCmd
|
|
8472
8769
|
);
|
|
8473
8770
|
if (!workspaceReady) {
|
|
8474
8771
|
throw new Error(
|
|
8475
|
-
`VS Code workspace '${
|
|
8772
|
+
`VS Code workspace '${path27.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
8476
8773
|
);
|
|
8477
8774
|
}
|
|
8478
8775
|
await sleep2(500);
|
|
@@ -8480,8 +8777,8 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
8480
8777
|
await raceSpawnError(child);
|
|
8481
8778
|
}
|
|
8482
8779
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
8483
|
-
const workspacePath =
|
|
8484
|
-
const messagesDir =
|
|
8780
|
+
const workspacePath = path27.join(subagentDir, `${path27.basename(subagentDir)}.code-workspace`);
|
|
8781
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
8485
8782
|
await mkdir9(messagesDir, { recursive: true });
|
|
8486
8783
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
8487
8784
|
for (const attachment of attachmentPaths) {
|
|
@@ -8490,13 +8787,13 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
8490
8787
|
chatArgs.push(chatInstruction);
|
|
8491
8788
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
8492
8789
|
workspacePath,
|
|
8493
|
-
|
|
8790
|
+
path27.basename(subagentDir),
|
|
8494
8791
|
subagentDir,
|
|
8495
8792
|
vscodeCmd
|
|
8496
8793
|
);
|
|
8497
8794
|
if (!workspaceReady) {
|
|
8498
8795
|
throw new Error(
|
|
8499
|
-
`VS Code workspace '${
|
|
8796
|
+
`VS Code workspace '${path27.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
8500
8797
|
);
|
|
8501
8798
|
}
|
|
8502
8799
|
await sleep2(500);
|
|
@@ -8505,11 +8802,11 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
8505
8802
|
}
|
|
8506
8803
|
|
|
8507
8804
|
// src/evaluation/providers/vscode/dispatch/workspaceManager.ts
|
|
8508
|
-
import { copyFile, mkdir as mkdir10, readFile as
|
|
8509
|
-
import
|
|
8805
|
+
import { copyFile, mkdir as mkdir10, readFile as readFile10, readdir as readdir3, stat as stat3, writeFile as writeFile3 } from "node:fs/promises";
|
|
8806
|
+
import path29 from "node:path";
|
|
8510
8807
|
|
|
8511
8808
|
// src/evaluation/providers/vscode/utils/workspace.ts
|
|
8512
|
-
import
|
|
8809
|
+
import path28 from "node:path";
|
|
8513
8810
|
import JSON5 from "json5";
|
|
8514
8811
|
function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
8515
8812
|
let workspace;
|
|
@@ -8526,10 +8823,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
8526
8823
|
}
|
|
8527
8824
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
8528
8825
|
const folderPath = folder.path;
|
|
8529
|
-
if (
|
|
8826
|
+
if (path28.isAbsolute(folderPath)) {
|
|
8530
8827
|
return folder;
|
|
8531
8828
|
}
|
|
8532
|
-
const absolutePath =
|
|
8829
|
+
const absolutePath = path28.resolve(templateDir, folderPath);
|
|
8533
8830
|
return {
|
|
8534
8831
|
...folder,
|
|
8535
8832
|
path: absolutePath
|
|
@@ -8551,19 +8848,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
8551
8848
|
if (locationMap && typeof locationMap === "object") {
|
|
8552
8849
|
const transformedMap = {};
|
|
8553
8850
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
8554
|
-
const isAbsolute =
|
|
8851
|
+
const isAbsolute = path28.isAbsolute(locationPath);
|
|
8555
8852
|
if (isAbsolute) {
|
|
8556
8853
|
transformedMap[locationPath] = value;
|
|
8557
8854
|
} else {
|
|
8558
8855
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
8559
8856
|
if (firstGlobIndex === -1) {
|
|
8560
|
-
const resolvedPath =
|
|
8857
|
+
const resolvedPath = path28.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
8561
8858
|
transformedMap[resolvedPath] = value;
|
|
8562
8859
|
} else {
|
|
8563
8860
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
8564
8861
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
8565
8862
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
8566
|
-
const resolvedPath = (
|
|
8863
|
+
const resolvedPath = (path28.resolve(templateDir, basePath) + patternPath).replace(
|
|
8567
8864
|
/\\/g,
|
|
8568
8865
|
"/"
|
|
8569
8866
|
);
|
|
@@ -8604,7 +8901,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
8604
8901
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
8605
8902
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
8606
8903
|
for (const subagent of subagents) {
|
|
8607
|
-
const lockFile =
|
|
8904
|
+
const lockFile = path29.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
8608
8905
|
if (!await pathExists(lockFile)) {
|
|
8609
8906
|
return subagent.absolutePath;
|
|
8610
8907
|
}
|
|
@@ -8614,26 +8911,26 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
8614
8911
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
8615
8912
|
let workspaceContent;
|
|
8616
8913
|
if (workspaceTemplate) {
|
|
8617
|
-
const workspaceSrc =
|
|
8914
|
+
const workspaceSrc = path29.resolve(workspaceTemplate);
|
|
8618
8915
|
if (!await pathExists(workspaceSrc)) {
|
|
8619
8916
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
8620
8917
|
}
|
|
8621
|
-
const stats = await
|
|
8918
|
+
const stats = await stat3(workspaceSrc);
|
|
8622
8919
|
if (!stats.isFile()) {
|
|
8623
8920
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
8624
8921
|
}
|
|
8625
|
-
const templateText = await
|
|
8922
|
+
const templateText = await readFile10(workspaceSrc, "utf8");
|
|
8626
8923
|
workspaceContent = JSON.parse(templateText);
|
|
8627
8924
|
} else {
|
|
8628
8925
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
8629
8926
|
}
|
|
8630
|
-
const workspaceName = `${
|
|
8631
|
-
const workspaceDst =
|
|
8632
|
-
const templateDir = workspaceTemplate ?
|
|
8927
|
+
const workspaceName = `${path29.basename(subagentDir)}.code-workspace`;
|
|
8928
|
+
const workspaceDst = path29.join(subagentDir, workspaceName);
|
|
8929
|
+
const templateDir = workspaceTemplate ? path29.dirname(path29.resolve(workspaceTemplate)) : subagentDir;
|
|
8633
8930
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
8634
8931
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
8635
8932
|
if (cwd) {
|
|
8636
|
-
const absCwd =
|
|
8933
|
+
const absCwd = path29.resolve(cwd);
|
|
8637
8934
|
const parsed = JSON.parse(transformedContent);
|
|
8638
8935
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
8639
8936
|
if (!alreadyPresent) {
|
|
@@ -8642,35 +8939,35 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
8642
8939
|
}
|
|
8643
8940
|
}
|
|
8644
8941
|
await writeFile3(workspaceDst, transformedContent, "utf8");
|
|
8645
|
-
const messagesDir =
|
|
8942
|
+
const messagesDir = path29.join(subagentDir, "messages");
|
|
8646
8943
|
await mkdir10(messagesDir, { recursive: true });
|
|
8647
8944
|
return { workspace: workspaceDst, messagesDir };
|
|
8648
8945
|
}
|
|
8649
8946
|
async function createSubagentLock(subagentDir) {
|
|
8650
|
-
const messagesDir =
|
|
8947
|
+
const messagesDir = path29.join(subagentDir, "messages");
|
|
8651
8948
|
if (await pathExists(messagesDir)) {
|
|
8652
|
-
const files = await
|
|
8949
|
+
const files = await readdir3(messagesDir);
|
|
8653
8950
|
await Promise.all(
|
|
8654
8951
|
files.map(async (file) => {
|
|
8655
|
-
const target =
|
|
8952
|
+
const target = path29.join(messagesDir, file);
|
|
8656
8953
|
await removeIfExists(target);
|
|
8657
8954
|
})
|
|
8658
8955
|
);
|
|
8659
8956
|
}
|
|
8660
|
-
const githubAgentsDir =
|
|
8957
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
8661
8958
|
if (await pathExists(githubAgentsDir)) {
|
|
8662
|
-
const agentFiles = await
|
|
8959
|
+
const agentFiles = await readdir3(githubAgentsDir);
|
|
8663
8960
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
8664
8961
|
await Promise.all(
|
|
8665
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
8962
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path29.join(githubAgentsDir, file)))
|
|
8666
8963
|
);
|
|
8667
8964
|
}
|
|
8668
|
-
const lockFile =
|
|
8965
|
+
const lockFile = path29.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
8669
8966
|
await writeFile3(lockFile, "", { encoding: "utf8" });
|
|
8670
8967
|
return lockFile;
|
|
8671
8968
|
}
|
|
8672
8969
|
async function removeSubagentLock(subagentDir) {
|
|
8673
|
-
const lockFile =
|
|
8970
|
+
const lockFile = path29.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
8674
8971
|
await removeIfExists(lockFile);
|
|
8675
8972
|
}
|
|
8676
8973
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -8690,9 +8987,9 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
8690
8987
|
return 1;
|
|
8691
8988
|
}
|
|
8692
8989
|
if (promptFile) {
|
|
8693
|
-
const githubAgentsDir =
|
|
8990
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
8694
8991
|
await mkdir10(githubAgentsDir, { recursive: true });
|
|
8695
|
-
const agentFile =
|
|
8992
|
+
const agentFile = path29.join(githubAgentsDir, `${chatId}.md`);
|
|
8696
8993
|
try {
|
|
8697
8994
|
await copyFile(promptFile, agentFile);
|
|
8698
8995
|
} catch (error) {
|
|
@@ -8711,11 +9008,11 @@ async function resolvePromptFile(promptFile) {
|
|
|
8711
9008
|
if (!promptFile) {
|
|
8712
9009
|
return void 0;
|
|
8713
9010
|
}
|
|
8714
|
-
const resolvedPrompt =
|
|
9011
|
+
const resolvedPrompt = path30.resolve(promptFile);
|
|
8715
9012
|
if (!await pathExists(resolvedPrompt)) {
|
|
8716
9013
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
8717
9014
|
}
|
|
8718
|
-
const promptStats = await
|
|
9015
|
+
const promptStats = await stat4(resolvedPrompt);
|
|
8719
9016
|
if (!promptStats.isFile()) {
|
|
8720
9017
|
throw new Error(`Prompt file must be a file, not a directory: ${resolvedPrompt}`);
|
|
8721
9018
|
}
|
|
@@ -8727,7 +9024,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
8727
9024
|
}
|
|
8728
9025
|
const resolved = [];
|
|
8729
9026
|
for (const attachment of extraAttachments) {
|
|
8730
|
-
const resolvedPath =
|
|
9027
|
+
const resolvedPath = path30.resolve(attachment);
|
|
8731
9028
|
if (!await pathExists(resolvedPath)) {
|
|
8732
9029
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
8733
9030
|
}
|
|
@@ -8769,7 +9066,7 @@ async function dispatchAgentSession(options) {
|
|
|
8769
9066
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
8770
9067
|
};
|
|
8771
9068
|
}
|
|
8772
|
-
const subagentName =
|
|
9069
|
+
const subagentName = path30.basename(subagentDir);
|
|
8773
9070
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
8774
9071
|
const preparationResult = await prepareSubagentDirectory(
|
|
8775
9072
|
subagentDir,
|
|
@@ -8797,9 +9094,9 @@ async function dispatchAgentSession(options) {
|
|
|
8797
9094
|
};
|
|
8798
9095
|
}
|
|
8799
9096
|
const timestamp = generateTimestamp();
|
|
8800
|
-
const messagesDir =
|
|
8801
|
-
const responseFileTmp =
|
|
8802
|
-
const responseFileFinal =
|
|
9097
|
+
const messagesDir = path30.join(subagentDir, "messages");
|
|
9098
|
+
const responseFileTmp = path30.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
9099
|
+
const responseFileFinal = path30.join(messagesDir, `${timestamp}_res.md`);
|
|
8803
9100
|
const requestInstructions = createRequestPrompt(
|
|
8804
9101
|
userQuery,
|
|
8805
9102
|
responseFileTmp,
|
|
@@ -8904,7 +9201,7 @@ async function dispatchBatchAgent(options) {
|
|
|
8904
9201
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
8905
9202
|
};
|
|
8906
9203
|
}
|
|
8907
|
-
subagentName =
|
|
9204
|
+
subagentName = path30.basename(subagentDir);
|
|
8908
9205
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
8909
9206
|
const preparationResult = await prepareSubagentDirectory(
|
|
8910
9207
|
subagentDir,
|
|
@@ -8935,17 +9232,17 @@ async function dispatchBatchAgent(options) {
|
|
|
8935
9232
|
};
|
|
8936
9233
|
}
|
|
8937
9234
|
const timestamp = generateTimestamp();
|
|
8938
|
-
const messagesDir =
|
|
9235
|
+
const messagesDir = path30.join(subagentDir, "messages");
|
|
8939
9236
|
requestFiles = userQueries.map(
|
|
8940
|
-
(_, index) =>
|
|
9237
|
+
(_, index) => path30.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
8941
9238
|
);
|
|
8942
9239
|
const responseTmpFiles = userQueries.map(
|
|
8943
|
-
(_, index) =>
|
|
9240
|
+
(_, index) => path30.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
8944
9241
|
);
|
|
8945
9242
|
responseFilesFinal = userQueries.map(
|
|
8946
|
-
(_, index) =>
|
|
9243
|
+
(_, index) => path30.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
8947
9244
|
);
|
|
8948
|
-
const orchestratorFile =
|
|
9245
|
+
const orchestratorFile = path30.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
8949
9246
|
if (!dryRun) {
|
|
8950
9247
|
await Promise.all(
|
|
8951
9248
|
userQueries.map((query, index) => {
|
|
@@ -9031,7 +9328,7 @@ async function dispatchBatchAgent(options) {
|
|
|
9031
9328
|
|
|
9032
9329
|
// src/evaluation/providers/vscode/dispatch/provision.ts
|
|
9033
9330
|
import { writeFile as writeFile5 } from "node:fs/promises";
|
|
9034
|
-
import
|
|
9331
|
+
import path31 from "node:path";
|
|
9035
9332
|
var DEFAULT_WORKSPACE_TEMPLATE2 = {
|
|
9036
9333
|
folders: [
|
|
9037
9334
|
{
|
|
@@ -9062,7 +9359,7 @@ async function provisionSubagents(options) {
|
|
|
9062
9359
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
9063
9360
|
throw new Error("subagents must be a positive integer");
|
|
9064
9361
|
}
|
|
9065
|
-
const targetPath =
|
|
9362
|
+
const targetPath = path31.resolve(targetRoot);
|
|
9066
9363
|
if (!dryRun) {
|
|
9067
9364
|
await ensureDir(targetPath);
|
|
9068
9365
|
}
|
|
@@ -9082,7 +9379,7 @@ async function provisionSubagents(options) {
|
|
|
9082
9379
|
continue;
|
|
9083
9380
|
}
|
|
9084
9381
|
highestNumber = Math.max(highestNumber, parsed);
|
|
9085
|
-
const lockFile =
|
|
9382
|
+
const lockFile = path31.join(entry.absolutePath, lockName);
|
|
9086
9383
|
const locked = await pathExists(lockFile);
|
|
9087
9384
|
if (locked) {
|
|
9088
9385
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -9099,10 +9396,10 @@ async function provisionSubagents(options) {
|
|
|
9099
9396
|
break;
|
|
9100
9397
|
}
|
|
9101
9398
|
const subagentDir = subagent.absolutePath;
|
|
9102
|
-
const githubAgentsDir =
|
|
9103
|
-
const lockFile =
|
|
9104
|
-
const workspaceDst =
|
|
9105
|
-
const wakeupDst =
|
|
9399
|
+
const githubAgentsDir = path31.join(subagentDir, ".github", "agents");
|
|
9400
|
+
const lockFile = path31.join(subagentDir, lockName);
|
|
9401
|
+
const workspaceDst = path31.join(subagentDir, `${path31.basename(subagentDir)}.code-workspace`);
|
|
9402
|
+
const wakeupDst = path31.join(githubAgentsDir, "wakeup.md");
|
|
9106
9403
|
const isLocked = await pathExists(lockFile);
|
|
9107
9404
|
if (isLocked && !force) {
|
|
9108
9405
|
continue;
|
|
@@ -9140,10 +9437,10 @@ async function provisionSubagents(options) {
|
|
|
9140
9437
|
let nextIndex = highestNumber;
|
|
9141
9438
|
while (subagentsProvisioned < subagents) {
|
|
9142
9439
|
nextIndex += 1;
|
|
9143
|
-
const subagentDir =
|
|
9144
|
-
const githubAgentsDir =
|
|
9145
|
-
const workspaceDst =
|
|
9146
|
-
const wakeupDst =
|
|
9440
|
+
const subagentDir = path31.join(targetPath, `subagent-${nextIndex}`);
|
|
9441
|
+
const githubAgentsDir = path31.join(subagentDir, ".github", "agents");
|
|
9442
|
+
const workspaceDst = path31.join(subagentDir, `${path31.basename(subagentDir)}.code-workspace`);
|
|
9443
|
+
const wakeupDst = path31.join(githubAgentsDir, "wakeup.md");
|
|
9147
9444
|
if (!dryRun) {
|
|
9148
9445
|
await ensureDir(subagentDir);
|
|
9149
9446
|
await ensureDir(githubAgentsDir);
|
|
@@ -9333,7 +9630,7 @@ var VSCodeProvider = class {
|
|
|
9333
9630
|
async function locateVSCodeExecutable(candidate) {
|
|
9334
9631
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
9335
9632
|
if (includesPathSeparator) {
|
|
9336
|
-
const resolved =
|
|
9633
|
+
const resolved = path32.isAbsolute(candidate) ? candidate : path32.resolve(candidate);
|
|
9337
9634
|
try {
|
|
9338
9635
|
await access3(resolved, constants3.F_OK);
|
|
9339
9636
|
return resolved;
|
|
@@ -9362,7 +9659,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
9362
9659
|
return void 0;
|
|
9363
9660
|
}
|
|
9364
9661
|
try {
|
|
9365
|
-
const stats = await
|
|
9662
|
+
const stats = await stat5(path32.resolve(template));
|
|
9366
9663
|
return stats.isFile() ? template : void 0;
|
|
9367
9664
|
} catch {
|
|
9368
9665
|
return template;
|
|
@@ -9386,7 +9683,7 @@ function buildMandatoryPrereadBlock2(attachmentFiles) {
|
|
|
9386
9683
|
return "";
|
|
9387
9684
|
}
|
|
9388
9685
|
const buildList = (files) => files.map((absolutePath) => {
|
|
9389
|
-
const fileName =
|
|
9686
|
+
const fileName = path32.basename(absolutePath);
|
|
9390
9687
|
const fileUri = pathToFileUri3(absolutePath);
|
|
9391
9688
|
return `* [${fileName}](${fileUri})`;
|
|
9392
9689
|
});
|
|
@@ -9407,7 +9704,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
9407
9704
|
}
|
|
9408
9705
|
const unique = /* @__PURE__ */ new Map();
|
|
9409
9706
|
for (const attachment of attachments) {
|
|
9410
|
-
const absolutePath =
|
|
9707
|
+
const absolutePath = path32.resolve(attachment);
|
|
9411
9708
|
if (!unique.has(absolutePath)) {
|
|
9412
9709
|
unique.set(absolutePath, absolutePath);
|
|
9413
9710
|
}
|
|
@@ -9415,7 +9712,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
9415
9712
|
return Array.from(unique.values());
|
|
9416
9713
|
}
|
|
9417
9714
|
function pathToFileUri3(filePath) {
|
|
9418
|
-
const absolutePath =
|
|
9715
|
+
const absolutePath = path32.isAbsolute(filePath) ? filePath : path32.resolve(filePath);
|
|
9419
9716
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
9420
9717
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
9421
9718
|
return `file:///${normalizedPath}`;
|
|
@@ -9428,7 +9725,7 @@ function normalizeAttachments(attachments) {
|
|
|
9428
9725
|
}
|
|
9429
9726
|
const deduped = /* @__PURE__ */ new Set();
|
|
9430
9727
|
for (const attachment of attachments) {
|
|
9431
|
-
deduped.add(
|
|
9728
|
+
deduped.add(path32.resolve(attachment));
|
|
9432
9729
|
}
|
|
9433
9730
|
return Array.from(deduped);
|
|
9434
9731
|
}
|
|
@@ -9437,7 +9734,7 @@ function mergeAttachments(all) {
|
|
|
9437
9734
|
for (const list of all) {
|
|
9438
9735
|
if (!list) continue;
|
|
9439
9736
|
for (const inputFile of list) {
|
|
9440
|
-
deduped.add(
|
|
9737
|
+
deduped.add(path32.resolve(inputFile));
|
|
9441
9738
|
}
|
|
9442
9739
|
}
|
|
9443
9740
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -9485,8 +9782,8 @@ total unlocked subagents available: ${result.created.length + result.skippedExis
|
|
|
9485
9782
|
|
|
9486
9783
|
// src/evaluation/providers/targets-file.ts
|
|
9487
9784
|
import { constants as constants4 } from "node:fs";
|
|
9488
|
-
import { access as access4, readFile as
|
|
9489
|
-
import
|
|
9785
|
+
import { access as access4, readFile as readFile11 } from "node:fs/promises";
|
|
9786
|
+
import path33 from "node:path";
|
|
9490
9787
|
import { parse as parse4 } from "yaml";
|
|
9491
9788
|
function isRecord(value) {
|
|
9492
9789
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -9523,11 +9820,11 @@ async function fileExists3(filePath) {
|
|
|
9523
9820
|
}
|
|
9524
9821
|
}
|
|
9525
9822
|
async function readTargetDefinitions(filePath) {
|
|
9526
|
-
const absolutePath =
|
|
9823
|
+
const absolutePath = path33.resolve(filePath);
|
|
9527
9824
|
if (!await fileExists3(absolutePath)) {
|
|
9528
9825
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
9529
9826
|
}
|
|
9530
|
-
const raw = await
|
|
9827
|
+
const raw = await readFile11(absolutePath, "utf8");
|
|
9531
9828
|
const parsed = parse4(raw);
|
|
9532
9829
|
if (!isRecord(parsed)) {
|
|
9533
9830
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -9543,16 +9840,16 @@ function listTargetNames(definitions) {
|
|
|
9543
9840
|
}
|
|
9544
9841
|
|
|
9545
9842
|
// src/evaluation/providers/provider-discovery.ts
|
|
9546
|
-
import
|
|
9843
|
+
import path34 from "node:path";
|
|
9547
9844
|
import fg from "fast-glob";
|
|
9548
9845
|
async function discoverProviders(registry, baseDir) {
|
|
9549
9846
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
9550
9847
|
const candidateDirs = [];
|
|
9551
|
-
let dir =
|
|
9552
|
-
const root =
|
|
9848
|
+
let dir = path34.resolve(baseDir);
|
|
9849
|
+
const root = path34.parse(dir).root;
|
|
9553
9850
|
while (dir !== root) {
|
|
9554
|
-
candidateDirs.push(
|
|
9555
|
-
dir =
|
|
9851
|
+
candidateDirs.push(path34.join(dir, ".agentv", "providers"));
|
|
9852
|
+
dir = path34.dirname(dir);
|
|
9556
9853
|
}
|
|
9557
9854
|
let files = [];
|
|
9558
9855
|
for (const providersDir of candidateDirs) {
|
|
@@ -9568,7 +9865,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
9568
9865
|
}
|
|
9569
9866
|
const discoveredKinds = [];
|
|
9570
9867
|
for (const filePath of files) {
|
|
9571
|
-
const basename =
|
|
9868
|
+
const basename = path34.basename(filePath);
|
|
9572
9869
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
9573
9870
|
if (registry.has(kindName)) {
|
|
9574
9871
|
continue;
|
|
@@ -9586,7 +9883,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
9586
9883
|
// src/evaluation/providers/index.ts
|
|
9587
9884
|
function createBuiltinProviderRegistry() {
|
|
9588
9885
|
const registry = new ProviderRegistry();
|
|
9589
|
-
registry.register("openai", (t) => new OpenAIProvider(t.name, t.config)).register("openrouter", (t) => new OpenRouterProvider(t.name, t.config)).register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-cli", (t) => new PiCliProvider(t.name, t.config)).register("claude-cli", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude-sdk", (t) => new ClaudeSdkProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("agentv", (t) => new AgentvProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
|
|
9886
|
+
registry.register("openai", (t) => new OpenAIProvider(t.name, t.config)).register("openrouter", (t) => new OpenRouterProvider(t.name, t.config)).register("azure", (t) => new AzureProvider(t.name, t.config)).register("anthropic", (t) => new AnthropicProvider(t.name, t.config)).register("gemini", (t) => new GeminiProvider(t.name, t.config)).register("cli", (t) => new CliProvider(t.name, t.config)).register("codex", (t) => new CodexProvider(t.name, t.config)).register("copilot-sdk", (t) => new CopilotSdkProvider(t.name, t.config)).register("copilot-cli", (t) => new CopilotCliProvider(t.name, t.config)).register("copilot-log", (t) => new CopilotLogProvider(t.name, t.config)).register("pi-coding-agent", (t) => new PiCodingAgentProvider(t.name, t.config)).register("pi-cli", (t) => new PiCliProvider(t.name, t.config)).register("claude-cli", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude", (t) => new ClaudeCliProvider(t.name, t.config)).register("claude-sdk", (t) => new ClaudeSdkProvider(t.name, t.config)).register("mock", (t) => new MockProvider(t.name, t.config)).register("agentv", (t) => new AgentvProvider(t.name, t.config)).register("vscode", (t) => new VSCodeProvider(t.name, t.config, "vscode")).register(
|
|
9590
9887
|
"vscode-insiders",
|
|
9591
9888
|
(t) => new VSCodeProvider(t.name, t.config, "vscode-insiders")
|
|
9592
9889
|
);
|
|
@@ -9793,15 +10090,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
9793
10090
|
});
|
|
9794
10091
|
}
|
|
9795
10092
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
9796
|
-
const { mkdir: mkdir16, readFile:
|
|
10093
|
+
const { mkdir: mkdir16, readFile: readFile14, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
|
|
9797
10094
|
const { tmpdir: tmpdir3 } = await import("node:os");
|
|
9798
|
-
const
|
|
10095
|
+
const path47 = await import("node:path");
|
|
9799
10096
|
const { randomUUID: randomUUID10 } = await import("node:crypto");
|
|
9800
|
-
const dir =
|
|
10097
|
+
const dir = path47.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
|
|
9801
10098
|
await mkdir16(dir, { recursive: true });
|
|
9802
|
-
const stdinPath =
|
|
9803
|
-
const stdoutPath =
|
|
9804
|
-
const stderrPath =
|
|
10099
|
+
const stdinPath = path47.join(dir, "stdin.txt");
|
|
10100
|
+
const stdoutPath = path47.join(dir, "stdout.txt");
|
|
10101
|
+
const stderrPath = path47.join(dir, "stderr.txt");
|
|
9805
10102
|
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
9806
10103
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
9807
10104
|
const { spawn: spawn5 } = await import("node:child_process");
|
|
@@ -9831,8 +10128,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
9831
10128
|
resolve(code ?? 0);
|
|
9832
10129
|
});
|
|
9833
10130
|
});
|
|
9834
|
-
const stdout = (await
|
|
9835
|
-
const stderr = (await
|
|
10131
|
+
const stdout = (await readFile14(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
10132
|
+
const stderr = (await readFile14(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
9836
10133
|
return { stdout, stderr, exitCode };
|
|
9837
10134
|
} finally {
|
|
9838
10135
|
await rm6(dir, { recursive: true, force: true });
|
|
@@ -10280,7 +10577,7 @@ import { generateText as generateText3 } from "ai";
|
|
|
10280
10577
|
|
|
10281
10578
|
// src/evaluation/evaluators/llm-grader.ts
|
|
10282
10579
|
import fs2 from "node:fs/promises";
|
|
10283
|
-
import
|
|
10580
|
+
import path35 from "node:path";
|
|
10284
10581
|
import { generateText as generateText2, stepCountIs, tool } from "ai";
|
|
10285
10582
|
import { z as z3 } from "zod";
|
|
10286
10583
|
var DEFAULT_MAX_STEPS = 10;
|
|
@@ -11135,8 +11432,8 @@ function calculateScoreRangeResult(result, rubrics) {
|
|
|
11135
11432
|
};
|
|
11136
11433
|
}
|
|
11137
11434
|
function resolveSandboxed(basePath, relativePath) {
|
|
11138
|
-
const resolved =
|
|
11139
|
-
if (!resolved.startsWith(basePath +
|
|
11435
|
+
const resolved = path35.resolve(basePath, relativePath);
|
|
11436
|
+
if (!resolved.startsWith(basePath + path35.sep) && resolved !== basePath) {
|
|
11140
11437
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
11141
11438
|
}
|
|
11142
11439
|
return resolved;
|
|
@@ -11169,11 +11466,11 @@ function createFilesystemTools(workspacePath) {
|
|
|
11169
11466
|
execute: async (input) => {
|
|
11170
11467
|
try {
|
|
11171
11468
|
const resolved = resolveSandboxed(workspacePath, input.path);
|
|
11172
|
-
const
|
|
11173
|
-
if (
|
|
11469
|
+
const stat9 = await fs2.stat(resolved);
|
|
11470
|
+
if (stat9.isDirectory()) {
|
|
11174
11471
|
return { error: `'${input.path}' is a directory, not a file` };
|
|
11175
11472
|
}
|
|
11176
|
-
const buffer = Buffer.alloc(Math.min(
|
|
11473
|
+
const buffer = Buffer.alloc(Math.min(stat9.size, MAX_FILE_SIZE));
|
|
11177
11474
|
const fd = await fs2.open(resolved, "r");
|
|
11178
11475
|
try {
|
|
11179
11476
|
await fd.read(buffer, 0, buffer.length, 0);
|
|
@@ -11181,8 +11478,8 @@ function createFilesystemTools(workspacePath) {
|
|
|
11181
11478
|
await fd.close();
|
|
11182
11479
|
}
|
|
11183
11480
|
const content = buffer.toString("utf-8");
|
|
11184
|
-
const truncated =
|
|
11185
|
-
return { content, truncated, size:
|
|
11481
|
+
const truncated = stat9.size > MAX_FILE_SIZE;
|
|
11482
|
+
return { content, truncated, size: stat9.size };
|
|
11186
11483
|
} catch (error) {
|
|
11187
11484
|
return { error: error instanceof Error ? error.message : String(error) };
|
|
11188
11485
|
}
|
|
@@ -11226,15 +11523,15 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
11226
11523
|
for (const entry of entries) {
|
|
11227
11524
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
11228
11525
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
11229
|
-
const fullPath =
|
|
11526
|
+
const fullPath = path35.join(dirPath, entry.name);
|
|
11230
11527
|
if (entry.isDirectory()) {
|
|
11231
11528
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
11232
11529
|
} else if (entry.isFile()) {
|
|
11233
|
-
const ext =
|
|
11530
|
+
const ext = path35.extname(entry.name).toLowerCase();
|
|
11234
11531
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
11235
11532
|
try {
|
|
11236
|
-
const
|
|
11237
|
-
if (
|
|
11533
|
+
const stat9 = await fs2.stat(fullPath);
|
|
11534
|
+
if (stat9.size > MAX_FILE_SIZE) continue;
|
|
11238
11535
|
const content = await fs2.readFile(fullPath, "utf-8");
|
|
11239
11536
|
const lines = content.split("\n");
|
|
11240
11537
|
for (let i = 0; i < lines.length; i++) {
|
|
@@ -11242,7 +11539,7 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
11242
11539
|
regex.lastIndex = 0;
|
|
11243
11540
|
if (regex.test(lines[i])) {
|
|
11244
11541
|
matches.push({
|
|
11245
|
-
file:
|
|
11542
|
+
file: path35.relative(workspacePath, fullPath),
|
|
11246
11543
|
line: i + 1,
|
|
11247
11544
|
text: lines[i].substring(0, 200)
|
|
11248
11545
|
});
|
|
@@ -11875,115 +12172,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
11875
12172
|
* Evaluate a single field against the expected value.
|
|
11876
12173
|
*/
|
|
11877
12174
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
11878
|
-
const { path:
|
|
11879
|
-
const candidateValue = resolvePath(candidateData,
|
|
11880
|
-
const expectedValue = resolvePath(expectedData,
|
|
12175
|
+
const { path: path47, match, required = true, weight = 1 } = fieldConfig;
|
|
12176
|
+
const candidateValue = resolvePath(candidateData, path47);
|
|
12177
|
+
const expectedValue = resolvePath(expectedData, path47);
|
|
11881
12178
|
if (expectedValue === void 0) {
|
|
11882
12179
|
return {
|
|
11883
|
-
path:
|
|
12180
|
+
path: path47,
|
|
11884
12181
|
score: 1,
|
|
11885
12182
|
// No expected value means no comparison needed
|
|
11886
12183
|
weight,
|
|
11887
12184
|
hit: true,
|
|
11888
|
-
message: `${
|
|
12185
|
+
message: `${path47}: no expected value`
|
|
11889
12186
|
};
|
|
11890
12187
|
}
|
|
11891
12188
|
if (candidateValue === void 0) {
|
|
11892
12189
|
if (required) {
|
|
11893
12190
|
return {
|
|
11894
|
-
path:
|
|
12191
|
+
path: path47,
|
|
11895
12192
|
score: 0,
|
|
11896
12193
|
weight,
|
|
11897
12194
|
hit: false,
|
|
11898
|
-
message: `${
|
|
12195
|
+
message: `${path47} (required, missing)`
|
|
11899
12196
|
};
|
|
11900
12197
|
}
|
|
11901
12198
|
return {
|
|
11902
|
-
path:
|
|
12199
|
+
path: path47,
|
|
11903
12200
|
score: 1,
|
|
11904
12201
|
// Don't penalize missing optional fields
|
|
11905
12202
|
weight: 0,
|
|
11906
12203
|
// Zero weight means it won't affect the score
|
|
11907
12204
|
hit: true,
|
|
11908
|
-
message: `${
|
|
12205
|
+
message: `${path47}: optional field missing`
|
|
11909
12206
|
};
|
|
11910
12207
|
}
|
|
11911
12208
|
switch (match) {
|
|
11912
12209
|
case "exact":
|
|
11913
|
-
return this.compareExact(
|
|
12210
|
+
return this.compareExact(path47, candidateValue, expectedValue, weight);
|
|
11914
12211
|
case "numeric_tolerance":
|
|
11915
12212
|
return this.compareNumericTolerance(
|
|
11916
|
-
|
|
12213
|
+
path47,
|
|
11917
12214
|
candidateValue,
|
|
11918
12215
|
expectedValue,
|
|
11919
12216
|
fieldConfig,
|
|
11920
12217
|
weight
|
|
11921
12218
|
);
|
|
11922
12219
|
case "date":
|
|
11923
|
-
return this.compareDate(
|
|
12220
|
+
return this.compareDate(path47, candidateValue, expectedValue, fieldConfig, weight);
|
|
11924
12221
|
default:
|
|
11925
12222
|
return {
|
|
11926
|
-
path:
|
|
12223
|
+
path: path47,
|
|
11927
12224
|
score: 0,
|
|
11928
12225
|
weight,
|
|
11929
12226
|
hit: false,
|
|
11930
|
-
message: `${
|
|
12227
|
+
message: `${path47}: unknown match type "${match}"`
|
|
11931
12228
|
};
|
|
11932
12229
|
}
|
|
11933
12230
|
}
|
|
11934
12231
|
/**
|
|
11935
12232
|
* Exact equality comparison.
|
|
11936
12233
|
*/
|
|
11937
|
-
compareExact(
|
|
12234
|
+
compareExact(path47, candidateValue, expectedValue, weight) {
|
|
11938
12235
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
11939
12236
|
return {
|
|
11940
|
-
path:
|
|
12237
|
+
path: path47,
|
|
11941
12238
|
score: 1,
|
|
11942
12239
|
weight,
|
|
11943
12240
|
hit: true,
|
|
11944
|
-
message:
|
|
12241
|
+
message: path47
|
|
11945
12242
|
};
|
|
11946
12243
|
}
|
|
11947
12244
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
11948
12245
|
return {
|
|
11949
|
-
path:
|
|
12246
|
+
path: path47,
|
|
11950
12247
|
score: 0,
|
|
11951
12248
|
weight,
|
|
11952
12249
|
hit: false,
|
|
11953
|
-
message: `${
|
|
12250
|
+
message: `${path47} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
11954
12251
|
};
|
|
11955
12252
|
}
|
|
11956
12253
|
return {
|
|
11957
|
-
path:
|
|
12254
|
+
path: path47,
|
|
11958
12255
|
score: 0,
|
|
11959
12256
|
weight,
|
|
11960
12257
|
hit: false,
|
|
11961
|
-
message: `${
|
|
12258
|
+
message: `${path47} (value mismatch)`
|
|
11962
12259
|
};
|
|
11963
12260
|
}
|
|
11964
12261
|
/**
|
|
11965
12262
|
* Numeric comparison with absolute or relative tolerance.
|
|
11966
12263
|
*/
|
|
11967
|
-
compareNumericTolerance(
|
|
12264
|
+
compareNumericTolerance(path47, candidateValue, expectedValue, fieldConfig, weight) {
|
|
11968
12265
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
11969
12266
|
const candidateNum = toNumber(candidateValue);
|
|
11970
12267
|
const expectedNum = toNumber(expectedValue);
|
|
11971
12268
|
if (candidateNum === null || expectedNum === null) {
|
|
11972
12269
|
return {
|
|
11973
|
-
path:
|
|
12270
|
+
path: path47,
|
|
11974
12271
|
score: 0,
|
|
11975
12272
|
weight,
|
|
11976
12273
|
hit: false,
|
|
11977
|
-
message: `${
|
|
12274
|
+
message: `${path47} (non-numeric value)`
|
|
11978
12275
|
};
|
|
11979
12276
|
}
|
|
11980
12277
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
11981
12278
|
return {
|
|
11982
|
-
path:
|
|
12279
|
+
path: path47,
|
|
11983
12280
|
score: 0,
|
|
11984
12281
|
weight,
|
|
11985
12282
|
hit: false,
|
|
11986
|
-
message: `${
|
|
12283
|
+
message: `${path47} (invalid numeric value)`
|
|
11987
12284
|
};
|
|
11988
12285
|
}
|
|
11989
12286
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -11996,61 +12293,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
11996
12293
|
}
|
|
11997
12294
|
if (withinTolerance) {
|
|
11998
12295
|
return {
|
|
11999
|
-
path:
|
|
12296
|
+
path: path47,
|
|
12000
12297
|
score: 1,
|
|
12001
12298
|
weight,
|
|
12002
12299
|
hit: true,
|
|
12003
|
-
message: `${
|
|
12300
|
+
message: `${path47} (within tolerance: diff=${diff.toFixed(2)})`
|
|
12004
12301
|
};
|
|
12005
12302
|
}
|
|
12006
12303
|
return {
|
|
12007
|
-
path:
|
|
12304
|
+
path: path47,
|
|
12008
12305
|
score: 0,
|
|
12009
12306
|
weight,
|
|
12010
12307
|
hit: false,
|
|
12011
|
-
message: `${
|
|
12308
|
+
message: `${path47} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
12012
12309
|
};
|
|
12013
12310
|
}
|
|
12014
12311
|
/**
|
|
12015
12312
|
* Date comparison with format normalization.
|
|
12016
12313
|
*/
|
|
12017
|
-
compareDate(
|
|
12314
|
+
compareDate(path47, candidateValue, expectedValue, fieldConfig, weight) {
|
|
12018
12315
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
12019
12316
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
12020
12317
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
12021
12318
|
if (candidateDate === null) {
|
|
12022
12319
|
return {
|
|
12023
|
-
path:
|
|
12320
|
+
path: path47,
|
|
12024
12321
|
score: 0,
|
|
12025
12322
|
weight,
|
|
12026
12323
|
hit: false,
|
|
12027
|
-
message: `${
|
|
12324
|
+
message: `${path47} (unparseable candidate date)`
|
|
12028
12325
|
};
|
|
12029
12326
|
}
|
|
12030
12327
|
if (expectedDate === null) {
|
|
12031
12328
|
return {
|
|
12032
|
-
path:
|
|
12329
|
+
path: path47,
|
|
12033
12330
|
score: 0,
|
|
12034
12331
|
weight,
|
|
12035
12332
|
hit: false,
|
|
12036
|
-
message: `${
|
|
12333
|
+
message: `${path47} (unparseable expected date)`
|
|
12037
12334
|
};
|
|
12038
12335
|
}
|
|
12039
12336
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
12040
12337
|
return {
|
|
12041
|
-
path:
|
|
12338
|
+
path: path47,
|
|
12042
12339
|
score: 1,
|
|
12043
12340
|
weight,
|
|
12044
12341
|
hit: true,
|
|
12045
|
-
message:
|
|
12342
|
+
message: path47
|
|
12046
12343
|
};
|
|
12047
12344
|
}
|
|
12048
12345
|
return {
|
|
12049
|
-
path:
|
|
12346
|
+
path: path47,
|
|
12050
12347
|
score: 0,
|
|
12051
12348
|
weight,
|
|
12052
12349
|
hit: false,
|
|
12053
|
-
message: `${
|
|
12350
|
+
message: `${path47} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
12054
12351
|
};
|
|
12055
12352
|
}
|
|
12056
12353
|
/**
|
|
@@ -12083,11 +12380,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
12083
12380
|
};
|
|
12084
12381
|
}
|
|
12085
12382
|
};
|
|
12086
|
-
function resolvePath(obj,
|
|
12087
|
-
if (!
|
|
12383
|
+
function resolvePath(obj, path47) {
|
|
12384
|
+
if (!path47 || !obj) {
|
|
12088
12385
|
return void 0;
|
|
12089
12386
|
}
|
|
12090
|
-
const parts =
|
|
12387
|
+
const parts = path47.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
12091
12388
|
let current = obj;
|
|
12092
12389
|
for (const part of parts) {
|
|
12093
12390
|
if (current === null || current === void 0) {
|
|
@@ -12255,6 +12552,7 @@ var PROVIDER_TOOL_SEMANTICS = {
|
|
|
12255
12552
|
"pi-coding-agent": PI_CODING_AGENT_MATCHER,
|
|
12256
12553
|
"pi-cli": PI_CODING_AGENT_MATCHER,
|
|
12257
12554
|
"copilot-cli": COPILOT_MATCHER,
|
|
12555
|
+
"copilot-log": COPILOT_MATCHER,
|
|
12258
12556
|
"copilot-sdk": COPILOT_MATCHER,
|
|
12259
12557
|
vscode: COPILOT_MATCHER,
|
|
12260
12558
|
"vscode-insiders": COPILOT_MATCHER
|
|
@@ -12281,8 +12579,9 @@ var SkillTriggerEvaluator = class {
|
|
|
12281
12579
|
let triggered = false;
|
|
12282
12580
|
let evidence = "";
|
|
12283
12581
|
for (const toolCall of allToolCalls) {
|
|
12582
|
+
const toolName = toolCall.tool ?? "";
|
|
12284
12583
|
const input = toolCall.input ?? {};
|
|
12285
|
-
if (matcher.skillTools.includes(
|
|
12584
|
+
if (matcher.skillTools.includes(toolName)) {
|
|
12286
12585
|
const skillArg = String(input[matcher.skillInputField] ?? "");
|
|
12287
12586
|
if (skillArg.includes(skillName)) {
|
|
12288
12587
|
triggered = true;
|
|
@@ -12290,12 +12589,12 @@ var SkillTriggerEvaluator = class {
|
|
|
12290
12589
|
break;
|
|
12291
12590
|
}
|
|
12292
12591
|
} else if (matcher.skillToolPrefixes?.some(
|
|
12293
|
-
(prefix) =>
|
|
12592
|
+
(prefix) => toolName.startsWith(prefix) && toolName.includes(skillName)
|
|
12294
12593
|
)) {
|
|
12295
12594
|
triggered = true;
|
|
12296
|
-
evidence = `Skill tool invoked via tool name "${
|
|
12595
|
+
evidence = `Skill tool invoked via tool name "${toolName}"`;
|
|
12297
12596
|
break;
|
|
12298
|
-
} else if (matcher.readTools.includes(
|
|
12597
|
+
} else if (matcher.readTools.includes(toolName)) {
|
|
12299
12598
|
const filePath = this.readPathFromInput(input, matcher);
|
|
12300
12599
|
if (filePath.includes(skillName)) {
|
|
12301
12600
|
triggered = true;
|
|
@@ -12303,10 +12602,10 @@ var SkillTriggerEvaluator = class {
|
|
|
12303
12602
|
break;
|
|
12304
12603
|
}
|
|
12305
12604
|
} else if (matcher.readToolPrefixes?.some(
|
|
12306
|
-
(prefix) =>
|
|
12605
|
+
(prefix) => toolName.startsWith(prefix) && toolName.includes(skillName)
|
|
12307
12606
|
)) {
|
|
12308
12607
|
triggered = true;
|
|
12309
|
-
evidence = `Read tool loaded skill file via tool name "${
|
|
12608
|
+
evidence = `Read tool loaded skill file via tool name "${toolName}"`;
|
|
12310
12609
|
break;
|
|
12311
12610
|
}
|
|
12312
12611
|
}
|
|
@@ -12568,8 +12867,8 @@ var TokenUsageEvaluator = class {
|
|
|
12568
12867
|
};
|
|
12569
12868
|
|
|
12570
12869
|
// src/evaluation/evaluators/tool-trajectory.ts
|
|
12571
|
-
function getNestedValue(obj,
|
|
12572
|
-
const parts =
|
|
12870
|
+
function getNestedValue(obj, path47) {
|
|
12871
|
+
const parts = path47.split(".");
|
|
12573
12872
|
let current = obj;
|
|
12574
12873
|
for (const part of parts) {
|
|
12575
12874
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -13190,8 +13489,8 @@ function runEqualsAssertion(output, value) {
|
|
|
13190
13489
|
|
|
13191
13490
|
// src/evaluation/orchestrator.ts
|
|
13192
13491
|
import { createHash as createHash2, randomUUID as randomUUID9 } from "node:crypto";
|
|
13193
|
-
import { copyFile as copyFile2, mkdir as mkdir14, readdir as
|
|
13194
|
-
import
|
|
13492
|
+
import { copyFile as copyFile2, mkdir as mkdir14, readdir as readdir7, stat as stat8 } from "node:fs/promises";
|
|
13493
|
+
import path44 from "node:path";
|
|
13195
13494
|
import micromatch3 from "micromatch";
|
|
13196
13495
|
|
|
13197
13496
|
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
@@ -13405,7 +13704,7 @@ var InlineAssertEvaluator = class {
|
|
|
13405
13704
|
};
|
|
13406
13705
|
|
|
13407
13706
|
// src/evaluation/evaluators/prompt-resolution.ts
|
|
13408
|
-
import
|
|
13707
|
+
import path36 from "node:path";
|
|
13409
13708
|
async function resolveCustomPrompt(promptConfig, context, timeoutMs) {
|
|
13410
13709
|
if (promptConfig.resolvedPromptScript && promptConfig.resolvedPromptScript.length > 0) {
|
|
13411
13710
|
if (!context) {
|
|
@@ -13451,7 +13750,7 @@ async function executePromptTemplate(script, context, config, timeoutMs) {
|
|
|
13451
13750
|
};
|
|
13452
13751
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
13453
13752
|
const scriptPath = script[script.length - 1];
|
|
13454
|
-
const cwd =
|
|
13753
|
+
const cwd = path36.dirname(scriptPath);
|
|
13455
13754
|
try {
|
|
13456
13755
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
13457
13756
|
const prompt = stdout.trim();
|
|
@@ -13723,16 +14022,16 @@ function createBuiltinRegistry() {
|
|
|
13723
14022
|
}
|
|
13724
14023
|
|
|
13725
14024
|
// src/evaluation/registry/assertion-discovery.ts
|
|
13726
|
-
import
|
|
14025
|
+
import path37 from "node:path";
|
|
13727
14026
|
import fg2 from "fast-glob";
|
|
13728
14027
|
async function discoverAssertions(registry, baseDir) {
|
|
13729
14028
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
13730
14029
|
const candidateDirs = [];
|
|
13731
|
-
let dir =
|
|
13732
|
-
const root =
|
|
14030
|
+
let dir = path37.resolve(baseDir);
|
|
14031
|
+
const root = path37.parse(dir).root;
|
|
13733
14032
|
while (dir !== root) {
|
|
13734
|
-
candidateDirs.push(
|
|
13735
|
-
dir =
|
|
14033
|
+
candidateDirs.push(path37.join(dir, ".agentv", "assertions"));
|
|
14034
|
+
dir = path37.dirname(dir);
|
|
13736
14035
|
}
|
|
13737
14036
|
let files = [];
|
|
13738
14037
|
for (const assertionsDir of candidateDirs) {
|
|
@@ -13748,7 +14047,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
13748
14047
|
}
|
|
13749
14048
|
const discoveredTypes = [];
|
|
13750
14049
|
for (const filePath of files) {
|
|
13751
|
-
const basename =
|
|
14050
|
+
const basename = path37.basename(filePath);
|
|
13752
14051
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
13753
14052
|
if (registry.has(typeName)) {
|
|
13754
14053
|
continue;
|
|
@@ -13766,17 +14065,17 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
13766
14065
|
}
|
|
13767
14066
|
|
|
13768
14067
|
// src/evaluation/registry/grader-discovery.ts
|
|
13769
|
-
import
|
|
14068
|
+
import path38 from "node:path";
|
|
13770
14069
|
import fg3 from "fast-glob";
|
|
13771
14070
|
async function discoverGraders(registry, baseDir) {
|
|
13772
14071
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
13773
14072
|
const candidateDirs = [];
|
|
13774
|
-
let dir =
|
|
13775
|
-
const root =
|
|
14073
|
+
let dir = path38.resolve(baseDir);
|
|
14074
|
+
const root = path38.parse(dir).root;
|
|
13776
14075
|
while (dir !== root) {
|
|
13777
|
-
candidateDirs.push(
|
|
13778
|
-
candidateDirs.push(
|
|
13779
|
-
dir =
|
|
14076
|
+
candidateDirs.push(path38.join(dir, ".agentv", "graders"));
|
|
14077
|
+
candidateDirs.push(path38.join(dir, ".agentv", "judges"));
|
|
14078
|
+
dir = path38.dirname(dir);
|
|
13780
14079
|
}
|
|
13781
14080
|
let files = [];
|
|
13782
14081
|
for (const gradersDir of candidateDirs) {
|
|
@@ -13792,7 +14091,7 @@ async function discoverGraders(registry, baseDir) {
|
|
|
13792
14091
|
}
|
|
13793
14092
|
const discoveredTypes = [];
|
|
13794
14093
|
for (const filePath of files) {
|
|
13795
|
-
const basename =
|
|
14094
|
+
const basename = path38.basename(filePath);
|
|
13796
14095
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
13797
14096
|
if (registry.has(typeName)) {
|
|
13798
14097
|
continue;
|
|
@@ -13952,7 +14251,7 @@ function getTCritical(df) {
|
|
|
13952
14251
|
// src/evaluation/workspace/file-changes.ts
|
|
13953
14252
|
import { exec as execCallback } from "node:child_process";
|
|
13954
14253
|
import { readdirSync as readdirSync2, statSync } from "node:fs";
|
|
13955
|
-
import
|
|
14254
|
+
import path39 from "node:path";
|
|
13956
14255
|
import { promisify as promisify4 } from "node:util";
|
|
13957
14256
|
var execAsync4 = promisify4(execCallback);
|
|
13958
14257
|
function gitExecOpts(workspacePath) {
|
|
@@ -13986,10 +14285,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
13986
14285
|
}
|
|
13987
14286
|
for (const entry of entries) {
|
|
13988
14287
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
13989
|
-
const childPath =
|
|
14288
|
+
const childPath = path39.join(workspacePath, entry);
|
|
13990
14289
|
try {
|
|
13991
14290
|
if (!statSync(childPath).isDirectory()) continue;
|
|
13992
|
-
if (!statSync(
|
|
14291
|
+
if (!statSync(path39.join(childPath, ".git")).isDirectory()) continue;
|
|
13993
14292
|
} catch {
|
|
13994
14293
|
continue;
|
|
13995
14294
|
}
|
|
@@ -13999,8 +14298,8 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
13999
14298
|
}
|
|
14000
14299
|
|
|
14001
14300
|
// src/evaluation/workspace/manager.ts
|
|
14002
|
-
import { cp, mkdir as mkdir12, readdir as
|
|
14003
|
-
import
|
|
14301
|
+
import { cp, mkdir as mkdir12, readdir as readdir4, rm as rm4, stat as stat6 } from "node:fs/promises";
|
|
14302
|
+
import path40 from "node:path";
|
|
14004
14303
|
var TemplateNotFoundError = class extends Error {
|
|
14005
14304
|
constructor(templatePath) {
|
|
14006
14305
|
super(`Workspace template not found: ${templatePath}`);
|
|
@@ -14022,7 +14321,7 @@ var WorkspaceCreationError = class extends Error {
|
|
|
14022
14321
|
};
|
|
14023
14322
|
async function isDirectory(filePath) {
|
|
14024
14323
|
try {
|
|
14025
|
-
const stats = await
|
|
14324
|
+
const stats = await stat6(filePath);
|
|
14026
14325
|
return stats.isDirectory();
|
|
14027
14326
|
} catch {
|
|
14028
14327
|
return false;
|
|
@@ -14030,14 +14329,14 @@ async function isDirectory(filePath) {
|
|
|
14030
14329
|
}
|
|
14031
14330
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
14032
14331
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
14033
|
-
return
|
|
14332
|
+
return path40.join(root, evalRunId, caseId);
|
|
14034
14333
|
}
|
|
14035
14334
|
async function copyDirectoryRecursive(src, dest) {
|
|
14036
14335
|
await mkdir12(dest, { recursive: true });
|
|
14037
|
-
const entries = await
|
|
14336
|
+
const entries = await readdir4(src, { withFileTypes: true });
|
|
14038
14337
|
for (const entry of entries) {
|
|
14039
|
-
const srcPath =
|
|
14040
|
-
const destPath =
|
|
14338
|
+
const srcPath = path40.join(src, entry.name);
|
|
14339
|
+
const destPath = path40.join(dest, entry.name);
|
|
14041
14340
|
if (entry.name === ".git") {
|
|
14042
14341
|
continue;
|
|
14043
14342
|
}
|
|
@@ -14049,7 +14348,7 @@ async function copyDirectoryRecursive(src, dest) {
|
|
|
14049
14348
|
}
|
|
14050
14349
|
}
|
|
14051
14350
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
14052
|
-
const resolvedTemplatePath =
|
|
14351
|
+
const resolvedTemplatePath = path40.resolve(templatePath);
|
|
14053
14352
|
if (!await fileExists(resolvedTemplatePath)) {
|
|
14054
14353
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
14055
14354
|
}
|
|
@@ -14098,7 +14397,7 @@ async function cleanupWorkspace(workspacePath) {
|
|
|
14098
14397
|
}
|
|
14099
14398
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
14100
14399
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
14101
|
-
const evalDir =
|
|
14400
|
+
const evalDir = path40.join(root, evalRunId);
|
|
14102
14401
|
if (await fileExists(evalDir)) {
|
|
14103
14402
|
await rm4(evalDir, { recursive: true, force: true });
|
|
14104
14403
|
}
|
|
@@ -14108,8 +14407,8 @@ async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
|
14108
14407
|
import { execFile } from "node:child_process";
|
|
14109
14408
|
import { createHash } from "node:crypto";
|
|
14110
14409
|
import { existsSync as existsSync2 } from "node:fs";
|
|
14111
|
-
import { cp as cp2, mkdir as mkdir13, readFile as
|
|
14112
|
-
import
|
|
14410
|
+
import { cp as cp2, mkdir as mkdir13, readFile as readFile12, readdir as readdir5, rm as rm5, unlink, writeFile as writeFile7 } from "node:fs/promises";
|
|
14411
|
+
import path41 from "node:path";
|
|
14113
14412
|
import { promisify as promisify5 } from "node:util";
|
|
14114
14413
|
var execFileAsync = promisify5(execFile);
|
|
14115
14414
|
function gitEnv() {
|
|
@@ -14161,10 +14460,10 @@ function computeWorkspaceFingerprint(repos) {
|
|
|
14161
14460
|
}
|
|
14162
14461
|
async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
14163
14462
|
await mkdir13(dest, { recursive: true });
|
|
14164
|
-
const entries = await
|
|
14463
|
+
const entries = await readdir5(src, { withFileTypes: true });
|
|
14165
14464
|
for (const entry of entries) {
|
|
14166
|
-
const srcPath =
|
|
14167
|
-
const destPath =
|
|
14465
|
+
const srcPath = path41.join(src, entry.name);
|
|
14466
|
+
const destPath = path41.join(dest, entry.name);
|
|
14168
14467
|
if (entry.name === ".git") {
|
|
14169
14468
|
continue;
|
|
14170
14469
|
}
|
|
@@ -14197,7 +14496,7 @@ var WorkspacePoolManager = class {
|
|
|
14197
14496
|
async acquireWorkspace(options) {
|
|
14198
14497
|
const { templatePath, repos, maxSlots, repoManager, poolReset } = options;
|
|
14199
14498
|
const fingerprint = computeWorkspaceFingerprint(repos);
|
|
14200
|
-
const poolDir =
|
|
14499
|
+
const poolDir = path41.join(this.poolRoot, fingerprint);
|
|
14201
14500
|
await mkdir13(poolDir, { recursive: true });
|
|
14202
14501
|
const drifted = await this.checkDrift(poolDir, fingerprint);
|
|
14203
14502
|
if (drifted) {
|
|
@@ -14207,7 +14506,7 @@ var WorkspacePoolManager = class {
|
|
|
14207
14506
|
await this.removeAllSlots(poolDir);
|
|
14208
14507
|
}
|
|
14209
14508
|
for (let i = 0; i < maxSlots; i++) {
|
|
14210
|
-
const slotPath =
|
|
14509
|
+
const slotPath = path41.join(poolDir, `slot-${i}`);
|
|
14211
14510
|
const lockPath = `${slotPath}.lock`;
|
|
14212
14511
|
const locked = await this.tryLock(lockPath);
|
|
14213
14512
|
if (!locked) {
|
|
@@ -14269,7 +14568,7 @@ var WorkspacePoolManager = class {
|
|
|
14269
14568
|
throw err;
|
|
14270
14569
|
}
|
|
14271
14570
|
try {
|
|
14272
|
-
const pidStr = await
|
|
14571
|
+
const pidStr = await readFile12(lockPath, "utf-8");
|
|
14273
14572
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
14274
14573
|
if (!Number.isNaN(pid)) {
|
|
14275
14574
|
try {
|
|
@@ -14294,9 +14593,9 @@ var WorkspacePoolManager = class {
|
|
|
14294
14593
|
* Returns false (no drift) if metadata.json doesn't exist (first use).
|
|
14295
14594
|
*/
|
|
14296
14595
|
async checkDrift(poolDir, fingerprint) {
|
|
14297
|
-
const metadataPath =
|
|
14596
|
+
const metadataPath = path41.join(poolDir, "metadata.json");
|
|
14298
14597
|
try {
|
|
14299
|
-
const raw = await
|
|
14598
|
+
const raw = await readFile12(metadataPath, "utf-8");
|
|
14300
14599
|
const metadata = JSON.parse(raw);
|
|
14301
14600
|
return metadata.fingerprint !== fingerprint;
|
|
14302
14601
|
} catch {
|
|
@@ -14311,17 +14610,17 @@ var WorkspacePoolManager = class {
|
|
|
14311
14610
|
repos,
|
|
14312
14611
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
14313
14612
|
};
|
|
14314
|
-
await writeFile7(
|
|
14613
|
+
await writeFile7(path41.join(poolDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
14315
14614
|
}
|
|
14316
14615
|
/** Remove all slot directories and their lock files from a pool directory. */
|
|
14317
14616
|
async removeAllSlots(poolDir) {
|
|
14318
|
-
const entries = await
|
|
14617
|
+
const entries = await readdir5(poolDir);
|
|
14319
14618
|
for (const entry of entries) {
|
|
14320
14619
|
if (entry.startsWith("slot-") && !entry.endsWith(".lock")) {
|
|
14321
|
-
const lockPath =
|
|
14620
|
+
const lockPath = path41.join(poolDir, `${entry}.lock`);
|
|
14322
14621
|
if (existsSync2(lockPath)) {
|
|
14323
14622
|
try {
|
|
14324
|
-
const pidStr = await
|
|
14623
|
+
const pidStr = await readFile12(lockPath, "utf-8");
|
|
14325
14624
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
14326
14625
|
if (!Number.isNaN(pid)) {
|
|
14327
14626
|
try {
|
|
@@ -14334,12 +14633,12 @@ var WorkspacePoolManager = class {
|
|
|
14334
14633
|
} catch {
|
|
14335
14634
|
}
|
|
14336
14635
|
}
|
|
14337
|
-
await rm5(
|
|
14636
|
+
await rm5(path41.join(poolDir, entry), { recursive: true, force: true });
|
|
14338
14637
|
await rm5(lockPath, { force: true }).catch(() => {
|
|
14339
14638
|
});
|
|
14340
14639
|
}
|
|
14341
14640
|
}
|
|
14342
|
-
await rm5(
|
|
14641
|
+
await rm5(path41.join(poolDir, "metadata.json"), { force: true }).catch(() => {
|
|
14343
14642
|
});
|
|
14344
14643
|
}
|
|
14345
14644
|
/**
|
|
@@ -14349,7 +14648,7 @@ var WorkspacePoolManager = class {
|
|
|
14349
14648
|
*/
|
|
14350
14649
|
async resetSlot(slotPath, templatePath, repos, poolReset = "fast") {
|
|
14351
14650
|
for (const repo of repos) {
|
|
14352
|
-
const repoDir =
|
|
14651
|
+
const repoDir = path41.join(slotPath, repo.path);
|
|
14353
14652
|
if (!existsSync2(repoDir)) {
|
|
14354
14653
|
continue;
|
|
14355
14654
|
}
|
|
@@ -14376,7 +14675,7 @@ var WorkspacePoolManager = class {
|
|
|
14376
14675
|
// src/evaluation/workspace/repo-manager.ts
|
|
14377
14676
|
import { execFile as execFile2 } from "node:child_process";
|
|
14378
14677
|
import { existsSync as existsSync3 } from "node:fs";
|
|
14379
|
-
import
|
|
14678
|
+
import path42 from "node:path";
|
|
14380
14679
|
import { promisify as promisify6 } from "node:util";
|
|
14381
14680
|
var execFileAsync2 = promisify6(execFile2);
|
|
14382
14681
|
var DEFAULT_TIMEOUT_MS2 = 3e5;
|
|
@@ -14476,7 +14775,7 @@ ${lines.join("\n")}`;
|
|
|
14476
14775
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
14477
14776
|
*/
|
|
14478
14777
|
async materialize(repo, workspacePath) {
|
|
14479
|
-
const targetDir =
|
|
14778
|
+
const targetDir = path42.join(workspacePath, repo.path);
|
|
14480
14779
|
const sourceUrl = getSourceUrl(repo.source);
|
|
14481
14780
|
const startedAt = Date.now();
|
|
14482
14781
|
if (this.verbose) {
|
|
@@ -14567,7 +14866,7 @@ ${lines.join("\n")}`;
|
|
|
14567
14866
|
async reset(repos, workspacePath, reset) {
|
|
14568
14867
|
const cleanFlag = reset === "strict" ? "-fdx" : "-fd";
|
|
14569
14868
|
for (const repo of repos) {
|
|
14570
|
-
const targetDir =
|
|
14869
|
+
const targetDir = path42.join(workspacePath, repo.path);
|
|
14571
14870
|
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
14572
14871
|
await this.runGit(["clean", cleanFlag], { cwd: targetDir });
|
|
14573
14872
|
}
|
|
@@ -14575,36 +14874,36 @@ ${lines.join("\n")}`;
|
|
|
14575
14874
|
};
|
|
14576
14875
|
|
|
14577
14876
|
// src/evaluation/workspace/resolve.ts
|
|
14578
|
-
import { readdir as
|
|
14579
|
-
import
|
|
14877
|
+
import { readdir as readdir6, stat as stat7 } from "node:fs/promises";
|
|
14878
|
+
import path43 from "node:path";
|
|
14580
14879
|
async function resolveWorkspaceTemplate(templatePath) {
|
|
14581
14880
|
if (!templatePath) {
|
|
14582
14881
|
return void 0;
|
|
14583
14882
|
}
|
|
14584
|
-
const resolved =
|
|
14585
|
-
const stats = await
|
|
14883
|
+
const resolved = path43.resolve(templatePath);
|
|
14884
|
+
const stats = await stat7(resolved);
|
|
14586
14885
|
if (stats.isFile()) {
|
|
14587
14886
|
return {
|
|
14588
|
-
dir:
|
|
14887
|
+
dir: path43.dirname(resolved),
|
|
14589
14888
|
workspaceFile: resolved
|
|
14590
14889
|
};
|
|
14591
14890
|
}
|
|
14592
14891
|
if (!stats.isDirectory()) {
|
|
14593
14892
|
throw new Error(`workspace template is neither a file nor a directory: ${resolved}`);
|
|
14594
14893
|
}
|
|
14595
|
-
const entries = await
|
|
14894
|
+
const entries = await readdir6(resolved);
|
|
14596
14895
|
const workspaceFiles = entries.filter((e) => e.endsWith(".code-workspace"));
|
|
14597
14896
|
if (workspaceFiles.length === 1) {
|
|
14598
14897
|
return {
|
|
14599
14898
|
dir: resolved,
|
|
14600
|
-
workspaceFile:
|
|
14899
|
+
workspaceFile: path43.join(resolved, workspaceFiles[0])
|
|
14601
14900
|
};
|
|
14602
14901
|
}
|
|
14603
14902
|
if (workspaceFiles.length > 1) {
|
|
14604
14903
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
14605
14904
|
return {
|
|
14606
14905
|
dir: resolved,
|
|
14607
|
-
workspaceFile: conventionFile ?
|
|
14906
|
+
workspaceFile: conventionFile ? path43.join(resolved, conventionFile) : void 0
|
|
14608
14907
|
};
|
|
14609
14908
|
}
|
|
14610
14909
|
return { dir: resolved };
|
|
@@ -14820,7 +15119,7 @@ async function runEvaluation(options) {
|
|
|
14820
15119
|
];
|
|
14821
15120
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveGraderProvider);
|
|
14822
15121
|
const typeRegistry = createBuiltinRegistry();
|
|
14823
|
-
const discoveryBaseDir = evalFilePath ?
|
|
15122
|
+
const discoveryBaseDir = evalFilePath ? path44.dirname(path44.resolve(evalFilePath)) : process.cwd();
|
|
14824
15123
|
const evalDir = discoveryBaseDir;
|
|
14825
15124
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
14826
15125
|
await discoverGraders(typeRegistry, discoveryBaseDir);
|
|
@@ -14960,11 +15259,11 @@ async function runEvaluation(options) {
|
|
|
14960
15259
|
let staticMaterialised = false;
|
|
14961
15260
|
if (useStaticWorkspace && configuredStaticPath) {
|
|
14962
15261
|
const isYamlConfiguredPath = !cliWorkspacePath && !!yamlWorkspacePath;
|
|
14963
|
-
const dirExists = await
|
|
15262
|
+
const dirExists = await stat8(configuredStaticPath).then(
|
|
14964
15263
|
(s) => s.isDirectory(),
|
|
14965
15264
|
() => false
|
|
14966
15265
|
);
|
|
14967
|
-
const isEmpty = dirExists ? (await
|
|
15266
|
+
const isEmpty = dirExists ? (await readdir7(configuredStaticPath)).length === 0 : false;
|
|
14968
15267
|
if (isYamlConfiguredPath && (!dirExists || isEmpty)) {
|
|
14969
15268
|
if (!dirExists) {
|
|
14970
15269
|
await mkdir14(configuredStaticPath, { recursive: true });
|
|
@@ -15017,9 +15316,9 @@ async function runEvaluation(options) {
|
|
|
15017
15316
|
}
|
|
15018
15317
|
try {
|
|
15019
15318
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
15020
|
-
const copiedWorkspaceFile =
|
|
15319
|
+
const copiedWorkspaceFile = path44.join(sharedWorkspacePath, path44.basename(suiteWorkspaceFile));
|
|
15021
15320
|
try {
|
|
15022
|
-
await
|
|
15321
|
+
await stat8(copiedWorkspaceFile);
|
|
15023
15322
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
15024
15323
|
} catch {
|
|
15025
15324
|
}
|
|
@@ -15599,9 +15898,9 @@ async function runEvalCase(options) {
|
|
|
15599
15898
|
);
|
|
15600
15899
|
}
|
|
15601
15900
|
if (caseWorkspaceFile && workspacePath) {
|
|
15602
|
-
const copiedFile =
|
|
15901
|
+
const copiedFile = path44.join(workspacePath, path44.basename(caseWorkspaceFile));
|
|
15603
15902
|
try {
|
|
15604
|
-
await
|
|
15903
|
+
await stat8(copiedFile);
|
|
15605
15904
|
caseWorkspaceFile = copiedFile;
|
|
15606
15905
|
} catch {
|
|
15607
15906
|
}
|
|
@@ -15661,10 +15960,10 @@ async function runEvalCase(options) {
|
|
|
15661
15960
|
const files = evalCase.metadata.agent_skills_files;
|
|
15662
15961
|
if (baseDir && files.length > 0) {
|
|
15663
15962
|
for (const relPath of files) {
|
|
15664
|
-
const srcPath =
|
|
15665
|
-
const destPath =
|
|
15963
|
+
const srcPath = path44.resolve(baseDir, relPath);
|
|
15964
|
+
const destPath = path44.resolve(workspacePath, relPath);
|
|
15666
15965
|
try {
|
|
15667
|
-
await mkdir14(
|
|
15966
|
+
await mkdir14(path44.dirname(destPath), { recursive: true });
|
|
15668
15967
|
await copyFile2(srcPath, destPath);
|
|
15669
15968
|
} catch (error) {
|
|
15670
15969
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -16310,7 +16609,7 @@ async function runEvaluatorList(options) {
|
|
|
16310
16609
|
fileChanges,
|
|
16311
16610
|
workspacePath
|
|
16312
16611
|
};
|
|
16313
|
-
const evalFileDir = evalCase.file_paths[0] ?
|
|
16612
|
+
const evalFileDir = evalCase.file_paths[0] ? path44.dirname(evalCase.file_paths[0]) : process.cwd();
|
|
16314
16613
|
const dispatchContext = {
|
|
16315
16614
|
graderProvider,
|
|
16316
16615
|
targetResolver,
|
|
@@ -16644,7 +16943,7 @@ function computeWeightedMean(entries) {
|
|
|
16644
16943
|
|
|
16645
16944
|
// src/evaluation/evaluate.ts
|
|
16646
16945
|
import { existsSync as existsSync4 } from "node:fs";
|
|
16647
|
-
import
|
|
16946
|
+
import path45 from "node:path";
|
|
16648
16947
|
|
|
16649
16948
|
// src/evaluation/providers/function-provider.ts
|
|
16650
16949
|
function createFunctionProvider(taskFn) {
|
|
@@ -16681,7 +16980,7 @@ async function evaluate(config) {
|
|
|
16681
16980
|
}
|
|
16682
16981
|
const gitRoot = await findGitRoot(process.cwd());
|
|
16683
16982
|
const repoRoot = gitRoot ?? process.cwd();
|
|
16684
|
-
const testFilePath = config.specFile ?
|
|
16983
|
+
const testFilePath = config.specFile ? path45.resolve(config.specFile) : path45.join(process.cwd(), "__programmatic__.yaml");
|
|
16685
16984
|
await loadEnvHierarchy(repoRoot, testFilePath);
|
|
16686
16985
|
let resolvedTarget;
|
|
16687
16986
|
let taskProvider;
|
|
@@ -16802,10 +17101,10 @@ function computeSummary(results, durationMs) {
|
|
|
16802
17101
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
16803
17102
|
async function discoverDefaultTarget(repoRoot) {
|
|
16804
17103
|
const cwd = process.cwd();
|
|
16805
|
-
const chain = buildDirectoryChain(
|
|
17104
|
+
const chain = buildDirectoryChain(path45.join(cwd, "_placeholder"), repoRoot);
|
|
16806
17105
|
for (const dir of chain) {
|
|
16807
17106
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
16808
|
-
const targetsPath =
|
|
17107
|
+
const targetsPath = path45.join(dir, candidate);
|
|
16809
17108
|
if (!existsSync4(targetsPath)) continue;
|
|
16810
17109
|
try {
|
|
16811
17110
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -16822,7 +17121,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
|
|
|
16822
17121
|
const chain = buildDirectoryChain(startPath, repoRoot);
|
|
16823
17122
|
const envFiles = [];
|
|
16824
17123
|
for (const dir of chain) {
|
|
16825
|
-
const envPath =
|
|
17124
|
+
const envPath = path45.join(dir, ".env");
|
|
16826
17125
|
if (existsSync4(envPath)) envFiles.push(envPath);
|
|
16827
17126
|
}
|
|
16828
17127
|
for (let i = 0; i < envFiles.length; i++) {
|
|
@@ -17001,8 +17300,8 @@ function buildPrompt(criteria, question, referenceAnswer) {
|
|
|
17001
17300
|
}
|
|
17002
17301
|
|
|
17003
17302
|
// src/evaluation/cache/response-cache.ts
|
|
17004
|
-
import { mkdir as mkdir15, readFile as
|
|
17005
|
-
import
|
|
17303
|
+
import { mkdir as mkdir15, readFile as readFile13, writeFile as writeFile8 } from "node:fs/promises";
|
|
17304
|
+
import path46 from "node:path";
|
|
17006
17305
|
var DEFAULT_CACHE_PATH = ".agentv/cache";
|
|
17007
17306
|
var ResponseCache = class {
|
|
17008
17307
|
cachePath;
|
|
@@ -17012,7 +17311,7 @@ var ResponseCache = class {
|
|
|
17012
17311
|
async get(key) {
|
|
17013
17312
|
const filePath = this.keyToPath(key);
|
|
17014
17313
|
try {
|
|
17015
|
-
const data = await
|
|
17314
|
+
const data = await readFile13(filePath, "utf8");
|
|
17016
17315
|
return JSON.parse(data);
|
|
17017
17316
|
} catch {
|
|
17018
17317
|
return void 0;
|
|
@@ -17020,13 +17319,13 @@ var ResponseCache = class {
|
|
|
17020
17319
|
}
|
|
17021
17320
|
async set(key, value) {
|
|
17022
17321
|
const filePath = this.keyToPath(key);
|
|
17023
|
-
const dir =
|
|
17322
|
+
const dir = path46.dirname(filePath);
|
|
17024
17323
|
await mkdir15(dir, { recursive: true });
|
|
17025
17324
|
await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
17026
17325
|
}
|
|
17027
17326
|
keyToPath(key) {
|
|
17028
17327
|
const prefix = key.slice(0, 2);
|
|
17029
|
-
return
|
|
17328
|
+
return path46.join(this.cachePath, prefix, `${key}.json`);
|
|
17030
17329
|
}
|
|
17031
17330
|
};
|
|
17032
17331
|
function shouldEnableCache(params) {
|
|
@@ -17647,6 +17946,7 @@ export {
|
|
|
17647
17946
|
defineConfig,
|
|
17648
17947
|
detectFormat,
|
|
17649
17948
|
discoverAssertions,
|
|
17949
|
+
discoverCopilotSessions,
|
|
17650
17950
|
discoverGraders,
|
|
17651
17951
|
discoverGraders as discoverJudges,
|
|
17652
17952
|
discoverProviders,
|