agentv 3.6.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-UU5N43YS.js → chunk-ASYRKFAI.js} +125 -13
- package/dist/chunk-ASYRKFAI.js.map +1 -0
- package/dist/{chunk-K4RXLQWV.js → chunk-F4UDJ7LG.js} +559 -556
- package/dist/chunk-F4UDJ7LG.js.map +1 -0
- package/dist/{chunk-IP5BO54H.js → chunk-YZRGQ6ZS.js} +15 -12
- package/dist/chunk-YZRGQ6ZS.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{dist-VWEFBDZ5.js → dist-4AQUJJAP.js} +2 -2
- package/dist/index.js +3 -3
- package/dist/{interactive-5S4ILY2Y.js → interactive-OPQGDF77.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-IP5BO54H.js.map +0 -1
- package/dist/chunk-K4RXLQWV.js.map +0 -1
- package/dist/chunk-UU5N43YS.js.map +0 -1
- /package/dist/{dist-VWEFBDZ5.js.map → dist-4AQUJJAP.js.map} +0 -0
- /package/dist/{interactive-5S4ILY2Y.js.map → interactive-OPQGDF77.js.map} +0 -0
|
@@ -301,7 +301,7 @@ var require_dist = __commonJS({
|
|
|
301
301
|
}
|
|
302
302
|
});
|
|
303
303
|
|
|
304
|
-
// ../../packages/core/dist/chunk-
|
|
304
|
+
// ../../packages/core/dist/chunk-3ZS3GCMI.js
|
|
305
305
|
import { constants } from "node:fs";
|
|
306
306
|
import { access, readFile } from "node:fs/promises";
|
|
307
307
|
import path from "node:path";
|
|
@@ -419,7 +419,11 @@ __export(external_exports2, {
|
|
|
419
419
|
void: () => voidType
|
|
420
420
|
});
|
|
421
421
|
|
|
422
|
-
// ../../packages/core/dist/chunk-
|
|
422
|
+
// ../../packages/core/dist/chunk-3ZS3GCMI.js
|
|
423
|
+
import { readFile as readFile2 } from "node:fs/promises";
|
|
424
|
+
import path3 from "node:path";
|
|
425
|
+
import fg from "fast-glob";
|
|
426
|
+
import { parse as parseYaml } from "yaml";
|
|
423
427
|
var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
|
|
424
428
|
var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
|
|
425
429
|
var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
|
|
@@ -1595,8 +1599,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
1595
1599
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
1596
1600
|
if (!parseResult.success) {
|
|
1597
1601
|
const firstError = parseResult.error.errors[0];
|
|
1598
|
-
const
|
|
1599
|
-
const prefix =
|
|
1602
|
+
const path45 = firstError?.path.join(".") || "";
|
|
1603
|
+
const prefix = path45 ? `${target.name} ${path45}: ` : `${target.name}: `;
|
|
1600
1604
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
1601
1605
|
}
|
|
1602
1606
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -1802,6 +1806,135 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
1802
1806
|
}
|
|
1803
1807
|
return resolved.length > 0 ? resolved : void 0;
|
|
1804
1808
|
}
|
|
1809
|
+
var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
1810
|
+
function interpolateEnv(value, env) {
|
|
1811
|
+
if (typeof value === "string") {
|
|
1812
|
+
return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
|
|
1813
|
+
}
|
|
1814
|
+
if (Array.isArray(value)) {
|
|
1815
|
+
return value.map((item) => interpolateEnv(item, env));
|
|
1816
|
+
}
|
|
1817
|
+
if (value !== null && typeof value === "object") {
|
|
1818
|
+
const result = {};
|
|
1819
|
+
for (const [key, val] of Object.entries(value)) {
|
|
1820
|
+
result[key] = interpolateEnv(val, env);
|
|
1821
|
+
}
|
|
1822
|
+
return result;
|
|
1823
|
+
}
|
|
1824
|
+
return value;
|
|
1825
|
+
}
|
|
1826
|
+
var ANSI_YELLOW = "\x1B[33m";
|
|
1827
|
+
var ANSI_RESET = "\x1B[0m";
|
|
1828
|
+
var FILE_PROTOCOL = "file://";
|
|
1829
|
+
function isFileReference(value) {
|
|
1830
|
+
return typeof value === "string" && value.startsWith(FILE_PROTOCOL);
|
|
1831
|
+
}
|
|
1832
|
+
function extractFilePath(ref) {
|
|
1833
|
+
return ref.slice(FILE_PROTOCOL.length);
|
|
1834
|
+
}
|
|
1835
|
+
function isGlobPattern(filePath) {
|
|
1836
|
+
return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
|
|
1837
|
+
}
|
|
1838
|
+
function parseYamlCases(content, filePath) {
|
|
1839
|
+
const raw = parseYaml(content);
|
|
1840
|
+
const parsed = interpolateEnv(raw, process.env);
|
|
1841
|
+
if (!Array.isArray(parsed)) {
|
|
1842
|
+
throw new Error(
|
|
1843
|
+
`External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
const results = [];
|
|
1847
|
+
for (const item of parsed) {
|
|
1848
|
+
if (!isJsonObject(item)) {
|
|
1849
|
+
throw new Error(`External test file contains non-object entry: ${filePath}`);
|
|
1850
|
+
}
|
|
1851
|
+
results.push(item);
|
|
1852
|
+
}
|
|
1853
|
+
return results;
|
|
1854
|
+
}
|
|
1855
|
+
function parseJsonlCases(content, filePath) {
|
|
1856
|
+
const lines = content.split("\n");
|
|
1857
|
+
const results = [];
|
|
1858
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1859
|
+
const line = lines[i].trim();
|
|
1860
|
+
if (line === "") continue;
|
|
1861
|
+
try {
|
|
1862
|
+
const raw = JSON.parse(line);
|
|
1863
|
+
const parsed = interpolateEnv(raw, process.env);
|
|
1864
|
+
if (!isJsonObject(parsed)) {
|
|
1865
|
+
throw new Error("Expected JSON object");
|
|
1866
|
+
}
|
|
1867
|
+
results.push(parsed);
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1870
|
+
throw new Error(`Malformed JSONL at line ${i + 1}: ${message}
|
|
1871
|
+
File: ${filePath}`);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
return results;
|
|
1875
|
+
}
|
|
1876
|
+
async function loadCasesFromFile(filePath) {
|
|
1877
|
+
const ext = path3.extname(filePath).toLowerCase();
|
|
1878
|
+
let content;
|
|
1879
|
+
try {
|
|
1880
|
+
content = await readFile2(filePath, "utf8");
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1883
|
+
throw new Error(`Cannot read external test file: ${filePath}
|
|
1884
|
+
${message}`);
|
|
1885
|
+
}
|
|
1886
|
+
if (content.trim() === "") {
|
|
1887
|
+
console.warn(
|
|
1888
|
+
`${ANSI_YELLOW}Warning: External test file is empty, skipping: ${filePath}${ANSI_RESET}`
|
|
1889
|
+
);
|
|
1890
|
+
return [];
|
|
1891
|
+
}
|
|
1892
|
+
if (ext === ".yaml" || ext === ".yml") {
|
|
1893
|
+
return parseYamlCases(content, filePath);
|
|
1894
|
+
}
|
|
1895
|
+
if (ext === ".jsonl") {
|
|
1896
|
+
return parseJsonlCases(content, filePath);
|
|
1897
|
+
}
|
|
1898
|
+
throw new Error(
|
|
1899
|
+
`Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
async function resolveFileReference2(ref, evalFileDir) {
|
|
1903
|
+
const rawPath = extractFilePath(ref);
|
|
1904
|
+
const absolutePattern = path3.resolve(evalFileDir, rawPath);
|
|
1905
|
+
if (isGlobPattern(rawPath)) {
|
|
1906
|
+
const matches = await fg(absolutePattern.replaceAll("\\", "/"), {
|
|
1907
|
+
onlyFiles: true,
|
|
1908
|
+
absolute: true
|
|
1909
|
+
});
|
|
1910
|
+
if (matches.length === 0) {
|
|
1911
|
+
console.warn(
|
|
1912
|
+
`${ANSI_YELLOW}Warning: Glob pattern matched no files: ${ref} (resolved to ${absolutePattern})${ANSI_RESET}`
|
|
1913
|
+
);
|
|
1914
|
+
return [];
|
|
1915
|
+
}
|
|
1916
|
+
matches.sort();
|
|
1917
|
+
const allCases = [];
|
|
1918
|
+
for (const match of matches) {
|
|
1919
|
+
const cases = await loadCasesFromFile(match);
|
|
1920
|
+
allCases.push(...cases);
|
|
1921
|
+
}
|
|
1922
|
+
return allCases;
|
|
1923
|
+
}
|
|
1924
|
+
return loadCasesFromFile(absolutePattern);
|
|
1925
|
+
}
|
|
1926
|
+
async function expandFileReferences(tests, evalFileDir) {
|
|
1927
|
+
const expanded = [];
|
|
1928
|
+
for (const entry of tests) {
|
|
1929
|
+
if (isFileReference(entry)) {
|
|
1930
|
+
const cases = await resolveFileReference2(entry, evalFileDir);
|
|
1931
|
+
expanded.push(...cases);
|
|
1932
|
+
} else {
|
|
1933
|
+
expanded.push(entry);
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
return expanded;
|
|
1937
|
+
}
|
|
1805
1938
|
var AGENT_PROVIDER_KINDS = [
|
|
1806
1939
|
"codex",
|
|
1807
1940
|
"copilot-sdk",
|
|
@@ -1876,36 +2009,32 @@ function isAgentProvider(provider) {
|
|
|
1876
2009
|
}
|
|
1877
2010
|
|
|
1878
2011
|
// ../../packages/core/dist/index.js
|
|
1879
|
-
import { readFile as
|
|
1880
|
-
import
|
|
2012
|
+
import { readFile as readFile7 } from "node:fs/promises";
|
|
2013
|
+
import path8 from "node:path";
|
|
1881
2014
|
import micromatch3 from "micromatch";
|
|
1882
2015
|
import { parse as parse2 } from "yaml";
|
|
1883
|
-
import { readFile as readFile2 } from "node:fs/promises";
|
|
1884
|
-
import path3 from "node:path";
|
|
1885
|
-
import { readFile as readFile22 } from "node:fs/promises";
|
|
1886
|
-
import path22 from "node:path";
|
|
1887
|
-
import fg from "fast-glob";
|
|
1888
|
-
import { parse as parseYaml } from "yaml";
|
|
1889
2016
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
1890
2017
|
import path4 from "node:path";
|
|
2018
|
+
import { readFile as readFile22 } from "node:fs/promises";
|
|
2019
|
+
import path32 from "node:path";
|
|
1891
2020
|
import micromatch from "micromatch";
|
|
1892
2021
|
import { parse } from "yaml";
|
|
1893
2022
|
import { constants as constants2 } from "node:fs";
|
|
1894
2023
|
import { access as access2 } from "node:fs/promises";
|
|
1895
|
-
import
|
|
2024
|
+
import path22 from "node:path";
|
|
1896
2025
|
import { fileURLToPath } from "node:url";
|
|
1897
|
-
import
|
|
2026
|
+
import path42 from "node:path";
|
|
2027
|
+
import { readFile as readFile32 } from "node:fs/promises";
|
|
2028
|
+
import { readFile as readFile5 } from "node:fs/promises";
|
|
2029
|
+
import path6 from "node:path";
|
|
2030
|
+
import micromatch2 from "micromatch";
|
|
2031
|
+
import { parse as parseYaml2 } from "yaml";
|
|
1898
2032
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
2033
|
+
import path5 from "node:path";
|
|
1899
2034
|
import { readFile as readFile6 } from "node:fs/promises";
|
|
1900
2035
|
import path7 from "node:path";
|
|
1901
|
-
import micromatch2 from "micromatch";
|
|
1902
|
-
import { parse as parseYaml2 } from "yaml";
|
|
1903
|
-
import { readFile as readFile5 } from "node:fs/promises";
|
|
1904
|
-
import path6 from "node:path";
|
|
1905
|
-
import { readFile as readFile7 } from "node:fs/promises";
|
|
1906
|
-
import path8 from "node:path";
|
|
1907
2036
|
import { readFileSync } from "node:fs";
|
|
1908
|
-
import
|
|
2037
|
+
import path9 from "node:path";
|
|
1909
2038
|
import { parse as parse3 } from "yaml";
|
|
1910
2039
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
1911
2040
|
|
|
@@ -6379,7 +6508,7 @@ function createOpenRouter(options = {}) {
|
|
|
6379
6508
|
);
|
|
6380
6509
|
const createChatModel = (modelId, settings = {}) => new OpenRouterChatLanguageModel(modelId, settings, {
|
|
6381
6510
|
provider: "openrouter.chat",
|
|
6382
|
-
url: ({ path:
|
|
6511
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6383
6512
|
headers: getHeaders,
|
|
6384
6513
|
compatibility,
|
|
6385
6514
|
fetch: options.fetch,
|
|
@@ -6387,7 +6516,7 @@ function createOpenRouter(options = {}) {
|
|
|
6387
6516
|
});
|
|
6388
6517
|
const createCompletionModel = (modelId, settings = {}) => new OpenRouterCompletionLanguageModel(modelId, settings, {
|
|
6389
6518
|
provider: "openrouter.completion",
|
|
6390
|
-
url: ({ path:
|
|
6519
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6391
6520
|
headers: getHeaders,
|
|
6392
6521
|
compatibility,
|
|
6393
6522
|
fetch: options.fetch,
|
|
@@ -6395,14 +6524,14 @@ function createOpenRouter(options = {}) {
|
|
|
6395
6524
|
});
|
|
6396
6525
|
const createEmbeddingModel = (modelId, settings = {}) => new OpenRouterEmbeddingModel(modelId, settings, {
|
|
6397
6526
|
provider: "openrouter.embedding",
|
|
6398
|
-
url: ({ path:
|
|
6527
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6399
6528
|
headers: getHeaders,
|
|
6400
6529
|
fetch: options.fetch,
|
|
6401
6530
|
extraBody: options.extraBody
|
|
6402
6531
|
});
|
|
6403
6532
|
const createImageModel = (modelId, settings = {}) => new OpenRouterImageModel(modelId, settings, {
|
|
6404
6533
|
provider: "openrouter.image",
|
|
6405
|
-
url: ({ path:
|
|
6534
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6406
6535
|
headers: getHeaders,
|
|
6407
6536
|
fetch: options.fetch,
|
|
6408
6537
|
extraBody: options.extraBody
|
|
@@ -12398,24 +12527,24 @@ import { spawn } from "node:child_process";
|
|
|
12398
12527
|
import { randomUUID } from "node:crypto";
|
|
12399
12528
|
import { createWriteStream } from "node:fs";
|
|
12400
12529
|
import { mkdir } from "node:fs/promises";
|
|
12401
|
-
import path12 from "node:path";
|
|
12402
12530
|
import path11 from "node:path";
|
|
12531
|
+
import path10 from "node:path";
|
|
12403
12532
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
12404
12533
|
import { createWriteStream as createWriteStream2 } from "node:fs";
|
|
12405
12534
|
import { mkdir as mkdir2 } from "node:fs/promises";
|
|
12406
|
-
import
|
|
12535
|
+
import path12 from "node:path";
|
|
12407
12536
|
import { exec as execWithCallback } from "node:child_process";
|
|
12408
12537
|
import fs from "node:fs/promises";
|
|
12409
12538
|
import os from "node:os";
|
|
12410
|
-
import
|
|
12539
|
+
import path13 from "node:path";
|
|
12411
12540
|
import { promisify } from "node:util";
|
|
12412
12541
|
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
12413
12542
|
import { createWriteStream as createWriteStream3 } from "node:fs";
|
|
12414
12543
|
import { mkdir as mkdir3 } from "node:fs/promises";
|
|
12415
|
-
import
|
|
12544
|
+
import path14 from "node:path";
|
|
12416
12545
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
12417
12546
|
import { mkdir as mkdir4 } from "node:fs/promises";
|
|
12418
|
-
import
|
|
12547
|
+
import path16 from "node:path";
|
|
12419
12548
|
import { Readable, Writable } from "node:stream";
|
|
12420
12549
|
import { spawn as spawn2 } from "node:child_process";
|
|
12421
12550
|
|
|
@@ -13900,48 +14029,48 @@ var RequestError = class _RequestError extends Error {
|
|
|
13900
14029
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
13901
14030
|
import { createWriteStream as createWriteStream4, existsSync, readdirSync } from "node:fs";
|
|
13902
14031
|
import { arch, platform } from "node:os";
|
|
13903
|
-
import
|
|
14032
|
+
import path15 from "node:path";
|
|
13904
14033
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13905
14034
|
import { randomUUID as randomUUID6 } from "node:crypto";
|
|
13906
14035
|
import { mkdir as mkdir5 } from "node:fs/promises";
|
|
13907
|
-
import
|
|
14036
|
+
import path17 from "node:path";
|
|
13908
14037
|
import { spawn as spawn3 } from "node:child_process";
|
|
13909
14038
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
13910
14039
|
import { createWriteStream as createWriteStream5 } from "node:fs";
|
|
13911
14040
|
import { mkdir as mkdir6, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
13912
14041
|
import { tmpdir } from "node:os";
|
|
13913
|
-
import
|
|
14042
|
+
import path18 from "node:path";
|
|
13914
14043
|
import { exec as exec2 } from "node:child_process";
|
|
13915
14044
|
import { constants as constants3, access as access3, stat as stat4 } from "node:fs/promises";
|
|
13916
|
-
import
|
|
14045
|
+
import path30 from "node:path";
|
|
13917
14046
|
import { promisify as promisify3 } from "node:util";
|
|
13918
14047
|
import { stat as stat3, writeFile as writeFile4 } from "node:fs/promises";
|
|
13919
|
-
import
|
|
14048
|
+
import path28 from "node:path";
|
|
13920
14049
|
import { constants as constants22 } from "node:fs";
|
|
13921
14050
|
import { access as access22, mkdir as mkdir7, readdir, rm as rm2, stat } from "node:fs/promises";
|
|
14051
|
+
import path19 from "node:path";
|
|
13922
14052
|
import path20 from "node:path";
|
|
13923
14053
|
import path21 from "node:path";
|
|
14054
|
+
import { readFile as readFile8 } from "node:fs/promises";
|
|
13924
14055
|
import path222 from "node:path";
|
|
13925
|
-
import { readFile as readFile9 } from "node:fs/promises";
|
|
13926
|
-
import path23 from "node:path";
|
|
13927
14056
|
import { exec, spawn as spawn4 } from "node:child_process";
|
|
13928
14057
|
import { mkdir as mkdir8, writeFile as writeFile2 } from "node:fs/promises";
|
|
13929
|
-
import path26 from "node:path";
|
|
13930
|
-
import { promisify as promisify2 } from "node:util";
|
|
13931
14058
|
import path25 from "node:path";
|
|
13932
|
-
import
|
|
14059
|
+
import { promisify as promisify2 } from "node:util";
|
|
13933
14060
|
import path24 from "node:path";
|
|
13934
|
-
import
|
|
13935
|
-
import
|
|
14061
|
+
import os2 from "node:os";
|
|
14062
|
+
import path23 from "node:path";
|
|
14063
|
+
import { copyFile, mkdir as mkdir9, readFile as readFile9, readdir as readdir2, stat as stat2, writeFile as writeFile3 } from "node:fs/promises";
|
|
13936
14064
|
import path27 from "node:path";
|
|
14065
|
+
import path26 from "node:path";
|
|
13937
14066
|
import JSON5 from "json5";
|
|
13938
14067
|
import { writeFile as writeFile5 } from "node:fs/promises";
|
|
13939
|
-
import
|
|
14068
|
+
import path29 from "node:path";
|
|
13940
14069
|
import { constants as constants4 } from "node:fs";
|
|
13941
|
-
import { access as access4, readFile as
|
|
13942
|
-
import
|
|
14070
|
+
import { access as access4, readFile as readFile10 } from "node:fs/promises";
|
|
14071
|
+
import path31 from "node:path";
|
|
13943
14072
|
import { parse as parse4 } from "yaml";
|
|
13944
|
-
import
|
|
14073
|
+
import path322 from "node:path";
|
|
13945
14074
|
import fg2 from "fast-glob";
|
|
13946
14075
|
import { mkdtemp as mkdtemp2, rm as rm3, writeFile as writeFile6 } from "node:fs/promises";
|
|
13947
14076
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
@@ -13949,38 +14078,38 @@ import { dirname, join } from "node:path";
|
|
|
13949
14078
|
import { randomBytes } from "node:crypto";
|
|
13950
14079
|
import { createServer } from "node:http";
|
|
13951
14080
|
import fs2 from "node:fs/promises";
|
|
13952
|
-
import
|
|
14081
|
+
import path33 from "node:path";
|
|
13953
14082
|
import { createHash as createHash2, randomUUID as randomUUID8 } from "node:crypto";
|
|
13954
14083
|
import { copyFile as copyFile2, mkdir as mkdir13, readdir as readdir6, stat as stat7 } from "node:fs/promises";
|
|
13955
|
-
import
|
|
14084
|
+
import path422 from "node:path";
|
|
13956
14085
|
import micromatch4 from "micromatch";
|
|
14086
|
+
import path34 from "node:path";
|
|
13957
14087
|
import path35 from "node:path";
|
|
14088
|
+
import fg22 from "fast-glob";
|
|
13958
14089
|
import path36 from "node:path";
|
|
13959
14090
|
import fg3 from "fast-glob";
|
|
13960
|
-
import path37 from "node:path";
|
|
13961
|
-
import fg4 from "fast-glob";
|
|
13962
14091
|
import { exec as execCallback } from "node:child_process";
|
|
13963
14092
|
import { readdirSync as readdirSync2, statSync } from "node:fs";
|
|
13964
|
-
import
|
|
14093
|
+
import path37 from "node:path";
|
|
13965
14094
|
import { promisify as promisify4 } from "node:util";
|
|
13966
14095
|
import { cp, mkdir as mkdir11, readdir as readdir3, rm as rm4, stat as stat5 } from "node:fs/promises";
|
|
13967
|
-
import
|
|
14096
|
+
import path38 from "node:path";
|
|
13968
14097
|
import { execFile } from "node:child_process";
|
|
13969
14098
|
import { createHash } from "node:crypto";
|
|
13970
14099
|
import { existsSync as existsSync2 } from "node:fs";
|
|
13971
|
-
import { cp as cp2, mkdir as mkdir12, readFile as
|
|
13972
|
-
import
|
|
14100
|
+
import { cp as cp2, mkdir as mkdir12, readFile as readFile11, readdir as readdir4, rm as rm5, unlink, writeFile as writeFile7 } from "node:fs/promises";
|
|
14101
|
+
import path39 from "node:path";
|
|
13973
14102
|
import { promisify as promisify5 } from "node:util";
|
|
13974
14103
|
import { execFile as execFile2 } from "node:child_process";
|
|
13975
14104
|
import { existsSync as existsSync3 } from "node:fs";
|
|
13976
|
-
import
|
|
14105
|
+
import path40 from "node:path";
|
|
13977
14106
|
import { promisify as promisify6 } from "node:util";
|
|
13978
14107
|
import { readdir as readdir5, stat as stat6 } from "node:fs/promises";
|
|
13979
|
-
import
|
|
14108
|
+
import path41 from "node:path";
|
|
13980
14109
|
import { existsSync as existsSync4 } from "node:fs";
|
|
14110
|
+
import path43 from "node:path";
|
|
14111
|
+
import { mkdir as mkdir14, readFile as readFile12, writeFile as writeFile8 } from "node:fs/promises";
|
|
13981
14112
|
import path44 from "node:path";
|
|
13982
|
-
import { mkdir as mkdir14, readFile as readFile13, writeFile as writeFile8 } from "node:fs/promises";
|
|
13983
|
-
import path45 from "node:path";
|
|
13984
14113
|
function computeTraceSummary(messages) {
|
|
13985
14114
|
const toolCallCounts = {};
|
|
13986
14115
|
const toolDurations = {};
|
|
@@ -14036,12 +14165,10 @@ function computeTraceSummary(messages) {
|
|
|
14036
14165
|
}
|
|
14037
14166
|
}
|
|
14038
14167
|
}
|
|
14039
|
-
const toolNames = Object.keys(toolCallCounts).sort();
|
|
14040
14168
|
return {
|
|
14041
14169
|
trace: {
|
|
14042
14170
|
eventCount: totalToolCalls,
|
|
14043
|
-
|
|
14044
|
-
toolCallsByName: toolCallCounts,
|
|
14171
|
+
toolCalls: toolCallCounts,
|
|
14045
14172
|
errorCount: 0,
|
|
14046
14173
|
llmCallCount,
|
|
14047
14174
|
...hasAnyDuration ? { toolDurations } : {}
|
|
@@ -14065,7 +14192,7 @@ var DEFAULT_EXPLORATION_TOOLS = [
|
|
|
14065
14192
|
function explorationRatio(summary, explorationTools = DEFAULT_EXPLORATION_TOOLS) {
|
|
14066
14193
|
if (summary.eventCount === 0) return void 0;
|
|
14067
14194
|
const explorationCalls = explorationTools.reduce(
|
|
14068
|
-
(sum, tool2) => sum + (summary.
|
|
14195
|
+
(sum, tool2) => sum + (summary.toolCalls[tool2] ?? 0),
|
|
14069
14196
|
0
|
|
14070
14197
|
);
|
|
14071
14198
|
return explorationCalls / summary.eventCount;
|
|
@@ -14099,27 +14226,10 @@ function mergeExecutionMetrics(computed, metrics) {
|
|
|
14099
14226
|
endTime: metrics.endTime ?? computed.endTime
|
|
14100
14227
|
};
|
|
14101
14228
|
}
|
|
14102
|
-
var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
14103
|
-
function interpolateEnv(value, env) {
|
|
14104
|
-
if (typeof value === "string") {
|
|
14105
|
-
return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
|
|
14106
|
-
}
|
|
14107
|
-
if (Array.isArray(value)) {
|
|
14108
|
-
return value.map((item) => interpolateEnv(item, env));
|
|
14109
|
-
}
|
|
14110
|
-
if (value !== null && typeof value === "object") {
|
|
14111
|
-
const result = {};
|
|
14112
|
-
for (const [key, val] of Object.entries(value)) {
|
|
14113
|
-
result[key] = interpolateEnv(val, env);
|
|
14114
|
-
}
|
|
14115
|
-
return result;
|
|
14116
|
-
}
|
|
14117
|
-
return value;
|
|
14118
|
-
}
|
|
14119
14229
|
var ANSI_RED = "\x1B[31m";
|
|
14120
|
-
var
|
|
14230
|
+
var ANSI_RESET2 = "\x1B[0m";
|
|
14121
14231
|
function logError(msg) {
|
|
14122
|
-
console.error(`${ANSI_RED}Error: ${msg}${
|
|
14232
|
+
console.error(`${ANSI_RED}Error: ${msg}${ANSI_RESET2}`);
|
|
14123
14233
|
}
|
|
14124
14234
|
function isAgentSkillsFormat(parsed) {
|
|
14125
14235
|
if (typeof parsed !== "object" || parsed === null) return false;
|
|
@@ -14127,14 +14237,14 @@ function isAgentSkillsFormat(parsed) {
|
|
|
14127
14237
|
return Array.isArray(obj.evals);
|
|
14128
14238
|
}
|
|
14129
14239
|
async function loadTestsFromAgentSkills(filePath) {
|
|
14130
|
-
const raw = await
|
|
14240
|
+
const raw = await readFile3(filePath, "utf8");
|
|
14131
14241
|
let parsed;
|
|
14132
14242
|
try {
|
|
14133
14243
|
parsed = JSON.parse(raw);
|
|
14134
14244
|
} catch {
|
|
14135
14245
|
throw new Error(`Invalid Agent Skills evals.json: failed to parse JSON in '${filePath}'`);
|
|
14136
14246
|
}
|
|
14137
|
-
return parseAgentSkillsEvals(parsed, filePath,
|
|
14247
|
+
return parseAgentSkillsEvals(parsed, filePath, path4.dirname(path4.resolve(filePath)));
|
|
14138
14248
|
}
|
|
14139
14249
|
function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
14140
14250
|
if (!isAgentSkillsFormat(parsed)) {
|
|
@@ -14172,7 +14282,7 @@ function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
|
14172
14282
|
if (baseDir) {
|
|
14173
14283
|
metadata.agent_skills_base_dir = baseDir;
|
|
14174
14284
|
for (const file of evalCase.files) {
|
|
14175
|
-
filePaths.push(
|
|
14285
|
+
filePaths.push(path4.resolve(baseDir, file));
|
|
14176
14286
|
}
|
|
14177
14287
|
}
|
|
14178
14288
|
}
|
|
@@ -14194,118 +14304,6 @@ function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
|
14194
14304
|
}
|
|
14195
14305
|
return tests;
|
|
14196
14306
|
}
|
|
14197
|
-
var ANSI_YELLOW = "\x1B[33m";
|
|
14198
|
-
var ANSI_RESET2 = "\x1B[0m";
|
|
14199
|
-
var FILE_PROTOCOL = "file://";
|
|
14200
|
-
function isFileReference(value) {
|
|
14201
|
-
return typeof value === "string" && value.startsWith(FILE_PROTOCOL);
|
|
14202
|
-
}
|
|
14203
|
-
function extractFilePath(ref) {
|
|
14204
|
-
return ref.slice(FILE_PROTOCOL.length);
|
|
14205
|
-
}
|
|
14206
|
-
function isGlobPattern(filePath) {
|
|
14207
|
-
return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
|
|
14208
|
-
}
|
|
14209
|
-
function parseYamlCases(content, filePath) {
|
|
14210
|
-
const raw = parseYaml(content);
|
|
14211
|
-
const parsed = interpolateEnv(raw, process.env);
|
|
14212
|
-
if (!Array.isArray(parsed)) {
|
|
14213
|
-
throw new Error(
|
|
14214
|
-
`External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
|
|
14215
|
-
);
|
|
14216
|
-
}
|
|
14217
|
-
const results = [];
|
|
14218
|
-
for (const item of parsed) {
|
|
14219
|
-
if (!isJsonObject(item)) {
|
|
14220
|
-
throw new Error(`External test file contains non-object entry: ${filePath}`);
|
|
14221
|
-
}
|
|
14222
|
-
results.push(item);
|
|
14223
|
-
}
|
|
14224
|
-
return results;
|
|
14225
|
-
}
|
|
14226
|
-
function parseJsonlCases(content, filePath) {
|
|
14227
|
-
const lines = content.split("\n");
|
|
14228
|
-
const results = [];
|
|
14229
|
-
for (let i = 0; i < lines.length; i++) {
|
|
14230
|
-
const line = lines[i].trim();
|
|
14231
|
-
if (line === "") continue;
|
|
14232
|
-
try {
|
|
14233
|
-
const raw = JSON.parse(line);
|
|
14234
|
-
const parsed = interpolateEnv(raw, process.env);
|
|
14235
|
-
if (!isJsonObject(parsed)) {
|
|
14236
|
-
throw new Error("Expected JSON object");
|
|
14237
|
-
}
|
|
14238
|
-
results.push(parsed);
|
|
14239
|
-
} catch (error) {
|
|
14240
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
14241
|
-
throw new Error(`Malformed JSONL at line ${i + 1}: ${message}
|
|
14242
|
-
File: ${filePath}`);
|
|
14243
|
-
}
|
|
14244
|
-
}
|
|
14245
|
-
return results;
|
|
14246
|
-
}
|
|
14247
|
-
async function loadCasesFromFile(filePath) {
|
|
14248
|
-
const ext = path22.extname(filePath).toLowerCase();
|
|
14249
|
-
let content;
|
|
14250
|
-
try {
|
|
14251
|
-
content = await readFile22(filePath, "utf8");
|
|
14252
|
-
} catch (error) {
|
|
14253
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
14254
|
-
throw new Error(`Cannot read external test file: ${filePath}
|
|
14255
|
-
${message}`);
|
|
14256
|
-
}
|
|
14257
|
-
if (content.trim() === "") {
|
|
14258
|
-
console.warn(
|
|
14259
|
-
`${ANSI_YELLOW}Warning: External test file is empty, skipping: ${filePath}${ANSI_RESET2}`
|
|
14260
|
-
);
|
|
14261
|
-
return [];
|
|
14262
|
-
}
|
|
14263
|
-
if (ext === ".yaml" || ext === ".yml") {
|
|
14264
|
-
return parseYamlCases(content, filePath);
|
|
14265
|
-
}
|
|
14266
|
-
if (ext === ".jsonl") {
|
|
14267
|
-
return parseJsonlCases(content, filePath);
|
|
14268
|
-
}
|
|
14269
|
-
throw new Error(
|
|
14270
|
-
`Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
|
|
14271
|
-
);
|
|
14272
|
-
}
|
|
14273
|
-
async function resolveFileReference2(ref, evalFileDir) {
|
|
14274
|
-
const rawPath = extractFilePath(ref);
|
|
14275
|
-
const absolutePattern = path22.resolve(evalFileDir, rawPath);
|
|
14276
|
-
if (isGlobPattern(rawPath)) {
|
|
14277
|
-
const matches = await fg(absolutePattern.replaceAll("\\", "/"), {
|
|
14278
|
-
onlyFiles: true,
|
|
14279
|
-
absolute: true
|
|
14280
|
-
});
|
|
14281
|
-
if (matches.length === 0) {
|
|
14282
|
-
console.warn(
|
|
14283
|
-
`${ANSI_YELLOW}Warning: Glob pattern matched no files: ${ref} (resolved to ${absolutePattern})${ANSI_RESET2}`
|
|
14284
|
-
);
|
|
14285
|
-
return [];
|
|
14286
|
-
}
|
|
14287
|
-
matches.sort();
|
|
14288
|
-
const allCases = [];
|
|
14289
|
-
for (const match of matches) {
|
|
14290
|
-
const cases = await loadCasesFromFile(match);
|
|
14291
|
-
allCases.push(...cases);
|
|
14292
|
-
}
|
|
14293
|
-
return allCases;
|
|
14294
|
-
}
|
|
14295
|
-
return loadCasesFromFile(absolutePattern);
|
|
14296
|
-
}
|
|
14297
|
-
async function expandFileReferences(tests, evalFileDir) {
|
|
14298
|
-
const expanded = [];
|
|
14299
|
-
for (const entry of tests) {
|
|
14300
|
-
if (isFileReference(entry)) {
|
|
14301
|
-
const cases = await resolveFileReference2(entry, evalFileDir);
|
|
14302
|
-
expanded.push(...cases);
|
|
14303
|
-
} else {
|
|
14304
|
-
expanded.push(entry);
|
|
14305
|
-
}
|
|
14306
|
-
}
|
|
14307
|
-
return expanded;
|
|
14308
|
-
}
|
|
14309
14307
|
async function fileExists2(absolutePath) {
|
|
14310
14308
|
try {
|
|
14311
14309
|
await access2(absolutePath, constants2.F_OK);
|
|
@@ -14322,15 +14320,15 @@ function resolveToAbsolutePath(candidate) {
|
|
|
14322
14320
|
if (candidate.startsWith("file:")) {
|
|
14323
14321
|
return fileURLToPath(candidate);
|
|
14324
14322
|
}
|
|
14325
|
-
return
|
|
14323
|
+
return path22.resolve(candidate);
|
|
14326
14324
|
}
|
|
14327
14325
|
throw new TypeError("Unsupported repoRoot value. Expected string or URL.");
|
|
14328
14326
|
}
|
|
14329
14327
|
function buildDirectoryChain2(filePath, repoRoot) {
|
|
14330
14328
|
const directories = [];
|
|
14331
14329
|
const seen = /* @__PURE__ */ new Set();
|
|
14332
|
-
const boundary =
|
|
14333
|
-
let current =
|
|
14330
|
+
const boundary = path22.resolve(repoRoot);
|
|
14331
|
+
let current = path22.resolve(path22.dirname(filePath));
|
|
14334
14332
|
while (current !== void 0) {
|
|
14335
14333
|
if (!seen.has(current)) {
|
|
14336
14334
|
directories.push(current);
|
|
@@ -14339,7 +14337,7 @@ function buildDirectoryChain2(filePath, repoRoot) {
|
|
|
14339
14337
|
if (current === boundary) {
|
|
14340
14338
|
break;
|
|
14341
14339
|
}
|
|
14342
|
-
const parent =
|
|
14340
|
+
const parent = path22.dirname(current);
|
|
14343
14341
|
if (parent === current) {
|
|
14344
14342
|
break;
|
|
14345
14343
|
}
|
|
@@ -14353,16 +14351,16 @@ function buildDirectoryChain2(filePath, repoRoot) {
|
|
|
14353
14351
|
function buildSearchRoots2(evalPath, repoRoot) {
|
|
14354
14352
|
const uniqueRoots = [];
|
|
14355
14353
|
const addRoot = (root) => {
|
|
14356
|
-
const normalized =
|
|
14354
|
+
const normalized = path22.resolve(root);
|
|
14357
14355
|
if (!uniqueRoots.includes(normalized)) {
|
|
14358
14356
|
uniqueRoots.push(normalized);
|
|
14359
14357
|
}
|
|
14360
14358
|
};
|
|
14361
|
-
let currentDir =
|
|
14359
|
+
let currentDir = path22.dirname(evalPath);
|
|
14362
14360
|
let reachedBoundary = false;
|
|
14363
14361
|
while (!reachedBoundary) {
|
|
14364
14362
|
addRoot(currentDir);
|
|
14365
|
-
const parentDir =
|
|
14363
|
+
const parentDir = path22.dirname(currentDir);
|
|
14366
14364
|
if (currentDir === repoRoot || parentDir === currentDir) {
|
|
14367
14365
|
reachedBoundary = true;
|
|
14368
14366
|
} else {
|
|
@@ -14377,19 +14375,19 @@ function trimLeadingSeparators2(value) {
|
|
|
14377
14375
|
const trimmed = value.replace(/^[/\\]+/, "");
|
|
14378
14376
|
return trimmed.length > 0 ? trimmed : value;
|
|
14379
14377
|
}
|
|
14380
|
-
async function
|
|
14378
|
+
async function resolveFileReference22(rawValue, searchRoots) {
|
|
14381
14379
|
const displayPath = trimLeadingSeparators2(rawValue);
|
|
14382
14380
|
const potentialPaths = [];
|
|
14383
|
-
if (
|
|
14384
|
-
potentialPaths.push(
|
|
14381
|
+
if (path22.isAbsolute(rawValue)) {
|
|
14382
|
+
potentialPaths.push(path22.normalize(rawValue));
|
|
14385
14383
|
}
|
|
14386
14384
|
for (const base of searchRoots) {
|
|
14387
|
-
potentialPaths.push(
|
|
14385
|
+
potentialPaths.push(path22.resolve(base, displayPath));
|
|
14388
14386
|
}
|
|
14389
14387
|
const attempted = [];
|
|
14390
14388
|
const seen = /* @__PURE__ */ new Set();
|
|
14391
14389
|
for (const candidate of potentialPaths) {
|
|
14392
|
-
const absoluteCandidate =
|
|
14390
|
+
const absoluteCandidate = path22.resolve(candidate);
|
|
14393
14391
|
if (seen.has(absoluteCandidate)) {
|
|
14394
14392
|
continue;
|
|
14395
14393
|
}
|
|
@@ -14402,7 +14400,7 @@ async function resolveFileReference3(rawValue, searchRoots) {
|
|
|
14402
14400
|
return { displayPath, attempted };
|
|
14403
14401
|
}
|
|
14404
14402
|
var ANSI_YELLOW2 = "\x1B[33m";
|
|
14405
|
-
var
|
|
14403
|
+
var ANSI_RESET22 = "\x1B[0m";
|
|
14406
14404
|
var DEFAULT_EVAL_PATTERNS = [
|
|
14407
14405
|
"**/evals/**/*.eval.yaml",
|
|
14408
14406
|
"**/evals/**/eval.yaml"
|
|
@@ -14410,12 +14408,12 @@ var DEFAULT_EVAL_PATTERNS = [
|
|
|
14410
14408
|
async function loadConfig(evalFilePath, repoRoot) {
|
|
14411
14409
|
const directories = buildDirectoryChain2(evalFilePath, repoRoot);
|
|
14412
14410
|
for (const directory of directories) {
|
|
14413
|
-
const configPath =
|
|
14411
|
+
const configPath = path32.join(directory, ".agentv", "config.yaml");
|
|
14414
14412
|
if (!await fileExists2(configPath)) {
|
|
14415
14413
|
continue;
|
|
14416
14414
|
}
|
|
14417
14415
|
try {
|
|
14418
|
-
const rawConfig = await
|
|
14416
|
+
const rawConfig = await readFile22(configPath, "utf8");
|
|
14419
14417
|
const parsed = parse(rawConfig);
|
|
14420
14418
|
if (!isJsonObject(parsed)) {
|
|
14421
14419
|
logWarning(`Invalid .agentv/config.yaml format at ${configPath}`);
|
|
@@ -14652,7 +14650,7 @@ function parseExecutionDefaults(raw, configPath) {
|
|
|
14652
14650
|
return Object.keys(result).length > 0 ? result : void 0;
|
|
14653
14651
|
}
|
|
14654
14652
|
function logWarning(message) {
|
|
14655
|
-
console.warn(`${ANSI_YELLOW2}Warning: ${message}${
|
|
14653
|
+
console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET22}`);
|
|
14656
14654
|
}
|
|
14657
14655
|
var TEMPLATE_VARIABLES = {
|
|
14658
14656
|
EXPECTED_OUTPUT: "expected_output",
|
|
@@ -14669,10 +14667,10 @@ var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
|
|
|
14669
14667
|
TEMPLATE_VARIABLES.OUTPUT_TEXT,
|
|
14670
14668
|
TEMPLATE_VARIABLES.EXPECTED_OUTPUT
|
|
14671
14669
|
]);
|
|
14672
|
-
var
|
|
14673
|
-
var
|
|
14670
|
+
var ANSI_YELLOW22 = "\x1B[33m";
|
|
14671
|
+
var ANSI_RESET3 = "\x1B[0m";
|
|
14674
14672
|
async function validateCustomPromptContent(promptPath) {
|
|
14675
|
-
const content = await
|
|
14673
|
+
const content = await readFile32(promptPath, "utf8");
|
|
14676
14674
|
validateTemplateVariables(content, promptPath);
|
|
14677
14675
|
}
|
|
14678
14676
|
function validateTemplateVariables(content, source) {
|
|
@@ -14699,14 +14697,14 @@ function validateTemplateVariables(content, source) {
|
|
|
14699
14697
|
);
|
|
14700
14698
|
}
|
|
14701
14699
|
if (invalidVariables.length > 0) {
|
|
14702
|
-
const warningMessage = `${
|
|
14700
|
+
const warningMessage = `${ANSI_YELLOW22}Warning: Custom evaluator template at ${source}
|
|
14703
14701
|
Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(", ")}
|
|
14704
|
-
Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES).map((v) => `{{ ${v} }}`).join(", ")}${
|
|
14702
|
+
Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES).map((v) => `{{ ${v} }}`).join(", ")}${ANSI_RESET3}`;
|
|
14705
14703
|
console.warn(warningMessage);
|
|
14706
14704
|
}
|
|
14707
14705
|
}
|
|
14708
|
-
var
|
|
14709
|
-
var
|
|
14706
|
+
var ANSI_YELLOW3 = "\x1B[33m";
|
|
14707
|
+
var ANSI_RESET4 = "\x1B[0m";
|
|
14710
14708
|
function normalizeEvaluatorType(type) {
|
|
14711
14709
|
return type.replace(/_/g, "-");
|
|
14712
14710
|
}
|
|
@@ -14808,7 +14806,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14808
14806
|
let command;
|
|
14809
14807
|
if (rawEvaluator.script !== void 0 && rawEvaluator.command === void 0) {
|
|
14810
14808
|
console.warn(
|
|
14811
|
-
`${
|
|
14809
|
+
`${ANSI_YELLOW3}Warning: 'script' is deprecated in evaluator '${name21}' in '${evalId}'. Use 'command' instead.${ANSI_RESET4}`
|
|
14812
14810
|
);
|
|
14813
14811
|
}
|
|
14814
14812
|
const rawCommand = rawEvaluator.command ?? rawEvaluator.script;
|
|
@@ -14834,9 +14832,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14834
14832
|
const cwd = asString(rawEvaluator.cwd);
|
|
14835
14833
|
let resolvedCwd;
|
|
14836
14834
|
if (cwd) {
|
|
14837
|
-
const resolved = await
|
|
14835
|
+
const resolved = await resolveFileReference22(cwd, searchRoots);
|
|
14838
14836
|
if (resolved.resolvedPath) {
|
|
14839
|
-
resolvedCwd =
|
|
14837
|
+
resolvedCwd = path42.resolve(resolved.resolvedPath);
|
|
14840
14838
|
} else {
|
|
14841
14839
|
logWarning2(
|
|
14842
14840
|
`Code-grader evaluator '${name21}' in '${evalId}': cwd not found (${resolved.displayPath})`,
|
|
@@ -14992,9 +14990,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14992
14990
|
const aggregatorPrompt = asString(rawAggregator.prompt);
|
|
14993
14991
|
let promptPath2;
|
|
14994
14992
|
if (aggregatorPrompt) {
|
|
14995
|
-
const resolved = await
|
|
14993
|
+
const resolved = await resolveFileReference22(aggregatorPrompt, searchRoots);
|
|
14996
14994
|
if (resolved.resolvedPath) {
|
|
14997
|
-
promptPath2 =
|
|
14995
|
+
promptPath2 = path42.resolve(resolved.resolvedPath);
|
|
14998
14996
|
}
|
|
14999
14997
|
}
|
|
15000
14998
|
aggregator = {
|
|
@@ -15551,7 +15549,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15551
15549
|
if (isJsonObject2(rawPrompt)) {
|
|
15552
15550
|
if (rawPrompt.script !== void 0 && rawPrompt.command === void 0) {
|
|
15553
15551
|
console.warn(
|
|
15554
|
-
`${
|
|
15552
|
+
`${ANSI_YELLOW3}Warning: 'prompt.script' is deprecated in evaluator '${name21}' in '${evalId}'. Use 'prompt.command' instead.${ANSI_RESET4}`
|
|
15555
15553
|
);
|
|
15556
15554
|
}
|
|
15557
15555
|
const commandArray = asStringArray(
|
|
@@ -15562,9 +15560,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15562
15560
|
throw new Error(`Evaluator '${name21}' in '${evalId}': prompt object requires command array`);
|
|
15563
15561
|
}
|
|
15564
15562
|
const commandPath = commandArray[commandArray.length - 1];
|
|
15565
|
-
const resolved = await
|
|
15563
|
+
const resolved = await resolveFileReference22(commandPath, searchRoots);
|
|
15566
15564
|
if (resolved.resolvedPath) {
|
|
15567
|
-
resolvedPromptScript = [...commandArray.slice(0, -1),
|
|
15565
|
+
resolvedPromptScript = [...commandArray.slice(0, -1), path42.resolve(resolved.resolvedPath)];
|
|
15568
15566
|
} else {
|
|
15569
15567
|
throw new Error(
|
|
15570
15568
|
`Evaluator '${name21}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
|
|
@@ -15575,9 +15573,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15575
15573
|
}
|
|
15576
15574
|
} else if (typeof rawPrompt === "string") {
|
|
15577
15575
|
prompt = rawPrompt;
|
|
15578
|
-
const resolved = await
|
|
15576
|
+
const resolved = await resolveFileReference22(prompt, searchRoots);
|
|
15579
15577
|
if (resolved.resolvedPath) {
|
|
15580
|
-
promptPath =
|
|
15578
|
+
promptPath = path42.resolve(resolved.resolvedPath);
|
|
15581
15579
|
try {
|
|
15582
15580
|
await validateCustomPromptContent(promptPath);
|
|
15583
15581
|
} catch (error) {
|
|
@@ -15777,10 +15775,10 @@ function warnUnconsumedCriteria(_criteria, _evaluators, _testId) {
|
|
|
15777
15775
|
function logWarning2(message, details) {
|
|
15778
15776
|
if (details && details.length > 0) {
|
|
15779
15777
|
const detailBlock = details.join("\n");
|
|
15780
|
-
console.warn(`${
|
|
15781
|
-
${detailBlock}${
|
|
15778
|
+
console.warn(`${ANSI_YELLOW3}Warning: ${message}
|
|
15779
|
+
${detailBlock}${ANSI_RESET4}`);
|
|
15782
15780
|
} else {
|
|
15783
|
-
console.warn(`${
|
|
15781
|
+
console.warn(`${ANSI_YELLOW3}Warning: ${message}${ANSI_RESET4}`);
|
|
15784
15782
|
}
|
|
15785
15783
|
}
|
|
15786
15784
|
function parseRequired(value) {
|
|
@@ -16085,8 +16083,8 @@ function hasVisibleContent(segments) {
|
|
|
16085
16083
|
function asString2(value) {
|
|
16086
16084
|
return typeof value === "string" ? value : void 0;
|
|
16087
16085
|
}
|
|
16088
|
-
var
|
|
16089
|
-
var
|
|
16086
|
+
var ANSI_YELLOW4 = "\x1B[33m";
|
|
16087
|
+
var ANSI_RESET5 = "\x1B[0m";
|
|
16090
16088
|
async function processMessages(options) {
|
|
16091
16089
|
const {
|
|
16092
16090
|
messages,
|
|
@@ -16130,7 +16128,7 @@ async function processMessages(options) {
|
|
|
16130
16128
|
if (!rawValue) {
|
|
16131
16129
|
continue;
|
|
16132
16130
|
}
|
|
16133
|
-
const { displayPath, resolvedPath, attempted } = await
|
|
16131
|
+
const { displayPath, resolvedPath, attempted } = await resolveFileReference22(
|
|
16134
16132
|
rawValue,
|
|
16135
16133
|
searchRoots
|
|
16136
16134
|
);
|
|
@@ -16141,7 +16139,7 @@ async function processMessages(options) {
|
|
|
16141
16139
|
continue;
|
|
16142
16140
|
}
|
|
16143
16141
|
try {
|
|
16144
|
-
const fileContent = (await
|
|
16142
|
+
const fileContent = (await readFile4(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
16145
16143
|
const classifyAsGuideline = shouldTreatAsGuideline({
|
|
16146
16144
|
messageType,
|
|
16147
16145
|
resolvedPath,
|
|
@@ -16150,7 +16148,7 @@ async function processMessages(options) {
|
|
|
16150
16148
|
treatFileSegmentsAsGuidelines
|
|
16151
16149
|
});
|
|
16152
16150
|
if (classifyAsGuideline && guidelinePaths) {
|
|
16153
|
-
guidelinePaths.push(
|
|
16151
|
+
guidelinePaths.push(path5.resolve(resolvedPath));
|
|
16154
16152
|
if (verbose) {
|
|
16155
16153
|
console.log(` [Guideline] Found: ${displayPath}`);
|
|
16156
16154
|
console.log(` Resolved to: ${resolvedPath}`);
|
|
@@ -16161,7 +16159,7 @@ async function processMessages(options) {
|
|
|
16161
16159
|
type: "file",
|
|
16162
16160
|
path: displayPath,
|
|
16163
16161
|
text: fileContent,
|
|
16164
|
-
resolvedPath:
|
|
16162
|
+
resolvedPath: path5.resolve(resolvedPath)
|
|
16165
16163
|
});
|
|
16166
16164
|
if (verbose) {
|
|
16167
16165
|
const label = messageType === "input" ? "[File]" : "[Expected Output File]";
|
|
@@ -16201,7 +16199,7 @@ function shouldTreatAsGuideline(options) {
|
|
|
16201
16199
|
if (!guidelinePatterns || guidelinePatterns.length === 0) {
|
|
16202
16200
|
return false;
|
|
16203
16201
|
}
|
|
16204
|
-
const relativeToRepo =
|
|
16202
|
+
const relativeToRepo = path5.relative(repoRootPath, resolvedPath);
|
|
16205
16203
|
return isGuidelineFile(relativeToRepo, guidelinePatterns);
|
|
16206
16204
|
}
|
|
16207
16205
|
function asString3(value) {
|
|
@@ -16229,10 +16227,10 @@ function cloneJsonValue(value) {
|
|
|
16229
16227
|
function logWarning3(message, details) {
|
|
16230
16228
|
if (details && details.length > 0) {
|
|
16231
16229
|
const detailBlock = details.join("\n");
|
|
16232
|
-
console.warn(`${
|
|
16233
|
-
${detailBlock}${
|
|
16230
|
+
console.warn(`${ANSI_YELLOW4}Warning: ${message}
|
|
16231
|
+
${detailBlock}${ANSI_RESET5}`);
|
|
16234
16232
|
} else {
|
|
16235
|
-
console.warn(`${
|
|
16233
|
+
console.warn(`${ANSI_YELLOW4}Warning: ${message}${ANSI_RESET5}`);
|
|
16236
16234
|
}
|
|
16237
16235
|
}
|
|
16238
16236
|
async function processExpectedMessages(options) {
|
|
@@ -16261,7 +16259,7 @@ async function processExpectedMessages(options) {
|
|
|
16261
16259
|
if (!rawValue) {
|
|
16262
16260
|
continue;
|
|
16263
16261
|
}
|
|
16264
|
-
const { displayPath, resolvedPath, attempted } = await
|
|
16262
|
+
const { displayPath, resolvedPath, attempted } = await resolveFileReference22(
|
|
16265
16263
|
rawValue,
|
|
16266
16264
|
searchRoots
|
|
16267
16265
|
);
|
|
@@ -16271,12 +16269,12 @@ async function processExpectedMessages(options) {
|
|
|
16271
16269
|
continue;
|
|
16272
16270
|
}
|
|
16273
16271
|
try {
|
|
16274
|
-
const fileContent = (await
|
|
16272
|
+
const fileContent = (await readFile4(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
16275
16273
|
processedContent.push({
|
|
16276
16274
|
type: "file",
|
|
16277
16275
|
path: displayPath,
|
|
16278
16276
|
text: fileContent,
|
|
16279
|
-
resolvedPath:
|
|
16277
|
+
resolvedPath: path5.resolve(resolvedPath)
|
|
16280
16278
|
});
|
|
16281
16279
|
if (verbose) {
|
|
16282
16280
|
console.log(` [Expected Output File] Found: ${displayPath}`);
|
|
@@ -16369,11 +16367,11 @@ function resolveInputMessages(raw, suiteInputFiles) {
|
|
|
16369
16367
|
function resolveExpectedMessages(raw) {
|
|
16370
16368
|
return expandExpectedOutputShorthand(raw.expected_output);
|
|
16371
16369
|
}
|
|
16372
|
-
var
|
|
16370
|
+
var ANSI_YELLOW5 = "\x1B[33m";
|
|
16373
16371
|
var ANSI_RED2 = "\x1B[31m";
|
|
16374
|
-
var
|
|
16372
|
+
var ANSI_RESET6 = "\x1B[0m";
|
|
16375
16373
|
function detectFormat(filePath) {
|
|
16376
|
-
const ext =
|
|
16374
|
+
const ext = path6.extname(filePath).toLowerCase();
|
|
16377
16375
|
if (ext === ".jsonl") return "jsonl";
|
|
16378
16376
|
if (ext === ".yaml" || ext === ".yml") return "yaml";
|
|
16379
16377
|
if (ext === ".json") return "agent-skills-json";
|
|
@@ -16382,9 +16380,9 @@ function detectFormat(filePath) {
|
|
|
16382
16380
|
);
|
|
16383
16381
|
}
|
|
16384
16382
|
async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
16385
|
-
const dir =
|
|
16386
|
-
const base =
|
|
16387
|
-
const sidecarPath =
|
|
16383
|
+
const dir = path6.dirname(jsonlPath);
|
|
16384
|
+
const base = path6.basename(jsonlPath, ".jsonl");
|
|
16385
|
+
const sidecarPath = path6.join(dir, `${base}.yaml`);
|
|
16388
16386
|
if (!await fileExists2(sidecarPath)) {
|
|
16389
16387
|
if (verbose) {
|
|
16390
16388
|
logWarning4(`Sidecar metadata file not found: ${sidecarPath} (using defaults)`);
|
|
@@ -16392,7 +16390,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
|
16392
16390
|
return {};
|
|
16393
16391
|
}
|
|
16394
16392
|
try {
|
|
16395
|
-
const content = await
|
|
16393
|
+
const content = await readFile5(sidecarPath, "utf8");
|
|
16396
16394
|
const parsed = interpolateEnv(parseYaml2(content), process.env);
|
|
16397
16395
|
if (!isJsonObject(parsed)) {
|
|
16398
16396
|
logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
|
|
@@ -16433,15 +16431,15 @@ function parseJsonlContent(content, filePath) {
|
|
|
16433
16431
|
async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
16434
16432
|
const verbose = options?.verbose ?? false;
|
|
16435
16433
|
const filterPattern = options?.filter;
|
|
16436
|
-
const absoluteTestPath =
|
|
16434
|
+
const absoluteTestPath = path6.resolve(evalFilePath);
|
|
16437
16435
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
16438
16436
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
16439
16437
|
const config = await loadConfig(absoluteTestPath, repoRootPath);
|
|
16440
16438
|
const guidelinePatterns = config?.guideline_patterns;
|
|
16441
16439
|
const sidecar = await loadSidecarMetadata(absoluteTestPath, verbose);
|
|
16442
|
-
const rawFile = await
|
|
16440
|
+
const rawFile = await readFile5(absoluteTestPath, "utf8");
|
|
16443
16441
|
const rawCases = parseJsonlContent(rawFile, evalFilePath);
|
|
16444
|
-
const fallbackDataset =
|
|
16442
|
+
const fallbackDataset = path6.basename(absoluteTestPath, ".jsonl") || "eval";
|
|
16445
16443
|
const datasetName = sidecar.dataset && sidecar.dataset.trim().length > 0 ? sidecar.dataset : fallbackDataset;
|
|
16446
16444
|
const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm-grader";
|
|
16447
16445
|
const globalExecution = sidecar.execution;
|
|
@@ -16540,7 +16538,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
|
16540
16538
|
}
|
|
16541
16539
|
}
|
|
16542
16540
|
const allFilePaths = [
|
|
16543
|
-
...guidelinePaths.map((guidelinePath) =>
|
|
16541
|
+
...guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
|
|
16544
16542
|
...userFilePaths
|
|
16545
16543
|
];
|
|
16546
16544
|
const testCase = {
|
|
@@ -16552,7 +16550,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
|
16552
16550
|
input_segments: inputSegments,
|
|
16553
16551
|
expected_output: outputSegments,
|
|
16554
16552
|
reference_answer: referenceAnswer,
|
|
16555
|
-
guideline_paths: guidelinePaths.map((guidelinePath) =>
|
|
16553
|
+
guideline_paths: guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
|
|
16556
16554
|
guideline_patterns: guidelinePatterns,
|
|
16557
16555
|
file_paths: allFilePaths,
|
|
16558
16556
|
criteria: outcome ?? "",
|
|
@@ -16583,19 +16581,19 @@ function asString4(value) {
|
|
|
16583
16581
|
function logWarning4(message, details) {
|
|
16584
16582
|
if (details && details.length > 0) {
|
|
16585
16583
|
const detailBlock = details.join("\n");
|
|
16586
|
-
console.warn(`${
|
|
16587
|
-
${detailBlock}${
|
|
16584
|
+
console.warn(`${ANSI_YELLOW5}Warning: ${message}
|
|
16585
|
+
${detailBlock}${ANSI_RESET6}`);
|
|
16588
16586
|
} else {
|
|
16589
|
-
console.warn(`${
|
|
16587
|
+
console.warn(`${ANSI_YELLOW5}Warning: ${message}${ANSI_RESET6}`);
|
|
16590
16588
|
}
|
|
16591
16589
|
}
|
|
16592
16590
|
function logError2(message, details) {
|
|
16593
16591
|
if (details && details.length > 0) {
|
|
16594
16592
|
const detailBlock = details.join("\n");
|
|
16595
16593
|
console.error(`${ANSI_RED2}Error: ${message}
|
|
16596
|
-
${detailBlock}${
|
|
16594
|
+
${detailBlock}${ANSI_RESET6}`);
|
|
16597
16595
|
} else {
|
|
16598
|
-
console.error(`${ANSI_RED2}Error: ${message}${
|
|
16596
|
+
console.error(`${ANSI_RED2}Error: ${message}${ANSI_RESET6}`);
|
|
16599
16597
|
}
|
|
16600
16598
|
}
|
|
16601
16599
|
var MetadataSchema = external_exports2.object({
|
|
@@ -16625,22 +16623,22 @@ function parseMetadata(suite) {
|
|
|
16625
16623
|
requires: suite.requires
|
|
16626
16624
|
});
|
|
16627
16625
|
}
|
|
16628
|
-
var
|
|
16629
|
-
var
|
|
16626
|
+
var ANSI_YELLOW6 = "\x1B[33m";
|
|
16627
|
+
var ANSI_RESET7 = "\x1B[0m";
|
|
16630
16628
|
async function buildPromptInputs(testCase, mode = "lm") {
|
|
16631
16629
|
const guidelineParts = [];
|
|
16632
16630
|
for (const rawPath of testCase.guideline_paths) {
|
|
16633
|
-
const absolutePath =
|
|
16631
|
+
const absolutePath = path7.resolve(rawPath);
|
|
16634
16632
|
if (!await fileExists2(absolutePath)) {
|
|
16635
16633
|
logWarning5(`Could not read guideline file ${absolutePath}: file does not exist`);
|
|
16636
16634
|
continue;
|
|
16637
16635
|
}
|
|
16638
16636
|
try {
|
|
16639
|
-
const content = (await
|
|
16637
|
+
const content = (await readFile6(absolutePath, "utf8")).replace(/\r\n/g, "\n").trim();
|
|
16640
16638
|
guidelineParts.push({
|
|
16641
16639
|
content,
|
|
16642
16640
|
isFile: true,
|
|
16643
|
-
displayPath:
|
|
16641
|
+
displayPath: path7.basename(absolutePath)
|
|
16644
16642
|
});
|
|
16645
16643
|
} catch (error) {
|
|
16646
16644
|
logWarning5(`Could not read guideline file ${absolutePath}: ${error.message}`);
|
|
@@ -16838,11 +16836,11 @@ function asString5(value) {
|
|
|
16838
16836
|
return typeof value === "string" ? value : void 0;
|
|
16839
16837
|
}
|
|
16840
16838
|
function logWarning5(message) {
|
|
16841
|
-
console.warn(`${
|
|
16839
|
+
console.warn(`${ANSI_YELLOW6}Warning: ${message}${ANSI_RESET7}`);
|
|
16842
16840
|
}
|
|
16843
|
-
var
|
|
16841
|
+
var ANSI_YELLOW7 = "\x1B[33m";
|
|
16844
16842
|
var ANSI_RED3 = "\x1B[31m";
|
|
16845
|
-
var
|
|
16843
|
+
var ANSI_RESET8 = "\x1B[0m";
|
|
16846
16844
|
function resolveTests(suite) {
|
|
16847
16845
|
if (suite.tests !== void 0) return suite.tests;
|
|
16848
16846
|
if (suite.eval_cases !== void 0) {
|
|
@@ -16857,8 +16855,8 @@ function resolveTests(suite) {
|
|
|
16857
16855
|
}
|
|
16858
16856
|
async function readTestSuiteMetadata(testFilePath) {
|
|
16859
16857
|
try {
|
|
16860
|
-
const absolutePath =
|
|
16861
|
-
const content = await
|
|
16858
|
+
const absolutePath = path8.resolve(testFilePath);
|
|
16859
|
+
const content = await readFile7(absolutePath, "utf8");
|
|
16862
16860
|
const parsed = interpolateEnv(parse2(content), process.env);
|
|
16863
16861
|
if (!isJsonObject(parsed)) {
|
|
16864
16862
|
return {};
|
|
@@ -16909,26 +16907,26 @@ var loadEvalCases = loadTests;
|
|
|
16909
16907
|
async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
16910
16908
|
const verbose = options?.verbose ?? false;
|
|
16911
16909
|
const filterPattern = options?.filter;
|
|
16912
|
-
const absoluteTestPath =
|
|
16910
|
+
const absoluteTestPath = path8.resolve(evalFilePath);
|
|
16913
16911
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
16914
16912
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
16915
16913
|
const config = await loadConfig(absoluteTestPath, repoRootPath);
|
|
16916
16914
|
const guidelinePatterns = config?.guideline_patterns;
|
|
16917
|
-
const rawFile = await
|
|
16915
|
+
const rawFile = await readFile7(absoluteTestPath, "utf8");
|
|
16918
16916
|
const interpolated = interpolateEnv(parse2(rawFile), process.env);
|
|
16919
16917
|
if (!isJsonObject(interpolated)) {
|
|
16920
16918
|
throw new Error(`Invalid test file format: ${evalFilePath}`);
|
|
16921
16919
|
}
|
|
16922
16920
|
const suite = interpolated;
|
|
16923
16921
|
const datasetNameFromSuite = asString6(suite.dataset)?.trim();
|
|
16924
|
-
const fallbackDataset =
|
|
16922
|
+
const fallbackDataset = path8.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
|
|
16925
16923
|
const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
|
|
16926
16924
|
const rawTestcases = resolveTests(suite);
|
|
16927
16925
|
const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-grader";
|
|
16928
|
-
const evalFileDir =
|
|
16926
|
+
const evalFileDir = path8.dirname(absoluteTestPath);
|
|
16929
16927
|
let expandedTestcases;
|
|
16930
16928
|
if (typeof rawTestcases === "string") {
|
|
16931
|
-
const externalPath =
|
|
16929
|
+
const externalPath = path8.resolve(evalFileDir, rawTestcases);
|
|
16932
16930
|
expandedTestcases = await loadCasesFromFile(externalPath);
|
|
16933
16931
|
} else if (Array.isArray(rawTestcases)) {
|
|
16934
16932
|
expandedTestcases = await expandFileReferences(rawTestcases, evalFileDir);
|
|
@@ -17049,7 +17047,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17049
17047
|
}
|
|
17050
17048
|
}
|
|
17051
17049
|
const allFilePaths = [
|
|
17052
|
-
...guidelinePaths.map((guidelinePath) =>
|
|
17050
|
+
...guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
17053
17051
|
...userFilePaths
|
|
17054
17052
|
];
|
|
17055
17053
|
const caseWorkspace = await resolveWorkspaceConfig(evalcase.workspace, evalFileDir);
|
|
@@ -17065,7 +17063,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17065
17063
|
input_segments: inputSegments,
|
|
17066
17064
|
expected_output: outputSegments,
|
|
17067
17065
|
reference_answer: referenceAnswer,
|
|
17068
|
-
guideline_paths: guidelinePaths.map((guidelinePath) =>
|
|
17066
|
+
guideline_paths: guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
17069
17067
|
guideline_patterns: guidelinePatterns,
|
|
17070
17068
|
file_paths: allFilePaths,
|
|
17071
17069
|
criteria: outcome ?? "",
|
|
@@ -17115,8 +17113,8 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
|
|
|
17115
17113
|
if (commandArr.length === 0) return void 0;
|
|
17116
17114
|
const timeoutMs = typeof obj.timeout_ms === "number" ? obj.timeout_ms : void 0;
|
|
17117
17115
|
let cwd = typeof obj.cwd === "string" ? obj.cwd : void 0;
|
|
17118
|
-
if (cwd && !
|
|
17119
|
-
cwd =
|
|
17116
|
+
if (cwd && !path8.isAbsolute(cwd)) {
|
|
17117
|
+
cwd = path8.resolve(evalFileDir, cwd);
|
|
17120
17118
|
}
|
|
17121
17119
|
const config = { command: commandArr };
|
|
17122
17120
|
if (timeoutMs !== void 0) {
|
|
@@ -17206,10 +17204,10 @@ function parseWorkspaceHooksConfig(raw, evalFileDir) {
|
|
|
17206
17204
|
}
|
|
17207
17205
|
async function resolveWorkspaceConfig(raw, evalFileDir) {
|
|
17208
17206
|
if (typeof raw === "string") {
|
|
17209
|
-
const workspaceFilePath =
|
|
17207
|
+
const workspaceFilePath = path8.resolve(evalFileDir, raw);
|
|
17210
17208
|
let content;
|
|
17211
17209
|
try {
|
|
17212
|
-
content = await
|
|
17210
|
+
content = await readFile7(workspaceFilePath, "utf8");
|
|
17213
17211
|
} catch {
|
|
17214
17212
|
throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
|
|
17215
17213
|
}
|
|
@@ -17219,7 +17217,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
|
|
|
17219
17217
|
`Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
|
|
17220
17218
|
);
|
|
17221
17219
|
}
|
|
17222
|
-
const workspaceFileDir =
|
|
17220
|
+
const workspaceFileDir = path8.dirname(workspaceFilePath);
|
|
17223
17221
|
return parseWorkspaceConfig(parsed, workspaceFileDir);
|
|
17224
17222
|
}
|
|
17225
17223
|
return parseWorkspaceConfig(raw, evalFileDir);
|
|
@@ -17239,8 +17237,8 @@ function parseWorkspaceConfig(raw, evalFileDir) {
|
|
|
17239
17237
|
throw new Error("workspace.static has been removed. Use workspace.mode='static'.");
|
|
17240
17238
|
}
|
|
17241
17239
|
let template = typeof obj.template === "string" ? obj.template : void 0;
|
|
17242
|
-
if (template && !
|
|
17243
|
-
template =
|
|
17240
|
+
if (template && !path8.isAbsolute(template)) {
|
|
17241
|
+
template = path8.resolve(evalFileDir, template);
|
|
17244
17242
|
}
|
|
17245
17243
|
const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
|
|
17246
17244
|
const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
|
|
@@ -17293,19 +17291,19 @@ function asString6(value) {
|
|
|
17293
17291
|
function logWarning6(message, details) {
|
|
17294
17292
|
if (details && details.length > 0) {
|
|
17295
17293
|
const detailBlock = details.join("\n");
|
|
17296
|
-
console.warn(`${
|
|
17297
|
-
${detailBlock}${
|
|
17294
|
+
console.warn(`${ANSI_YELLOW7}Warning: ${message}
|
|
17295
|
+
${detailBlock}${ANSI_RESET8}`);
|
|
17298
17296
|
} else {
|
|
17299
|
-
console.warn(`${
|
|
17297
|
+
console.warn(`${ANSI_YELLOW7}Warning: ${message}${ANSI_RESET8}`);
|
|
17300
17298
|
}
|
|
17301
17299
|
}
|
|
17302
17300
|
function logError3(message, details) {
|
|
17303
17301
|
if (details && details.length > 0) {
|
|
17304
17302
|
const detailBlock = details.join("\n");
|
|
17305
17303
|
console.error(`${ANSI_RED3}Error: ${message}
|
|
17306
|
-
${detailBlock}${
|
|
17304
|
+
${detailBlock}${ANSI_RESET8}`);
|
|
17307
17305
|
} else {
|
|
17308
|
-
console.error(`${ANSI_RED3}Error: ${message}${
|
|
17306
|
+
console.error(`${ANSI_RED3}Error: ${message}${ANSI_RESET8}`);
|
|
17309
17307
|
}
|
|
17310
17308
|
}
|
|
17311
17309
|
function codeGraderInstruction(graderName, description) {
|
|
@@ -17551,7 +17549,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
17551
17549
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
17552
17550
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
17553
17551
|
const parsed = parse3(content);
|
|
17554
|
-
return transpileEvalYaml(parsed,
|
|
17552
|
+
return transpileEvalYaml(parsed, path9.basename(evalYamlPath));
|
|
17555
17553
|
}
|
|
17556
17554
|
function getOutputFilenames(result) {
|
|
17557
17555
|
const names = /* @__PURE__ */ new Map();
|
|
@@ -18051,7 +18049,7 @@ function normalizeInputFiles(inputFiles) {
|
|
|
18051
18049
|
}
|
|
18052
18050
|
const deduped = /* @__PURE__ */ new Map();
|
|
18053
18051
|
for (const inputFile of inputFiles) {
|
|
18054
|
-
const absolutePath =
|
|
18052
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18055
18053
|
if (!deduped.has(absolutePath)) {
|
|
18056
18054
|
deduped.set(absolutePath, absolutePath);
|
|
18057
18055
|
}
|
|
@@ -18064,14 +18062,14 @@ function collectGuidelineFiles(inputFiles, guidelinePatterns, overrides) {
|
|
|
18064
18062
|
}
|
|
18065
18063
|
const unique = /* @__PURE__ */ new Map();
|
|
18066
18064
|
for (const inputFile of inputFiles) {
|
|
18067
|
-
const absolutePath =
|
|
18065
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18068
18066
|
if (overrides?.has(absolutePath)) {
|
|
18069
18067
|
if (!unique.has(absolutePath)) {
|
|
18070
18068
|
unique.set(absolutePath, absolutePath);
|
|
18071
18069
|
}
|
|
18072
18070
|
continue;
|
|
18073
18071
|
}
|
|
18074
|
-
const normalized = absolutePath.split(
|
|
18072
|
+
const normalized = absolutePath.split(path10.sep).join("/");
|
|
18075
18073
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
18076
18074
|
if (!unique.has(absolutePath)) {
|
|
18077
18075
|
unique.set(absolutePath, absolutePath);
|
|
@@ -18086,7 +18084,7 @@ function collectInputFiles(inputFiles) {
|
|
|
18086
18084
|
}
|
|
18087
18085
|
const unique = /* @__PURE__ */ new Map();
|
|
18088
18086
|
for (const inputFile of inputFiles) {
|
|
18089
|
-
const absolutePath =
|
|
18087
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18090
18088
|
if (!unique.has(absolutePath)) {
|
|
18091
18089
|
unique.set(absolutePath, absolutePath);
|
|
18092
18090
|
}
|
|
@@ -18098,7 +18096,7 @@ function buildMandatoryPrereadBlock(guidelineFiles, inputFiles) {
|
|
|
18098
18096
|
return "";
|
|
18099
18097
|
}
|
|
18100
18098
|
const buildList = (files) => files.map((absolutePath) => {
|
|
18101
|
-
const fileName =
|
|
18099
|
+
const fileName = path10.basename(absolutePath);
|
|
18102
18100
|
const fileUri = pathToFileUri(absolutePath);
|
|
18103
18101
|
return `* [${fileName}](${fileUri})`;
|
|
18104
18102
|
});
|
|
@@ -18118,7 +18116,7 @@ ${buildList(inputFiles).join("\n")}.`);
|
|
|
18118
18116
|
return sections.join("\n");
|
|
18119
18117
|
}
|
|
18120
18118
|
function pathToFileUri(filePath) {
|
|
18121
|
-
const absolutePath =
|
|
18119
|
+
const absolutePath = path10.isAbsolute(filePath) ? filePath : path10.resolve(filePath);
|
|
18122
18120
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
18123
18121
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
18124
18122
|
return `file:///${normalizedPath}`;
|
|
@@ -18263,10 +18261,10 @@ var ClaudeCliProvider = class {
|
|
|
18263
18261
|
}
|
|
18264
18262
|
resolveCwd(cwdOverride) {
|
|
18265
18263
|
if (cwdOverride) {
|
|
18266
|
-
return
|
|
18264
|
+
return path11.resolve(cwdOverride);
|
|
18267
18265
|
}
|
|
18268
18266
|
if (this.config.cwd) {
|
|
18269
|
-
return
|
|
18267
|
+
return path11.resolve(this.config.cwd);
|
|
18270
18268
|
}
|
|
18271
18269
|
return void 0;
|
|
18272
18270
|
}
|
|
@@ -18276,9 +18274,9 @@ var ClaudeCliProvider = class {
|
|
|
18276
18274
|
return void 0;
|
|
18277
18275
|
}
|
|
18278
18276
|
if (this.config.logDir) {
|
|
18279
|
-
return
|
|
18277
|
+
return path11.resolve(this.config.logDir);
|
|
18280
18278
|
}
|
|
18281
|
-
return
|
|
18279
|
+
return path11.join(process.cwd(), ".agentv", "logs", "claude-cli");
|
|
18282
18280
|
}
|
|
18283
18281
|
async createStreamLogger(request) {
|
|
18284
18282
|
const logDir = this.resolveLogDirectory();
|
|
@@ -18292,7 +18290,7 @@ var ClaudeCliProvider = class {
|
|
|
18292
18290
|
console.warn(`Skipping Claude CLI stream logging (could not create ${logDir}): ${message}`);
|
|
18293
18291
|
return void 0;
|
|
18294
18292
|
}
|
|
18295
|
-
const filePath =
|
|
18293
|
+
const filePath = path11.join(logDir, buildLogFilename(request, this.targetName));
|
|
18296
18294
|
try {
|
|
18297
18295
|
const logger = await ClaudeCliStreamLogger.create({
|
|
18298
18296
|
filePath,
|
|
@@ -18767,10 +18765,10 @@ var ClaudeSdkProvider = class {
|
|
|
18767
18765
|
}
|
|
18768
18766
|
resolveCwd(cwdOverride) {
|
|
18769
18767
|
if (cwdOverride) {
|
|
18770
|
-
return
|
|
18768
|
+
return path12.resolve(cwdOverride);
|
|
18771
18769
|
}
|
|
18772
18770
|
if (this.config.cwd) {
|
|
18773
|
-
return
|
|
18771
|
+
return path12.resolve(this.config.cwd);
|
|
18774
18772
|
}
|
|
18775
18773
|
return void 0;
|
|
18776
18774
|
}
|
|
@@ -18780,9 +18778,9 @@ var ClaudeSdkProvider = class {
|
|
|
18780
18778
|
return void 0;
|
|
18781
18779
|
}
|
|
18782
18780
|
if (this.config.logDir) {
|
|
18783
|
-
return
|
|
18781
|
+
return path12.resolve(this.config.logDir);
|
|
18784
18782
|
}
|
|
18785
|
-
return
|
|
18783
|
+
return path12.join(process.cwd(), ".agentv", "logs", "claude");
|
|
18786
18784
|
}
|
|
18787
18785
|
async createStreamLogger(request) {
|
|
18788
18786
|
const logDir = this.resolveLogDirectory();
|
|
@@ -18796,7 +18794,7 @@ var ClaudeSdkProvider = class {
|
|
|
18796
18794
|
console.warn(`Skipping Claude stream logging (could not create ${logDir}): ${message}`);
|
|
18797
18795
|
return void 0;
|
|
18798
18796
|
}
|
|
18799
|
-
const filePath =
|
|
18797
|
+
const filePath = path12.join(logDir, buildLogFilename2(request, this.targetName));
|
|
18800
18798
|
try {
|
|
18801
18799
|
const logger = await ClaudeStreamLogger.create({
|
|
18802
18800
|
filePath,
|
|
@@ -19500,7 +19498,7 @@ function normalizeInputFiles2(inputFiles) {
|
|
|
19500
19498
|
}
|
|
19501
19499
|
const unique = /* @__PURE__ */ new Map();
|
|
19502
19500
|
for (const inputFile of inputFiles) {
|
|
19503
|
-
const absolutePath =
|
|
19501
|
+
const absolutePath = path13.resolve(inputFile);
|
|
19504
19502
|
if (!unique.has(absolutePath)) {
|
|
19505
19503
|
unique.set(absolutePath, absolutePath);
|
|
19506
19504
|
}
|
|
@@ -19514,7 +19512,7 @@ function formatFileList(files, template) {
|
|
|
19514
19512
|
const formatter = template ?? "{path}";
|
|
19515
19513
|
return files.map((filePath) => {
|
|
19516
19514
|
const escapedPath = shellEscape(filePath);
|
|
19517
|
-
const escapedName = shellEscape(
|
|
19515
|
+
const escapedName = shellEscape(path13.basename(filePath));
|
|
19518
19516
|
return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
|
|
19519
19517
|
}).join(" ");
|
|
19520
19518
|
}
|
|
@@ -19538,7 +19536,7 @@ function generateOutputFilePath(evalCaseId, extension = ".json") {
|
|
|
19538
19536
|
const safeEvalId = evalCaseId || "unknown";
|
|
19539
19537
|
const timestamp = Date.now();
|
|
19540
19538
|
const random = Math.random().toString(36).substring(2, 9);
|
|
19541
|
-
return
|
|
19539
|
+
return path13.join(os.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
|
|
19542
19540
|
}
|
|
19543
19541
|
function formatTimeoutSuffix2(timeoutMs) {
|
|
19544
19542
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
@@ -19777,10 +19775,10 @@ ${basePrompt}` : basePrompt;
|
|
|
19777
19775
|
}
|
|
19778
19776
|
resolveCwd(cwdOverride) {
|
|
19779
19777
|
if (cwdOverride) {
|
|
19780
|
-
return
|
|
19778
|
+
return path14.resolve(cwdOverride);
|
|
19781
19779
|
}
|
|
19782
19780
|
if (this.config.cwd) {
|
|
19783
|
-
return
|
|
19781
|
+
return path14.resolve(this.config.cwd);
|
|
19784
19782
|
}
|
|
19785
19783
|
return void 0;
|
|
19786
19784
|
}
|
|
@@ -19790,9 +19788,9 @@ ${basePrompt}` : basePrompt;
|
|
|
19790
19788
|
return void 0;
|
|
19791
19789
|
}
|
|
19792
19790
|
if (this.config.logDir) {
|
|
19793
|
-
return
|
|
19791
|
+
return path14.resolve(this.config.logDir);
|
|
19794
19792
|
}
|
|
19795
|
-
return
|
|
19793
|
+
return path14.join(process.cwd(), ".agentv", "logs", "codex");
|
|
19796
19794
|
}
|
|
19797
19795
|
async createStreamLogger(request) {
|
|
19798
19796
|
const logDir = this.resolveLogDirectory();
|
|
@@ -19806,7 +19804,7 @@ ${basePrompt}` : basePrompt;
|
|
|
19806
19804
|
console.warn(`Skipping Codex SDK stream logging (could not create ${logDir}): ${message}`);
|
|
19807
19805
|
return void 0;
|
|
19808
19806
|
}
|
|
19809
|
-
const filePath =
|
|
19807
|
+
const filePath = path14.join(logDir, buildLogFilename3(request, this.targetName));
|
|
19810
19808
|
try {
|
|
19811
19809
|
const logger = await CodexSdkStreamLogger.create({
|
|
19812
19810
|
filePath,
|
|
@@ -20019,7 +20017,7 @@ function resolvePlatformCliPath() {
|
|
|
20019
20017
|
try {
|
|
20020
20018
|
const resolved = import.meta.resolve(`${packageName}/package.json`);
|
|
20021
20019
|
const packageJsonPath = resolved.startsWith("file:") ? fileURLToPath2(resolved) : resolved;
|
|
20022
|
-
const binaryPath =
|
|
20020
|
+
const binaryPath = path15.join(path15.dirname(packageJsonPath), binaryName);
|
|
20023
20021
|
if (existsSync(binaryPath)) {
|
|
20024
20022
|
return binaryPath;
|
|
20025
20023
|
}
|
|
@@ -20027,7 +20025,7 @@ function resolvePlatformCliPath() {
|
|
|
20027
20025
|
}
|
|
20028
20026
|
let searchDir = process.cwd();
|
|
20029
20027
|
for (let i = 0; i < 10; i++) {
|
|
20030
|
-
const standardPath =
|
|
20028
|
+
const standardPath = path15.join(
|
|
20031
20029
|
searchDir,
|
|
20032
20030
|
"node_modules",
|
|
20033
20031
|
...packageName.split("/"),
|
|
@@ -20036,13 +20034,13 @@ function resolvePlatformCliPath() {
|
|
|
20036
20034
|
if (existsSync(standardPath)) {
|
|
20037
20035
|
return standardPath;
|
|
20038
20036
|
}
|
|
20039
|
-
const bunDir =
|
|
20037
|
+
const bunDir = path15.join(searchDir, "node_modules", ".bun");
|
|
20040
20038
|
const prefix = `@github+copilot-${osPart}-${archPart}@`;
|
|
20041
20039
|
try {
|
|
20042
20040
|
const entries = readdirSync(bunDir);
|
|
20043
20041
|
for (const entry of entries) {
|
|
20044
20042
|
if (entry.startsWith(prefix)) {
|
|
20045
|
-
const candidate =
|
|
20043
|
+
const candidate = path15.join(
|
|
20046
20044
|
bunDir,
|
|
20047
20045
|
entry,
|
|
20048
20046
|
"node_modules",
|
|
@@ -20057,7 +20055,7 @@ function resolvePlatformCliPath() {
|
|
|
20057
20055
|
}
|
|
20058
20056
|
} catch {
|
|
20059
20057
|
}
|
|
20060
|
-
const parent =
|
|
20058
|
+
const parent = path15.dirname(searchDir);
|
|
20061
20059
|
if (parent === searchDir) break;
|
|
20062
20060
|
searchDir = parent;
|
|
20063
20061
|
}
|
|
@@ -20393,10 +20391,10 @@ var CopilotCliProvider = class {
|
|
|
20393
20391
|
}
|
|
20394
20392
|
resolveCwd(cwdOverride) {
|
|
20395
20393
|
if (cwdOverride) {
|
|
20396
|
-
return
|
|
20394
|
+
return path16.resolve(cwdOverride);
|
|
20397
20395
|
}
|
|
20398
20396
|
if (this.config.cwd) {
|
|
20399
|
-
return
|
|
20397
|
+
return path16.resolve(this.config.cwd);
|
|
20400
20398
|
}
|
|
20401
20399
|
return void 0;
|
|
20402
20400
|
}
|
|
@@ -20415,9 +20413,9 @@ var CopilotCliProvider = class {
|
|
|
20415
20413
|
return void 0;
|
|
20416
20414
|
}
|
|
20417
20415
|
if (this.config.logDir) {
|
|
20418
|
-
return
|
|
20416
|
+
return path16.resolve(this.config.logDir);
|
|
20419
20417
|
}
|
|
20420
|
-
return
|
|
20418
|
+
return path16.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
20421
20419
|
}
|
|
20422
20420
|
async createStreamLogger(request) {
|
|
20423
20421
|
const logDir = this.resolveLogDirectory();
|
|
@@ -20431,7 +20429,7 @@ var CopilotCliProvider = class {
|
|
|
20431
20429
|
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
20432
20430
|
return void 0;
|
|
20433
20431
|
}
|
|
20434
|
-
const filePath =
|
|
20432
|
+
const filePath = path16.join(logDir, buildLogFilename4(request, this.targetName, "copilot-cli"));
|
|
20435
20433
|
try {
|
|
20436
20434
|
const logger = await CopilotStreamLogger.create(
|
|
20437
20435
|
{
|
|
@@ -20796,10 +20794,10 @@ var CopilotSdkProvider = class {
|
|
|
20796
20794
|
}
|
|
20797
20795
|
resolveCwd(cwdOverride) {
|
|
20798
20796
|
if (cwdOverride) {
|
|
20799
|
-
return
|
|
20797
|
+
return path17.resolve(cwdOverride);
|
|
20800
20798
|
}
|
|
20801
20799
|
if (this.config.cwd) {
|
|
20802
|
-
return
|
|
20800
|
+
return path17.resolve(this.config.cwd);
|
|
20803
20801
|
}
|
|
20804
20802
|
return void 0;
|
|
20805
20803
|
}
|
|
@@ -20808,9 +20806,9 @@ var CopilotSdkProvider = class {
|
|
|
20808
20806
|
return void 0;
|
|
20809
20807
|
}
|
|
20810
20808
|
if (this.config.logDir) {
|
|
20811
|
-
return
|
|
20809
|
+
return path17.resolve(this.config.logDir);
|
|
20812
20810
|
}
|
|
20813
|
-
return
|
|
20811
|
+
return path17.join(process.cwd(), ".agentv", "logs", "copilot-sdk");
|
|
20814
20812
|
}
|
|
20815
20813
|
async createStreamLogger(request) {
|
|
20816
20814
|
const logDir = this.resolveLogDirectory();
|
|
@@ -20824,7 +20822,7 @@ var CopilotSdkProvider = class {
|
|
|
20824
20822
|
console.warn(`Skipping Copilot SDK stream logging (could not create ${logDir}): ${message}`);
|
|
20825
20823
|
return void 0;
|
|
20826
20824
|
}
|
|
20827
|
-
const filePath =
|
|
20825
|
+
const filePath = path17.join(logDir, buildLogFilename4(request, this.targetName, "copilot-sdk"));
|
|
20828
20826
|
try {
|
|
20829
20827
|
const logger = await CopilotStreamLogger.create(
|
|
20830
20828
|
{
|
|
@@ -21243,7 +21241,7 @@ var PiCodingAgentProvider = class {
|
|
|
21243
21241
|
const workspaceRoot = await this.createWorkspace();
|
|
21244
21242
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
21245
21243
|
try {
|
|
21246
|
-
const promptFile =
|
|
21244
|
+
const promptFile = path18.join(workspaceRoot, PROMPT_FILENAME);
|
|
21247
21245
|
await writeFile(promptFile, request.question, "utf8");
|
|
21248
21246
|
const args = this.buildPiArgs(request.question, inputFiles, request.captureFileChanges);
|
|
21249
21247
|
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
@@ -21305,12 +21303,12 @@ var PiCodingAgentProvider = class {
|
|
|
21305
21303
|
}
|
|
21306
21304
|
resolveCwd(workspaceRoot, cwdOverride) {
|
|
21307
21305
|
if (cwdOverride) {
|
|
21308
|
-
return
|
|
21306
|
+
return path18.resolve(cwdOverride);
|
|
21309
21307
|
}
|
|
21310
21308
|
if (!this.config.cwd) {
|
|
21311
21309
|
return workspaceRoot;
|
|
21312
21310
|
}
|
|
21313
|
-
return
|
|
21311
|
+
return path18.resolve(this.config.cwd);
|
|
21314
21312
|
}
|
|
21315
21313
|
buildPiArgs(prompt, inputFiles, _captureFileChanges) {
|
|
21316
21314
|
const args = [];
|
|
@@ -21399,7 +21397,7 @@ ${prompt}` : prompt;
|
|
|
21399
21397
|
return env;
|
|
21400
21398
|
}
|
|
21401
21399
|
async createWorkspace() {
|
|
21402
|
-
return await mkdtemp(
|
|
21400
|
+
return await mkdtemp(path18.join(tmpdir(), WORKSPACE_PREFIX));
|
|
21403
21401
|
}
|
|
21404
21402
|
async cleanupWorkspace(workspaceRoot) {
|
|
21405
21403
|
try {
|
|
@@ -21409,9 +21407,9 @@ ${prompt}` : prompt;
|
|
|
21409
21407
|
}
|
|
21410
21408
|
resolveLogDirectory() {
|
|
21411
21409
|
if (this.config.logDir) {
|
|
21412
|
-
return
|
|
21410
|
+
return path18.resolve(this.config.logDir);
|
|
21413
21411
|
}
|
|
21414
|
-
return
|
|
21412
|
+
return path18.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
21415
21413
|
}
|
|
21416
21414
|
async createStreamLogger(request) {
|
|
21417
21415
|
const logDir = this.resolveLogDirectory();
|
|
@@ -21425,7 +21423,7 @@ ${prompt}` : prompt;
|
|
|
21425
21423
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
21426
21424
|
return void 0;
|
|
21427
21425
|
}
|
|
21428
|
-
const filePath =
|
|
21426
|
+
const filePath = path18.join(logDir, buildLogFilename5(request, this.targetName));
|
|
21429
21427
|
try {
|
|
21430
21428
|
const logger = await PiStreamLogger.create({
|
|
21431
21429
|
filePath,
|
|
@@ -21935,7 +21933,7 @@ async function readDirEntries(target) {
|
|
|
21935
21933
|
const entries = await readdir(target, { withFileTypes: true });
|
|
21936
21934
|
return entries.map((entry) => ({
|
|
21937
21935
|
name: entry.name,
|
|
21938
|
-
absolutePath:
|
|
21936
|
+
absolutePath: path19.join(target, entry.name),
|
|
21939
21937
|
isDirectory: entry.isDirectory()
|
|
21940
21938
|
}));
|
|
21941
21939
|
}
|
|
@@ -21949,7 +21947,7 @@ async function removeIfExists(target) {
|
|
|
21949
21947
|
}
|
|
21950
21948
|
}
|
|
21951
21949
|
function pathToFileUri2(filePath) {
|
|
21952
|
-
const absolutePath =
|
|
21950
|
+
const absolutePath = path20.isAbsolute(filePath) ? filePath : path20.resolve(filePath);
|
|
21953
21951
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
21954
21952
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
21955
21953
|
return `file:///${normalizedPath}`;
|
|
@@ -22041,8 +22039,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
22041
22039
|
});
|
|
22042
22040
|
}
|
|
22043
22041
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
22044
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
22045
|
-
const responseList = responseFiles.map((file) => `"${
|
|
22042
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path21.basename(file)}`).join("\n");
|
|
22043
|
+
const responseList = responseFiles.map((file) => `"${path21.basename(file)}"`).join(", ");
|
|
22046
22044
|
return renderTemplate2(templateContent, {
|
|
22047
22045
|
requestFiles: requestLines,
|
|
22048
22046
|
responseList
|
|
@@ -22081,7 +22079,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
22081
22079
|
const maxAttempts = 10;
|
|
22082
22080
|
while (attempts < maxAttempts) {
|
|
22083
22081
|
try {
|
|
22084
|
-
const content = await
|
|
22082
|
+
const content = await readFile8(responseFileFinal, { encoding: "utf8" });
|
|
22085
22083
|
if (!silent) {
|
|
22086
22084
|
process.stdout.write(`${content}
|
|
22087
22085
|
`);
|
|
@@ -22102,7 +22100,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
22102
22100
|
}
|
|
22103
22101
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
22104
22102
|
if (!silent) {
|
|
22105
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
22103
|
+
const fileList = responseFilesFinal.map((file) => path222.basename(file)).join(", ");
|
|
22106
22104
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
22107
22105
|
}
|
|
22108
22106
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -22111,7 +22109,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
22111
22109
|
while (pending.size > 0) {
|
|
22112
22110
|
if (Date.now() >= deadline) {
|
|
22113
22111
|
if (!silent) {
|
|
22114
|
-
const remaining = [...pending].map((f) =>
|
|
22112
|
+
const remaining = [...pending].map((f) => path222.basename(f)).join(", ");
|
|
22115
22113
|
console.error(
|
|
22116
22114
|
`error: timed out after ${Math.round(timeoutMs / 1e3)}s waiting for batch responses. Still pending: ${remaining}`
|
|
22117
22115
|
);
|
|
@@ -22138,7 +22136,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
22138
22136
|
const maxAttempts = 10;
|
|
22139
22137
|
while (attempts < maxAttempts) {
|
|
22140
22138
|
try {
|
|
22141
|
-
const content = await
|
|
22139
|
+
const content = await readFile8(file, { encoding: "utf8" });
|
|
22142
22140
|
if (!silent) {
|
|
22143
22141
|
process.stdout.write(`${content}
|
|
22144
22142
|
`);
|
|
@@ -22168,25 +22166,25 @@ function getAgentvHome() {
|
|
|
22168
22166
|
}
|
|
22169
22167
|
return envHome;
|
|
22170
22168
|
}
|
|
22171
|
-
return
|
|
22169
|
+
return path23.join(os2.homedir(), ".agentv");
|
|
22172
22170
|
}
|
|
22173
22171
|
function getWorkspacesRoot() {
|
|
22174
|
-
return
|
|
22172
|
+
return path23.join(getAgentvHome(), "workspaces");
|
|
22175
22173
|
}
|
|
22176
22174
|
function getSubagentsRoot() {
|
|
22177
|
-
return
|
|
22175
|
+
return path23.join(getAgentvHome(), "subagents");
|
|
22178
22176
|
}
|
|
22179
22177
|
function getTraceStateRoot() {
|
|
22180
|
-
return
|
|
22178
|
+
return path23.join(getAgentvHome(), "trace-state");
|
|
22181
22179
|
}
|
|
22182
22180
|
function getWorkspacePoolRoot() {
|
|
22183
|
-
return
|
|
22181
|
+
return path23.join(getAgentvHome(), "workspace-pool");
|
|
22184
22182
|
}
|
|
22185
22183
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
22186
22184
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
22187
22185
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
22188
22186
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
22189
|
-
return
|
|
22187
|
+
return path24.join(getSubagentsRoot(), folder);
|
|
22190
22188
|
}
|
|
22191
22189
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
22192
22190
|
var execAsync2 = promisify2(exec);
|
|
@@ -22251,11 +22249,11 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22251
22249
|
await raceSpawnError(child);
|
|
22252
22250
|
return true;
|
|
22253
22251
|
}
|
|
22254
|
-
const aliveFile =
|
|
22252
|
+
const aliveFile = path25.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
22255
22253
|
await removeIfExists(aliveFile);
|
|
22256
|
-
const githubAgentsDir =
|
|
22254
|
+
const githubAgentsDir = path25.join(subagentDir, ".github", "agents");
|
|
22257
22255
|
await mkdir8(githubAgentsDir, { recursive: true });
|
|
22258
|
-
const wakeupDst =
|
|
22256
|
+
const wakeupDst = path25.join(githubAgentsDir, "wakeup.md");
|
|
22259
22257
|
await writeFile2(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
22260
22258
|
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
22261
22259
|
label: "open-workspace"
|
|
@@ -22268,7 +22266,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22268
22266
|
"chat",
|
|
22269
22267
|
"-m",
|
|
22270
22268
|
wakeupChatId,
|
|
22271
|
-
`create a file named .alive in the ${
|
|
22269
|
+
`create a file named .alive in the ${path25.basename(subagentDir)} folder`
|
|
22272
22270
|
];
|
|
22273
22271
|
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
22274
22272
|
await raceSpawnError(wakeupChild);
|
|
@@ -22283,10 +22281,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22283
22281
|
return true;
|
|
22284
22282
|
}
|
|
22285
22283
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
22286
|
-
const workspacePath =
|
|
22287
|
-
const messagesDir =
|
|
22284
|
+
const workspacePath = path25.join(subagentDir, `${path25.basename(subagentDir)}.code-workspace`);
|
|
22285
|
+
const messagesDir = path25.join(subagentDir, "messages");
|
|
22288
22286
|
await mkdir8(messagesDir, { recursive: true });
|
|
22289
|
-
const reqFile =
|
|
22287
|
+
const reqFile = path25.join(messagesDir, `${timestamp}_req.md`);
|
|
22290
22288
|
await writeFile2(reqFile, requestInstructions, { encoding: "utf8" });
|
|
22291
22289
|
const reqUri = pathToFileUri2(reqFile);
|
|
22292
22290
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
@@ -22294,16 +22292,16 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
22294
22292
|
chatArgs.push("-a", attachment);
|
|
22295
22293
|
}
|
|
22296
22294
|
chatArgs.push("-a", reqFile);
|
|
22297
|
-
chatArgs.push(`Follow instructions in [${
|
|
22295
|
+
chatArgs.push(`Follow instructions in [${path25.basename(reqFile)}](${reqUri})`);
|
|
22298
22296
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
22299
22297
|
workspacePath,
|
|
22300
|
-
|
|
22298
|
+
path25.basename(subagentDir),
|
|
22301
22299
|
subagentDir,
|
|
22302
22300
|
vscodeCmd
|
|
22303
22301
|
);
|
|
22304
22302
|
if (!workspaceReady) {
|
|
22305
22303
|
throw new Error(
|
|
22306
|
-
`VS Code workspace '${
|
|
22304
|
+
`VS Code workspace '${path25.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
22307
22305
|
);
|
|
22308
22306
|
}
|
|
22309
22307
|
await sleep2(500);
|
|
@@ -22311,8 +22309,8 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
22311
22309
|
await raceSpawnError(child);
|
|
22312
22310
|
}
|
|
22313
22311
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
22314
|
-
const workspacePath =
|
|
22315
|
-
const messagesDir =
|
|
22312
|
+
const workspacePath = path25.join(subagentDir, `${path25.basename(subagentDir)}.code-workspace`);
|
|
22313
|
+
const messagesDir = path25.join(subagentDir, "messages");
|
|
22316
22314
|
await mkdir8(messagesDir, { recursive: true });
|
|
22317
22315
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
22318
22316
|
for (const attachment of attachmentPaths) {
|
|
@@ -22321,13 +22319,13 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
22321
22319
|
chatArgs.push(chatInstruction);
|
|
22322
22320
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
22323
22321
|
workspacePath,
|
|
22324
|
-
|
|
22322
|
+
path25.basename(subagentDir),
|
|
22325
22323
|
subagentDir,
|
|
22326
22324
|
vscodeCmd
|
|
22327
22325
|
);
|
|
22328
22326
|
if (!workspaceReady) {
|
|
22329
22327
|
throw new Error(
|
|
22330
|
-
`VS Code workspace '${
|
|
22328
|
+
`VS Code workspace '${path25.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
22331
22329
|
);
|
|
22332
22330
|
}
|
|
22333
22331
|
await sleep2(500);
|
|
@@ -22349,10 +22347,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
22349
22347
|
}
|
|
22350
22348
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
22351
22349
|
const folderPath = folder.path;
|
|
22352
|
-
if (
|
|
22350
|
+
if (path26.isAbsolute(folderPath)) {
|
|
22353
22351
|
return folder;
|
|
22354
22352
|
}
|
|
22355
|
-
const absolutePath =
|
|
22353
|
+
const absolutePath = path26.resolve(templateDir, folderPath);
|
|
22356
22354
|
return {
|
|
22357
22355
|
...folder,
|
|
22358
22356
|
path: absolutePath
|
|
@@ -22374,19 +22372,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
22374
22372
|
if (locationMap && typeof locationMap === "object") {
|
|
22375
22373
|
const transformedMap = {};
|
|
22376
22374
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
22377
|
-
const isAbsolute =
|
|
22375
|
+
const isAbsolute = path26.isAbsolute(locationPath);
|
|
22378
22376
|
if (isAbsolute) {
|
|
22379
22377
|
transformedMap[locationPath] = value;
|
|
22380
22378
|
} else {
|
|
22381
22379
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
22382
22380
|
if (firstGlobIndex === -1) {
|
|
22383
|
-
const resolvedPath =
|
|
22381
|
+
const resolvedPath = path26.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
22384
22382
|
transformedMap[resolvedPath] = value;
|
|
22385
22383
|
} else {
|
|
22386
22384
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
22387
22385
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
22388
22386
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
22389
|
-
const resolvedPath = (
|
|
22387
|
+
const resolvedPath = (path26.resolve(templateDir, basePath) + patternPath).replace(
|
|
22390
22388
|
/\\/g,
|
|
22391
22389
|
"/"
|
|
22392
22390
|
);
|
|
@@ -22425,7 +22423,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
22425
22423
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
22426
22424
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
22427
22425
|
for (const subagent of subagents) {
|
|
22428
|
-
const lockFile =
|
|
22426
|
+
const lockFile = path27.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
22429
22427
|
if (!await pathExists(lockFile)) {
|
|
22430
22428
|
return subagent.absolutePath;
|
|
22431
22429
|
}
|
|
@@ -22435,7 +22433,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
22435
22433
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
22436
22434
|
let workspaceContent;
|
|
22437
22435
|
if (workspaceTemplate) {
|
|
22438
|
-
const workspaceSrc =
|
|
22436
|
+
const workspaceSrc = path27.resolve(workspaceTemplate);
|
|
22439
22437
|
if (!await pathExists(workspaceSrc)) {
|
|
22440
22438
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
22441
22439
|
}
|
|
@@ -22443,18 +22441,18 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
22443
22441
|
if (!stats.isFile()) {
|
|
22444
22442
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
22445
22443
|
}
|
|
22446
|
-
const templateText = await
|
|
22444
|
+
const templateText = await readFile9(workspaceSrc, "utf8");
|
|
22447
22445
|
workspaceContent = JSON.parse(templateText);
|
|
22448
22446
|
} else {
|
|
22449
22447
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
22450
22448
|
}
|
|
22451
|
-
const workspaceName = `${
|
|
22452
|
-
const workspaceDst =
|
|
22453
|
-
const templateDir = workspaceTemplate ?
|
|
22449
|
+
const workspaceName = `${path27.basename(subagentDir)}.code-workspace`;
|
|
22450
|
+
const workspaceDst = path27.join(subagentDir, workspaceName);
|
|
22451
|
+
const templateDir = workspaceTemplate ? path27.dirname(path27.resolve(workspaceTemplate)) : subagentDir;
|
|
22454
22452
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
22455
22453
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
22456
22454
|
if (cwd) {
|
|
22457
|
-
const absCwd =
|
|
22455
|
+
const absCwd = path27.resolve(cwd);
|
|
22458
22456
|
const parsed = JSON.parse(transformedContent);
|
|
22459
22457
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
22460
22458
|
if (!alreadyPresent) {
|
|
@@ -22463,35 +22461,35 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
22463
22461
|
}
|
|
22464
22462
|
}
|
|
22465
22463
|
await writeFile3(workspaceDst, transformedContent, "utf8");
|
|
22466
|
-
const messagesDir =
|
|
22464
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
22467
22465
|
await mkdir9(messagesDir, { recursive: true });
|
|
22468
22466
|
return { workspace: workspaceDst, messagesDir };
|
|
22469
22467
|
}
|
|
22470
22468
|
async function createSubagentLock(subagentDir) {
|
|
22471
|
-
const messagesDir =
|
|
22469
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
22472
22470
|
if (await pathExists(messagesDir)) {
|
|
22473
22471
|
const files = await readdir2(messagesDir);
|
|
22474
22472
|
await Promise.all(
|
|
22475
22473
|
files.map(async (file) => {
|
|
22476
|
-
const target =
|
|
22474
|
+
const target = path27.join(messagesDir, file);
|
|
22477
22475
|
await removeIfExists(target);
|
|
22478
22476
|
})
|
|
22479
22477
|
);
|
|
22480
22478
|
}
|
|
22481
|
-
const githubAgentsDir =
|
|
22479
|
+
const githubAgentsDir = path27.join(subagentDir, ".github", "agents");
|
|
22482
22480
|
if (await pathExists(githubAgentsDir)) {
|
|
22483
22481
|
const agentFiles = await readdir2(githubAgentsDir);
|
|
22484
22482
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
22485
22483
|
await Promise.all(
|
|
22486
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
22484
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path27.join(githubAgentsDir, file)))
|
|
22487
22485
|
);
|
|
22488
22486
|
}
|
|
22489
|
-
const lockFile =
|
|
22487
|
+
const lockFile = path27.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
22490
22488
|
await writeFile3(lockFile, "", { encoding: "utf8" });
|
|
22491
22489
|
return lockFile;
|
|
22492
22490
|
}
|
|
22493
22491
|
async function removeSubagentLock(subagentDir) {
|
|
22494
|
-
const lockFile =
|
|
22492
|
+
const lockFile = path27.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
22495
22493
|
await removeIfExists(lockFile);
|
|
22496
22494
|
}
|
|
22497
22495
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -22511,9 +22509,9 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
22511
22509
|
return 1;
|
|
22512
22510
|
}
|
|
22513
22511
|
if (promptFile) {
|
|
22514
|
-
const githubAgentsDir =
|
|
22512
|
+
const githubAgentsDir = path27.join(subagentDir, ".github", "agents");
|
|
22515
22513
|
await mkdir9(githubAgentsDir, { recursive: true });
|
|
22516
|
-
const agentFile =
|
|
22514
|
+
const agentFile = path27.join(githubAgentsDir, `${chatId}.md`);
|
|
22517
22515
|
try {
|
|
22518
22516
|
await copyFile(promptFile, agentFile);
|
|
22519
22517
|
} catch (error) {
|
|
@@ -22530,7 +22528,7 @@ async function resolvePromptFile(promptFile) {
|
|
|
22530
22528
|
if (!promptFile) {
|
|
22531
22529
|
return void 0;
|
|
22532
22530
|
}
|
|
22533
|
-
const resolvedPrompt =
|
|
22531
|
+
const resolvedPrompt = path28.resolve(promptFile);
|
|
22534
22532
|
if (!await pathExists(resolvedPrompt)) {
|
|
22535
22533
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
22536
22534
|
}
|
|
@@ -22546,7 +22544,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
22546
22544
|
}
|
|
22547
22545
|
const resolved = [];
|
|
22548
22546
|
for (const attachment of extraAttachments) {
|
|
22549
|
-
const resolvedPath =
|
|
22547
|
+
const resolvedPath = path28.resolve(attachment);
|
|
22550
22548
|
if (!await pathExists(resolvedPath)) {
|
|
22551
22549
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
22552
22550
|
}
|
|
@@ -22588,7 +22586,7 @@ async function dispatchAgentSession(options) {
|
|
|
22588
22586
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
22589
22587
|
};
|
|
22590
22588
|
}
|
|
22591
|
-
const subagentName =
|
|
22589
|
+
const subagentName = path28.basename(subagentDir);
|
|
22592
22590
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
22593
22591
|
const preparationResult = await prepareSubagentDirectory(
|
|
22594
22592
|
subagentDir,
|
|
@@ -22616,9 +22614,9 @@ async function dispatchAgentSession(options) {
|
|
|
22616
22614
|
};
|
|
22617
22615
|
}
|
|
22618
22616
|
const timestamp = generateTimestamp();
|
|
22619
|
-
const messagesDir =
|
|
22620
|
-
const responseFileTmp =
|
|
22621
|
-
const responseFileFinal =
|
|
22617
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
22618
|
+
const responseFileTmp = path28.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
22619
|
+
const responseFileFinal = path28.join(messagesDir, `${timestamp}_res.md`);
|
|
22622
22620
|
const requestInstructions = createRequestPrompt(
|
|
22623
22621
|
userQuery,
|
|
22624
22622
|
responseFileTmp,
|
|
@@ -22723,7 +22721,7 @@ async function dispatchBatchAgent(options) {
|
|
|
22723
22721
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
22724
22722
|
};
|
|
22725
22723
|
}
|
|
22726
|
-
subagentName =
|
|
22724
|
+
subagentName = path28.basename(subagentDir);
|
|
22727
22725
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
22728
22726
|
const preparationResult = await prepareSubagentDirectory(
|
|
22729
22727
|
subagentDir,
|
|
@@ -22754,17 +22752,17 @@ async function dispatchBatchAgent(options) {
|
|
|
22754
22752
|
};
|
|
22755
22753
|
}
|
|
22756
22754
|
const timestamp = generateTimestamp();
|
|
22757
|
-
const messagesDir =
|
|
22755
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
22758
22756
|
requestFiles = userQueries.map(
|
|
22759
|
-
(_, index) =>
|
|
22757
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
22760
22758
|
);
|
|
22761
22759
|
const responseTmpFiles = userQueries.map(
|
|
22762
|
-
(_, index) =>
|
|
22760
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
22763
22761
|
);
|
|
22764
22762
|
responseFilesFinal = userQueries.map(
|
|
22765
|
-
(_, index) =>
|
|
22763
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
22766
22764
|
);
|
|
22767
|
-
const orchestratorFile =
|
|
22765
|
+
const orchestratorFile = path28.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
22768
22766
|
if (!dryRun) {
|
|
22769
22767
|
await Promise.all(
|
|
22770
22768
|
userQueries.map((query, index) => {
|
|
@@ -22877,7 +22875,7 @@ async function provisionSubagents(options) {
|
|
|
22877
22875
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
22878
22876
|
throw new Error("subagents must be a positive integer");
|
|
22879
22877
|
}
|
|
22880
|
-
const targetPath =
|
|
22878
|
+
const targetPath = path29.resolve(targetRoot);
|
|
22881
22879
|
if (!dryRun) {
|
|
22882
22880
|
await ensureDir(targetPath);
|
|
22883
22881
|
}
|
|
@@ -22897,7 +22895,7 @@ async function provisionSubagents(options) {
|
|
|
22897
22895
|
continue;
|
|
22898
22896
|
}
|
|
22899
22897
|
highestNumber = Math.max(highestNumber, parsed);
|
|
22900
|
-
const lockFile =
|
|
22898
|
+
const lockFile = path29.join(entry.absolutePath, lockName);
|
|
22901
22899
|
const locked = await pathExists(lockFile);
|
|
22902
22900
|
if (locked) {
|
|
22903
22901
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -22914,10 +22912,10 @@ async function provisionSubagents(options) {
|
|
|
22914
22912
|
break;
|
|
22915
22913
|
}
|
|
22916
22914
|
const subagentDir = subagent.absolutePath;
|
|
22917
|
-
const githubAgentsDir =
|
|
22918
|
-
const lockFile =
|
|
22919
|
-
const workspaceDst =
|
|
22920
|
-
const wakeupDst =
|
|
22915
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
22916
|
+
const lockFile = path29.join(subagentDir, lockName);
|
|
22917
|
+
const workspaceDst = path29.join(subagentDir, `${path29.basename(subagentDir)}.code-workspace`);
|
|
22918
|
+
const wakeupDst = path29.join(githubAgentsDir, "wakeup.md");
|
|
22921
22919
|
const isLocked = await pathExists(lockFile);
|
|
22922
22920
|
if (isLocked && !force) {
|
|
22923
22921
|
continue;
|
|
@@ -22955,10 +22953,10 @@ async function provisionSubagents(options) {
|
|
|
22955
22953
|
let nextIndex = highestNumber;
|
|
22956
22954
|
while (subagentsProvisioned < subagents) {
|
|
22957
22955
|
nextIndex += 1;
|
|
22958
|
-
const subagentDir =
|
|
22959
|
-
const githubAgentsDir =
|
|
22960
|
-
const workspaceDst =
|
|
22961
|
-
const wakeupDst =
|
|
22956
|
+
const subagentDir = path29.join(targetPath, `subagent-${nextIndex}`);
|
|
22957
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
22958
|
+
const workspaceDst = path29.join(subagentDir, `${path29.basename(subagentDir)}.code-workspace`);
|
|
22959
|
+
const wakeupDst = path29.join(githubAgentsDir, "wakeup.md");
|
|
22962
22960
|
if (!dryRun) {
|
|
22963
22961
|
await ensureDir(subagentDir);
|
|
22964
22962
|
await ensureDir(githubAgentsDir);
|
|
@@ -23144,7 +23142,7 @@ var VSCodeProvider = class {
|
|
|
23144
23142
|
async function locateVSCodeExecutable(candidate) {
|
|
23145
23143
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
23146
23144
|
if (includesPathSeparator) {
|
|
23147
|
-
const resolved =
|
|
23145
|
+
const resolved = path30.isAbsolute(candidate) ? candidate : path30.resolve(candidate);
|
|
23148
23146
|
try {
|
|
23149
23147
|
await access3(resolved, constants3.F_OK);
|
|
23150
23148
|
return resolved;
|
|
@@ -23173,7 +23171,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
23173
23171
|
return void 0;
|
|
23174
23172
|
}
|
|
23175
23173
|
try {
|
|
23176
|
-
const stats = await stat4(
|
|
23174
|
+
const stats = await stat4(path30.resolve(template));
|
|
23177
23175
|
return stats.isFile() ? template : void 0;
|
|
23178
23176
|
} catch {
|
|
23179
23177
|
return template;
|
|
@@ -23199,7 +23197,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
23199
23197
|
return "";
|
|
23200
23198
|
}
|
|
23201
23199
|
const buildList = (files) => files.map((absolutePath) => {
|
|
23202
|
-
const fileName =
|
|
23200
|
+
const fileName = path30.basename(absolutePath);
|
|
23203
23201
|
const fileUri = pathToFileUri3(absolutePath);
|
|
23204
23202
|
return `* [${fileName}](${fileUri})`;
|
|
23205
23203
|
});
|
|
@@ -23224,8 +23222,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
23224
23222
|
}
|
|
23225
23223
|
const unique = /* @__PURE__ */ new Map();
|
|
23226
23224
|
for (const attachment of attachments) {
|
|
23227
|
-
const absolutePath =
|
|
23228
|
-
const normalized = absolutePath.split(
|
|
23225
|
+
const absolutePath = path30.resolve(attachment);
|
|
23226
|
+
const normalized = absolutePath.split(path30.sep).join("/");
|
|
23229
23227
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
23230
23228
|
if (!unique.has(absolutePath)) {
|
|
23231
23229
|
unique.set(absolutePath, absolutePath);
|
|
@@ -23240,7 +23238,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
23240
23238
|
}
|
|
23241
23239
|
const unique = /* @__PURE__ */ new Map();
|
|
23242
23240
|
for (const attachment of attachments) {
|
|
23243
|
-
const absolutePath =
|
|
23241
|
+
const absolutePath = path30.resolve(attachment);
|
|
23244
23242
|
if (!unique.has(absolutePath)) {
|
|
23245
23243
|
unique.set(absolutePath, absolutePath);
|
|
23246
23244
|
}
|
|
@@ -23248,7 +23246,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
23248
23246
|
return Array.from(unique.values());
|
|
23249
23247
|
}
|
|
23250
23248
|
function pathToFileUri3(filePath) {
|
|
23251
|
-
const absolutePath =
|
|
23249
|
+
const absolutePath = path30.isAbsolute(filePath) ? filePath : path30.resolve(filePath);
|
|
23252
23250
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
23253
23251
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
23254
23252
|
return `file:///${normalizedPath}`;
|
|
@@ -23261,7 +23259,7 @@ function normalizeAttachments(attachments) {
|
|
|
23261
23259
|
}
|
|
23262
23260
|
const deduped = /* @__PURE__ */ new Set();
|
|
23263
23261
|
for (const attachment of attachments) {
|
|
23264
|
-
deduped.add(
|
|
23262
|
+
deduped.add(path30.resolve(attachment));
|
|
23265
23263
|
}
|
|
23266
23264
|
return Array.from(deduped);
|
|
23267
23265
|
}
|
|
@@ -23270,7 +23268,7 @@ function mergeAttachments(all) {
|
|
|
23270
23268
|
for (const list of all) {
|
|
23271
23269
|
if (!list) continue;
|
|
23272
23270
|
for (const inputFile of list) {
|
|
23273
|
-
deduped.add(
|
|
23271
|
+
deduped.add(path30.resolve(inputFile));
|
|
23274
23272
|
}
|
|
23275
23273
|
}
|
|
23276
23274
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -23350,11 +23348,11 @@ async function fileExists3(filePath) {
|
|
|
23350
23348
|
}
|
|
23351
23349
|
}
|
|
23352
23350
|
async function readTargetDefinitions(filePath) {
|
|
23353
|
-
const absolutePath =
|
|
23351
|
+
const absolutePath = path31.resolve(filePath);
|
|
23354
23352
|
if (!await fileExists3(absolutePath)) {
|
|
23355
23353
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
23356
23354
|
}
|
|
23357
|
-
const raw = await
|
|
23355
|
+
const raw = await readFile10(absolutePath, "utf8");
|
|
23358
23356
|
const parsed = parse4(raw);
|
|
23359
23357
|
if (!isRecord(parsed)) {
|
|
23360
23358
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -23371,11 +23369,11 @@ function listTargetNames(definitions) {
|
|
|
23371
23369
|
async function discoverProviders(registry, baseDir) {
|
|
23372
23370
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
23373
23371
|
const candidateDirs = [];
|
|
23374
|
-
let dir =
|
|
23375
|
-
const root =
|
|
23372
|
+
let dir = path322.resolve(baseDir);
|
|
23373
|
+
const root = path322.parse(dir).root;
|
|
23376
23374
|
while (dir !== root) {
|
|
23377
|
-
candidateDirs.push(
|
|
23378
|
-
dir =
|
|
23375
|
+
candidateDirs.push(path322.join(dir, ".agentv", "providers"));
|
|
23376
|
+
dir = path322.dirname(dir);
|
|
23379
23377
|
}
|
|
23380
23378
|
let files = [];
|
|
23381
23379
|
for (const providersDir of candidateDirs) {
|
|
@@ -23391,7 +23389,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
23391
23389
|
}
|
|
23392
23390
|
const discoveredKinds = [];
|
|
23393
23391
|
for (const filePath of files) {
|
|
23394
|
-
const basename =
|
|
23392
|
+
const basename = path322.basename(filePath);
|
|
23395
23393
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
23396
23394
|
if (registry.has(kindName)) {
|
|
23397
23395
|
continue;
|
|
@@ -23589,15 +23587,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
23589
23587
|
});
|
|
23590
23588
|
}
|
|
23591
23589
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
23592
|
-
const { mkdir: mkdir15, readFile:
|
|
23590
|
+
const { mkdir: mkdir15, readFile: readFile13, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
|
|
23593
23591
|
const { tmpdir: tmpdir3 } = await import("node:os");
|
|
23594
|
-
const
|
|
23592
|
+
const path45 = await import("node:path");
|
|
23595
23593
|
const { randomUUID: randomUUID9 } = await import("node:crypto");
|
|
23596
|
-
const dir =
|
|
23594
|
+
const dir = path45.join(tmpdir3(), `agentv-exec-${randomUUID9()}`);
|
|
23597
23595
|
await mkdir15(dir, { recursive: true });
|
|
23598
|
-
const stdinPath =
|
|
23599
|
-
const stdoutPath =
|
|
23600
|
-
const stderrPath =
|
|
23596
|
+
const stdinPath = path45.join(dir, "stdin.txt");
|
|
23597
|
+
const stdoutPath = path45.join(dir, "stdout.txt");
|
|
23598
|
+
const stderrPath = path45.join(dir, "stderr.txt");
|
|
23601
23599
|
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
23602
23600
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
23603
23601
|
const { spawn: spawn5 } = await import("node:child_process");
|
|
@@ -23627,8 +23625,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
23627
23625
|
resolve2(code ?? 0);
|
|
23628
23626
|
});
|
|
23629
23627
|
});
|
|
23630
|
-
const stdout = (await
|
|
23631
|
-
const stderr = (await
|
|
23628
|
+
const stdout = (await readFile13(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
23629
|
+
const stderr = (await readFile13(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
23632
23630
|
return { stdout, stderr, exitCode };
|
|
23633
23631
|
} finally {
|
|
23634
23632
|
await rm6(dir, { recursive: true, force: true });
|
|
@@ -23939,7 +23937,7 @@ var CodeEvaluator = class {
|
|
|
23939
23937
|
outputPath,
|
|
23940
23938
|
guidelineFiles: context2.evalCase.guideline_paths,
|
|
23941
23939
|
inputFiles: context2.evalCase.file_paths.filter(
|
|
23942
|
-
(
|
|
23940
|
+
(path45) => !context2.evalCase.guideline_paths.includes(path45)
|
|
23943
23941
|
),
|
|
23944
23942
|
input: context2.evalCase.input,
|
|
23945
23943
|
trace: context2.trace ?? null,
|
|
@@ -24894,8 +24892,8 @@ function calculateScoreRangeResult(result, rubrics) {
|
|
|
24894
24892
|
};
|
|
24895
24893
|
}
|
|
24896
24894
|
function resolveSandboxed(basePath, relativePath) {
|
|
24897
|
-
const resolved =
|
|
24898
|
-
if (!resolved.startsWith(basePath +
|
|
24895
|
+
const resolved = path33.resolve(basePath, relativePath);
|
|
24896
|
+
if (!resolved.startsWith(basePath + path33.sep) && resolved !== basePath) {
|
|
24899
24897
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
24900
24898
|
}
|
|
24901
24899
|
return resolved;
|
|
@@ -24985,11 +24983,11 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
24985
24983
|
for (const entry of entries) {
|
|
24986
24984
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
24987
24985
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
24988
|
-
const fullPath =
|
|
24986
|
+
const fullPath = path33.join(dirPath, entry.name);
|
|
24989
24987
|
if (entry.isDirectory()) {
|
|
24990
24988
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
24991
24989
|
} else if (entry.isFile()) {
|
|
24992
|
-
const ext =
|
|
24990
|
+
const ext = path33.extname(entry.name).toLowerCase();
|
|
24993
24991
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
24994
24992
|
try {
|
|
24995
24993
|
const stat8 = await fs2.stat(fullPath);
|
|
@@ -25001,7 +24999,7 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
25001
24999
|
regex.lastIndex = 0;
|
|
25002
25000
|
if (regex.test(lines[i])) {
|
|
25003
25001
|
matches.push({
|
|
25004
|
-
file:
|
|
25002
|
+
file: path33.relative(workspacePath, fullPath),
|
|
25005
25003
|
line: i + 1,
|
|
25006
25004
|
text: lines[i].substring(0, 200)
|
|
25007
25005
|
});
|
|
@@ -25628,115 +25626,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
25628
25626
|
* Evaluate a single field against the expected value.
|
|
25629
25627
|
*/
|
|
25630
25628
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
25631
|
-
const { path:
|
|
25632
|
-
const candidateValue = resolvePath(candidateData,
|
|
25633
|
-
const expectedValue = resolvePath(expectedData,
|
|
25629
|
+
const { path: path45, match, required = true, weight = 1 } = fieldConfig;
|
|
25630
|
+
const candidateValue = resolvePath(candidateData, path45);
|
|
25631
|
+
const expectedValue = resolvePath(expectedData, path45);
|
|
25634
25632
|
if (expectedValue === void 0) {
|
|
25635
25633
|
return {
|
|
25636
|
-
path:
|
|
25634
|
+
path: path45,
|
|
25637
25635
|
score: 1,
|
|
25638
25636
|
// No expected value means no comparison needed
|
|
25639
25637
|
weight,
|
|
25640
25638
|
hit: true,
|
|
25641
|
-
message: `${
|
|
25639
|
+
message: `${path45}: no expected value`
|
|
25642
25640
|
};
|
|
25643
25641
|
}
|
|
25644
25642
|
if (candidateValue === void 0) {
|
|
25645
25643
|
if (required) {
|
|
25646
25644
|
return {
|
|
25647
|
-
path:
|
|
25645
|
+
path: path45,
|
|
25648
25646
|
score: 0,
|
|
25649
25647
|
weight,
|
|
25650
25648
|
hit: false,
|
|
25651
|
-
message: `${
|
|
25649
|
+
message: `${path45} (required, missing)`
|
|
25652
25650
|
};
|
|
25653
25651
|
}
|
|
25654
25652
|
return {
|
|
25655
|
-
path:
|
|
25653
|
+
path: path45,
|
|
25656
25654
|
score: 1,
|
|
25657
25655
|
// Don't penalize missing optional fields
|
|
25658
25656
|
weight: 0,
|
|
25659
25657
|
// Zero weight means it won't affect the score
|
|
25660
25658
|
hit: true,
|
|
25661
|
-
message: `${
|
|
25659
|
+
message: `${path45}: optional field missing`
|
|
25662
25660
|
};
|
|
25663
25661
|
}
|
|
25664
25662
|
switch (match) {
|
|
25665
25663
|
case "exact":
|
|
25666
|
-
return this.compareExact(
|
|
25664
|
+
return this.compareExact(path45, candidateValue, expectedValue, weight);
|
|
25667
25665
|
case "numeric_tolerance":
|
|
25668
25666
|
return this.compareNumericTolerance(
|
|
25669
|
-
|
|
25667
|
+
path45,
|
|
25670
25668
|
candidateValue,
|
|
25671
25669
|
expectedValue,
|
|
25672
25670
|
fieldConfig,
|
|
25673
25671
|
weight
|
|
25674
25672
|
);
|
|
25675
25673
|
case "date":
|
|
25676
|
-
return this.compareDate(
|
|
25674
|
+
return this.compareDate(path45, candidateValue, expectedValue, fieldConfig, weight);
|
|
25677
25675
|
default:
|
|
25678
25676
|
return {
|
|
25679
|
-
path:
|
|
25677
|
+
path: path45,
|
|
25680
25678
|
score: 0,
|
|
25681
25679
|
weight,
|
|
25682
25680
|
hit: false,
|
|
25683
|
-
message: `${
|
|
25681
|
+
message: `${path45}: unknown match type "${match}"`
|
|
25684
25682
|
};
|
|
25685
25683
|
}
|
|
25686
25684
|
}
|
|
25687
25685
|
/**
|
|
25688
25686
|
* Exact equality comparison.
|
|
25689
25687
|
*/
|
|
25690
|
-
compareExact(
|
|
25688
|
+
compareExact(path45, candidateValue, expectedValue, weight) {
|
|
25691
25689
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
25692
25690
|
return {
|
|
25693
|
-
path:
|
|
25691
|
+
path: path45,
|
|
25694
25692
|
score: 1,
|
|
25695
25693
|
weight,
|
|
25696
25694
|
hit: true,
|
|
25697
|
-
message:
|
|
25695
|
+
message: path45
|
|
25698
25696
|
};
|
|
25699
25697
|
}
|
|
25700
25698
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
25701
25699
|
return {
|
|
25702
|
-
path:
|
|
25700
|
+
path: path45,
|
|
25703
25701
|
score: 0,
|
|
25704
25702
|
weight,
|
|
25705
25703
|
hit: false,
|
|
25706
|
-
message: `${
|
|
25704
|
+
message: `${path45} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
25707
25705
|
};
|
|
25708
25706
|
}
|
|
25709
25707
|
return {
|
|
25710
|
-
path:
|
|
25708
|
+
path: path45,
|
|
25711
25709
|
score: 0,
|
|
25712
25710
|
weight,
|
|
25713
25711
|
hit: false,
|
|
25714
|
-
message: `${
|
|
25712
|
+
message: `${path45} (value mismatch)`
|
|
25715
25713
|
};
|
|
25716
25714
|
}
|
|
25717
25715
|
/**
|
|
25718
25716
|
* Numeric comparison with absolute or relative tolerance.
|
|
25719
25717
|
*/
|
|
25720
|
-
compareNumericTolerance(
|
|
25718
|
+
compareNumericTolerance(path45, candidateValue, expectedValue, fieldConfig, weight) {
|
|
25721
25719
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
25722
25720
|
const candidateNum = toNumber(candidateValue);
|
|
25723
25721
|
const expectedNum = toNumber(expectedValue);
|
|
25724
25722
|
if (candidateNum === null || expectedNum === null) {
|
|
25725
25723
|
return {
|
|
25726
|
-
path:
|
|
25724
|
+
path: path45,
|
|
25727
25725
|
score: 0,
|
|
25728
25726
|
weight,
|
|
25729
25727
|
hit: false,
|
|
25730
|
-
message: `${
|
|
25728
|
+
message: `${path45} (non-numeric value)`
|
|
25731
25729
|
};
|
|
25732
25730
|
}
|
|
25733
25731
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
25734
25732
|
return {
|
|
25735
|
-
path:
|
|
25733
|
+
path: path45,
|
|
25736
25734
|
score: 0,
|
|
25737
25735
|
weight,
|
|
25738
25736
|
hit: false,
|
|
25739
|
-
message: `${
|
|
25737
|
+
message: `${path45} (invalid numeric value)`
|
|
25740
25738
|
};
|
|
25741
25739
|
}
|
|
25742
25740
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -25749,61 +25747,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
25749
25747
|
}
|
|
25750
25748
|
if (withinTolerance) {
|
|
25751
25749
|
return {
|
|
25752
|
-
path:
|
|
25750
|
+
path: path45,
|
|
25753
25751
|
score: 1,
|
|
25754
25752
|
weight,
|
|
25755
25753
|
hit: true,
|
|
25756
|
-
message: `${
|
|
25754
|
+
message: `${path45} (within tolerance: diff=${diff.toFixed(2)})`
|
|
25757
25755
|
};
|
|
25758
25756
|
}
|
|
25759
25757
|
return {
|
|
25760
|
-
path:
|
|
25758
|
+
path: path45,
|
|
25761
25759
|
score: 0,
|
|
25762
25760
|
weight,
|
|
25763
25761
|
hit: false,
|
|
25764
|
-
message: `${
|
|
25762
|
+
message: `${path45} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
25765
25763
|
};
|
|
25766
25764
|
}
|
|
25767
25765
|
/**
|
|
25768
25766
|
* Date comparison with format normalization.
|
|
25769
25767
|
*/
|
|
25770
|
-
compareDate(
|
|
25768
|
+
compareDate(path45, candidateValue, expectedValue, fieldConfig, weight) {
|
|
25771
25769
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
25772
25770
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
25773
25771
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
25774
25772
|
if (candidateDate === null) {
|
|
25775
25773
|
return {
|
|
25776
|
-
path:
|
|
25774
|
+
path: path45,
|
|
25777
25775
|
score: 0,
|
|
25778
25776
|
weight,
|
|
25779
25777
|
hit: false,
|
|
25780
|
-
message: `${
|
|
25778
|
+
message: `${path45} (unparseable candidate date)`
|
|
25781
25779
|
};
|
|
25782
25780
|
}
|
|
25783
25781
|
if (expectedDate === null) {
|
|
25784
25782
|
return {
|
|
25785
|
-
path:
|
|
25783
|
+
path: path45,
|
|
25786
25784
|
score: 0,
|
|
25787
25785
|
weight,
|
|
25788
25786
|
hit: false,
|
|
25789
|
-
message: `${
|
|
25787
|
+
message: `${path45} (unparseable expected date)`
|
|
25790
25788
|
};
|
|
25791
25789
|
}
|
|
25792
25790
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
25793
25791
|
return {
|
|
25794
|
-
path:
|
|
25792
|
+
path: path45,
|
|
25795
25793
|
score: 1,
|
|
25796
25794
|
weight,
|
|
25797
25795
|
hit: true,
|
|
25798
|
-
message:
|
|
25796
|
+
message: path45
|
|
25799
25797
|
};
|
|
25800
25798
|
}
|
|
25801
25799
|
return {
|
|
25802
|
-
path:
|
|
25800
|
+
path: path45,
|
|
25803
25801
|
score: 0,
|
|
25804
25802
|
weight,
|
|
25805
25803
|
hit: false,
|
|
25806
|
-
message: `${
|
|
25804
|
+
message: `${path45} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
25807
25805
|
};
|
|
25808
25806
|
}
|
|
25809
25807
|
/**
|
|
@@ -25836,11 +25834,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
25836
25834
|
};
|
|
25837
25835
|
}
|
|
25838
25836
|
};
|
|
25839
|
-
function resolvePath(obj,
|
|
25840
|
-
if (!
|
|
25837
|
+
function resolvePath(obj, path45) {
|
|
25838
|
+
if (!path45 || !obj) {
|
|
25841
25839
|
return void 0;
|
|
25842
25840
|
}
|
|
25843
|
-
const parts =
|
|
25841
|
+
const parts = path45.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
25844
25842
|
let current = obj;
|
|
25845
25843
|
for (const part of parts) {
|
|
25846
25844
|
if (current === null || current === void 0) {
|
|
@@ -26290,8 +26288,8 @@ var TokenUsageEvaluator = class {
|
|
|
26290
26288
|
};
|
|
26291
26289
|
}
|
|
26292
26290
|
};
|
|
26293
|
-
function getNestedValue(obj,
|
|
26294
|
-
const parts =
|
|
26291
|
+
function getNestedValue(obj, path45) {
|
|
26292
|
+
const parts = path45.split(".");
|
|
26295
26293
|
let current = obj;
|
|
26296
26294
|
for (const part of parts) {
|
|
26297
26295
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -26435,11 +26433,9 @@ var ToolTrajectoryEvaluator = class {
|
|
|
26435
26433
|
for (const call of toolCalls) {
|
|
26436
26434
|
toolCallsByName[call.name] = (toolCallsByName[call.name] ?? 0) + 1;
|
|
26437
26435
|
}
|
|
26438
|
-
const toolNames = Object.keys(toolCallsByName).sort();
|
|
26439
26436
|
return {
|
|
26440
26437
|
eventCount: toolCalls.length,
|
|
26441
|
-
|
|
26442
|
-
toolCallsByName,
|
|
26438
|
+
toolCalls: toolCallsByName,
|
|
26443
26439
|
errorCount: 0
|
|
26444
26440
|
};
|
|
26445
26441
|
}
|
|
@@ -26457,7 +26453,7 @@ var ToolTrajectoryEvaluator = class {
|
|
|
26457
26453
|
const assertions = [];
|
|
26458
26454
|
for (const toolName of toolNames) {
|
|
26459
26455
|
const required = minimums[toolName];
|
|
26460
|
-
const actual = summary.
|
|
26456
|
+
const actual = summary.toolCalls[toolName] ?? 0;
|
|
26461
26457
|
if (actual >= required) {
|
|
26462
26458
|
assertions.push({
|
|
26463
26459
|
text: `${toolName}: called ${actual} times (required >=${required})`,
|
|
@@ -27159,7 +27155,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
|
27159
27155
|
};
|
|
27160
27156
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
27161
27157
|
const scriptPath = script[script.length - 1];
|
|
27162
|
-
const cwd =
|
|
27158
|
+
const cwd = path34.dirname(scriptPath);
|
|
27163
27159
|
try {
|
|
27164
27160
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
27165
27161
|
const prompt = stdout.trim();
|
|
@@ -27430,16 +27426,16 @@ function createBuiltinRegistry() {
|
|
|
27430
27426
|
async function discoverAssertions(registry, baseDir) {
|
|
27431
27427
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
27432
27428
|
const candidateDirs = [];
|
|
27433
|
-
let dir =
|
|
27434
|
-
const root =
|
|
27429
|
+
let dir = path35.resolve(baseDir);
|
|
27430
|
+
const root = path35.parse(dir).root;
|
|
27435
27431
|
while (dir !== root) {
|
|
27436
|
-
candidateDirs.push(
|
|
27437
|
-
dir =
|
|
27432
|
+
candidateDirs.push(path35.join(dir, ".agentv", "assertions"));
|
|
27433
|
+
dir = path35.dirname(dir);
|
|
27438
27434
|
}
|
|
27439
27435
|
let files = [];
|
|
27440
27436
|
for (const assertionsDir of candidateDirs) {
|
|
27441
27437
|
try {
|
|
27442
|
-
const found = await
|
|
27438
|
+
const found = await fg22(patterns, {
|
|
27443
27439
|
cwd: assertionsDir,
|
|
27444
27440
|
absolute: true,
|
|
27445
27441
|
onlyFiles: true
|
|
@@ -27450,7 +27446,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
27450
27446
|
}
|
|
27451
27447
|
const discoveredTypes = [];
|
|
27452
27448
|
for (const filePath of files) {
|
|
27453
|
-
const basename =
|
|
27449
|
+
const basename = path35.basename(filePath);
|
|
27454
27450
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
27455
27451
|
if (registry.has(typeName)) {
|
|
27456
27452
|
continue;
|
|
@@ -27469,17 +27465,17 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
27469
27465
|
async function discoverGraders(registry, baseDir) {
|
|
27470
27466
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
27471
27467
|
const candidateDirs = [];
|
|
27472
|
-
let dir =
|
|
27473
|
-
const root =
|
|
27468
|
+
let dir = path36.resolve(baseDir);
|
|
27469
|
+
const root = path36.parse(dir).root;
|
|
27474
27470
|
while (dir !== root) {
|
|
27475
|
-
candidateDirs.push(
|
|
27476
|
-
candidateDirs.push(
|
|
27477
|
-
dir =
|
|
27471
|
+
candidateDirs.push(path36.join(dir, ".agentv", "graders"));
|
|
27472
|
+
candidateDirs.push(path36.join(dir, ".agentv", "judges"));
|
|
27473
|
+
dir = path36.dirname(dir);
|
|
27478
27474
|
}
|
|
27479
27475
|
let files = [];
|
|
27480
27476
|
for (const gradersDir of candidateDirs) {
|
|
27481
27477
|
try {
|
|
27482
|
-
const found = await
|
|
27478
|
+
const found = await fg3(patterns, {
|
|
27483
27479
|
cwd: gradersDir,
|
|
27484
27480
|
absolute: true,
|
|
27485
27481
|
onlyFiles: true
|
|
@@ -27490,7 +27486,7 @@ async function discoverGraders(registry, baseDir) {
|
|
|
27490
27486
|
}
|
|
27491
27487
|
const discoveredTypes = [];
|
|
27492
27488
|
for (const filePath of files) {
|
|
27493
|
-
const basename =
|
|
27489
|
+
const basename = path36.basename(filePath);
|
|
27494
27490
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
27495
27491
|
if (registry.has(typeName)) {
|
|
27496
27492
|
continue;
|
|
@@ -27676,10 +27672,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
27676
27672
|
}
|
|
27677
27673
|
for (const entry of entries) {
|
|
27678
27674
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
27679
|
-
const childPath =
|
|
27675
|
+
const childPath = path37.join(workspacePath, entry);
|
|
27680
27676
|
try {
|
|
27681
27677
|
if (!statSync(childPath).isDirectory()) continue;
|
|
27682
|
-
if (!statSync(
|
|
27678
|
+
if (!statSync(path37.join(childPath, ".git")).isDirectory()) continue;
|
|
27683
27679
|
} catch {
|
|
27684
27680
|
continue;
|
|
27685
27681
|
}
|
|
@@ -27716,14 +27712,14 @@ async function isDirectory(filePath) {
|
|
|
27716
27712
|
}
|
|
27717
27713
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
27718
27714
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
27719
|
-
return
|
|
27715
|
+
return path38.join(root, evalRunId, caseId);
|
|
27720
27716
|
}
|
|
27721
27717
|
async function copyDirectoryRecursive(src, dest) {
|
|
27722
27718
|
await mkdir11(dest, { recursive: true });
|
|
27723
27719
|
const entries = await readdir3(src, { withFileTypes: true });
|
|
27724
27720
|
for (const entry of entries) {
|
|
27725
|
-
const srcPath =
|
|
27726
|
-
const destPath =
|
|
27721
|
+
const srcPath = path38.join(src, entry.name);
|
|
27722
|
+
const destPath = path38.join(dest, entry.name);
|
|
27727
27723
|
if (entry.name === ".git") {
|
|
27728
27724
|
continue;
|
|
27729
27725
|
}
|
|
@@ -27735,7 +27731,7 @@ async function copyDirectoryRecursive(src, dest) {
|
|
|
27735
27731
|
}
|
|
27736
27732
|
}
|
|
27737
27733
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
27738
|
-
const resolvedTemplatePath =
|
|
27734
|
+
const resolvedTemplatePath = path38.resolve(templatePath);
|
|
27739
27735
|
if (!await fileExists(resolvedTemplatePath)) {
|
|
27740
27736
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
27741
27737
|
}
|
|
@@ -27784,7 +27780,7 @@ async function cleanupWorkspace(workspacePath) {
|
|
|
27784
27780
|
}
|
|
27785
27781
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
27786
27782
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
27787
|
-
const evalDir =
|
|
27783
|
+
const evalDir = path38.join(root, evalRunId);
|
|
27788
27784
|
if (await fileExists(evalDir)) {
|
|
27789
27785
|
await rm4(evalDir, { recursive: true, force: true });
|
|
27790
27786
|
}
|
|
@@ -27841,8 +27837,8 @@ async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
|
27841
27837
|
await mkdir12(dest, { recursive: true });
|
|
27842
27838
|
const entries = await readdir4(src, { withFileTypes: true });
|
|
27843
27839
|
for (const entry of entries) {
|
|
27844
|
-
const srcPath =
|
|
27845
|
-
const destPath =
|
|
27840
|
+
const srcPath = path39.join(src, entry.name);
|
|
27841
|
+
const destPath = path39.join(dest, entry.name);
|
|
27846
27842
|
if (entry.name === ".git") {
|
|
27847
27843
|
continue;
|
|
27848
27844
|
}
|
|
@@ -27875,7 +27871,7 @@ var WorkspacePoolManager = class {
|
|
|
27875
27871
|
async acquireWorkspace(options) {
|
|
27876
27872
|
const { templatePath, repos, maxSlots, repoManager, poolReset } = options;
|
|
27877
27873
|
const fingerprint = computeWorkspaceFingerprint(repos);
|
|
27878
|
-
const poolDir =
|
|
27874
|
+
const poolDir = path39.join(this.poolRoot, fingerprint);
|
|
27879
27875
|
await mkdir12(poolDir, { recursive: true });
|
|
27880
27876
|
const drifted = await this.checkDrift(poolDir, fingerprint);
|
|
27881
27877
|
if (drifted) {
|
|
@@ -27885,7 +27881,7 @@ var WorkspacePoolManager = class {
|
|
|
27885
27881
|
await this.removeAllSlots(poolDir);
|
|
27886
27882
|
}
|
|
27887
27883
|
for (let i = 0; i < maxSlots; i++) {
|
|
27888
|
-
const slotPath =
|
|
27884
|
+
const slotPath = path39.join(poolDir, `slot-${i}`);
|
|
27889
27885
|
const lockPath = `${slotPath}.lock`;
|
|
27890
27886
|
const locked = await this.tryLock(lockPath);
|
|
27891
27887
|
if (!locked) {
|
|
@@ -27947,7 +27943,7 @@ var WorkspacePoolManager = class {
|
|
|
27947
27943
|
throw err;
|
|
27948
27944
|
}
|
|
27949
27945
|
try {
|
|
27950
|
-
const pidStr = await
|
|
27946
|
+
const pidStr = await readFile11(lockPath, "utf-8");
|
|
27951
27947
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
27952
27948
|
if (!Number.isNaN(pid)) {
|
|
27953
27949
|
try {
|
|
@@ -27972,9 +27968,9 @@ var WorkspacePoolManager = class {
|
|
|
27972
27968
|
* Returns false (no drift) if metadata.json doesn't exist (first use).
|
|
27973
27969
|
*/
|
|
27974
27970
|
async checkDrift(poolDir, fingerprint) {
|
|
27975
|
-
const metadataPath =
|
|
27971
|
+
const metadataPath = path39.join(poolDir, "metadata.json");
|
|
27976
27972
|
try {
|
|
27977
|
-
const raw = await
|
|
27973
|
+
const raw = await readFile11(metadataPath, "utf-8");
|
|
27978
27974
|
const metadata = JSON.parse(raw);
|
|
27979
27975
|
return metadata.fingerprint !== fingerprint;
|
|
27980
27976
|
} catch {
|
|
@@ -27989,17 +27985,17 @@ var WorkspacePoolManager = class {
|
|
|
27989
27985
|
repos,
|
|
27990
27986
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
27991
27987
|
};
|
|
27992
|
-
await writeFile7(
|
|
27988
|
+
await writeFile7(path39.join(poolDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
27993
27989
|
}
|
|
27994
27990
|
/** Remove all slot directories and their lock files from a pool directory. */
|
|
27995
27991
|
async removeAllSlots(poolDir) {
|
|
27996
27992
|
const entries = await readdir4(poolDir);
|
|
27997
27993
|
for (const entry of entries) {
|
|
27998
27994
|
if (entry.startsWith("slot-") && !entry.endsWith(".lock")) {
|
|
27999
|
-
const lockPath =
|
|
27995
|
+
const lockPath = path39.join(poolDir, `${entry}.lock`);
|
|
28000
27996
|
if (existsSync2(lockPath)) {
|
|
28001
27997
|
try {
|
|
28002
|
-
const pidStr = await
|
|
27998
|
+
const pidStr = await readFile11(lockPath, "utf-8");
|
|
28003
27999
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
28004
28000
|
if (!Number.isNaN(pid)) {
|
|
28005
28001
|
try {
|
|
@@ -28012,12 +28008,12 @@ var WorkspacePoolManager = class {
|
|
|
28012
28008
|
} catch {
|
|
28013
28009
|
}
|
|
28014
28010
|
}
|
|
28015
|
-
await rm5(
|
|
28011
|
+
await rm5(path39.join(poolDir, entry), { recursive: true, force: true });
|
|
28016
28012
|
await rm5(lockPath, { force: true }).catch(() => {
|
|
28017
28013
|
});
|
|
28018
28014
|
}
|
|
28019
28015
|
}
|
|
28020
|
-
await rm5(
|
|
28016
|
+
await rm5(path39.join(poolDir, "metadata.json"), { force: true }).catch(() => {
|
|
28021
28017
|
});
|
|
28022
28018
|
}
|
|
28023
28019
|
/**
|
|
@@ -28027,7 +28023,7 @@ var WorkspacePoolManager = class {
|
|
|
28027
28023
|
*/
|
|
28028
28024
|
async resetSlot(slotPath, templatePath, repos, poolReset = "fast") {
|
|
28029
28025
|
for (const repo of repos) {
|
|
28030
|
-
const repoDir =
|
|
28026
|
+
const repoDir = path39.join(slotPath, repo.path);
|
|
28031
28027
|
if (!existsSync2(repoDir)) {
|
|
28032
28028
|
continue;
|
|
28033
28029
|
}
|
|
@@ -28148,7 +28144,7 @@ ${lines.join("\n")}`;
|
|
|
28148
28144
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
28149
28145
|
*/
|
|
28150
28146
|
async materialize(repo, workspacePath) {
|
|
28151
|
-
const targetDir =
|
|
28147
|
+
const targetDir = path40.join(workspacePath, repo.path);
|
|
28152
28148
|
const sourceUrl = getSourceUrl(repo.source);
|
|
28153
28149
|
const startedAt = Date.now();
|
|
28154
28150
|
if (this.verbose) {
|
|
@@ -28239,7 +28235,7 @@ ${lines.join("\n")}`;
|
|
|
28239
28235
|
async reset(repos, workspacePath, reset) {
|
|
28240
28236
|
const cleanFlag = reset === "strict" ? "-fdx" : "-fd";
|
|
28241
28237
|
for (const repo of repos) {
|
|
28242
|
-
const targetDir =
|
|
28238
|
+
const targetDir = path40.join(workspacePath, repo.path);
|
|
28243
28239
|
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
28244
28240
|
await this.runGit(["clean", cleanFlag], { cwd: targetDir });
|
|
28245
28241
|
}
|
|
@@ -28249,11 +28245,11 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
28249
28245
|
if (!templatePath) {
|
|
28250
28246
|
return void 0;
|
|
28251
28247
|
}
|
|
28252
|
-
const resolved =
|
|
28248
|
+
const resolved = path41.resolve(templatePath);
|
|
28253
28249
|
const stats = await stat6(resolved);
|
|
28254
28250
|
if (stats.isFile()) {
|
|
28255
28251
|
return {
|
|
28256
|
-
dir:
|
|
28252
|
+
dir: path41.dirname(resolved),
|
|
28257
28253
|
workspaceFile: resolved
|
|
28258
28254
|
};
|
|
28259
28255
|
}
|
|
@@ -28265,14 +28261,14 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
28265
28261
|
if (workspaceFiles.length === 1) {
|
|
28266
28262
|
return {
|
|
28267
28263
|
dir: resolved,
|
|
28268
|
-
workspaceFile:
|
|
28264
|
+
workspaceFile: path41.join(resolved, workspaceFiles[0])
|
|
28269
28265
|
};
|
|
28270
28266
|
}
|
|
28271
28267
|
if (workspaceFiles.length > 1) {
|
|
28272
28268
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
28273
28269
|
return {
|
|
28274
28270
|
dir: resolved,
|
|
28275
|
-
workspaceFile: conventionFile ?
|
|
28271
|
+
workspaceFile: conventionFile ? path41.join(resolved, conventionFile) : void 0
|
|
28276
28272
|
};
|
|
28277
28273
|
}
|
|
28278
28274
|
return { dir: resolved };
|
|
@@ -28472,7 +28468,7 @@ async function runEvaluation(options) {
|
|
|
28472
28468
|
];
|
|
28473
28469
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveGraderProvider);
|
|
28474
28470
|
const typeRegistry = createBuiltinRegistry();
|
|
28475
|
-
const discoveryBaseDir = evalFilePath ?
|
|
28471
|
+
const discoveryBaseDir = evalFilePath ? path422.dirname(path422.resolve(evalFilePath)) : process.cwd();
|
|
28476
28472
|
const evalDir = discoveryBaseDir;
|
|
28477
28473
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
28478
28474
|
await discoverGraders(typeRegistry, discoveryBaseDir);
|
|
@@ -28661,7 +28657,7 @@ async function runEvaluation(options) {
|
|
|
28661
28657
|
}
|
|
28662
28658
|
try {
|
|
28663
28659
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
28664
|
-
const copiedWorkspaceFile =
|
|
28660
|
+
const copiedWorkspaceFile = path422.join(sharedWorkspacePath, path422.basename(suiteWorkspaceFile));
|
|
28665
28661
|
try {
|
|
28666
28662
|
await stat7(copiedWorkspaceFile);
|
|
28667
28663
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
@@ -28774,7 +28770,7 @@ async function runEvaluation(options) {
|
|
|
28774
28770
|
dataset: evalCase.dataset,
|
|
28775
28771
|
score: 0,
|
|
28776
28772
|
assertions: [],
|
|
28777
|
-
|
|
28773
|
+
output: [],
|
|
28778
28774
|
target: target.name,
|
|
28779
28775
|
error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
|
|
28780
28776
|
budgetExceeded: true,
|
|
@@ -28810,7 +28806,7 @@ async function runEvaluation(options) {
|
|
|
28810
28806
|
dataset: evalCase.dataset,
|
|
28811
28807
|
score: 0,
|
|
28812
28808
|
assertions: [],
|
|
28813
|
-
|
|
28809
|
+
output: [],
|
|
28814
28810
|
target: target.name,
|
|
28815
28811
|
error: errorMsg,
|
|
28816
28812
|
executionStatus: "execution_error",
|
|
@@ -29075,7 +29071,7 @@ async function runBatchEvaluation(options) {
|
|
|
29075
29071
|
const providerResponse = batchResponse[i];
|
|
29076
29072
|
const output = providerResponse.output;
|
|
29077
29073
|
const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
|
|
29078
|
-
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0,
|
|
29074
|
+
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
|
|
29079
29075
|
const merged = computed ? mergeExecutionMetrics(computed, {
|
|
29080
29076
|
tokenUsage: providerResponse.tokenUsage,
|
|
29081
29077
|
costUsd: providerResponse.costUsd,
|
|
@@ -29238,7 +29234,7 @@ async function runEvalCase(options) {
|
|
|
29238
29234
|
);
|
|
29239
29235
|
}
|
|
29240
29236
|
if (caseWorkspaceFile && workspacePath) {
|
|
29241
|
-
const copiedFile =
|
|
29237
|
+
const copiedFile = path422.join(workspacePath, path422.basename(caseWorkspaceFile));
|
|
29242
29238
|
try {
|
|
29243
29239
|
await stat7(copiedFile);
|
|
29244
29240
|
caseWorkspaceFile = copiedFile;
|
|
@@ -29298,10 +29294,10 @@ async function runEvalCase(options) {
|
|
|
29298
29294
|
const files = evalCase.metadata.agent_skills_files;
|
|
29299
29295
|
if (baseDir && files.length > 0) {
|
|
29300
29296
|
for (const relPath of files) {
|
|
29301
|
-
const srcPath =
|
|
29302
|
-
const destPath =
|
|
29297
|
+
const srcPath = path422.resolve(baseDir, relPath);
|
|
29298
|
+
const destPath = path422.resolve(workspacePath, relPath);
|
|
29303
29299
|
try {
|
|
29304
|
-
await mkdir13(
|
|
29300
|
+
await mkdir13(path422.dirname(destPath), { recursive: true });
|
|
29305
29301
|
await copyFile2(srcPath, destPath);
|
|
29306
29302
|
} catch (error) {
|
|
29307
29303
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -29472,7 +29468,7 @@ async function runEvalCase(options) {
|
|
|
29472
29468
|
}
|
|
29473
29469
|
const output = providerResponse.output;
|
|
29474
29470
|
const hasExecutionMetrics = providerResponse.tokenUsage !== void 0 || providerResponse.costUsd !== void 0 || providerResponse.durationMs !== void 0;
|
|
29475
|
-
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0,
|
|
29471
|
+
const computed = output ? computeTraceSummary(output) : hasExecutionMetrics ? { trace: { eventCount: 0, toolCalls: {}, errorCount: 0 } } : void 0;
|
|
29476
29472
|
const merged = computed ? mergeExecutionMetrics(computed, {
|
|
29477
29473
|
tokenUsage: providerResponse.tokenUsage,
|
|
29478
29474
|
costUsd: providerResponse.costUsd,
|
|
@@ -29777,7 +29773,6 @@ async function evaluateCandidate(options) {
|
|
|
29777
29773
|
conversationId: evalCase.conversation_id,
|
|
29778
29774
|
score: score.score,
|
|
29779
29775
|
assertions: score.assertions,
|
|
29780
|
-
outputText: candidate,
|
|
29781
29776
|
target: target.name,
|
|
29782
29777
|
tokenUsage,
|
|
29783
29778
|
costUsd,
|
|
@@ -29788,7 +29783,7 @@ async function evaluateCandidate(options) {
|
|
|
29788
29783
|
input,
|
|
29789
29784
|
scores,
|
|
29790
29785
|
trace: trace2,
|
|
29791
|
-
output,
|
|
29786
|
+
output: output ?? [{ role: "assistant", content: candidate }],
|
|
29792
29787
|
fileChanges,
|
|
29793
29788
|
executionStatus: classifyQualityStatus(score.score)
|
|
29794
29789
|
};
|
|
@@ -29922,7 +29917,7 @@ async function runEvaluatorList(options) {
|
|
|
29922
29917
|
fileChanges,
|
|
29923
29918
|
workspacePath
|
|
29924
29919
|
};
|
|
29925
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
29920
|
+
const evalFileDir = evalCase.guideline_paths[0] ? path422.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
29926
29921
|
const dispatchContext = {
|
|
29927
29922
|
graderProvider,
|
|
29928
29923
|
targetResolver,
|
|
@@ -29953,7 +29948,7 @@ async function runEvaluatorList(options) {
|
|
|
29953
29948
|
weight,
|
|
29954
29949
|
verdict: score2.verdict,
|
|
29955
29950
|
assertions: score2.assertions,
|
|
29956
|
-
|
|
29951
|
+
input: score2.evaluatorRawRequest,
|
|
29957
29952
|
details: score2.details,
|
|
29958
29953
|
scores: mapChildResults(score2.scores),
|
|
29959
29954
|
tokenUsage: score2.tokenUsage,
|
|
@@ -30133,7 +30128,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
|
|
|
30133
30128
|
conversationId: evalCase.conversation_id,
|
|
30134
30129
|
score: 0,
|
|
30135
30130
|
assertions: [{ text: `Error: ${message}`, passed: false }],
|
|
30136
|
-
|
|
30131
|
+
output: [{ role: "assistant", content: `Error occurred: ${message}` }],
|
|
30137
30132
|
target: targetName,
|
|
30138
30133
|
requests,
|
|
30139
30134
|
input,
|
|
@@ -30177,7 +30172,7 @@ function buildResultInput(promptInputs) {
|
|
|
30177
30172
|
content: message.content
|
|
30178
30173
|
}));
|
|
30179
30174
|
}
|
|
30180
|
-
return promptInputs.question;
|
|
30175
|
+
return [{ role: "user", content: promptInputs.question }];
|
|
30181
30176
|
}
|
|
30182
30177
|
function aggregateEvaluatorTokenUsage(scores) {
|
|
30183
30178
|
if (!scores || scores.length === 0) return void 0;
|
|
@@ -30243,7 +30238,7 @@ function mapChildResults(children) {
|
|
|
30243
30238
|
weight: child.weight,
|
|
30244
30239
|
verdict: child.verdict,
|
|
30245
30240
|
assertions: child.assertions,
|
|
30246
|
-
|
|
30241
|
+
input: child.evaluatorRawRequest,
|
|
30247
30242
|
scores: mapChildResults(child.scores),
|
|
30248
30243
|
details: child.details,
|
|
30249
30244
|
tokenUsage: child.tokenUsage
|
|
@@ -30291,7 +30286,7 @@ async function evaluate(config) {
|
|
|
30291
30286
|
}
|
|
30292
30287
|
const gitRoot = await findGitRoot(process.cwd());
|
|
30293
30288
|
const repoRoot = gitRoot ?? process.cwd();
|
|
30294
|
-
const testFilePath = config.specFile ?
|
|
30289
|
+
const testFilePath = config.specFile ? path43.resolve(config.specFile) : path43.join(process.cwd(), "__programmatic__.yaml");
|
|
30295
30290
|
await loadEnvHierarchy(repoRoot, testFilePath);
|
|
30296
30291
|
let resolvedTarget;
|
|
30297
30292
|
let taskProvider;
|
|
@@ -30420,10 +30415,10 @@ function computeSummary(results, durationMs) {
|
|
|
30420
30415
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
30421
30416
|
async function discoverDefaultTarget(repoRoot) {
|
|
30422
30417
|
const cwd = process.cwd();
|
|
30423
|
-
const chain = buildDirectoryChain(
|
|
30418
|
+
const chain = buildDirectoryChain(path43.join(cwd, "_placeholder"), repoRoot);
|
|
30424
30419
|
for (const dir of chain) {
|
|
30425
30420
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
30426
|
-
const targetsPath =
|
|
30421
|
+
const targetsPath = path43.join(dir, candidate);
|
|
30427
30422
|
if (!existsSync4(targetsPath)) continue;
|
|
30428
30423
|
try {
|
|
30429
30424
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -30440,7 +30435,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
|
|
|
30440
30435
|
const chain = buildDirectoryChain(startPath, repoRoot);
|
|
30441
30436
|
const envFiles = [];
|
|
30442
30437
|
for (const dir of chain) {
|
|
30443
|
-
const envPath =
|
|
30438
|
+
const envPath = path43.join(dir, ".env");
|
|
30444
30439
|
if (existsSync4(envPath)) envFiles.push(envPath);
|
|
30445
30440
|
}
|
|
30446
30441
|
for (let i = 0; i < envFiles.length; i++) {
|
|
@@ -30621,7 +30616,7 @@ var ResponseCache = class {
|
|
|
30621
30616
|
async get(key) {
|
|
30622
30617
|
const filePath = this.keyToPath(key);
|
|
30623
30618
|
try {
|
|
30624
|
-
const data = await
|
|
30619
|
+
const data = await readFile12(filePath, "utf8");
|
|
30625
30620
|
return JSON.parse(data);
|
|
30626
30621
|
} catch {
|
|
30627
30622
|
return void 0;
|
|
@@ -30629,13 +30624,13 @@ var ResponseCache = class {
|
|
|
30629
30624
|
}
|
|
30630
30625
|
async set(key, value) {
|
|
30631
30626
|
const filePath = this.keyToPath(key);
|
|
30632
|
-
const dir =
|
|
30627
|
+
const dir = path44.dirname(filePath);
|
|
30633
30628
|
await mkdir14(dir, { recursive: true });
|
|
30634
30629
|
await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
30635
30630
|
}
|
|
30636
30631
|
keyToPath(key) {
|
|
30637
30632
|
const prefix = key.slice(0, 2);
|
|
30638
|
-
return
|
|
30633
|
+
return path44.join(this.cachePath, prefix, `${key}.json`);
|
|
30639
30634
|
}
|
|
30640
30635
|
};
|
|
30641
30636
|
function shouldEnableCache(params) {
|
|
@@ -30650,7 +30645,6 @@ function shouldSkipCacheForTemperature(targetConfig) {
|
|
|
30650
30645
|
return false;
|
|
30651
30646
|
}
|
|
30652
30647
|
var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
30653
|
-
"outputText",
|
|
30654
30648
|
"requests",
|
|
30655
30649
|
"trace",
|
|
30656
30650
|
"workspacePath",
|
|
@@ -30667,7 +30661,7 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
30667
30661
|
"startTime",
|
|
30668
30662
|
"endTime"
|
|
30669
30663
|
]);
|
|
30670
|
-
var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "
|
|
30664
|
+
var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "input"]);
|
|
30671
30665
|
function trimEvaluatorResult(result) {
|
|
30672
30666
|
const trimmed = {};
|
|
30673
30667
|
for (const [key, value] of Object.entries(result)) {
|
|
@@ -30822,14 +30816,21 @@ var OtelTraceExporter = class {
|
|
|
30822
30816
|
rootSpan.setAttribute("agentv.target", result.target);
|
|
30823
30817
|
if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
|
|
30824
30818
|
rootSpan.setAttribute("agentv.score", result.score);
|
|
30825
|
-
if (captureContent
|
|
30819
|
+
if (captureContent && result.output.length > 0) {
|
|
30820
|
+
const lastMsg = result.output[result.output.length - 1];
|
|
30821
|
+
const text2 = typeof lastMsg.content === "string" ? lastMsg.content : JSON.stringify(lastMsg.content);
|
|
30822
|
+
rootSpan.setAttribute("agentv.output_text", text2);
|
|
30823
|
+
}
|
|
30826
30824
|
if (result.durationMs != null)
|
|
30827
30825
|
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
30828
30826
|
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
30829
30827
|
if (result.trace) {
|
|
30830
30828
|
const t = result.trace;
|
|
30831
30829
|
rootSpan.setAttribute("agentv.trace.event_count", t.eventCount);
|
|
30832
|
-
rootSpan.setAttribute(
|
|
30830
|
+
rootSpan.setAttribute(
|
|
30831
|
+
"agentv.trace.tool_names",
|
|
30832
|
+
Object.keys(t.toolCalls).sort().join(",")
|
|
30833
|
+
);
|
|
30833
30834
|
if (t.llmCallCount != null)
|
|
30834
30835
|
rootSpan.setAttribute("agentv.trace.llm_call_count", t.llmCallCount);
|
|
30835
30836
|
}
|
|
@@ -31120,6 +31121,8 @@ export {
|
|
|
31120
31121
|
resolveFileReference,
|
|
31121
31122
|
CLI_PLACEHOLDERS,
|
|
31122
31123
|
resolveTargetDefinition,
|
|
31124
|
+
interpolateEnv,
|
|
31125
|
+
loadCasesFromFile,
|
|
31123
31126
|
KNOWN_PROVIDERS,
|
|
31124
31127
|
PROVIDER_ALIASES,
|
|
31125
31128
|
computeTraceSummary,
|
|
@@ -31249,4 +31252,4 @@ export {
|
|
|
31249
31252
|
OtelStreamingObserver,
|
|
31250
31253
|
createAgentKernel
|
|
31251
31254
|
};
|
|
31252
|
-
//# sourceMappingURL=chunk-
|
|
31255
|
+
//# sourceMappingURL=chunk-F4UDJ7LG.js.map
|