agentv 3.7.0 → 3.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7YS6YNJZ.js → chunk-ASYRKFAI.js} +125 -13
- package/dist/chunk-ASYRKFAI.js.map +1 -0
- package/dist/{chunk-XGG64VIY.js → chunk-F4UDJ7LG.js} +549 -545
- package/dist/chunk-F4UDJ7LG.js.map +1 -0
- package/dist/{chunk-TR6H437M.js → chunk-YZRGQ6ZS.js} +13 -9
- package/dist/chunk-YZRGQ6ZS.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{dist-VP6AXX6B.js → dist-4AQUJJAP.js} +2 -2
- package/dist/index.js +3 -3
- package/dist/{interactive-F6XECJ33.js → interactive-OPQGDF77.js} +3 -3
- package/package.json +1 -1
- package/dist/chunk-7YS6YNJZ.js.map +0 -1
- package/dist/chunk-TR6H437M.js.map +0 -1
- package/dist/chunk-XGG64VIY.js.map +0 -1
- /package/dist/{dist-VP6AXX6B.js.map → dist-4AQUJJAP.js.map} +0 -0
- /package/dist/{interactive-F6XECJ33.js.map → interactive-OPQGDF77.js.map} +0 -0
|
@@ -301,7 +301,7 @@ var require_dist = __commonJS({
|
|
|
301
301
|
}
|
|
302
302
|
});
|
|
303
303
|
|
|
304
|
-
// ../../packages/core/dist/chunk-
|
|
304
|
+
// ../../packages/core/dist/chunk-3ZS3GCMI.js
|
|
305
305
|
import { constants } from "node:fs";
|
|
306
306
|
import { access, readFile } from "node:fs/promises";
|
|
307
307
|
import path from "node:path";
|
|
@@ -419,7 +419,11 @@ __export(external_exports2, {
|
|
|
419
419
|
void: () => voidType
|
|
420
420
|
});
|
|
421
421
|
|
|
422
|
-
// ../../packages/core/dist/chunk-
|
|
422
|
+
// ../../packages/core/dist/chunk-3ZS3GCMI.js
|
|
423
|
+
import { readFile as readFile2 } from "node:fs/promises";
|
|
424
|
+
import path3 from "node:path";
|
|
425
|
+
import fg from "fast-glob";
|
|
426
|
+
import { parse as parseYaml } from "yaml";
|
|
423
427
|
var TEST_MESSAGE_ROLE_VALUES = ["system", "user", "assistant", "tool"];
|
|
424
428
|
var TEST_MESSAGE_ROLES = TEST_MESSAGE_ROLE_VALUES;
|
|
425
429
|
var TEST_MESSAGE_ROLE_SET = new Set(TEST_MESSAGE_ROLE_VALUES);
|
|
@@ -1595,8 +1599,8 @@ function resolveCliConfig(target, env, evalFilePath) {
|
|
|
1595
1599
|
const parseResult = CliTargetInputSchema.safeParse(target, { errorMap: cliErrorMap });
|
|
1596
1600
|
if (!parseResult.success) {
|
|
1597
1601
|
const firstError = parseResult.error.errors[0];
|
|
1598
|
-
const
|
|
1599
|
-
const prefix =
|
|
1602
|
+
const path45 = firstError?.path.join(".") || "";
|
|
1603
|
+
const prefix = path45 ? `${target.name} ${path45}: ` : `${target.name}: `;
|
|
1600
1604
|
throw new Error(`${prefix}${firstError?.message}`);
|
|
1601
1605
|
}
|
|
1602
1606
|
const normalized = normalizeCliTargetInput(parseResult.data, env, evalFilePath);
|
|
@@ -1802,6 +1806,135 @@ function resolveOptionalNumberArray(source, description) {
|
|
|
1802
1806
|
}
|
|
1803
1807
|
return resolved.length > 0 ? resolved : void 0;
|
|
1804
1808
|
}
|
|
1809
|
+
var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
1810
|
+
function interpolateEnv(value, env) {
|
|
1811
|
+
if (typeof value === "string") {
|
|
1812
|
+
return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
|
|
1813
|
+
}
|
|
1814
|
+
if (Array.isArray(value)) {
|
|
1815
|
+
return value.map((item) => interpolateEnv(item, env));
|
|
1816
|
+
}
|
|
1817
|
+
if (value !== null && typeof value === "object") {
|
|
1818
|
+
const result = {};
|
|
1819
|
+
for (const [key, val] of Object.entries(value)) {
|
|
1820
|
+
result[key] = interpolateEnv(val, env);
|
|
1821
|
+
}
|
|
1822
|
+
return result;
|
|
1823
|
+
}
|
|
1824
|
+
return value;
|
|
1825
|
+
}
|
|
1826
|
+
var ANSI_YELLOW = "\x1B[33m";
|
|
1827
|
+
var ANSI_RESET = "\x1B[0m";
|
|
1828
|
+
var FILE_PROTOCOL = "file://";
|
|
1829
|
+
function isFileReference(value) {
|
|
1830
|
+
return typeof value === "string" && value.startsWith(FILE_PROTOCOL);
|
|
1831
|
+
}
|
|
1832
|
+
function extractFilePath(ref) {
|
|
1833
|
+
return ref.slice(FILE_PROTOCOL.length);
|
|
1834
|
+
}
|
|
1835
|
+
function isGlobPattern(filePath) {
|
|
1836
|
+
return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
|
|
1837
|
+
}
|
|
1838
|
+
function parseYamlCases(content, filePath) {
|
|
1839
|
+
const raw = parseYaml(content);
|
|
1840
|
+
const parsed = interpolateEnv(raw, process.env);
|
|
1841
|
+
if (!Array.isArray(parsed)) {
|
|
1842
|
+
throw new Error(
|
|
1843
|
+
`External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
|
|
1844
|
+
);
|
|
1845
|
+
}
|
|
1846
|
+
const results = [];
|
|
1847
|
+
for (const item of parsed) {
|
|
1848
|
+
if (!isJsonObject(item)) {
|
|
1849
|
+
throw new Error(`External test file contains non-object entry: ${filePath}`);
|
|
1850
|
+
}
|
|
1851
|
+
results.push(item);
|
|
1852
|
+
}
|
|
1853
|
+
return results;
|
|
1854
|
+
}
|
|
1855
|
+
function parseJsonlCases(content, filePath) {
|
|
1856
|
+
const lines = content.split("\n");
|
|
1857
|
+
const results = [];
|
|
1858
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1859
|
+
const line = lines[i].trim();
|
|
1860
|
+
if (line === "") continue;
|
|
1861
|
+
try {
|
|
1862
|
+
const raw = JSON.parse(line);
|
|
1863
|
+
const parsed = interpolateEnv(raw, process.env);
|
|
1864
|
+
if (!isJsonObject(parsed)) {
|
|
1865
|
+
throw new Error("Expected JSON object");
|
|
1866
|
+
}
|
|
1867
|
+
results.push(parsed);
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1870
|
+
throw new Error(`Malformed JSONL at line ${i + 1}: ${message}
|
|
1871
|
+
File: ${filePath}`);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
return results;
|
|
1875
|
+
}
|
|
1876
|
+
async function loadCasesFromFile(filePath) {
|
|
1877
|
+
const ext = path3.extname(filePath).toLowerCase();
|
|
1878
|
+
let content;
|
|
1879
|
+
try {
|
|
1880
|
+
content = await readFile2(filePath, "utf8");
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1883
|
+
throw new Error(`Cannot read external test file: ${filePath}
|
|
1884
|
+
${message}`);
|
|
1885
|
+
}
|
|
1886
|
+
if (content.trim() === "") {
|
|
1887
|
+
console.warn(
|
|
1888
|
+
`${ANSI_YELLOW}Warning: External test file is empty, skipping: ${filePath}${ANSI_RESET}`
|
|
1889
|
+
);
|
|
1890
|
+
return [];
|
|
1891
|
+
}
|
|
1892
|
+
if (ext === ".yaml" || ext === ".yml") {
|
|
1893
|
+
return parseYamlCases(content, filePath);
|
|
1894
|
+
}
|
|
1895
|
+
if (ext === ".jsonl") {
|
|
1896
|
+
return parseJsonlCases(content, filePath);
|
|
1897
|
+
}
|
|
1898
|
+
throw new Error(
|
|
1899
|
+
`Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
|
|
1900
|
+
);
|
|
1901
|
+
}
|
|
1902
|
+
async function resolveFileReference2(ref, evalFileDir) {
|
|
1903
|
+
const rawPath = extractFilePath(ref);
|
|
1904
|
+
const absolutePattern = path3.resolve(evalFileDir, rawPath);
|
|
1905
|
+
if (isGlobPattern(rawPath)) {
|
|
1906
|
+
const matches = await fg(absolutePattern.replaceAll("\\", "/"), {
|
|
1907
|
+
onlyFiles: true,
|
|
1908
|
+
absolute: true
|
|
1909
|
+
});
|
|
1910
|
+
if (matches.length === 0) {
|
|
1911
|
+
console.warn(
|
|
1912
|
+
`${ANSI_YELLOW}Warning: Glob pattern matched no files: ${ref} (resolved to ${absolutePattern})${ANSI_RESET}`
|
|
1913
|
+
);
|
|
1914
|
+
return [];
|
|
1915
|
+
}
|
|
1916
|
+
matches.sort();
|
|
1917
|
+
const allCases = [];
|
|
1918
|
+
for (const match of matches) {
|
|
1919
|
+
const cases = await loadCasesFromFile(match);
|
|
1920
|
+
allCases.push(...cases);
|
|
1921
|
+
}
|
|
1922
|
+
return allCases;
|
|
1923
|
+
}
|
|
1924
|
+
return loadCasesFromFile(absolutePattern);
|
|
1925
|
+
}
|
|
1926
|
+
async function expandFileReferences(tests, evalFileDir) {
|
|
1927
|
+
const expanded = [];
|
|
1928
|
+
for (const entry of tests) {
|
|
1929
|
+
if (isFileReference(entry)) {
|
|
1930
|
+
const cases = await resolveFileReference2(entry, evalFileDir);
|
|
1931
|
+
expanded.push(...cases);
|
|
1932
|
+
} else {
|
|
1933
|
+
expanded.push(entry);
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
return expanded;
|
|
1937
|
+
}
|
|
1805
1938
|
var AGENT_PROVIDER_KINDS = [
|
|
1806
1939
|
"codex",
|
|
1807
1940
|
"copilot-sdk",
|
|
@@ -1876,36 +2009,32 @@ function isAgentProvider(provider) {
|
|
|
1876
2009
|
}
|
|
1877
2010
|
|
|
1878
2011
|
// ../../packages/core/dist/index.js
|
|
1879
|
-
import { readFile as
|
|
1880
|
-
import
|
|
2012
|
+
import { readFile as readFile7 } from "node:fs/promises";
|
|
2013
|
+
import path8 from "node:path";
|
|
1881
2014
|
import micromatch3 from "micromatch";
|
|
1882
2015
|
import { parse as parse2 } from "yaml";
|
|
1883
|
-
import { readFile as readFile2 } from "node:fs/promises";
|
|
1884
|
-
import path3 from "node:path";
|
|
1885
|
-
import { readFile as readFile22 } from "node:fs/promises";
|
|
1886
|
-
import path22 from "node:path";
|
|
1887
|
-
import fg from "fast-glob";
|
|
1888
|
-
import { parse as parseYaml } from "yaml";
|
|
1889
2016
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
1890
2017
|
import path4 from "node:path";
|
|
2018
|
+
import { readFile as readFile22 } from "node:fs/promises";
|
|
2019
|
+
import path32 from "node:path";
|
|
1891
2020
|
import micromatch from "micromatch";
|
|
1892
2021
|
import { parse } from "yaml";
|
|
1893
2022
|
import { constants as constants2 } from "node:fs";
|
|
1894
2023
|
import { access as access2 } from "node:fs/promises";
|
|
1895
|
-
import
|
|
2024
|
+
import path22 from "node:path";
|
|
1896
2025
|
import { fileURLToPath } from "node:url";
|
|
1897
|
-
import
|
|
2026
|
+
import path42 from "node:path";
|
|
2027
|
+
import { readFile as readFile32 } from "node:fs/promises";
|
|
2028
|
+
import { readFile as readFile5 } from "node:fs/promises";
|
|
2029
|
+
import path6 from "node:path";
|
|
2030
|
+
import micromatch2 from "micromatch";
|
|
2031
|
+
import { parse as parseYaml2 } from "yaml";
|
|
1898
2032
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
2033
|
+
import path5 from "node:path";
|
|
1899
2034
|
import { readFile as readFile6 } from "node:fs/promises";
|
|
1900
2035
|
import path7 from "node:path";
|
|
1901
|
-
import micromatch2 from "micromatch";
|
|
1902
|
-
import { parse as parseYaml2 } from "yaml";
|
|
1903
|
-
import { readFile as readFile5 } from "node:fs/promises";
|
|
1904
|
-
import path6 from "node:path";
|
|
1905
|
-
import { readFile as readFile7 } from "node:fs/promises";
|
|
1906
|
-
import path8 from "node:path";
|
|
1907
2036
|
import { readFileSync } from "node:fs";
|
|
1908
|
-
import
|
|
2037
|
+
import path9 from "node:path";
|
|
1909
2038
|
import { parse as parse3 } from "yaml";
|
|
1910
2039
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
1911
2040
|
|
|
@@ -6379,7 +6508,7 @@ function createOpenRouter(options = {}) {
|
|
|
6379
6508
|
);
|
|
6380
6509
|
const createChatModel = (modelId, settings = {}) => new OpenRouterChatLanguageModel(modelId, settings, {
|
|
6381
6510
|
provider: "openrouter.chat",
|
|
6382
|
-
url: ({ path:
|
|
6511
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6383
6512
|
headers: getHeaders,
|
|
6384
6513
|
compatibility,
|
|
6385
6514
|
fetch: options.fetch,
|
|
@@ -6387,7 +6516,7 @@ function createOpenRouter(options = {}) {
|
|
|
6387
6516
|
});
|
|
6388
6517
|
const createCompletionModel = (modelId, settings = {}) => new OpenRouterCompletionLanguageModel(modelId, settings, {
|
|
6389
6518
|
provider: "openrouter.completion",
|
|
6390
|
-
url: ({ path:
|
|
6519
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6391
6520
|
headers: getHeaders,
|
|
6392
6521
|
compatibility,
|
|
6393
6522
|
fetch: options.fetch,
|
|
@@ -6395,14 +6524,14 @@ function createOpenRouter(options = {}) {
|
|
|
6395
6524
|
});
|
|
6396
6525
|
const createEmbeddingModel = (modelId, settings = {}) => new OpenRouterEmbeddingModel(modelId, settings, {
|
|
6397
6526
|
provider: "openrouter.embedding",
|
|
6398
|
-
url: ({ path:
|
|
6527
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6399
6528
|
headers: getHeaders,
|
|
6400
6529
|
fetch: options.fetch,
|
|
6401
6530
|
extraBody: options.extraBody
|
|
6402
6531
|
});
|
|
6403
6532
|
const createImageModel = (modelId, settings = {}) => new OpenRouterImageModel(modelId, settings, {
|
|
6404
6533
|
provider: "openrouter.image",
|
|
6405
|
-
url: ({ path:
|
|
6534
|
+
url: ({ path: path45 }) => `${baseURL}${path45}`,
|
|
6406
6535
|
headers: getHeaders,
|
|
6407
6536
|
fetch: options.fetch,
|
|
6408
6537
|
extraBody: options.extraBody
|
|
@@ -12398,24 +12527,24 @@ import { spawn } from "node:child_process";
|
|
|
12398
12527
|
import { randomUUID } from "node:crypto";
|
|
12399
12528
|
import { createWriteStream } from "node:fs";
|
|
12400
12529
|
import { mkdir } from "node:fs/promises";
|
|
12401
|
-
import path12 from "node:path";
|
|
12402
12530
|
import path11 from "node:path";
|
|
12531
|
+
import path10 from "node:path";
|
|
12403
12532
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
12404
12533
|
import { createWriteStream as createWriteStream2 } from "node:fs";
|
|
12405
12534
|
import { mkdir as mkdir2 } from "node:fs/promises";
|
|
12406
|
-
import
|
|
12535
|
+
import path12 from "node:path";
|
|
12407
12536
|
import { exec as execWithCallback } from "node:child_process";
|
|
12408
12537
|
import fs from "node:fs/promises";
|
|
12409
12538
|
import os from "node:os";
|
|
12410
|
-
import
|
|
12539
|
+
import path13 from "node:path";
|
|
12411
12540
|
import { promisify } from "node:util";
|
|
12412
12541
|
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
12413
12542
|
import { createWriteStream as createWriteStream3 } from "node:fs";
|
|
12414
12543
|
import { mkdir as mkdir3 } from "node:fs/promises";
|
|
12415
|
-
import
|
|
12544
|
+
import path14 from "node:path";
|
|
12416
12545
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
12417
12546
|
import { mkdir as mkdir4 } from "node:fs/promises";
|
|
12418
|
-
import
|
|
12547
|
+
import path16 from "node:path";
|
|
12419
12548
|
import { Readable, Writable } from "node:stream";
|
|
12420
12549
|
import { spawn as spawn2 } from "node:child_process";
|
|
12421
12550
|
|
|
@@ -13900,48 +14029,48 @@ var RequestError = class _RequestError extends Error {
|
|
|
13900
14029
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
13901
14030
|
import { createWriteStream as createWriteStream4, existsSync, readdirSync } from "node:fs";
|
|
13902
14031
|
import { arch, platform } from "node:os";
|
|
13903
|
-
import
|
|
14032
|
+
import path15 from "node:path";
|
|
13904
14033
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
13905
14034
|
import { randomUUID as randomUUID6 } from "node:crypto";
|
|
13906
14035
|
import { mkdir as mkdir5 } from "node:fs/promises";
|
|
13907
|
-
import
|
|
14036
|
+
import path17 from "node:path";
|
|
13908
14037
|
import { spawn as spawn3 } from "node:child_process";
|
|
13909
14038
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
13910
14039
|
import { createWriteStream as createWriteStream5 } from "node:fs";
|
|
13911
14040
|
import { mkdir as mkdir6, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
13912
14041
|
import { tmpdir } from "node:os";
|
|
13913
|
-
import
|
|
14042
|
+
import path18 from "node:path";
|
|
13914
14043
|
import { exec as exec2 } from "node:child_process";
|
|
13915
14044
|
import { constants as constants3, access as access3, stat as stat4 } from "node:fs/promises";
|
|
13916
|
-
import
|
|
14045
|
+
import path30 from "node:path";
|
|
13917
14046
|
import { promisify as promisify3 } from "node:util";
|
|
13918
14047
|
import { stat as stat3, writeFile as writeFile4 } from "node:fs/promises";
|
|
13919
|
-
import
|
|
14048
|
+
import path28 from "node:path";
|
|
13920
14049
|
import { constants as constants22 } from "node:fs";
|
|
13921
14050
|
import { access as access22, mkdir as mkdir7, readdir, rm as rm2, stat } from "node:fs/promises";
|
|
14051
|
+
import path19 from "node:path";
|
|
13922
14052
|
import path20 from "node:path";
|
|
13923
14053
|
import path21 from "node:path";
|
|
14054
|
+
import { readFile as readFile8 } from "node:fs/promises";
|
|
13924
14055
|
import path222 from "node:path";
|
|
13925
|
-
import { readFile as readFile9 } from "node:fs/promises";
|
|
13926
|
-
import path23 from "node:path";
|
|
13927
14056
|
import { exec, spawn as spawn4 } from "node:child_process";
|
|
13928
14057
|
import { mkdir as mkdir8, writeFile as writeFile2 } from "node:fs/promises";
|
|
13929
|
-
import path26 from "node:path";
|
|
13930
|
-
import { promisify as promisify2 } from "node:util";
|
|
13931
14058
|
import path25 from "node:path";
|
|
13932
|
-
import
|
|
14059
|
+
import { promisify as promisify2 } from "node:util";
|
|
13933
14060
|
import path24 from "node:path";
|
|
13934
|
-
import
|
|
13935
|
-
import
|
|
14061
|
+
import os2 from "node:os";
|
|
14062
|
+
import path23 from "node:path";
|
|
14063
|
+
import { copyFile, mkdir as mkdir9, readFile as readFile9, readdir as readdir2, stat as stat2, writeFile as writeFile3 } from "node:fs/promises";
|
|
13936
14064
|
import path27 from "node:path";
|
|
14065
|
+
import path26 from "node:path";
|
|
13937
14066
|
import JSON5 from "json5";
|
|
13938
14067
|
import { writeFile as writeFile5 } from "node:fs/promises";
|
|
13939
|
-
import
|
|
14068
|
+
import path29 from "node:path";
|
|
13940
14069
|
import { constants as constants4 } from "node:fs";
|
|
13941
|
-
import { access as access4, readFile as
|
|
13942
|
-
import
|
|
14070
|
+
import { access as access4, readFile as readFile10 } from "node:fs/promises";
|
|
14071
|
+
import path31 from "node:path";
|
|
13943
14072
|
import { parse as parse4 } from "yaml";
|
|
13944
|
-
import
|
|
14073
|
+
import path322 from "node:path";
|
|
13945
14074
|
import fg2 from "fast-glob";
|
|
13946
14075
|
import { mkdtemp as mkdtemp2, rm as rm3, writeFile as writeFile6 } from "node:fs/promises";
|
|
13947
14076
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
@@ -13949,38 +14078,38 @@ import { dirname, join } from "node:path";
|
|
|
13949
14078
|
import { randomBytes } from "node:crypto";
|
|
13950
14079
|
import { createServer } from "node:http";
|
|
13951
14080
|
import fs2 from "node:fs/promises";
|
|
13952
|
-
import
|
|
14081
|
+
import path33 from "node:path";
|
|
13953
14082
|
import { createHash as createHash2, randomUUID as randomUUID8 } from "node:crypto";
|
|
13954
14083
|
import { copyFile as copyFile2, mkdir as mkdir13, readdir as readdir6, stat as stat7 } from "node:fs/promises";
|
|
13955
|
-
import
|
|
14084
|
+
import path422 from "node:path";
|
|
13956
14085
|
import micromatch4 from "micromatch";
|
|
14086
|
+
import path34 from "node:path";
|
|
13957
14087
|
import path35 from "node:path";
|
|
14088
|
+
import fg22 from "fast-glob";
|
|
13958
14089
|
import path36 from "node:path";
|
|
13959
14090
|
import fg3 from "fast-glob";
|
|
13960
|
-
import path37 from "node:path";
|
|
13961
|
-
import fg4 from "fast-glob";
|
|
13962
14091
|
import { exec as execCallback } from "node:child_process";
|
|
13963
14092
|
import { readdirSync as readdirSync2, statSync } from "node:fs";
|
|
13964
|
-
import
|
|
14093
|
+
import path37 from "node:path";
|
|
13965
14094
|
import { promisify as promisify4 } from "node:util";
|
|
13966
14095
|
import { cp, mkdir as mkdir11, readdir as readdir3, rm as rm4, stat as stat5 } from "node:fs/promises";
|
|
13967
|
-
import
|
|
14096
|
+
import path38 from "node:path";
|
|
13968
14097
|
import { execFile } from "node:child_process";
|
|
13969
14098
|
import { createHash } from "node:crypto";
|
|
13970
14099
|
import { existsSync as existsSync2 } from "node:fs";
|
|
13971
|
-
import { cp as cp2, mkdir as mkdir12, readFile as
|
|
13972
|
-
import
|
|
14100
|
+
import { cp as cp2, mkdir as mkdir12, readFile as readFile11, readdir as readdir4, rm as rm5, unlink, writeFile as writeFile7 } from "node:fs/promises";
|
|
14101
|
+
import path39 from "node:path";
|
|
13973
14102
|
import { promisify as promisify5 } from "node:util";
|
|
13974
14103
|
import { execFile as execFile2 } from "node:child_process";
|
|
13975
14104
|
import { existsSync as existsSync3 } from "node:fs";
|
|
13976
|
-
import
|
|
14105
|
+
import path40 from "node:path";
|
|
13977
14106
|
import { promisify as promisify6 } from "node:util";
|
|
13978
14107
|
import { readdir as readdir5, stat as stat6 } from "node:fs/promises";
|
|
13979
|
-
import
|
|
14108
|
+
import path41 from "node:path";
|
|
13980
14109
|
import { existsSync as existsSync4 } from "node:fs";
|
|
14110
|
+
import path43 from "node:path";
|
|
14111
|
+
import { mkdir as mkdir14, readFile as readFile12, writeFile as writeFile8 } from "node:fs/promises";
|
|
13981
14112
|
import path44 from "node:path";
|
|
13982
|
-
import { mkdir as mkdir14, readFile as readFile13, writeFile as writeFile8 } from "node:fs/promises";
|
|
13983
|
-
import path45 from "node:path";
|
|
13984
14113
|
function computeTraceSummary(messages) {
|
|
13985
14114
|
const toolCallCounts = {};
|
|
13986
14115
|
const toolDurations = {};
|
|
@@ -14097,27 +14226,10 @@ function mergeExecutionMetrics(computed, metrics) {
|
|
|
14097
14226
|
endTime: metrics.endTime ?? computed.endTime
|
|
14098
14227
|
};
|
|
14099
14228
|
}
|
|
14100
|
-
var ENV_VAR_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
14101
|
-
function interpolateEnv(value, env) {
|
|
14102
|
-
if (typeof value === "string") {
|
|
14103
|
-
return value.replace(ENV_VAR_PATTERN, (_, varName) => env[varName] ?? "");
|
|
14104
|
-
}
|
|
14105
|
-
if (Array.isArray(value)) {
|
|
14106
|
-
return value.map((item) => interpolateEnv(item, env));
|
|
14107
|
-
}
|
|
14108
|
-
if (value !== null && typeof value === "object") {
|
|
14109
|
-
const result = {};
|
|
14110
|
-
for (const [key, val] of Object.entries(value)) {
|
|
14111
|
-
result[key] = interpolateEnv(val, env);
|
|
14112
|
-
}
|
|
14113
|
-
return result;
|
|
14114
|
-
}
|
|
14115
|
-
return value;
|
|
14116
|
-
}
|
|
14117
14229
|
var ANSI_RED = "\x1B[31m";
|
|
14118
|
-
var
|
|
14230
|
+
var ANSI_RESET2 = "\x1B[0m";
|
|
14119
14231
|
function logError(msg) {
|
|
14120
|
-
console.error(`${ANSI_RED}Error: ${msg}${
|
|
14232
|
+
console.error(`${ANSI_RED}Error: ${msg}${ANSI_RESET2}`);
|
|
14121
14233
|
}
|
|
14122
14234
|
function isAgentSkillsFormat(parsed) {
|
|
14123
14235
|
if (typeof parsed !== "object" || parsed === null) return false;
|
|
@@ -14125,14 +14237,14 @@ function isAgentSkillsFormat(parsed) {
|
|
|
14125
14237
|
return Array.isArray(obj.evals);
|
|
14126
14238
|
}
|
|
14127
14239
|
async function loadTestsFromAgentSkills(filePath) {
|
|
14128
|
-
const raw = await
|
|
14240
|
+
const raw = await readFile3(filePath, "utf8");
|
|
14129
14241
|
let parsed;
|
|
14130
14242
|
try {
|
|
14131
14243
|
parsed = JSON.parse(raw);
|
|
14132
14244
|
} catch {
|
|
14133
14245
|
throw new Error(`Invalid Agent Skills evals.json: failed to parse JSON in '${filePath}'`);
|
|
14134
14246
|
}
|
|
14135
|
-
return parseAgentSkillsEvals(parsed, filePath,
|
|
14247
|
+
return parseAgentSkillsEvals(parsed, filePath, path4.dirname(path4.resolve(filePath)));
|
|
14136
14248
|
}
|
|
14137
14249
|
function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
14138
14250
|
if (!isAgentSkillsFormat(parsed)) {
|
|
@@ -14170,7 +14282,7 @@ function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
|
14170
14282
|
if (baseDir) {
|
|
14171
14283
|
metadata.agent_skills_base_dir = baseDir;
|
|
14172
14284
|
for (const file of evalCase.files) {
|
|
14173
|
-
filePaths.push(
|
|
14285
|
+
filePaths.push(path4.resolve(baseDir, file));
|
|
14174
14286
|
}
|
|
14175
14287
|
}
|
|
14176
14288
|
}
|
|
@@ -14192,118 +14304,6 @@ function parseAgentSkillsEvals(parsed, source = "evals.json", baseDir) {
|
|
|
14192
14304
|
}
|
|
14193
14305
|
return tests;
|
|
14194
14306
|
}
|
|
14195
|
-
var ANSI_YELLOW = "\x1B[33m";
|
|
14196
|
-
var ANSI_RESET2 = "\x1B[0m";
|
|
14197
|
-
var FILE_PROTOCOL = "file://";
|
|
14198
|
-
function isFileReference(value) {
|
|
14199
|
-
return typeof value === "string" && value.startsWith(FILE_PROTOCOL);
|
|
14200
|
-
}
|
|
14201
|
-
function extractFilePath(ref) {
|
|
14202
|
-
return ref.slice(FILE_PROTOCOL.length);
|
|
14203
|
-
}
|
|
14204
|
-
function isGlobPattern(filePath) {
|
|
14205
|
-
return filePath.includes("*") || filePath.includes("?") || filePath.includes("{");
|
|
14206
|
-
}
|
|
14207
|
-
function parseYamlCases(content, filePath) {
|
|
14208
|
-
const raw = parseYaml(content);
|
|
14209
|
-
const parsed = interpolateEnv(raw, process.env);
|
|
14210
|
-
if (!Array.isArray(parsed)) {
|
|
14211
|
-
throw new Error(
|
|
14212
|
-
`External test file must contain a YAML array, got ${typeof parsed}: ${filePath}`
|
|
14213
|
-
);
|
|
14214
|
-
}
|
|
14215
|
-
const results = [];
|
|
14216
|
-
for (const item of parsed) {
|
|
14217
|
-
if (!isJsonObject(item)) {
|
|
14218
|
-
throw new Error(`External test file contains non-object entry: ${filePath}`);
|
|
14219
|
-
}
|
|
14220
|
-
results.push(item);
|
|
14221
|
-
}
|
|
14222
|
-
return results;
|
|
14223
|
-
}
|
|
14224
|
-
function parseJsonlCases(content, filePath) {
|
|
14225
|
-
const lines = content.split("\n");
|
|
14226
|
-
const results = [];
|
|
14227
|
-
for (let i = 0; i < lines.length; i++) {
|
|
14228
|
-
const line = lines[i].trim();
|
|
14229
|
-
if (line === "") continue;
|
|
14230
|
-
try {
|
|
14231
|
-
const raw = JSON.parse(line);
|
|
14232
|
-
const parsed = interpolateEnv(raw, process.env);
|
|
14233
|
-
if (!isJsonObject(parsed)) {
|
|
14234
|
-
throw new Error("Expected JSON object");
|
|
14235
|
-
}
|
|
14236
|
-
results.push(parsed);
|
|
14237
|
-
} catch (error) {
|
|
14238
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
14239
|
-
throw new Error(`Malformed JSONL at line ${i + 1}: ${message}
|
|
14240
|
-
File: ${filePath}`);
|
|
14241
|
-
}
|
|
14242
|
-
}
|
|
14243
|
-
return results;
|
|
14244
|
-
}
|
|
14245
|
-
async function loadCasesFromFile(filePath) {
|
|
14246
|
-
const ext = path22.extname(filePath).toLowerCase();
|
|
14247
|
-
let content;
|
|
14248
|
-
try {
|
|
14249
|
-
content = await readFile22(filePath, "utf8");
|
|
14250
|
-
} catch (error) {
|
|
14251
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
14252
|
-
throw new Error(`Cannot read external test file: ${filePath}
|
|
14253
|
-
${message}`);
|
|
14254
|
-
}
|
|
14255
|
-
if (content.trim() === "") {
|
|
14256
|
-
console.warn(
|
|
14257
|
-
`${ANSI_YELLOW}Warning: External test file is empty, skipping: ${filePath}${ANSI_RESET2}`
|
|
14258
|
-
);
|
|
14259
|
-
return [];
|
|
14260
|
-
}
|
|
14261
|
-
if (ext === ".yaml" || ext === ".yml") {
|
|
14262
|
-
return parseYamlCases(content, filePath);
|
|
14263
|
-
}
|
|
14264
|
-
if (ext === ".jsonl") {
|
|
14265
|
-
return parseJsonlCases(content, filePath);
|
|
14266
|
-
}
|
|
14267
|
-
throw new Error(
|
|
14268
|
-
`Unsupported external test file format '${ext}': ${filePath}. Supported: .yaml, .yml, .jsonl`
|
|
14269
|
-
);
|
|
14270
|
-
}
|
|
14271
|
-
async function resolveFileReference2(ref, evalFileDir) {
|
|
14272
|
-
const rawPath = extractFilePath(ref);
|
|
14273
|
-
const absolutePattern = path22.resolve(evalFileDir, rawPath);
|
|
14274
|
-
if (isGlobPattern(rawPath)) {
|
|
14275
|
-
const matches = await fg(absolutePattern.replaceAll("\\", "/"), {
|
|
14276
|
-
onlyFiles: true,
|
|
14277
|
-
absolute: true
|
|
14278
|
-
});
|
|
14279
|
-
if (matches.length === 0) {
|
|
14280
|
-
console.warn(
|
|
14281
|
-
`${ANSI_YELLOW}Warning: Glob pattern matched no files: ${ref} (resolved to ${absolutePattern})${ANSI_RESET2}`
|
|
14282
|
-
);
|
|
14283
|
-
return [];
|
|
14284
|
-
}
|
|
14285
|
-
matches.sort();
|
|
14286
|
-
const allCases = [];
|
|
14287
|
-
for (const match of matches) {
|
|
14288
|
-
const cases = await loadCasesFromFile(match);
|
|
14289
|
-
allCases.push(...cases);
|
|
14290
|
-
}
|
|
14291
|
-
return allCases;
|
|
14292
|
-
}
|
|
14293
|
-
return loadCasesFromFile(absolutePattern);
|
|
14294
|
-
}
|
|
14295
|
-
async function expandFileReferences(tests, evalFileDir) {
|
|
14296
|
-
const expanded = [];
|
|
14297
|
-
for (const entry of tests) {
|
|
14298
|
-
if (isFileReference(entry)) {
|
|
14299
|
-
const cases = await resolveFileReference2(entry, evalFileDir);
|
|
14300
|
-
expanded.push(...cases);
|
|
14301
|
-
} else {
|
|
14302
|
-
expanded.push(entry);
|
|
14303
|
-
}
|
|
14304
|
-
}
|
|
14305
|
-
return expanded;
|
|
14306
|
-
}
|
|
14307
14307
|
async function fileExists2(absolutePath) {
|
|
14308
14308
|
try {
|
|
14309
14309
|
await access2(absolutePath, constants2.F_OK);
|
|
@@ -14320,15 +14320,15 @@ function resolveToAbsolutePath(candidate) {
|
|
|
14320
14320
|
if (candidate.startsWith("file:")) {
|
|
14321
14321
|
return fileURLToPath(candidate);
|
|
14322
14322
|
}
|
|
14323
|
-
return
|
|
14323
|
+
return path22.resolve(candidate);
|
|
14324
14324
|
}
|
|
14325
14325
|
throw new TypeError("Unsupported repoRoot value. Expected string or URL.");
|
|
14326
14326
|
}
|
|
14327
14327
|
function buildDirectoryChain2(filePath, repoRoot) {
|
|
14328
14328
|
const directories = [];
|
|
14329
14329
|
const seen = /* @__PURE__ */ new Set();
|
|
14330
|
-
const boundary =
|
|
14331
|
-
let current =
|
|
14330
|
+
const boundary = path22.resolve(repoRoot);
|
|
14331
|
+
let current = path22.resolve(path22.dirname(filePath));
|
|
14332
14332
|
while (current !== void 0) {
|
|
14333
14333
|
if (!seen.has(current)) {
|
|
14334
14334
|
directories.push(current);
|
|
@@ -14337,7 +14337,7 @@ function buildDirectoryChain2(filePath, repoRoot) {
|
|
|
14337
14337
|
if (current === boundary) {
|
|
14338
14338
|
break;
|
|
14339
14339
|
}
|
|
14340
|
-
const parent =
|
|
14340
|
+
const parent = path22.dirname(current);
|
|
14341
14341
|
if (parent === current) {
|
|
14342
14342
|
break;
|
|
14343
14343
|
}
|
|
@@ -14351,16 +14351,16 @@ function buildDirectoryChain2(filePath, repoRoot) {
|
|
|
14351
14351
|
function buildSearchRoots2(evalPath, repoRoot) {
|
|
14352
14352
|
const uniqueRoots = [];
|
|
14353
14353
|
const addRoot = (root) => {
|
|
14354
|
-
const normalized =
|
|
14354
|
+
const normalized = path22.resolve(root);
|
|
14355
14355
|
if (!uniqueRoots.includes(normalized)) {
|
|
14356
14356
|
uniqueRoots.push(normalized);
|
|
14357
14357
|
}
|
|
14358
14358
|
};
|
|
14359
|
-
let currentDir =
|
|
14359
|
+
let currentDir = path22.dirname(evalPath);
|
|
14360
14360
|
let reachedBoundary = false;
|
|
14361
14361
|
while (!reachedBoundary) {
|
|
14362
14362
|
addRoot(currentDir);
|
|
14363
|
-
const parentDir =
|
|
14363
|
+
const parentDir = path22.dirname(currentDir);
|
|
14364
14364
|
if (currentDir === repoRoot || parentDir === currentDir) {
|
|
14365
14365
|
reachedBoundary = true;
|
|
14366
14366
|
} else {
|
|
@@ -14375,19 +14375,19 @@ function trimLeadingSeparators2(value) {
|
|
|
14375
14375
|
const trimmed = value.replace(/^[/\\]+/, "");
|
|
14376
14376
|
return trimmed.length > 0 ? trimmed : value;
|
|
14377
14377
|
}
|
|
14378
|
-
async function
|
|
14378
|
+
async function resolveFileReference22(rawValue, searchRoots) {
|
|
14379
14379
|
const displayPath = trimLeadingSeparators2(rawValue);
|
|
14380
14380
|
const potentialPaths = [];
|
|
14381
|
-
if (
|
|
14382
|
-
potentialPaths.push(
|
|
14381
|
+
if (path22.isAbsolute(rawValue)) {
|
|
14382
|
+
potentialPaths.push(path22.normalize(rawValue));
|
|
14383
14383
|
}
|
|
14384
14384
|
for (const base of searchRoots) {
|
|
14385
|
-
potentialPaths.push(
|
|
14385
|
+
potentialPaths.push(path22.resolve(base, displayPath));
|
|
14386
14386
|
}
|
|
14387
14387
|
const attempted = [];
|
|
14388
14388
|
const seen = /* @__PURE__ */ new Set();
|
|
14389
14389
|
for (const candidate of potentialPaths) {
|
|
14390
|
-
const absoluteCandidate =
|
|
14390
|
+
const absoluteCandidate = path22.resolve(candidate);
|
|
14391
14391
|
if (seen.has(absoluteCandidate)) {
|
|
14392
14392
|
continue;
|
|
14393
14393
|
}
|
|
@@ -14400,7 +14400,7 @@ async function resolveFileReference3(rawValue, searchRoots) {
|
|
|
14400
14400
|
return { displayPath, attempted };
|
|
14401
14401
|
}
|
|
14402
14402
|
var ANSI_YELLOW2 = "\x1B[33m";
|
|
14403
|
-
var
|
|
14403
|
+
var ANSI_RESET22 = "\x1B[0m";
|
|
14404
14404
|
var DEFAULT_EVAL_PATTERNS = [
|
|
14405
14405
|
"**/evals/**/*.eval.yaml",
|
|
14406
14406
|
"**/evals/**/eval.yaml"
|
|
@@ -14408,12 +14408,12 @@ var DEFAULT_EVAL_PATTERNS = [
|
|
|
14408
14408
|
async function loadConfig(evalFilePath, repoRoot) {
|
|
14409
14409
|
const directories = buildDirectoryChain2(evalFilePath, repoRoot);
|
|
14410
14410
|
for (const directory of directories) {
|
|
14411
|
-
const configPath =
|
|
14411
|
+
const configPath = path32.join(directory, ".agentv", "config.yaml");
|
|
14412
14412
|
if (!await fileExists2(configPath)) {
|
|
14413
14413
|
continue;
|
|
14414
14414
|
}
|
|
14415
14415
|
try {
|
|
14416
|
-
const rawConfig = await
|
|
14416
|
+
const rawConfig = await readFile22(configPath, "utf8");
|
|
14417
14417
|
const parsed = parse(rawConfig);
|
|
14418
14418
|
if (!isJsonObject(parsed)) {
|
|
14419
14419
|
logWarning(`Invalid .agentv/config.yaml format at ${configPath}`);
|
|
@@ -14650,7 +14650,7 @@ function parseExecutionDefaults(raw, configPath) {
|
|
|
14650
14650
|
return Object.keys(result).length > 0 ? result : void 0;
|
|
14651
14651
|
}
|
|
14652
14652
|
function logWarning(message) {
|
|
14653
|
-
console.warn(`${ANSI_YELLOW2}Warning: ${message}${
|
|
14653
|
+
console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET22}`);
|
|
14654
14654
|
}
|
|
14655
14655
|
var TEMPLATE_VARIABLES = {
|
|
14656
14656
|
EXPECTED_OUTPUT: "expected_output",
|
|
@@ -14667,10 +14667,10 @@ var REQUIRED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Set([
|
|
|
14667
14667
|
TEMPLATE_VARIABLES.OUTPUT_TEXT,
|
|
14668
14668
|
TEMPLATE_VARIABLES.EXPECTED_OUTPUT
|
|
14669
14669
|
]);
|
|
14670
|
-
var
|
|
14671
|
-
var
|
|
14670
|
+
var ANSI_YELLOW22 = "\x1B[33m";
|
|
14671
|
+
var ANSI_RESET3 = "\x1B[0m";
|
|
14672
14672
|
async function validateCustomPromptContent(promptPath) {
|
|
14673
|
-
const content = await
|
|
14673
|
+
const content = await readFile32(promptPath, "utf8");
|
|
14674
14674
|
validateTemplateVariables(content, promptPath);
|
|
14675
14675
|
}
|
|
14676
14676
|
function validateTemplateVariables(content, source) {
|
|
@@ -14697,14 +14697,14 @@ function validateTemplateVariables(content, source) {
|
|
|
14697
14697
|
);
|
|
14698
14698
|
}
|
|
14699
14699
|
if (invalidVariables.length > 0) {
|
|
14700
|
-
const warningMessage = `${
|
|
14700
|
+
const warningMessage = `${ANSI_YELLOW22}Warning: Custom evaluator template at ${source}
|
|
14701
14701
|
Contains invalid variables: ${invalidVariables.map((v) => `{{ ${v} }}`).join(", ")}
|
|
14702
|
-
Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES).map((v) => `{{ ${v} }}`).join(", ")}${
|
|
14702
|
+
Valid variables: ${Array.from(VALID_TEMPLATE_VARIABLES).map((v) => `{{ ${v} }}`).join(", ")}${ANSI_RESET3}`;
|
|
14703
14703
|
console.warn(warningMessage);
|
|
14704
14704
|
}
|
|
14705
14705
|
}
|
|
14706
|
-
var
|
|
14707
|
-
var
|
|
14706
|
+
var ANSI_YELLOW3 = "\x1B[33m";
|
|
14707
|
+
var ANSI_RESET4 = "\x1B[0m";
|
|
14708
14708
|
function normalizeEvaluatorType(type) {
|
|
14709
14709
|
return type.replace(/_/g, "-");
|
|
14710
14710
|
}
|
|
@@ -14806,7 +14806,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14806
14806
|
let command;
|
|
14807
14807
|
if (rawEvaluator.script !== void 0 && rawEvaluator.command === void 0) {
|
|
14808
14808
|
console.warn(
|
|
14809
|
-
`${
|
|
14809
|
+
`${ANSI_YELLOW3}Warning: 'script' is deprecated in evaluator '${name21}' in '${evalId}'. Use 'command' instead.${ANSI_RESET4}`
|
|
14810
14810
|
);
|
|
14811
14811
|
}
|
|
14812
14812
|
const rawCommand = rawEvaluator.command ?? rawEvaluator.script;
|
|
@@ -14832,9 +14832,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14832
14832
|
const cwd = asString(rawEvaluator.cwd);
|
|
14833
14833
|
let resolvedCwd;
|
|
14834
14834
|
if (cwd) {
|
|
14835
|
-
const resolved = await
|
|
14835
|
+
const resolved = await resolveFileReference22(cwd, searchRoots);
|
|
14836
14836
|
if (resolved.resolvedPath) {
|
|
14837
|
-
resolvedCwd =
|
|
14837
|
+
resolvedCwd = path42.resolve(resolved.resolvedPath);
|
|
14838
14838
|
} else {
|
|
14839
14839
|
logWarning2(
|
|
14840
14840
|
`Code-grader evaluator '${name21}' in '${evalId}': cwd not found (${resolved.displayPath})`,
|
|
@@ -14990,9 +14990,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
14990
14990
|
const aggregatorPrompt = asString(rawAggregator.prompt);
|
|
14991
14991
|
let promptPath2;
|
|
14992
14992
|
if (aggregatorPrompt) {
|
|
14993
|
-
const resolved = await
|
|
14993
|
+
const resolved = await resolveFileReference22(aggregatorPrompt, searchRoots);
|
|
14994
14994
|
if (resolved.resolvedPath) {
|
|
14995
|
-
promptPath2 =
|
|
14995
|
+
promptPath2 = path42.resolve(resolved.resolvedPath);
|
|
14996
14996
|
}
|
|
14997
14997
|
}
|
|
14998
14998
|
aggregator = {
|
|
@@ -15549,7 +15549,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15549
15549
|
if (isJsonObject2(rawPrompt)) {
|
|
15550
15550
|
if (rawPrompt.script !== void 0 && rawPrompt.command === void 0) {
|
|
15551
15551
|
console.warn(
|
|
15552
|
-
`${
|
|
15552
|
+
`${ANSI_YELLOW3}Warning: 'prompt.script' is deprecated in evaluator '${name21}' in '${evalId}'. Use 'prompt.command' instead.${ANSI_RESET4}`
|
|
15553
15553
|
);
|
|
15554
15554
|
}
|
|
15555
15555
|
const commandArray = asStringArray(
|
|
@@ -15560,9 +15560,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15560
15560
|
throw new Error(`Evaluator '${name21}' in '${evalId}': prompt object requires command array`);
|
|
15561
15561
|
}
|
|
15562
15562
|
const commandPath = commandArray[commandArray.length - 1];
|
|
15563
|
-
const resolved = await
|
|
15563
|
+
const resolved = await resolveFileReference22(commandPath, searchRoots);
|
|
15564
15564
|
if (resolved.resolvedPath) {
|
|
15565
|
-
resolvedPromptScript = [...commandArray.slice(0, -1),
|
|
15565
|
+
resolvedPromptScript = [...commandArray.slice(0, -1), path42.resolve(resolved.resolvedPath)];
|
|
15566
15566
|
} else {
|
|
15567
15567
|
throw new Error(
|
|
15568
15568
|
`Evaluator '${name21}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
|
|
@@ -15573,9 +15573,9 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15573
15573
|
}
|
|
15574
15574
|
} else if (typeof rawPrompt === "string") {
|
|
15575
15575
|
prompt = rawPrompt;
|
|
15576
|
-
const resolved = await
|
|
15576
|
+
const resolved = await resolveFileReference22(prompt, searchRoots);
|
|
15577
15577
|
if (resolved.resolvedPath) {
|
|
15578
|
-
promptPath =
|
|
15578
|
+
promptPath = path42.resolve(resolved.resolvedPath);
|
|
15579
15579
|
try {
|
|
15580
15580
|
await validateCustomPromptContent(promptPath);
|
|
15581
15581
|
} catch (error) {
|
|
@@ -15775,10 +15775,10 @@ function warnUnconsumedCriteria(_criteria, _evaluators, _testId) {
|
|
|
15775
15775
|
function logWarning2(message, details) {
|
|
15776
15776
|
if (details && details.length > 0) {
|
|
15777
15777
|
const detailBlock = details.join("\n");
|
|
15778
|
-
console.warn(`${
|
|
15779
|
-
${detailBlock}${
|
|
15778
|
+
console.warn(`${ANSI_YELLOW3}Warning: ${message}
|
|
15779
|
+
${detailBlock}${ANSI_RESET4}`);
|
|
15780
15780
|
} else {
|
|
15781
|
-
console.warn(`${
|
|
15781
|
+
console.warn(`${ANSI_YELLOW3}Warning: ${message}${ANSI_RESET4}`);
|
|
15782
15782
|
}
|
|
15783
15783
|
}
|
|
15784
15784
|
function parseRequired(value) {
|
|
@@ -16083,8 +16083,8 @@ function hasVisibleContent(segments) {
|
|
|
16083
16083
|
function asString2(value) {
|
|
16084
16084
|
return typeof value === "string" ? value : void 0;
|
|
16085
16085
|
}
|
|
16086
|
-
var
|
|
16087
|
-
var
|
|
16086
|
+
var ANSI_YELLOW4 = "\x1B[33m";
|
|
16087
|
+
var ANSI_RESET5 = "\x1B[0m";
|
|
16088
16088
|
async function processMessages(options) {
|
|
16089
16089
|
const {
|
|
16090
16090
|
messages,
|
|
@@ -16128,7 +16128,7 @@ async function processMessages(options) {
|
|
|
16128
16128
|
if (!rawValue) {
|
|
16129
16129
|
continue;
|
|
16130
16130
|
}
|
|
16131
|
-
const { displayPath, resolvedPath, attempted } = await
|
|
16131
|
+
const { displayPath, resolvedPath, attempted } = await resolveFileReference22(
|
|
16132
16132
|
rawValue,
|
|
16133
16133
|
searchRoots
|
|
16134
16134
|
);
|
|
@@ -16139,7 +16139,7 @@ async function processMessages(options) {
|
|
|
16139
16139
|
continue;
|
|
16140
16140
|
}
|
|
16141
16141
|
try {
|
|
16142
|
-
const fileContent = (await
|
|
16142
|
+
const fileContent = (await readFile4(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
16143
16143
|
const classifyAsGuideline = shouldTreatAsGuideline({
|
|
16144
16144
|
messageType,
|
|
16145
16145
|
resolvedPath,
|
|
@@ -16148,7 +16148,7 @@ async function processMessages(options) {
|
|
|
16148
16148
|
treatFileSegmentsAsGuidelines
|
|
16149
16149
|
});
|
|
16150
16150
|
if (classifyAsGuideline && guidelinePaths) {
|
|
16151
|
-
guidelinePaths.push(
|
|
16151
|
+
guidelinePaths.push(path5.resolve(resolvedPath));
|
|
16152
16152
|
if (verbose) {
|
|
16153
16153
|
console.log(` [Guideline] Found: ${displayPath}`);
|
|
16154
16154
|
console.log(` Resolved to: ${resolvedPath}`);
|
|
@@ -16159,7 +16159,7 @@ async function processMessages(options) {
|
|
|
16159
16159
|
type: "file",
|
|
16160
16160
|
path: displayPath,
|
|
16161
16161
|
text: fileContent,
|
|
16162
|
-
resolvedPath:
|
|
16162
|
+
resolvedPath: path5.resolve(resolvedPath)
|
|
16163
16163
|
});
|
|
16164
16164
|
if (verbose) {
|
|
16165
16165
|
const label = messageType === "input" ? "[File]" : "[Expected Output File]";
|
|
@@ -16199,7 +16199,7 @@ function shouldTreatAsGuideline(options) {
|
|
|
16199
16199
|
if (!guidelinePatterns || guidelinePatterns.length === 0) {
|
|
16200
16200
|
return false;
|
|
16201
16201
|
}
|
|
16202
|
-
const relativeToRepo =
|
|
16202
|
+
const relativeToRepo = path5.relative(repoRootPath, resolvedPath);
|
|
16203
16203
|
return isGuidelineFile(relativeToRepo, guidelinePatterns);
|
|
16204
16204
|
}
|
|
16205
16205
|
function asString3(value) {
|
|
@@ -16227,10 +16227,10 @@ function cloneJsonValue(value) {
|
|
|
16227
16227
|
function logWarning3(message, details) {
|
|
16228
16228
|
if (details && details.length > 0) {
|
|
16229
16229
|
const detailBlock = details.join("\n");
|
|
16230
|
-
console.warn(`${
|
|
16231
|
-
${detailBlock}${
|
|
16230
|
+
console.warn(`${ANSI_YELLOW4}Warning: ${message}
|
|
16231
|
+
${detailBlock}${ANSI_RESET5}`);
|
|
16232
16232
|
} else {
|
|
16233
|
-
console.warn(`${
|
|
16233
|
+
console.warn(`${ANSI_YELLOW4}Warning: ${message}${ANSI_RESET5}`);
|
|
16234
16234
|
}
|
|
16235
16235
|
}
|
|
16236
16236
|
async function processExpectedMessages(options) {
|
|
@@ -16259,7 +16259,7 @@ async function processExpectedMessages(options) {
|
|
|
16259
16259
|
if (!rawValue) {
|
|
16260
16260
|
continue;
|
|
16261
16261
|
}
|
|
16262
|
-
const { displayPath, resolvedPath, attempted } = await
|
|
16262
|
+
const { displayPath, resolvedPath, attempted } = await resolveFileReference22(
|
|
16263
16263
|
rawValue,
|
|
16264
16264
|
searchRoots
|
|
16265
16265
|
);
|
|
@@ -16269,12 +16269,12 @@ async function processExpectedMessages(options) {
|
|
|
16269
16269
|
continue;
|
|
16270
16270
|
}
|
|
16271
16271
|
try {
|
|
16272
|
-
const fileContent = (await
|
|
16272
|
+
const fileContent = (await readFile4(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
16273
16273
|
processedContent.push({
|
|
16274
16274
|
type: "file",
|
|
16275
16275
|
path: displayPath,
|
|
16276
16276
|
text: fileContent,
|
|
16277
|
-
resolvedPath:
|
|
16277
|
+
resolvedPath: path5.resolve(resolvedPath)
|
|
16278
16278
|
});
|
|
16279
16279
|
if (verbose) {
|
|
16280
16280
|
console.log(` [Expected Output File] Found: ${displayPath}`);
|
|
@@ -16367,11 +16367,11 @@ function resolveInputMessages(raw, suiteInputFiles) {
|
|
|
16367
16367
|
function resolveExpectedMessages(raw) {
|
|
16368
16368
|
return expandExpectedOutputShorthand(raw.expected_output);
|
|
16369
16369
|
}
|
|
16370
|
-
var
|
|
16370
|
+
var ANSI_YELLOW5 = "\x1B[33m";
|
|
16371
16371
|
var ANSI_RED2 = "\x1B[31m";
|
|
16372
|
-
var
|
|
16372
|
+
var ANSI_RESET6 = "\x1B[0m";
|
|
16373
16373
|
function detectFormat(filePath) {
|
|
16374
|
-
const ext =
|
|
16374
|
+
const ext = path6.extname(filePath).toLowerCase();
|
|
16375
16375
|
if (ext === ".jsonl") return "jsonl";
|
|
16376
16376
|
if (ext === ".yaml" || ext === ".yml") return "yaml";
|
|
16377
16377
|
if (ext === ".json") return "agent-skills-json";
|
|
@@ -16380,9 +16380,9 @@ function detectFormat(filePath) {
|
|
|
16380
16380
|
);
|
|
16381
16381
|
}
|
|
16382
16382
|
async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
16383
|
-
const dir =
|
|
16384
|
-
const base =
|
|
16385
|
-
const sidecarPath =
|
|
16383
|
+
const dir = path6.dirname(jsonlPath);
|
|
16384
|
+
const base = path6.basename(jsonlPath, ".jsonl");
|
|
16385
|
+
const sidecarPath = path6.join(dir, `${base}.yaml`);
|
|
16386
16386
|
if (!await fileExists2(sidecarPath)) {
|
|
16387
16387
|
if (verbose) {
|
|
16388
16388
|
logWarning4(`Sidecar metadata file not found: ${sidecarPath} (using defaults)`);
|
|
@@ -16390,7 +16390,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
|
16390
16390
|
return {};
|
|
16391
16391
|
}
|
|
16392
16392
|
try {
|
|
16393
|
-
const content = await
|
|
16393
|
+
const content = await readFile5(sidecarPath, "utf8");
|
|
16394
16394
|
const parsed = interpolateEnv(parseYaml2(content), process.env);
|
|
16395
16395
|
if (!isJsonObject(parsed)) {
|
|
16396
16396
|
logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
|
|
@@ -16431,15 +16431,15 @@ function parseJsonlContent(content, filePath) {
|
|
|
16431
16431
|
async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
16432
16432
|
const verbose = options?.verbose ?? false;
|
|
16433
16433
|
const filterPattern = options?.filter;
|
|
16434
|
-
const absoluteTestPath =
|
|
16434
|
+
const absoluteTestPath = path6.resolve(evalFilePath);
|
|
16435
16435
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
16436
16436
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
16437
16437
|
const config = await loadConfig(absoluteTestPath, repoRootPath);
|
|
16438
16438
|
const guidelinePatterns = config?.guideline_patterns;
|
|
16439
16439
|
const sidecar = await loadSidecarMetadata(absoluteTestPath, verbose);
|
|
16440
|
-
const rawFile = await
|
|
16440
|
+
const rawFile = await readFile5(absoluteTestPath, "utf8");
|
|
16441
16441
|
const rawCases = parseJsonlContent(rawFile, evalFilePath);
|
|
16442
|
-
const fallbackDataset =
|
|
16442
|
+
const fallbackDataset = path6.basename(absoluteTestPath, ".jsonl") || "eval";
|
|
16443
16443
|
const datasetName = sidecar.dataset && sidecar.dataset.trim().length > 0 ? sidecar.dataset : fallbackDataset;
|
|
16444
16444
|
const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm-grader";
|
|
16445
16445
|
const globalExecution = sidecar.execution;
|
|
@@ -16538,7 +16538,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
|
16538
16538
|
}
|
|
16539
16539
|
}
|
|
16540
16540
|
const allFilePaths = [
|
|
16541
|
-
...guidelinePaths.map((guidelinePath) =>
|
|
16541
|
+
...guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
|
|
16542
16542
|
...userFilePaths
|
|
16543
16543
|
];
|
|
16544
16544
|
const testCase = {
|
|
@@ -16550,7 +16550,7 @@ async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
|
16550
16550
|
input_segments: inputSegments,
|
|
16551
16551
|
expected_output: outputSegments,
|
|
16552
16552
|
reference_answer: referenceAnswer,
|
|
16553
|
-
guideline_paths: guidelinePaths.map((guidelinePath) =>
|
|
16553
|
+
guideline_paths: guidelinePaths.map((guidelinePath) => path6.resolve(guidelinePath)),
|
|
16554
16554
|
guideline_patterns: guidelinePatterns,
|
|
16555
16555
|
file_paths: allFilePaths,
|
|
16556
16556
|
criteria: outcome ?? "",
|
|
@@ -16581,19 +16581,19 @@ function asString4(value) {
|
|
|
16581
16581
|
function logWarning4(message, details) {
|
|
16582
16582
|
if (details && details.length > 0) {
|
|
16583
16583
|
const detailBlock = details.join("\n");
|
|
16584
|
-
console.warn(`${
|
|
16585
|
-
${detailBlock}${
|
|
16584
|
+
console.warn(`${ANSI_YELLOW5}Warning: ${message}
|
|
16585
|
+
${detailBlock}${ANSI_RESET6}`);
|
|
16586
16586
|
} else {
|
|
16587
|
-
console.warn(`${
|
|
16587
|
+
console.warn(`${ANSI_YELLOW5}Warning: ${message}${ANSI_RESET6}`);
|
|
16588
16588
|
}
|
|
16589
16589
|
}
|
|
16590
16590
|
function logError2(message, details) {
|
|
16591
16591
|
if (details && details.length > 0) {
|
|
16592
16592
|
const detailBlock = details.join("\n");
|
|
16593
16593
|
console.error(`${ANSI_RED2}Error: ${message}
|
|
16594
|
-
${detailBlock}${
|
|
16594
|
+
${detailBlock}${ANSI_RESET6}`);
|
|
16595
16595
|
} else {
|
|
16596
|
-
console.error(`${ANSI_RED2}Error: ${message}${
|
|
16596
|
+
console.error(`${ANSI_RED2}Error: ${message}${ANSI_RESET6}`);
|
|
16597
16597
|
}
|
|
16598
16598
|
}
|
|
16599
16599
|
var MetadataSchema = external_exports2.object({
|
|
@@ -16623,22 +16623,22 @@ function parseMetadata(suite) {
|
|
|
16623
16623
|
requires: suite.requires
|
|
16624
16624
|
});
|
|
16625
16625
|
}
|
|
16626
|
-
var
|
|
16627
|
-
var
|
|
16626
|
+
var ANSI_YELLOW6 = "\x1B[33m";
|
|
16627
|
+
var ANSI_RESET7 = "\x1B[0m";
|
|
16628
16628
|
async function buildPromptInputs(testCase, mode = "lm") {
|
|
16629
16629
|
const guidelineParts = [];
|
|
16630
16630
|
for (const rawPath of testCase.guideline_paths) {
|
|
16631
|
-
const absolutePath =
|
|
16631
|
+
const absolutePath = path7.resolve(rawPath);
|
|
16632
16632
|
if (!await fileExists2(absolutePath)) {
|
|
16633
16633
|
logWarning5(`Could not read guideline file ${absolutePath}: file does not exist`);
|
|
16634
16634
|
continue;
|
|
16635
16635
|
}
|
|
16636
16636
|
try {
|
|
16637
|
-
const content = (await
|
|
16637
|
+
const content = (await readFile6(absolutePath, "utf8")).replace(/\r\n/g, "\n").trim();
|
|
16638
16638
|
guidelineParts.push({
|
|
16639
16639
|
content,
|
|
16640
16640
|
isFile: true,
|
|
16641
|
-
displayPath:
|
|
16641
|
+
displayPath: path7.basename(absolutePath)
|
|
16642
16642
|
});
|
|
16643
16643
|
} catch (error) {
|
|
16644
16644
|
logWarning5(`Could not read guideline file ${absolutePath}: ${error.message}`);
|
|
@@ -16836,11 +16836,11 @@ function asString5(value) {
|
|
|
16836
16836
|
return typeof value === "string" ? value : void 0;
|
|
16837
16837
|
}
|
|
16838
16838
|
function logWarning5(message) {
|
|
16839
|
-
console.warn(`${
|
|
16839
|
+
console.warn(`${ANSI_YELLOW6}Warning: ${message}${ANSI_RESET7}`);
|
|
16840
16840
|
}
|
|
16841
|
-
var
|
|
16841
|
+
var ANSI_YELLOW7 = "\x1B[33m";
|
|
16842
16842
|
var ANSI_RED3 = "\x1B[31m";
|
|
16843
|
-
var
|
|
16843
|
+
var ANSI_RESET8 = "\x1B[0m";
|
|
16844
16844
|
function resolveTests(suite) {
|
|
16845
16845
|
if (suite.tests !== void 0) return suite.tests;
|
|
16846
16846
|
if (suite.eval_cases !== void 0) {
|
|
@@ -16855,8 +16855,8 @@ function resolveTests(suite) {
|
|
|
16855
16855
|
}
|
|
16856
16856
|
async function readTestSuiteMetadata(testFilePath) {
|
|
16857
16857
|
try {
|
|
16858
|
-
const absolutePath =
|
|
16859
|
-
const content = await
|
|
16858
|
+
const absolutePath = path8.resolve(testFilePath);
|
|
16859
|
+
const content = await readFile7(absolutePath, "utf8");
|
|
16860
16860
|
const parsed = interpolateEnv(parse2(content), process.env);
|
|
16861
16861
|
if (!isJsonObject(parsed)) {
|
|
16862
16862
|
return {};
|
|
@@ -16907,26 +16907,26 @@ var loadEvalCases = loadTests;
|
|
|
16907
16907
|
async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
16908
16908
|
const verbose = options?.verbose ?? false;
|
|
16909
16909
|
const filterPattern = options?.filter;
|
|
16910
|
-
const absoluteTestPath =
|
|
16910
|
+
const absoluteTestPath = path8.resolve(evalFilePath);
|
|
16911
16911
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
16912
16912
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
16913
16913
|
const config = await loadConfig(absoluteTestPath, repoRootPath);
|
|
16914
16914
|
const guidelinePatterns = config?.guideline_patterns;
|
|
16915
|
-
const rawFile = await
|
|
16915
|
+
const rawFile = await readFile7(absoluteTestPath, "utf8");
|
|
16916
16916
|
const interpolated = interpolateEnv(parse2(rawFile), process.env);
|
|
16917
16917
|
if (!isJsonObject(interpolated)) {
|
|
16918
16918
|
throw new Error(`Invalid test file format: ${evalFilePath}`);
|
|
16919
16919
|
}
|
|
16920
16920
|
const suite = interpolated;
|
|
16921
16921
|
const datasetNameFromSuite = asString6(suite.dataset)?.trim();
|
|
16922
|
-
const fallbackDataset =
|
|
16922
|
+
const fallbackDataset = path8.basename(absoluteTestPath).replace(/\.ya?ml$/i, "") || "eval";
|
|
16923
16923
|
const datasetName = datasetNameFromSuite && datasetNameFromSuite.length > 0 ? datasetNameFromSuite : fallbackDataset;
|
|
16924
16924
|
const rawTestcases = resolveTests(suite);
|
|
16925
16925
|
const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-grader";
|
|
16926
|
-
const evalFileDir =
|
|
16926
|
+
const evalFileDir = path8.dirname(absoluteTestPath);
|
|
16927
16927
|
let expandedTestcases;
|
|
16928
16928
|
if (typeof rawTestcases === "string") {
|
|
16929
|
-
const externalPath =
|
|
16929
|
+
const externalPath = path8.resolve(evalFileDir, rawTestcases);
|
|
16930
16930
|
expandedTestcases = await loadCasesFromFile(externalPath);
|
|
16931
16931
|
} else if (Array.isArray(rawTestcases)) {
|
|
16932
16932
|
expandedTestcases = await expandFileReferences(rawTestcases, evalFileDir);
|
|
@@ -17047,7 +17047,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17047
17047
|
}
|
|
17048
17048
|
}
|
|
17049
17049
|
const allFilePaths = [
|
|
17050
|
-
...guidelinePaths.map((guidelinePath) =>
|
|
17050
|
+
...guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
17051
17051
|
...userFilePaths
|
|
17052
17052
|
];
|
|
17053
17053
|
const caseWorkspace = await resolveWorkspaceConfig(evalcase.workspace, evalFileDir);
|
|
@@ -17063,7 +17063,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17063
17063
|
input_segments: inputSegments,
|
|
17064
17064
|
expected_output: outputSegments,
|
|
17065
17065
|
reference_answer: referenceAnswer,
|
|
17066
|
-
guideline_paths: guidelinePaths.map((guidelinePath) =>
|
|
17066
|
+
guideline_paths: guidelinePaths.map((guidelinePath) => path8.resolve(guidelinePath)),
|
|
17067
17067
|
guideline_patterns: guidelinePatterns,
|
|
17068
17068
|
file_paths: allFilePaths,
|
|
17069
17069
|
criteria: outcome ?? "",
|
|
@@ -17113,8 +17113,8 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
|
|
|
17113
17113
|
if (commandArr.length === 0) return void 0;
|
|
17114
17114
|
const timeoutMs = typeof obj.timeout_ms === "number" ? obj.timeout_ms : void 0;
|
|
17115
17115
|
let cwd = typeof obj.cwd === "string" ? obj.cwd : void 0;
|
|
17116
|
-
if (cwd && !
|
|
17117
|
-
cwd =
|
|
17116
|
+
if (cwd && !path8.isAbsolute(cwd)) {
|
|
17117
|
+
cwd = path8.resolve(evalFileDir, cwd);
|
|
17118
17118
|
}
|
|
17119
17119
|
const config = { command: commandArr };
|
|
17120
17120
|
if (timeoutMs !== void 0) {
|
|
@@ -17204,10 +17204,10 @@ function parseWorkspaceHooksConfig(raw, evalFileDir) {
|
|
|
17204
17204
|
}
|
|
17205
17205
|
async function resolveWorkspaceConfig(raw, evalFileDir) {
|
|
17206
17206
|
if (typeof raw === "string") {
|
|
17207
|
-
const workspaceFilePath =
|
|
17207
|
+
const workspaceFilePath = path8.resolve(evalFileDir, raw);
|
|
17208
17208
|
let content;
|
|
17209
17209
|
try {
|
|
17210
|
-
content = await
|
|
17210
|
+
content = await readFile7(workspaceFilePath, "utf8");
|
|
17211
17211
|
} catch {
|
|
17212
17212
|
throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
|
|
17213
17213
|
}
|
|
@@ -17217,7 +17217,7 @@ async function resolveWorkspaceConfig(raw, evalFileDir) {
|
|
|
17217
17217
|
`Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
|
|
17218
17218
|
);
|
|
17219
17219
|
}
|
|
17220
|
-
const workspaceFileDir =
|
|
17220
|
+
const workspaceFileDir = path8.dirname(workspaceFilePath);
|
|
17221
17221
|
return parseWorkspaceConfig(parsed, workspaceFileDir);
|
|
17222
17222
|
}
|
|
17223
17223
|
return parseWorkspaceConfig(raw, evalFileDir);
|
|
@@ -17237,8 +17237,8 @@ function parseWorkspaceConfig(raw, evalFileDir) {
|
|
|
17237
17237
|
throw new Error("workspace.static has been removed. Use workspace.mode='static'.");
|
|
17238
17238
|
}
|
|
17239
17239
|
let template = typeof obj.template === "string" ? obj.template : void 0;
|
|
17240
|
-
if (template && !
|
|
17241
|
-
template =
|
|
17240
|
+
if (template && !path8.isAbsolute(template)) {
|
|
17241
|
+
template = path8.resolve(evalFileDir, template);
|
|
17242
17242
|
}
|
|
17243
17243
|
const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
|
|
17244
17244
|
const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
|
|
@@ -17291,19 +17291,19 @@ function asString6(value) {
|
|
|
17291
17291
|
function logWarning6(message, details) {
|
|
17292
17292
|
if (details && details.length > 0) {
|
|
17293
17293
|
const detailBlock = details.join("\n");
|
|
17294
|
-
console.warn(`${
|
|
17295
|
-
${detailBlock}${
|
|
17294
|
+
console.warn(`${ANSI_YELLOW7}Warning: ${message}
|
|
17295
|
+
${detailBlock}${ANSI_RESET8}`);
|
|
17296
17296
|
} else {
|
|
17297
|
-
console.warn(`${
|
|
17297
|
+
console.warn(`${ANSI_YELLOW7}Warning: ${message}${ANSI_RESET8}`);
|
|
17298
17298
|
}
|
|
17299
17299
|
}
|
|
17300
17300
|
function logError3(message, details) {
|
|
17301
17301
|
if (details && details.length > 0) {
|
|
17302
17302
|
const detailBlock = details.join("\n");
|
|
17303
17303
|
console.error(`${ANSI_RED3}Error: ${message}
|
|
17304
|
-
${detailBlock}${
|
|
17304
|
+
${detailBlock}${ANSI_RESET8}`);
|
|
17305
17305
|
} else {
|
|
17306
|
-
console.error(`${ANSI_RED3}Error: ${message}${
|
|
17306
|
+
console.error(`${ANSI_RED3}Error: ${message}${ANSI_RESET8}`);
|
|
17307
17307
|
}
|
|
17308
17308
|
}
|
|
17309
17309
|
function codeGraderInstruction(graderName, description) {
|
|
@@ -17549,7 +17549,7 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
17549
17549
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
17550
17550
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
17551
17551
|
const parsed = parse3(content);
|
|
17552
|
-
return transpileEvalYaml(parsed,
|
|
17552
|
+
return transpileEvalYaml(parsed, path9.basename(evalYamlPath));
|
|
17553
17553
|
}
|
|
17554
17554
|
function getOutputFilenames(result) {
|
|
17555
17555
|
const names = /* @__PURE__ */ new Map();
|
|
@@ -18049,7 +18049,7 @@ function normalizeInputFiles(inputFiles) {
|
|
|
18049
18049
|
}
|
|
18050
18050
|
const deduped = /* @__PURE__ */ new Map();
|
|
18051
18051
|
for (const inputFile of inputFiles) {
|
|
18052
|
-
const absolutePath =
|
|
18052
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18053
18053
|
if (!deduped.has(absolutePath)) {
|
|
18054
18054
|
deduped.set(absolutePath, absolutePath);
|
|
18055
18055
|
}
|
|
@@ -18062,14 +18062,14 @@ function collectGuidelineFiles(inputFiles, guidelinePatterns, overrides) {
|
|
|
18062
18062
|
}
|
|
18063
18063
|
const unique = /* @__PURE__ */ new Map();
|
|
18064
18064
|
for (const inputFile of inputFiles) {
|
|
18065
|
-
const absolutePath =
|
|
18065
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18066
18066
|
if (overrides?.has(absolutePath)) {
|
|
18067
18067
|
if (!unique.has(absolutePath)) {
|
|
18068
18068
|
unique.set(absolutePath, absolutePath);
|
|
18069
18069
|
}
|
|
18070
18070
|
continue;
|
|
18071
18071
|
}
|
|
18072
|
-
const normalized = absolutePath.split(
|
|
18072
|
+
const normalized = absolutePath.split(path10.sep).join("/");
|
|
18073
18073
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
18074
18074
|
if (!unique.has(absolutePath)) {
|
|
18075
18075
|
unique.set(absolutePath, absolutePath);
|
|
@@ -18084,7 +18084,7 @@ function collectInputFiles(inputFiles) {
|
|
|
18084
18084
|
}
|
|
18085
18085
|
const unique = /* @__PURE__ */ new Map();
|
|
18086
18086
|
for (const inputFile of inputFiles) {
|
|
18087
|
-
const absolutePath =
|
|
18087
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18088
18088
|
if (!unique.has(absolutePath)) {
|
|
18089
18089
|
unique.set(absolutePath, absolutePath);
|
|
18090
18090
|
}
|
|
@@ -18096,7 +18096,7 @@ function buildMandatoryPrereadBlock(guidelineFiles, inputFiles) {
|
|
|
18096
18096
|
return "";
|
|
18097
18097
|
}
|
|
18098
18098
|
const buildList = (files) => files.map((absolutePath) => {
|
|
18099
|
-
const fileName =
|
|
18099
|
+
const fileName = path10.basename(absolutePath);
|
|
18100
18100
|
const fileUri = pathToFileUri(absolutePath);
|
|
18101
18101
|
return `* [${fileName}](${fileUri})`;
|
|
18102
18102
|
});
|
|
@@ -18116,7 +18116,7 @@ ${buildList(inputFiles).join("\n")}.`);
|
|
|
18116
18116
|
return sections.join("\n");
|
|
18117
18117
|
}
|
|
18118
18118
|
function pathToFileUri(filePath) {
|
|
18119
|
-
const absolutePath =
|
|
18119
|
+
const absolutePath = path10.isAbsolute(filePath) ? filePath : path10.resolve(filePath);
|
|
18120
18120
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
18121
18121
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
18122
18122
|
return `file:///${normalizedPath}`;
|
|
@@ -18261,10 +18261,10 @@ var ClaudeCliProvider = class {
|
|
|
18261
18261
|
}
|
|
18262
18262
|
resolveCwd(cwdOverride) {
|
|
18263
18263
|
if (cwdOverride) {
|
|
18264
|
-
return
|
|
18264
|
+
return path11.resolve(cwdOverride);
|
|
18265
18265
|
}
|
|
18266
18266
|
if (this.config.cwd) {
|
|
18267
|
-
return
|
|
18267
|
+
return path11.resolve(this.config.cwd);
|
|
18268
18268
|
}
|
|
18269
18269
|
return void 0;
|
|
18270
18270
|
}
|
|
@@ -18274,9 +18274,9 @@ var ClaudeCliProvider = class {
|
|
|
18274
18274
|
return void 0;
|
|
18275
18275
|
}
|
|
18276
18276
|
if (this.config.logDir) {
|
|
18277
|
-
return
|
|
18277
|
+
return path11.resolve(this.config.logDir);
|
|
18278
18278
|
}
|
|
18279
|
-
return
|
|
18279
|
+
return path11.join(process.cwd(), ".agentv", "logs", "claude-cli");
|
|
18280
18280
|
}
|
|
18281
18281
|
async createStreamLogger(request) {
|
|
18282
18282
|
const logDir = this.resolveLogDirectory();
|
|
@@ -18290,7 +18290,7 @@ var ClaudeCliProvider = class {
|
|
|
18290
18290
|
console.warn(`Skipping Claude CLI stream logging (could not create ${logDir}): ${message}`);
|
|
18291
18291
|
return void 0;
|
|
18292
18292
|
}
|
|
18293
|
-
const filePath =
|
|
18293
|
+
const filePath = path11.join(logDir, buildLogFilename(request, this.targetName));
|
|
18294
18294
|
try {
|
|
18295
18295
|
const logger = await ClaudeCliStreamLogger.create({
|
|
18296
18296
|
filePath,
|
|
@@ -18765,10 +18765,10 @@ var ClaudeSdkProvider = class {
|
|
|
18765
18765
|
}
|
|
18766
18766
|
resolveCwd(cwdOverride) {
|
|
18767
18767
|
if (cwdOverride) {
|
|
18768
|
-
return
|
|
18768
|
+
return path12.resolve(cwdOverride);
|
|
18769
18769
|
}
|
|
18770
18770
|
if (this.config.cwd) {
|
|
18771
|
-
return
|
|
18771
|
+
return path12.resolve(this.config.cwd);
|
|
18772
18772
|
}
|
|
18773
18773
|
return void 0;
|
|
18774
18774
|
}
|
|
@@ -18778,9 +18778,9 @@ var ClaudeSdkProvider = class {
|
|
|
18778
18778
|
return void 0;
|
|
18779
18779
|
}
|
|
18780
18780
|
if (this.config.logDir) {
|
|
18781
|
-
return
|
|
18781
|
+
return path12.resolve(this.config.logDir);
|
|
18782
18782
|
}
|
|
18783
|
-
return
|
|
18783
|
+
return path12.join(process.cwd(), ".agentv", "logs", "claude");
|
|
18784
18784
|
}
|
|
18785
18785
|
async createStreamLogger(request) {
|
|
18786
18786
|
const logDir = this.resolveLogDirectory();
|
|
@@ -18794,7 +18794,7 @@ var ClaudeSdkProvider = class {
|
|
|
18794
18794
|
console.warn(`Skipping Claude stream logging (could not create ${logDir}): ${message}`);
|
|
18795
18795
|
return void 0;
|
|
18796
18796
|
}
|
|
18797
|
-
const filePath =
|
|
18797
|
+
const filePath = path12.join(logDir, buildLogFilename2(request, this.targetName));
|
|
18798
18798
|
try {
|
|
18799
18799
|
const logger = await ClaudeStreamLogger.create({
|
|
18800
18800
|
filePath,
|
|
@@ -19498,7 +19498,7 @@ function normalizeInputFiles2(inputFiles) {
|
|
|
19498
19498
|
}
|
|
19499
19499
|
const unique = /* @__PURE__ */ new Map();
|
|
19500
19500
|
for (const inputFile of inputFiles) {
|
|
19501
|
-
const absolutePath =
|
|
19501
|
+
const absolutePath = path13.resolve(inputFile);
|
|
19502
19502
|
if (!unique.has(absolutePath)) {
|
|
19503
19503
|
unique.set(absolutePath, absolutePath);
|
|
19504
19504
|
}
|
|
@@ -19512,7 +19512,7 @@ function formatFileList(files, template) {
|
|
|
19512
19512
|
const formatter = template ?? "{path}";
|
|
19513
19513
|
return files.map((filePath) => {
|
|
19514
19514
|
const escapedPath = shellEscape(filePath);
|
|
19515
|
-
const escapedName = shellEscape(
|
|
19515
|
+
const escapedName = shellEscape(path13.basename(filePath));
|
|
19516
19516
|
return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
|
|
19517
19517
|
}).join(" ");
|
|
19518
19518
|
}
|
|
@@ -19536,7 +19536,7 @@ function generateOutputFilePath(evalCaseId, extension = ".json") {
|
|
|
19536
19536
|
const safeEvalId = evalCaseId || "unknown";
|
|
19537
19537
|
const timestamp = Date.now();
|
|
19538
19538
|
const random = Math.random().toString(36).substring(2, 9);
|
|
19539
|
-
return
|
|
19539
|
+
return path13.join(os.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
|
|
19540
19540
|
}
|
|
19541
19541
|
function formatTimeoutSuffix2(timeoutMs) {
|
|
19542
19542
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
@@ -19775,10 +19775,10 @@ ${basePrompt}` : basePrompt;
|
|
|
19775
19775
|
}
|
|
19776
19776
|
resolveCwd(cwdOverride) {
|
|
19777
19777
|
if (cwdOverride) {
|
|
19778
|
-
return
|
|
19778
|
+
return path14.resolve(cwdOverride);
|
|
19779
19779
|
}
|
|
19780
19780
|
if (this.config.cwd) {
|
|
19781
|
-
return
|
|
19781
|
+
return path14.resolve(this.config.cwd);
|
|
19782
19782
|
}
|
|
19783
19783
|
return void 0;
|
|
19784
19784
|
}
|
|
@@ -19788,9 +19788,9 @@ ${basePrompt}` : basePrompt;
|
|
|
19788
19788
|
return void 0;
|
|
19789
19789
|
}
|
|
19790
19790
|
if (this.config.logDir) {
|
|
19791
|
-
return
|
|
19791
|
+
return path14.resolve(this.config.logDir);
|
|
19792
19792
|
}
|
|
19793
|
-
return
|
|
19793
|
+
return path14.join(process.cwd(), ".agentv", "logs", "codex");
|
|
19794
19794
|
}
|
|
19795
19795
|
async createStreamLogger(request) {
|
|
19796
19796
|
const logDir = this.resolveLogDirectory();
|
|
@@ -19804,7 +19804,7 @@ ${basePrompt}` : basePrompt;
|
|
|
19804
19804
|
console.warn(`Skipping Codex SDK stream logging (could not create ${logDir}): ${message}`);
|
|
19805
19805
|
return void 0;
|
|
19806
19806
|
}
|
|
19807
|
-
const filePath =
|
|
19807
|
+
const filePath = path14.join(logDir, buildLogFilename3(request, this.targetName));
|
|
19808
19808
|
try {
|
|
19809
19809
|
const logger = await CodexSdkStreamLogger.create({
|
|
19810
19810
|
filePath,
|
|
@@ -20017,7 +20017,7 @@ function resolvePlatformCliPath() {
|
|
|
20017
20017
|
try {
|
|
20018
20018
|
const resolved = import.meta.resolve(`${packageName}/package.json`);
|
|
20019
20019
|
const packageJsonPath = resolved.startsWith("file:") ? fileURLToPath2(resolved) : resolved;
|
|
20020
|
-
const binaryPath =
|
|
20020
|
+
const binaryPath = path15.join(path15.dirname(packageJsonPath), binaryName);
|
|
20021
20021
|
if (existsSync(binaryPath)) {
|
|
20022
20022
|
return binaryPath;
|
|
20023
20023
|
}
|
|
@@ -20025,7 +20025,7 @@ function resolvePlatformCliPath() {
|
|
|
20025
20025
|
}
|
|
20026
20026
|
let searchDir = process.cwd();
|
|
20027
20027
|
for (let i = 0; i < 10; i++) {
|
|
20028
|
-
const standardPath =
|
|
20028
|
+
const standardPath = path15.join(
|
|
20029
20029
|
searchDir,
|
|
20030
20030
|
"node_modules",
|
|
20031
20031
|
...packageName.split("/"),
|
|
@@ -20034,13 +20034,13 @@ function resolvePlatformCliPath() {
|
|
|
20034
20034
|
if (existsSync(standardPath)) {
|
|
20035
20035
|
return standardPath;
|
|
20036
20036
|
}
|
|
20037
|
-
const bunDir =
|
|
20037
|
+
const bunDir = path15.join(searchDir, "node_modules", ".bun");
|
|
20038
20038
|
const prefix = `@github+copilot-${osPart}-${archPart}@`;
|
|
20039
20039
|
try {
|
|
20040
20040
|
const entries = readdirSync(bunDir);
|
|
20041
20041
|
for (const entry of entries) {
|
|
20042
20042
|
if (entry.startsWith(prefix)) {
|
|
20043
|
-
const candidate =
|
|
20043
|
+
const candidate = path15.join(
|
|
20044
20044
|
bunDir,
|
|
20045
20045
|
entry,
|
|
20046
20046
|
"node_modules",
|
|
@@ -20055,7 +20055,7 @@ function resolvePlatformCliPath() {
|
|
|
20055
20055
|
}
|
|
20056
20056
|
} catch {
|
|
20057
20057
|
}
|
|
20058
|
-
const parent =
|
|
20058
|
+
const parent = path15.dirname(searchDir);
|
|
20059
20059
|
if (parent === searchDir) break;
|
|
20060
20060
|
searchDir = parent;
|
|
20061
20061
|
}
|
|
@@ -20391,10 +20391,10 @@ var CopilotCliProvider = class {
|
|
|
20391
20391
|
}
|
|
20392
20392
|
resolveCwd(cwdOverride) {
|
|
20393
20393
|
if (cwdOverride) {
|
|
20394
|
-
return
|
|
20394
|
+
return path16.resolve(cwdOverride);
|
|
20395
20395
|
}
|
|
20396
20396
|
if (this.config.cwd) {
|
|
20397
|
-
return
|
|
20397
|
+
return path16.resolve(this.config.cwd);
|
|
20398
20398
|
}
|
|
20399
20399
|
return void 0;
|
|
20400
20400
|
}
|
|
@@ -20413,9 +20413,9 @@ var CopilotCliProvider = class {
|
|
|
20413
20413
|
return void 0;
|
|
20414
20414
|
}
|
|
20415
20415
|
if (this.config.logDir) {
|
|
20416
|
-
return
|
|
20416
|
+
return path16.resolve(this.config.logDir);
|
|
20417
20417
|
}
|
|
20418
|
-
return
|
|
20418
|
+
return path16.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
20419
20419
|
}
|
|
20420
20420
|
async createStreamLogger(request) {
|
|
20421
20421
|
const logDir = this.resolveLogDirectory();
|
|
@@ -20429,7 +20429,7 @@ var CopilotCliProvider = class {
|
|
|
20429
20429
|
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
20430
20430
|
return void 0;
|
|
20431
20431
|
}
|
|
20432
|
-
const filePath =
|
|
20432
|
+
const filePath = path16.join(logDir, buildLogFilename4(request, this.targetName, "copilot-cli"));
|
|
20433
20433
|
try {
|
|
20434
20434
|
const logger = await CopilotStreamLogger.create(
|
|
20435
20435
|
{
|
|
@@ -20794,10 +20794,10 @@ var CopilotSdkProvider = class {
|
|
|
20794
20794
|
}
|
|
20795
20795
|
resolveCwd(cwdOverride) {
|
|
20796
20796
|
if (cwdOverride) {
|
|
20797
|
-
return
|
|
20797
|
+
return path17.resolve(cwdOverride);
|
|
20798
20798
|
}
|
|
20799
20799
|
if (this.config.cwd) {
|
|
20800
|
-
return
|
|
20800
|
+
return path17.resolve(this.config.cwd);
|
|
20801
20801
|
}
|
|
20802
20802
|
return void 0;
|
|
20803
20803
|
}
|
|
@@ -20806,9 +20806,9 @@ var CopilotSdkProvider = class {
|
|
|
20806
20806
|
return void 0;
|
|
20807
20807
|
}
|
|
20808
20808
|
if (this.config.logDir) {
|
|
20809
|
-
return
|
|
20809
|
+
return path17.resolve(this.config.logDir);
|
|
20810
20810
|
}
|
|
20811
|
-
return
|
|
20811
|
+
return path17.join(process.cwd(), ".agentv", "logs", "copilot-sdk");
|
|
20812
20812
|
}
|
|
20813
20813
|
async createStreamLogger(request) {
|
|
20814
20814
|
const logDir = this.resolveLogDirectory();
|
|
@@ -20822,7 +20822,7 @@ var CopilotSdkProvider = class {
|
|
|
20822
20822
|
console.warn(`Skipping Copilot SDK stream logging (could not create ${logDir}): ${message}`);
|
|
20823
20823
|
return void 0;
|
|
20824
20824
|
}
|
|
20825
|
-
const filePath =
|
|
20825
|
+
const filePath = path17.join(logDir, buildLogFilename4(request, this.targetName, "copilot-sdk"));
|
|
20826
20826
|
try {
|
|
20827
20827
|
const logger = await CopilotStreamLogger.create(
|
|
20828
20828
|
{
|
|
@@ -21241,7 +21241,7 @@ var PiCodingAgentProvider = class {
|
|
|
21241
21241
|
const workspaceRoot = await this.createWorkspace();
|
|
21242
21242
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
21243
21243
|
try {
|
|
21244
|
-
const promptFile =
|
|
21244
|
+
const promptFile = path18.join(workspaceRoot, PROMPT_FILENAME);
|
|
21245
21245
|
await writeFile(promptFile, request.question, "utf8");
|
|
21246
21246
|
const args = this.buildPiArgs(request.question, inputFiles, request.captureFileChanges);
|
|
21247
21247
|
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
@@ -21303,12 +21303,12 @@ var PiCodingAgentProvider = class {
|
|
|
21303
21303
|
}
|
|
21304
21304
|
resolveCwd(workspaceRoot, cwdOverride) {
|
|
21305
21305
|
if (cwdOverride) {
|
|
21306
|
-
return
|
|
21306
|
+
return path18.resolve(cwdOverride);
|
|
21307
21307
|
}
|
|
21308
21308
|
if (!this.config.cwd) {
|
|
21309
21309
|
return workspaceRoot;
|
|
21310
21310
|
}
|
|
21311
|
-
return
|
|
21311
|
+
return path18.resolve(this.config.cwd);
|
|
21312
21312
|
}
|
|
21313
21313
|
buildPiArgs(prompt, inputFiles, _captureFileChanges) {
|
|
21314
21314
|
const args = [];
|
|
@@ -21397,7 +21397,7 @@ ${prompt}` : prompt;
|
|
|
21397
21397
|
return env;
|
|
21398
21398
|
}
|
|
21399
21399
|
async createWorkspace() {
|
|
21400
|
-
return await mkdtemp(
|
|
21400
|
+
return await mkdtemp(path18.join(tmpdir(), WORKSPACE_PREFIX));
|
|
21401
21401
|
}
|
|
21402
21402
|
async cleanupWorkspace(workspaceRoot) {
|
|
21403
21403
|
try {
|
|
@@ -21407,9 +21407,9 @@ ${prompt}` : prompt;
|
|
|
21407
21407
|
}
|
|
21408
21408
|
resolveLogDirectory() {
|
|
21409
21409
|
if (this.config.logDir) {
|
|
21410
|
-
return
|
|
21410
|
+
return path18.resolve(this.config.logDir);
|
|
21411
21411
|
}
|
|
21412
|
-
return
|
|
21412
|
+
return path18.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
21413
21413
|
}
|
|
21414
21414
|
async createStreamLogger(request) {
|
|
21415
21415
|
const logDir = this.resolveLogDirectory();
|
|
@@ -21423,7 +21423,7 @@ ${prompt}` : prompt;
|
|
|
21423
21423
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
21424
21424
|
return void 0;
|
|
21425
21425
|
}
|
|
21426
|
-
const filePath =
|
|
21426
|
+
const filePath = path18.join(logDir, buildLogFilename5(request, this.targetName));
|
|
21427
21427
|
try {
|
|
21428
21428
|
const logger = await PiStreamLogger.create({
|
|
21429
21429
|
filePath,
|
|
@@ -21933,7 +21933,7 @@ async function readDirEntries(target) {
|
|
|
21933
21933
|
const entries = await readdir(target, { withFileTypes: true });
|
|
21934
21934
|
return entries.map((entry) => ({
|
|
21935
21935
|
name: entry.name,
|
|
21936
|
-
absolutePath:
|
|
21936
|
+
absolutePath: path19.join(target, entry.name),
|
|
21937
21937
|
isDirectory: entry.isDirectory()
|
|
21938
21938
|
}));
|
|
21939
21939
|
}
|
|
@@ -21947,7 +21947,7 @@ async function removeIfExists(target) {
|
|
|
21947
21947
|
}
|
|
21948
21948
|
}
|
|
21949
21949
|
function pathToFileUri2(filePath) {
|
|
21950
|
-
const absolutePath =
|
|
21950
|
+
const absolutePath = path20.isAbsolute(filePath) ? filePath : path20.resolve(filePath);
|
|
21951
21951
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
21952
21952
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
21953
21953
|
return `file:///${normalizedPath}`;
|
|
@@ -22039,8 +22039,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
22039
22039
|
});
|
|
22040
22040
|
}
|
|
22041
22041
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
22042
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
22043
|
-
const responseList = responseFiles.map((file) => `"${
|
|
22042
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path21.basename(file)}`).join("\n");
|
|
22043
|
+
const responseList = responseFiles.map((file) => `"${path21.basename(file)}"`).join(", ");
|
|
22044
22044
|
return renderTemplate2(templateContent, {
|
|
22045
22045
|
requestFiles: requestLines,
|
|
22046
22046
|
responseList
|
|
@@ -22079,7 +22079,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
22079
22079
|
const maxAttempts = 10;
|
|
22080
22080
|
while (attempts < maxAttempts) {
|
|
22081
22081
|
try {
|
|
22082
|
-
const content = await
|
|
22082
|
+
const content = await readFile8(responseFileFinal, { encoding: "utf8" });
|
|
22083
22083
|
if (!silent) {
|
|
22084
22084
|
process.stdout.write(`${content}
|
|
22085
22085
|
`);
|
|
@@ -22100,7 +22100,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
22100
22100
|
}
|
|
22101
22101
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
22102
22102
|
if (!silent) {
|
|
22103
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
22103
|
+
const fileList = responseFilesFinal.map((file) => path222.basename(file)).join(", ");
|
|
22104
22104
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
22105
22105
|
}
|
|
22106
22106
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -22109,7 +22109,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
22109
22109
|
while (pending.size > 0) {
|
|
22110
22110
|
if (Date.now() >= deadline) {
|
|
22111
22111
|
if (!silent) {
|
|
22112
|
-
const remaining = [...pending].map((f) =>
|
|
22112
|
+
const remaining = [...pending].map((f) => path222.basename(f)).join(", ");
|
|
22113
22113
|
console.error(
|
|
22114
22114
|
`error: timed out after ${Math.round(timeoutMs / 1e3)}s waiting for batch responses. Still pending: ${remaining}`
|
|
22115
22115
|
);
|
|
@@ -22136,7 +22136,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
22136
22136
|
const maxAttempts = 10;
|
|
22137
22137
|
while (attempts < maxAttempts) {
|
|
22138
22138
|
try {
|
|
22139
|
-
const content = await
|
|
22139
|
+
const content = await readFile8(file, { encoding: "utf8" });
|
|
22140
22140
|
if (!silent) {
|
|
22141
22141
|
process.stdout.write(`${content}
|
|
22142
22142
|
`);
|
|
@@ -22166,25 +22166,25 @@ function getAgentvHome() {
|
|
|
22166
22166
|
}
|
|
22167
22167
|
return envHome;
|
|
22168
22168
|
}
|
|
22169
|
-
return
|
|
22169
|
+
return path23.join(os2.homedir(), ".agentv");
|
|
22170
22170
|
}
|
|
22171
22171
|
function getWorkspacesRoot() {
|
|
22172
|
-
return
|
|
22172
|
+
return path23.join(getAgentvHome(), "workspaces");
|
|
22173
22173
|
}
|
|
22174
22174
|
function getSubagentsRoot() {
|
|
22175
|
-
return
|
|
22175
|
+
return path23.join(getAgentvHome(), "subagents");
|
|
22176
22176
|
}
|
|
22177
22177
|
function getTraceStateRoot() {
|
|
22178
|
-
return
|
|
22178
|
+
return path23.join(getAgentvHome(), "trace-state");
|
|
22179
22179
|
}
|
|
22180
22180
|
function getWorkspacePoolRoot() {
|
|
22181
|
-
return
|
|
22181
|
+
return path23.join(getAgentvHome(), "workspace-pool");
|
|
22182
22182
|
}
|
|
22183
22183
|
var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
22184
22184
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
22185
22185
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
22186
22186
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
22187
|
-
return
|
|
22187
|
+
return path24.join(getSubagentsRoot(), folder);
|
|
22188
22188
|
}
|
|
22189
22189
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
22190
22190
|
var execAsync2 = promisify2(exec);
|
|
@@ -22249,11 +22249,11 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22249
22249
|
await raceSpawnError(child);
|
|
22250
22250
|
return true;
|
|
22251
22251
|
}
|
|
22252
|
-
const aliveFile =
|
|
22252
|
+
const aliveFile = path25.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
22253
22253
|
await removeIfExists(aliveFile);
|
|
22254
|
-
const githubAgentsDir =
|
|
22254
|
+
const githubAgentsDir = path25.join(subagentDir, ".github", "agents");
|
|
22255
22255
|
await mkdir8(githubAgentsDir, { recursive: true });
|
|
22256
|
-
const wakeupDst =
|
|
22256
|
+
const wakeupDst = path25.join(githubAgentsDir, "wakeup.md");
|
|
22257
22257
|
await writeFile2(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
22258
22258
|
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
22259
22259
|
label: "open-workspace"
|
|
@@ -22266,7 +22266,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22266
22266
|
"chat",
|
|
22267
22267
|
"-m",
|
|
22268
22268
|
wakeupChatId,
|
|
22269
|
-
`create a file named .alive in the ${
|
|
22269
|
+
`create a file named .alive in the ${path25.basename(subagentDir)} folder`
|
|
22270
22270
|
];
|
|
22271
22271
|
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
22272
22272
|
await raceSpawnError(wakeupChild);
|
|
@@ -22281,10 +22281,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
22281
22281
|
return true;
|
|
22282
22282
|
}
|
|
22283
22283
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
22284
|
-
const workspacePath =
|
|
22285
|
-
const messagesDir =
|
|
22284
|
+
const workspacePath = path25.join(subagentDir, `${path25.basename(subagentDir)}.code-workspace`);
|
|
22285
|
+
const messagesDir = path25.join(subagentDir, "messages");
|
|
22286
22286
|
await mkdir8(messagesDir, { recursive: true });
|
|
22287
|
-
const reqFile =
|
|
22287
|
+
const reqFile = path25.join(messagesDir, `${timestamp}_req.md`);
|
|
22288
22288
|
await writeFile2(reqFile, requestInstructions, { encoding: "utf8" });
|
|
22289
22289
|
const reqUri = pathToFileUri2(reqFile);
|
|
22290
22290
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
@@ -22292,16 +22292,16 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
22292
22292
|
chatArgs.push("-a", attachment);
|
|
22293
22293
|
}
|
|
22294
22294
|
chatArgs.push("-a", reqFile);
|
|
22295
|
-
chatArgs.push(`Follow instructions in [${
|
|
22295
|
+
chatArgs.push(`Follow instructions in [${path25.basename(reqFile)}](${reqUri})`);
|
|
22296
22296
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
22297
22297
|
workspacePath,
|
|
22298
|
-
|
|
22298
|
+
path25.basename(subagentDir),
|
|
22299
22299
|
subagentDir,
|
|
22300
22300
|
vscodeCmd
|
|
22301
22301
|
);
|
|
22302
22302
|
if (!workspaceReady) {
|
|
22303
22303
|
throw new Error(
|
|
22304
|
-
`VS Code workspace '${
|
|
22304
|
+
`VS Code workspace '${path25.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
22305
22305
|
);
|
|
22306
22306
|
}
|
|
22307
22307
|
await sleep2(500);
|
|
@@ -22309,8 +22309,8 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
22309
22309
|
await raceSpawnError(child);
|
|
22310
22310
|
}
|
|
22311
22311
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
22312
|
-
const workspacePath =
|
|
22313
|
-
const messagesDir =
|
|
22312
|
+
const workspacePath = path25.join(subagentDir, `${path25.basename(subagentDir)}.code-workspace`);
|
|
22313
|
+
const messagesDir = path25.join(subagentDir, "messages");
|
|
22314
22314
|
await mkdir8(messagesDir, { recursive: true });
|
|
22315
22315
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
22316
22316
|
for (const attachment of attachmentPaths) {
|
|
@@ -22319,13 +22319,13 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
22319
22319
|
chatArgs.push(chatInstruction);
|
|
22320
22320
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
22321
22321
|
workspacePath,
|
|
22322
|
-
|
|
22322
|
+
path25.basename(subagentDir),
|
|
22323
22323
|
subagentDir,
|
|
22324
22324
|
vscodeCmd
|
|
22325
22325
|
);
|
|
22326
22326
|
if (!workspaceReady) {
|
|
22327
22327
|
throw new Error(
|
|
22328
|
-
`VS Code workspace '${
|
|
22328
|
+
`VS Code workspace '${path25.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
22329
22329
|
);
|
|
22330
22330
|
}
|
|
22331
22331
|
await sleep2(500);
|
|
@@ -22347,10 +22347,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
22347
22347
|
}
|
|
22348
22348
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
22349
22349
|
const folderPath = folder.path;
|
|
22350
|
-
if (
|
|
22350
|
+
if (path26.isAbsolute(folderPath)) {
|
|
22351
22351
|
return folder;
|
|
22352
22352
|
}
|
|
22353
|
-
const absolutePath =
|
|
22353
|
+
const absolutePath = path26.resolve(templateDir, folderPath);
|
|
22354
22354
|
return {
|
|
22355
22355
|
...folder,
|
|
22356
22356
|
path: absolutePath
|
|
@@ -22372,19 +22372,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
22372
22372
|
if (locationMap && typeof locationMap === "object") {
|
|
22373
22373
|
const transformedMap = {};
|
|
22374
22374
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
22375
|
-
const isAbsolute =
|
|
22375
|
+
const isAbsolute = path26.isAbsolute(locationPath);
|
|
22376
22376
|
if (isAbsolute) {
|
|
22377
22377
|
transformedMap[locationPath] = value;
|
|
22378
22378
|
} else {
|
|
22379
22379
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
22380
22380
|
if (firstGlobIndex === -1) {
|
|
22381
|
-
const resolvedPath =
|
|
22381
|
+
const resolvedPath = path26.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
22382
22382
|
transformedMap[resolvedPath] = value;
|
|
22383
22383
|
} else {
|
|
22384
22384
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
22385
22385
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
22386
22386
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
22387
|
-
const resolvedPath = (
|
|
22387
|
+
const resolvedPath = (path26.resolve(templateDir, basePath) + patternPath).replace(
|
|
22388
22388
|
/\\/g,
|
|
22389
22389
|
"/"
|
|
22390
22390
|
);
|
|
@@ -22423,7 +22423,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
22423
22423
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
22424
22424
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
22425
22425
|
for (const subagent of subagents) {
|
|
22426
|
-
const lockFile =
|
|
22426
|
+
const lockFile = path27.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
22427
22427
|
if (!await pathExists(lockFile)) {
|
|
22428
22428
|
return subagent.absolutePath;
|
|
22429
22429
|
}
|
|
@@ -22433,7 +22433,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
22433
22433
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
22434
22434
|
let workspaceContent;
|
|
22435
22435
|
if (workspaceTemplate) {
|
|
22436
|
-
const workspaceSrc =
|
|
22436
|
+
const workspaceSrc = path27.resolve(workspaceTemplate);
|
|
22437
22437
|
if (!await pathExists(workspaceSrc)) {
|
|
22438
22438
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
22439
22439
|
}
|
|
@@ -22441,18 +22441,18 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
22441
22441
|
if (!stats.isFile()) {
|
|
22442
22442
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
22443
22443
|
}
|
|
22444
|
-
const templateText = await
|
|
22444
|
+
const templateText = await readFile9(workspaceSrc, "utf8");
|
|
22445
22445
|
workspaceContent = JSON.parse(templateText);
|
|
22446
22446
|
} else {
|
|
22447
22447
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
22448
22448
|
}
|
|
22449
|
-
const workspaceName = `${
|
|
22450
|
-
const workspaceDst =
|
|
22451
|
-
const templateDir = workspaceTemplate ?
|
|
22449
|
+
const workspaceName = `${path27.basename(subagentDir)}.code-workspace`;
|
|
22450
|
+
const workspaceDst = path27.join(subagentDir, workspaceName);
|
|
22451
|
+
const templateDir = workspaceTemplate ? path27.dirname(path27.resolve(workspaceTemplate)) : subagentDir;
|
|
22452
22452
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
22453
22453
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
22454
22454
|
if (cwd) {
|
|
22455
|
-
const absCwd =
|
|
22455
|
+
const absCwd = path27.resolve(cwd);
|
|
22456
22456
|
const parsed = JSON.parse(transformedContent);
|
|
22457
22457
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
22458
22458
|
if (!alreadyPresent) {
|
|
@@ -22461,35 +22461,35 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
22461
22461
|
}
|
|
22462
22462
|
}
|
|
22463
22463
|
await writeFile3(workspaceDst, transformedContent, "utf8");
|
|
22464
|
-
const messagesDir =
|
|
22464
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
22465
22465
|
await mkdir9(messagesDir, { recursive: true });
|
|
22466
22466
|
return { workspace: workspaceDst, messagesDir };
|
|
22467
22467
|
}
|
|
22468
22468
|
async function createSubagentLock(subagentDir) {
|
|
22469
|
-
const messagesDir =
|
|
22469
|
+
const messagesDir = path27.join(subagentDir, "messages");
|
|
22470
22470
|
if (await pathExists(messagesDir)) {
|
|
22471
22471
|
const files = await readdir2(messagesDir);
|
|
22472
22472
|
await Promise.all(
|
|
22473
22473
|
files.map(async (file) => {
|
|
22474
|
-
const target =
|
|
22474
|
+
const target = path27.join(messagesDir, file);
|
|
22475
22475
|
await removeIfExists(target);
|
|
22476
22476
|
})
|
|
22477
22477
|
);
|
|
22478
22478
|
}
|
|
22479
|
-
const githubAgentsDir =
|
|
22479
|
+
const githubAgentsDir = path27.join(subagentDir, ".github", "agents");
|
|
22480
22480
|
if (await pathExists(githubAgentsDir)) {
|
|
22481
22481
|
const agentFiles = await readdir2(githubAgentsDir);
|
|
22482
22482
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
22483
22483
|
await Promise.all(
|
|
22484
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
22484
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path27.join(githubAgentsDir, file)))
|
|
22485
22485
|
);
|
|
22486
22486
|
}
|
|
22487
|
-
const lockFile =
|
|
22487
|
+
const lockFile = path27.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
22488
22488
|
await writeFile3(lockFile, "", { encoding: "utf8" });
|
|
22489
22489
|
return lockFile;
|
|
22490
22490
|
}
|
|
22491
22491
|
async function removeSubagentLock(subagentDir) {
|
|
22492
|
-
const lockFile =
|
|
22492
|
+
const lockFile = path27.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
22493
22493
|
await removeIfExists(lockFile);
|
|
22494
22494
|
}
|
|
22495
22495
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -22509,9 +22509,9 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
22509
22509
|
return 1;
|
|
22510
22510
|
}
|
|
22511
22511
|
if (promptFile) {
|
|
22512
|
-
const githubAgentsDir =
|
|
22512
|
+
const githubAgentsDir = path27.join(subagentDir, ".github", "agents");
|
|
22513
22513
|
await mkdir9(githubAgentsDir, { recursive: true });
|
|
22514
|
-
const agentFile =
|
|
22514
|
+
const agentFile = path27.join(githubAgentsDir, `${chatId}.md`);
|
|
22515
22515
|
try {
|
|
22516
22516
|
await copyFile(promptFile, agentFile);
|
|
22517
22517
|
} catch (error) {
|
|
@@ -22528,7 +22528,7 @@ async function resolvePromptFile(promptFile) {
|
|
|
22528
22528
|
if (!promptFile) {
|
|
22529
22529
|
return void 0;
|
|
22530
22530
|
}
|
|
22531
|
-
const resolvedPrompt =
|
|
22531
|
+
const resolvedPrompt = path28.resolve(promptFile);
|
|
22532
22532
|
if (!await pathExists(resolvedPrompt)) {
|
|
22533
22533
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
22534
22534
|
}
|
|
@@ -22544,7 +22544,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
22544
22544
|
}
|
|
22545
22545
|
const resolved = [];
|
|
22546
22546
|
for (const attachment of extraAttachments) {
|
|
22547
|
-
const resolvedPath =
|
|
22547
|
+
const resolvedPath = path28.resolve(attachment);
|
|
22548
22548
|
if (!await pathExists(resolvedPath)) {
|
|
22549
22549
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
22550
22550
|
}
|
|
@@ -22586,7 +22586,7 @@ async function dispatchAgentSession(options) {
|
|
|
22586
22586
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
22587
22587
|
};
|
|
22588
22588
|
}
|
|
22589
|
-
const subagentName =
|
|
22589
|
+
const subagentName = path28.basename(subagentDir);
|
|
22590
22590
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
22591
22591
|
const preparationResult = await prepareSubagentDirectory(
|
|
22592
22592
|
subagentDir,
|
|
@@ -22614,9 +22614,9 @@ async function dispatchAgentSession(options) {
|
|
|
22614
22614
|
};
|
|
22615
22615
|
}
|
|
22616
22616
|
const timestamp = generateTimestamp();
|
|
22617
|
-
const messagesDir =
|
|
22618
|
-
const responseFileTmp =
|
|
22619
|
-
const responseFileFinal =
|
|
22617
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
22618
|
+
const responseFileTmp = path28.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
22619
|
+
const responseFileFinal = path28.join(messagesDir, `${timestamp}_res.md`);
|
|
22620
22620
|
const requestInstructions = createRequestPrompt(
|
|
22621
22621
|
userQuery,
|
|
22622
22622
|
responseFileTmp,
|
|
@@ -22721,7 +22721,7 @@ async function dispatchBatchAgent(options) {
|
|
|
22721
22721
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
22722
22722
|
};
|
|
22723
22723
|
}
|
|
22724
|
-
subagentName =
|
|
22724
|
+
subagentName = path28.basename(subagentDir);
|
|
22725
22725
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
22726
22726
|
const preparationResult = await prepareSubagentDirectory(
|
|
22727
22727
|
subagentDir,
|
|
@@ -22752,17 +22752,17 @@ async function dispatchBatchAgent(options) {
|
|
|
22752
22752
|
};
|
|
22753
22753
|
}
|
|
22754
22754
|
const timestamp = generateTimestamp();
|
|
22755
|
-
const messagesDir =
|
|
22755
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
22756
22756
|
requestFiles = userQueries.map(
|
|
22757
|
-
(_, index) =>
|
|
22757
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
22758
22758
|
);
|
|
22759
22759
|
const responseTmpFiles = userQueries.map(
|
|
22760
|
-
(_, index) =>
|
|
22760
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
22761
22761
|
);
|
|
22762
22762
|
responseFilesFinal = userQueries.map(
|
|
22763
|
-
(_, index) =>
|
|
22763
|
+
(_, index) => path28.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
22764
22764
|
);
|
|
22765
|
-
const orchestratorFile =
|
|
22765
|
+
const orchestratorFile = path28.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
22766
22766
|
if (!dryRun) {
|
|
22767
22767
|
await Promise.all(
|
|
22768
22768
|
userQueries.map((query, index) => {
|
|
@@ -22875,7 +22875,7 @@ async function provisionSubagents(options) {
|
|
|
22875
22875
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
22876
22876
|
throw new Error("subagents must be a positive integer");
|
|
22877
22877
|
}
|
|
22878
|
-
const targetPath =
|
|
22878
|
+
const targetPath = path29.resolve(targetRoot);
|
|
22879
22879
|
if (!dryRun) {
|
|
22880
22880
|
await ensureDir(targetPath);
|
|
22881
22881
|
}
|
|
@@ -22895,7 +22895,7 @@ async function provisionSubagents(options) {
|
|
|
22895
22895
|
continue;
|
|
22896
22896
|
}
|
|
22897
22897
|
highestNumber = Math.max(highestNumber, parsed);
|
|
22898
|
-
const lockFile =
|
|
22898
|
+
const lockFile = path29.join(entry.absolutePath, lockName);
|
|
22899
22899
|
const locked = await pathExists(lockFile);
|
|
22900
22900
|
if (locked) {
|
|
22901
22901
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -22912,10 +22912,10 @@ async function provisionSubagents(options) {
|
|
|
22912
22912
|
break;
|
|
22913
22913
|
}
|
|
22914
22914
|
const subagentDir = subagent.absolutePath;
|
|
22915
|
-
const githubAgentsDir =
|
|
22916
|
-
const lockFile =
|
|
22917
|
-
const workspaceDst =
|
|
22918
|
-
const wakeupDst =
|
|
22915
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
22916
|
+
const lockFile = path29.join(subagentDir, lockName);
|
|
22917
|
+
const workspaceDst = path29.join(subagentDir, `${path29.basename(subagentDir)}.code-workspace`);
|
|
22918
|
+
const wakeupDst = path29.join(githubAgentsDir, "wakeup.md");
|
|
22919
22919
|
const isLocked = await pathExists(lockFile);
|
|
22920
22920
|
if (isLocked && !force) {
|
|
22921
22921
|
continue;
|
|
@@ -22953,10 +22953,10 @@ async function provisionSubagents(options) {
|
|
|
22953
22953
|
let nextIndex = highestNumber;
|
|
22954
22954
|
while (subagentsProvisioned < subagents) {
|
|
22955
22955
|
nextIndex += 1;
|
|
22956
|
-
const subagentDir =
|
|
22957
|
-
const githubAgentsDir =
|
|
22958
|
-
const workspaceDst =
|
|
22959
|
-
const wakeupDst =
|
|
22956
|
+
const subagentDir = path29.join(targetPath, `subagent-${nextIndex}`);
|
|
22957
|
+
const githubAgentsDir = path29.join(subagentDir, ".github", "agents");
|
|
22958
|
+
const workspaceDst = path29.join(subagentDir, `${path29.basename(subagentDir)}.code-workspace`);
|
|
22959
|
+
const wakeupDst = path29.join(githubAgentsDir, "wakeup.md");
|
|
22960
22960
|
if (!dryRun) {
|
|
22961
22961
|
await ensureDir(subagentDir);
|
|
22962
22962
|
await ensureDir(githubAgentsDir);
|
|
@@ -23142,7 +23142,7 @@ var VSCodeProvider = class {
|
|
|
23142
23142
|
async function locateVSCodeExecutable(candidate) {
|
|
23143
23143
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
23144
23144
|
if (includesPathSeparator) {
|
|
23145
|
-
const resolved =
|
|
23145
|
+
const resolved = path30.isAbsolute(candidate) ? candidate : path30.resolve(candidate);
|
|
23146
23146
|
try {
|
|
23147
23147
|
await access3(resolved, constants3.F_OK);
|
|
23148
23148
|
return resolved;
|
|
@@ -23171,7 +23171,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
23171
23171
|
return void 0;
|
|
23172
23172
|
}
|
|
23173
23173
|
try {
|
|
23174
|
-
const stats = await stat4(
|
|
23174
|
+
const stats = await stat4(path30.resolve(template));
|
|
23175
23175
|
return stats.isFile() ? template : void 0;
|
|
23176
23176
|
} catch {
|
|
23177
23177
|
return template;
|
|
@@ -23197,7 +23197,7 @@ function buildMandatoryPrereadBlock2(guidelineFiles, attachmentFiles) {
|
|
|
23197
23197
|
return "";
|
|
23198
23198
|
}
|
|
23199
23199
|
const buildList = (files) => files.map((absolutePath) => {
|
|
23200
|
-
const fileName =
|
|
23200
|
+
const fileName = path30.basename(absolutePath);
|
|
23201
23201
|
const fileUri = pathToFileUri3(absolutePath);
|
|
23202
23202
|
return `* [${fileName}](${fileUri})`;
|
|
23203
23203
|
});
|
|
@@ -23222,8 +23222,8 @@ function collectGuidelineFiles2(attachments, guidelinePatterns) {
|
|
|
23222
23222
|
}
|
|
23223
23223
|
const unique = /* @__PURE__ */ new Map();
|
|
23224
23224
|
for (const attachment of attachments) {
|
|
23225
|
-
const absolutePath =
|
|
23226
|
-
const normalized = absolutePath.split(
|
|
23225
|
+
const absolutePath = path30.resolve(attachment);
|
|
23226
|
+
const normalized = absolutePath.split(path30.sep).join("/");
|
|
23227
23227
|
if (isGuidelineFile(normalized, guidelinePatterns)) {
|
|
23228
23228
|
if (!unique.has(absolutePath)) {
|
|
23229
23229
|
unique.set(absolutePath, absolutePath);
|
|
@@ -23238,7 +23238,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
23238
23238
|
}
|
|
23239
23239
|
const unique = /* @__PURE__ */ new Map();
|
|
23240
23240
|
for (const attachment of attachments) {
|
|
23241
|
-
const absolutePath =
|
|
23241
|
+
const absolutePath = path30.resolve(attachment);
|
|
23242
23242
|
if (!unique.has(absolutePath)) {
|
|
23243
23243
|
unique.set(absolutePath, absolutePath);
|
|
23244
23244
|
}
|
|
@@ -23246,7 +23246,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
23246
23246
|
return Array.from(unique.values());
|
|
23247
23247
|
}
|
|
23248
23248
|
function pathToFileUri3(filePath) {
|
|
23249
|
-
const absolutePath =
|
|
23249
|
+
const absolutePath = path30.isAbsolute(filePath) ? filePath : path30.resolve(filePath);
|
|
23250
23250
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
23251
23251
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
23252
23252
|
return `file:///${normalizedPath}`;
|
|
@@ -23259,7 +23259,7 @@ function normalizeAttachments(attachments) {
|
|
|
23259
23259
|
}
|
|
23260
23260
|
const deduped = /* @__PURE__ */ new Set();
|
|
23261
23261
|
for (const attachment of attachments) {
|
|
23262
|
-
deduped.add(
|
|
23262
|
+
deduped.add(path30.resolve(attachment));
|
|
23263
23263
|
}
|
|
23264
23264
|
return Array.from(deduped);
|
|
23265
23265
|
}
|
|
@@ -23268,7 +23268,7 @@ function mergeAttachments(all) {
|
|
|
23268
23268
|
for (const list of all) {
|
|
23269
23269
|
if (!list) continue;
|
|
23270
23270
|
for (const inputFile of list) {
|
|
23271
|
-
deduped.add(
|
|
23271
|
+
deduped.add(path30.resolve(inputFile));
|
|
23272
23272
|
}
|
|
23273
23273
|
}
|
|
23274
23274
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -23348,11 +23348,11 @@ async function fileExists3(filePath) {
|
|
|
23348
23348
|
}
|
|
23349
23349
|
}
|
|
23350
23350
|
async function readTargetDefinitions(filePath) {
|
|
23351
|
-
const absolutePath =
|
|
23351
|
+
const absolutePath = path31.resolve(filePath);
|
|
23352
23352
|
if (!await fileExists3(absolutePath)) {
|
|
23353
23353
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
23354
23354
|
}
|
|
23355
|
-
const raw = await
|
|
23355
|
+
const raw = await readFile10(absolutePath, "utf8");
|
|
23356
23356
|
const parsed = parse4(raw);
|
|
23357
23357
|
if (!isRecord(parsed)) {
|
|
23358
23358
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
@@ -23369,11 +23369,11 @@ function listTargetNames(definitions) {
|
|
|
23369
23369
|
async function discoverProviders(registry, baseDir) {
|
|
23370
23370
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
23371
23371
|
const candidateDirs = [];
|
|
23372
|
-
let dir =
|
|
23373
|
-
const root =
|
|
23372
|
+
let dir = path322.resolve(baseDir);
|
|
23373
|
+
const root = path322.parse(dir).root;
|
|
23374
23374
|
while (dir !== root) {
|
|
23375
|
-
candidateDirs.push(
|
|
23376
|
-
dir =
|
|
23375
|
+
candidateDirs.push(path322.join(dir, ".agentv", "providers"));
|
|
23376
|
+
dir = path322.dirname(dir);
|
|
23377
23377
|
}
|
|
23378
23378
|
let files = [];
|
|
23379
23379
|
for (const providersDir of candidateDirs) {
|
|
@@ -23389,7 +23389,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
23389
23389
|
}
|
|
23390
23390
|
const discoveredKinds = [];
|
|
23391
23391
|
for (const filePath of files) {
|
|
23392
|
-
const basename =
|
|
23392
|
+
const basename = path322.basename(filePath);
|
|
23393
23393
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
23394
23394
|
if (registry.has(kindName)) {
|
|
23395
23395
|
continue;
|
|
@@ -23587,15 +23587,15 @@ async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
|
23587
23587
|
});
|
|
23588
23588
|
}
|
|
23589
23589
|
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
23590
|
-
const { mkdir: mkdir15, readFile:
|
|
23590
|
+
const { mkdir: mkdir15, readFile: readFile13, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
|
|
23591
23591
|
const { tmpdir: tmpdir3 } = await import("node:os");
|
|
23592
|
-
const
|
|
23592
|
+
const path45 = await import("node:path");
|
|
23593
23593
|
const { randomUUID: randomUUID9 } = await import("node:crypto");
|
|
23594
|
-
const dir =
|
|
23594
|
+
const dir = path45.join(tmpdir3(), `agentv-exec-${randomUUID9()}`);
|
|
23595
23595
|
await mkdir15(dir, { recursive: true });
|
|
23596
|
-
const stdinPath =
|
|
23597
|
-
const stdoutPath =
|
|
23598
|
-
const stderrPath =
|
|
23596
|
+
const stdinPath = path45.join(dir, "stdin.txt");
|
|
23597
|
+
const stdoutPath = path45.join(dir, "stdout.txt");
|
|
23598
|
+
const stderrPath = path45.join(dir, "stderr.txt");
|
|
23599
23599
|
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
23600
23600
|
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
23601
23601
|
const { spawn: spawn5 } = await import("node:child_process");
|
|
@@ -23625,8 +23625,8 @@ async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
|
23625
23625
|
resolve2(code ?? 0);
|
|
23626
23626
|
});
|
|
23627
23627
|
});
|
|
23628
|
-
const stdout = (await
|
|
23629
|
-
const stderr = (await
|
|
23628
|
+
const stdout = (await readFile13(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
23629
|
+
const stderr = (await readFile13(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
23630
23630
|
return { stdout, stderr, exitCode };
|
|
23631
23631
|
} finally {
|
|
23632
23632
|
await rm6(dir, { recursive: true, force: true });
|
|
@@ -23937,7 +23937,7 @@ var CodeEvaluator = class {
|
|
|
23937
23937
|
outputPath,
|
|
23938
23938
|
guidelineFiles: context2.evalCase.guideline_paths,
|
|
23939
23939
|
inputFiles: context2.evalCase.file_paths.filter(
|
|
23940
|
-
(
|
|
23940
|
+
(path45) => !context2.evalCase.guideline_paths.includes(path45)
|
|
23941
23941
|
),
|
|
23942
23942
|
input: context2.evalCase.input,
|
|
23943
23943
|
trace: context2.trace ?? null,
|
|
@@ -24892,8 +24892,8 @@ function calculateScoreRangeResult(result, rubrics) {
|
|
|
24892
24892
|
};
|
|
24893
24893
|
}
|
|
24894
24894
|
function resolveSandboxed(basePath, relativePath) {
|
|
24895
|
-
const resolved =
|
|
24896
|
-
if (!resolved.startsWith(basePath +
|
|
24895
|
+
const resolved = path33.resolve(basePath, relativePath);
|
|
24896
|
+
if (!resolved.startsWith(basePath + path33.sep) && resolved !== basePath) {
|
|
24897
24897
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
24898
24898
|
}
|
|
24899
24899
|
return resolved;
|
|
@@ -24983,11 +24983,11 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
24983
24983
|
for (const entry of entries) {
|
|
24984
24984
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
24985
24985
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
24986
|
-
const fullPath =
|
|
24986
|
+
const fullPath = path33.join(dirPath, entry.name);
|
|
24987
24987
|
if (entry.isDirectory()) {
|
|
24988
24988
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
24989
24989
|
} else if (entry.isFile()) {
|
|
24990
|
-
const ext =
|
|
24990
|
+
const ext = path33.extname(entry.name).toLowerCase();
|
|
24991
24991
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
24992
24992
|
try {
|
|
24993
24993
|
const stat8 = await fs2.stat(fullPath);
|
|
@@ -24999,7 +24999,7 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
24999
24999
|
regex.lastIndex = 0;
|
|
25000
25000
|
if (regex.test(lines[i])) {
|
|
25001
25001
|
matches.push({
|
|
25002
|
-
file:
|
|
25002
|
+
file: path33.relative(workspacePath, fullPath),
|
|
25003
25003
|
line: i + 1,
|
|
25004
25004
|
text: lines[i].substring(0, 200)
|
|
25005
25005
|
});
|
|
@@ -25626,115 +25626,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
25626
25626
|
* Evaluate a single field against the expected value.
|
|
25627
25627
|
*/
|
|
25628
25628
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
25629
|
-
const { path:
|
|
25630
|
-
const candidateValue = resolvePath(candidateData,
|
|
25631
|
-
const expectedValue = resolvePath(expectedData,
|
|
25629
|
+
const { path: path45, match, required = true, weight = 1 } = fieldConfig;
|
|
25630
|
+
const candidateValue = resolvePath(candidateData, path45);
|
|
25631
|
+
const expectedValue = resolvePath(expectedData, path45);
|
|
25632
25632
|
if (expectedValue === void 0) {
|
|
25633
25633
|
return {
|
|
25634
|
-
path:
|
|
25634
|
+
path: path45,
|
|
25635
25635
|
score: 1,
|
|
25636
25636
|
// No expected value means no comparison needed
|
|
25637
25637
|
weight,
|
|
25638
25638
|
hit: true,
|
|
25639
|
-
message: `${
|
|
25639
|
+
message: `${path45}: no expected value`
|
|
25640
25640
|
};
|
|
25641
25641
|
}
|
|
25642
25642
|
if (candidateValue === void 0) {
|
|
25643
25643
|
if (required) {
|
|
25644
25644
|
return {
|
|
25645
|
-
path:
|
|
25645
|
+
path: path45,
|
|
25646
25646
|
score: 0,
|
|
25647
25647
|
weight,
|
|
25648
25648
|
hit: false,
|
|
25649
|
-
message: `${
|
|
25649
|
+
message: `${path45} (required, missing)`
|
|
25650
25650
|
};
|
|
25651
25651
|
}
|
|
25652
25652
|
return {
|
|
25653
|
-
path:
|
|
25653
|
+
path: path45,
|
|
25654
25654
|
score: 1,
|
|
25655
25655
|
// Don't penalize missing optional fields
|
|
25656
25656
|
weight: 0,
|
|
25657
25657
|
// Zero weight means it won't affect the score
|
|
25658
25658
|
hit: true,
|
|
25659
|
-
message: `${
|
|
25659
|
+
message: `${path45}: optional field missing`
|
|
25660
25660
|
};
|
|
25661
25661
|
}
|
|
25662
25662
|
switch (match) {
|
|
25663
25663
|
case "exact":
|
|
25664
|
-
return this.compareExact(
|
|
25664
|
+
return this.compareExact(path45, candidateValue, expectedValue, weight);
|
|
25665
25665
|
case "numeric_tolerance":
|
|
25666
25666
|
return this.compareNumericTolerance(
|
|
25667
|
-
|
|
25667
|
+
path45,
|
|
25668
25668
|
candidateValue,
|
|
25669
25669
|
expectedValue,
|
|
25670
25670
|
fieldConfig,
|
|
25671
25671
|
weight
|
|
25672
25672
|
);
|
|
25673
25673
|
case "date":
|
|
25674
|
-
return this.compareDate(
|
|
25674
|
+
return this.compareDate(path45, candidateValue, expectedValue, fieldConfig, weight);
|
|
25675
25675
|
default:
|
|
25676
25676
|
return {
|
|
25677
|
-
path:
|
|
25677
|
+
path: path45,
|
|
25678
25678
|
score: 0,
|
|
25679
25679
|
weight,
|
|
25680
25680
|
hit: false,
|
|
25681
|
-
message: `${
|
|
25681
|
+
message: `${path45}: unknown match type "${match}"`
|
|
25682
25682
|
};
|
|
25683
25683
|
}
|
|
25684
25684
|
}
|
|
25685
25685
|
/**
|
|
25686
25686
|
* Exact equality comparison.
|
|
25687
25687
|
*/
|
|
25688
|
-
compareExact(
|
|
25688
|
+
compareExact(path45, candidateValue, expectedValue, weight) {
|
|
25689
25689
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
25690
25690
|
return {
|
|
25691
|
-
path:
|
|
25691
|
+
path: path45,
|
|
25692
25692
|
score: 1,
|
|
25693
25693
|
weight,
|
|
25694
25694
|
hit: true,
|
|
25695
|
-
message:
|
|
25695
|
+
message: path45
|
|
25696
25696
|
};
|
|
25697
25697
|
}
|
|
25698
25698
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
25699
25699
|
return {
|
|
25700
|
-
path:
|
|
25700
|
+
path: path45,
|
|
25701
25701
|
score: 0,
|
|
25702
25702
|
weight,
|
|
25703
25703
|
hit: false,
|
|
25704
|
-
message: `${
|
|
25704
|
+
message: `${path45} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
25705
25705
|
};
|
|
25706
25706
|
}
|
|
25707
25707
|
return {
|
|
25708
|
-
path:
|
|
25708
|
+
path: path45,
|
|
25709
25709
|
score: 0,
|
|
25710
25710
|
weight,
|
|
25711
25711
|
hit: false,
|
|
25712
|
-
message: `${
|
|
25712
|
+
message: `${path45} (value mismatch)`
|
|
25713
25713
|
};
|
|
25714
25714
|
}
|
|
25715
25715
|
/**
|
|
25716
25716
|
* Numeric comparison with absolute or relative tolerance.
|
|
25717
25717
|
*/
|
|
25718
|
-
compareNumericTolerance(
|
|
25718
|
+
compareNumericTolerance(path45, candidateValue, expectedValue, fieldConfig, weight) {
|
|
25719
25719
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
25720
25720
|
const candidateNum = toNumber(candidateValue);
|
|
25721
25721
|
const expectedNum = toNumber(expectedValue);
|
|
25722
25722
|
if (candidateNum === null || expectedNum === null) {
|
|
25723
25723
|
return {
|
|
25724
|
-
path:
|
|
25724
|
+
path: path45,
|
|
25725
25725
|
score: 0,
|
|
25726
25726
|
weight,
|
|
25727
25727
|
hit: false,
|
|
25728
|
-
message: `${
|
|
25728
|
+
message: `${path45} (non-numeric value)`
|
|
25729
25729
|
};
|
|
25730
25730
|
}
|
|
25731
25731
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
25732
25732
|
return {
|
|
25733
|
-
path:
|
|
25733
|
+
path: path45,
|
|
25734
25734
|
score: 0,
|
|
25735
25735
|
weight,
|
|
25736
25736
|
hit: false,
|
|
25737
|
-
message: `${
|
|
25737
|
+
message: `${path45} (invalid numeric value)`
|
|
25738
25738
|
};
|
|
25739
25739
|
}
|
|
25740
25740
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -25747,61 +25747,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
25747
25747
|
}
|
|
25748
25748
|
if (withinTolerance) {
|
|
25749
25749
|
return {
|
|
25750
|
-
path:
|
|
25750
|
+
path: path45,
|
|
25751
25751
|
score: 1,
|
|
25752
25752
|
weight,
|
|
25753
25753
|
hit: true,
|
|
25754
|
-
message: `${
|
|
25754
|
+
message: `${path45} (within tolerance: diff=${diff.toFixed(2)})`
|
|
25755
25755
|
};
|
|
25756
25756
|
}
|
|
25757
25757
|
return {
|
|
25758
|
-
path:
|
|
25758
|
+
path: path45,
|
|
25759
25759
|
score: 0,
|
|
25760
25760
|
weight,
|
|
25761
25761
|
hit: false,
|
|
25762
|
-
message: `${
|
|
25762
|
+
message: `${path45} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
25763
25763
|
};
|
|
25764
25764
|
}
|
|
25765
25765
|
/**
|
|
25766
25766
|
* Date comparison with format normalization.
|
|
25767
25767
|
*/
|
|
25768
|
-
compareDate(
|
|
25768
|
+
compareDate(path45, candidateValue, expectedValue, fieldConfig, weight) {
|
|
25769
25769
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
25770
25770
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
25771
25771
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
25772
25772
|
if (candidateDate === null) {
|
|
25773
25773
|
return {
|
|
25774
|
-
path:
|
|
25774
|
+
path: path45,
|
|
25775
25775
|
score: 0,
|
|
25776
25776
|
weight,
|
|
25777
25777
|
hit: false,
|
|
25778
|
-
message: `${
|
|
25778
|
+
message: `${path45} (unparseable candidate date)`
|
|
25779
25779
|
};
|
|
25780
25780
|
}
|
|
25781
25781
|
if (expectedDate === null) {
|
|
25782
25782
|
return {
|
|
25783
|
-
path:
|
|
25783
|
+
path: path45,
|
|
25784
25784
|
score: 0,
|
|
25785
25785
|
weight,
|
|
25786
25786
|
hit: false,
|
|
25787
|
-
message: `${
|
|
25787
|
+
message: `${path45} (unparseable expected date)`
|
|
25788
25788
|
};
|
|
25789
25789
|
}
|
|
25790
25790
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
25791
25791
|
return {
|
|
25792
|
-
path:
|
|
25792
|
+
path: path45,
|
|
25793
25793
|
score: 1,
|
|
25794
25794
|
weight,
|
|
25795
25795
|
hit: true,
|
|
25796
|
-
message:
|
|
25796
|
+
message: path45
|
|
25797
25797
|
};
|
|
25798
25798
|
}
|
|
25799
25799
|
return {
|
|
25800
|
-
path:
|
|
25800
|
+
path: path45,
|
|
25801
25801
|
score: 0,
|
|
25802
25802
|
weight,
|
|
25803
25803
|
hit: false,
|
|
25804
|
-
message: `${
|
|
25804
|
+
message: `${path45} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
25805
25805
|
};
|
|
25806
25806
|
}
|
|
25807
25807
|
/**
|
|
@@ -25834,11 +25834,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
25834
25834
|
};
|
|
25835
25835
|
}
|
|
25836
25836
|
};
|
|
25837
|
-
function resolvePath(obj,
|
|
25838
|
-
if (!
|
|
25837
|
+
function resolvePath(obj, path45) {
|
|
25838
|
+
if (!path45 || !obj) {
|
|
25839
25839
|
return void 0;
|
|
25840
25840
|
}
|
|
25841
|
-
const parts =
|
|
25841
|
+
const parts = path45.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
25842
25842
|
let current = obj;
|
|
25843
25843
|
for (const part of parts) {
|
|
25844
25844
|
if (current === null || current === void 0) {
|
|
@@ -26288,8 +26288,8 @@ var TokenUsageEvaluator = class {
|
|
|
26288
26288
|
};
|
|
26289
26289
|
}
|
|
26290
26290
|
};
|
|
26291
|
-
function getNestedValue(obj,
|
|
26292
|
-
const parts =
|
|
26291
|
+
function getNestedValue(obj, path45) {
|
|
26292
|
+
const parts = path45.split(".");
|
|
26293
26293
|
let current = obj;
|
|
26294
26294
|
for (const part of parts) {
|
|
26295
26295
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -27155,7 +27155,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
|
27155
27155
|
};
|
|
27156
27156
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
27157
27157
|
const scriptPath = script[script.length - 1];
|
|
27158
|
-
const cwd =
|
|
27158
|
+
const cwd = path34.dirname(scriptPath);
|
|
27159
27159
|
try {
|
|
27160
27160
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
27161
27161
|
const prompt = stdout.trim();
|
|
@@ -27426,16 +27426,16 @@ function createBuiltinRegistry() {
|
|
|
27426
27426
|
async function discoverAssertions(registry, baseDir) {
|
|
27427
27427
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
27428
27428
|
const candidateDirs = [];
|
|
27429
|
-
let dir =
|
|
27430
|
-
const root =
|
|
27429
|
+
let dir = path35.resolve(baseDir);
|
|
27430
|
+
const root = path35.parse(dir).root;
|
|
27431
27431
|
while (dir !== root) {
|
|
27432
|
-
candidateDirs.push(
|
|
27433
|
-
dir =
|
|
27432
|
+
candidateDirs.push(path35.join(dir, ".agentv", "assertions"));
|
|
27433
|
+
dir = path35.dirname(dir);
|
|
27434
27434
|
}
|
|
27435
27435
|
let files = [];
|
|
27436
27436
|
for (const assertionsDir of candidateDirs) {
|
|
27437
27437
|
try {
|
|
27438
|
-
const found = await
|
|
27438
|
+
const found = await fg22(patterns, {
|
|
27439
27439
|
cwd: assertionsDir,
|
|
27440
27440
|
absolute: true,
|
|
27441
27441
|
onlyFiles: true
|
|
@@ -27446,7 +27446,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
27446
27446
|
}
|
|
27447
27447
|
const discoveredTypes = [];
|
|
27448
27448
|
for (const filePath of files) {
|
|
27449
|
-
const basename =
|
|
27449
|
+
const basename = path35.basename(filePath);
|
|
27450
27450
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
27451
27451
|
if (registry.has(typeName)) {
|
|
27452
27452
|
continue;
|
|
@@ -27465,17 +27465,17 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
27465
27465
|
async function discoverGraders(registry, baseDir) {
|
|
27466
27466
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
27467
27467
|
const candidateDirs = [];
|
|
27468
|
-
let dir =
|
|
27469
|
-
const root =
|
|
27468
|
+
let dir = path36.resolve(baseDir);
|
|
27469
|
+
const root = path36.parse(dir).root;
|
|
27470
27470
|
while (dir !== root) {
|
|
27471
|
-
candidateDirs.push(
|
|
27472
|
-
candidateDirs.push(
|
|
27473
|
-
dir =
|
|
27471
|
+
candidateDirs.push(path36.join(dir, ".agentv", "graders"));
|
|
27472
|
+
candidateDirs.push(path36.join(dir, ".agentv", "judges"));
|
|
27473
|
+
dir = path36.dirname(dir);
|
|
27474
27474
|
}
|
|
27475
27475
|
let files = [];
|
|
27476
27476
|
for (const gradersDir of candidateDirs) {
|
|
27477
27477
|
try {
|
|
27478
|
-
const found = await
|
|
27478
|
+
const found = await fg3(patterns, {
|
|
27479
27479
|
cwd: gradersDir,
|
|
27480
27480
|
absolute: true,
|
|
27481
27481
|
onlyFiles: true
|
|
@@ -27486,7 +27486,7 @@ async function discoverGraders(registry, baseDir) {
|
|
|
27486
27486
|
}
|
|
27487
27487
|
const discoveredTypes = [];
|
|
27488
27488
|
for (const filePath of files) {
|
|
27489
|
-
const basename =
|
|
27489
|
+
const basename = path36.basename(filePath);
|
|
27490
27490
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
27491
27491
|
if (registry.has(typeName)) {
|
|
27492
27492
|
continue;
|
|
@@ -27672,10 +27672,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
27672
27672
|
}
|
|
27673
27673
|
for (const entry of entries) {
|
|
27674
27674
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
27675
|
-
const childPath =
|
|
27675
|
+
const childPath = path37.join(workspacePath, entry);
|
|
27676
27676
|
try {
|
|
27677
27677
|
if (!statSync(childPath).isDirectory()) continue;
|
|
27678
|
-
if (!statSync(
|
|
27678
|
+
if (!statSync(path37.join(childPath, ".git")).isDirectory()) continue;
|
|
27679
27679
|
} catch {
|
|
27680
27680
|
continue;
|
|
27681
27681
|
}
|
|
@@ -27712,14 +27712,14 @@ async function isDirectory(filePath) {
|
|
|
27712
27712
|
}
|
|
27713
27713
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
27714
27714
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
27715
|
-
return
|
|
27715
|
+
return path38.join(root, evalRunId, caseId);
|
|
27716
27716
|
}
|
|
27717
27717
|
async function copyDirectoryRecursive(src, dest) {
|
|
27718
27718
|
await mkdir11(dest, { recursive: true });
|
|
27719
27719
|
const entries = await readdir3(src, { withFileTypes: true });
|
|
27720
27720
|
for (const entry of entries) {
|
|
27721
|
-
const srcPath =
|
|
27722
|
-
const destPath =
|
|
27721
|
+
const srcPath = path38.join(src, entry.name);
|
|
27722
|
+
const destPath = path38.join(dest, entry.name);
|
|
27723
27723
|
if (entry.name === ".git") {
|
|
27724
27724
|
continue;
|
|
27725
27725
|
}
|
|
@@ -27731,7 +27731,7 @@ async function copyDirectoryRecursive(src, dest) {
|
|
|
27731
27731
|
}
|
|
27732
27732
|
}
|
|
27733
27733
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
27734
|
-
const resolvedTemplatePath =
|
|
27734
|
+
const resolvedTemplatePath = path38.resolve(templatePath);
|
|
27735
27735
|
if (!await fileExists(resolvedTemplatePath)) {
|
|
27736
27736
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
27737
27737
|
}
|
|
@@ -27780,7 +27780,7 @@ async function cleanupWorkspace(workspacePath) {
|
|
|
27780
27780
|
}
|
|
27781
27781
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
27782
27782
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
27783
|
-
const evalDir =
|
|
27783
|
+
const evalDir = path38.join(root, evalRunId);
|
|
27784
27784
|
if (await fileExists(evalDir)) {
|
|
27785
27785
|
await rm4(evalDir, { recursive: true, force: true });
|
|
27786
27786
|
}
|
|
@@ -27837,8 +27837,8 @@ async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
|
27837
27837
|
await mkdir12(dest, { recursive: true });
|
|
27838
27838
|
const entries = await readdir4(src, { withFileTypes: true });
|
|
27839
27839
|
for (const entry of entries) {
|
|
27840
|
-
const srcPath =
|
|
27841
|
-
const destPath =
|
|
27840
|
+
const srcPath = path39.join(src, entry.name);
|
|
27841
|
+
const destPath = path39.join(dest, entry.name);
|
|
27842
27842
|
if (entry.name === ".git") {
|
|
27843
27843
|
continue;
|
|
27844
27844
|
}
|
|
@@ -27871,7 +27871,7 @@ var WorkspacePoolManager = class {
|
|
|
27871
27871
|
async acquireWorkspace(options) {
|
|
27872
27872
|
const { templatePath, repos, maxSlots, repoManager, poolReset } = options;
|
|
27873
27873
|
const fingerprint = computeWorkspaceFingerprint(repos);
|
|
27874
|
-
const poolDir =
|
|
27874
|
+
const poolDir = path39.join(this.poolRoot, fingerprint);
|
|
27875
27875
|
await mkdir12(poolDir, { recursive: true });
|
|
27876
27876
|
const drifted = await this.checkDrift(poolDir, fingerprint);
|
|
27877
27877
|
if (drifted) {
|
|
@@ -27881,7 +27881,7 @@ var WorkspacePoolManager = class {
|
|
|
27881
27881
|
await this.removeAllSlots(poolDir);
|
|
27882
27882
|
}
|
|
27883
27883
|
for (let i = 0; i < maxSlots; i++) {
|
|
27884
|
-
const slotPath =
|
|
27884
|
+
const slotPath = path39.join(poolDir, `slot-${i}`);
|
|
27885
27885
|
const lockPath = `${slotPath}.lock`;
|
|
27886
27886
|
const locked = await this.tryLock(lockPath);
|
|
27887
27887
|
if (!locked) {
|
|
@@ -27943,7 +27943,7 @@ var WorkspacePoolManager = class {
|
|
|
27943
27943
|
throw err;
|
|
27944
27944
|
}
|
|
27945
27945
|
try {
|
|
27946
|
-
const pidStr = await
|
|
27946
|
+
const pidStr = await readFile11(lockPath, "utf-8");
|
|
27947
27947
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
27948
27948
|
if (!Number.isNaN(pid)) {
|
|
27949
27949
|
try {
|
|
@@ -27968,9 +27968,9 @@ var WorkspacePoolManager = class {
|
|
|
27968
27968
|
* Returns false (no drift) if metadata.json doesn't exist (first use).
|
|
27969
27969
|
*/
|
|
27970
27970
|
async checkDrift(poolDir, fingerprint) {
|
|
27971
|
-
const metadataPath =
|
|
27971
|
+
const metadataPath = path39.join(poolDir, "metadata.json");
|
|
27972
27972
|
try {
|
|
27973
|
-
const raw = await
|
|
27973
|
+
const raw = await readFile11(metadataPath, "utf-8");
|
|
27974
27974
|
const metadata = JSON.parse(raw);
|
|
27975
27975
|
return metadata.fingerprint !== fingerprint;
|
|
27976
27976
|
} catch {
|
|
@@ -27985,17 +27985,17 @@ var WorkspacePoolManager = class {
|
|
|
27985
27985
|
repos,
|
|
27986
27986
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
27987
27987
|
};
|
|
27988
|
-
await writeFile7(
|
|
27988
|
+
await writeFile7(path39.join(poolDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
27989
27989
|
}
|
|
27990
27990
|
/** Remove all slot directories and their lock files from a pool directory. */
|
|
27991
27991
|
async removeAllSlots(poolDir) {
|
|
27992
27992
|
const entries = await readdir4(poolDir);
|
|
27993
27993
|
for (const entry of entries) {
|
|
27994
27994
|
if (entry.startsWith("slot-") && !entry.endsWith(".lock")) {
|
|
27995
|
-
const lockPath =
|
|
27995
|
+
const lockPath = path39.join(poolDir, `${entry}.lock`);
|
|
27996
27996
|
if (existsSync2(lockPath)) {
|
|
27997
27997
|
try {
|
|
27998
|
-
const pidStr = await
|
|
27998
|
+
const pidStr = await readFile11(lockPath, "utf-8");
|
|
27999
27999
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
28000
28000
|
if (!Number.isNaN(pid)) {
|
|
28001
28001
|
try {
|
|
@@ -28008,12 +28008,12 @@ var WorkspacePoolManager = class {
|
|
|
28008
28008
|
} catch {
|
|
28009
28009
|
}
|
|
28010
28010
|
}
|
|
28011
|
-
await rm5(
|
|
28011
|
+
await rm5(path39.join(poolDir, entry), { recursive: true, force: true });
|
|
28012
28012
|
await rm5(lockPath, { force: true }).catch(() => {
|
|
28013
28013
|
});
|
|
28014
28014
|
}
|
|
28015
28015
|
}
|
|
28016
|
-
await rm5(
|
|
28016
|
+
await rm5(path39.join(poolDir, "metadata.json"), { force: true }).catch(() => {
|
|
28017
28017
|
});
|
|
28018
28018
|
}
|
|
28019
28019
|
/**
|
|
@@ -28023,7 +28023,7 @@ var WorkspacePoolManager = class {
|
|
|
28023
28023
|
*/
|
|
28024
28024
|
async resetSlot(slotPath, templatePath, repos, poolReset = "fast") {
|
|
28025
28025
|
for (const repo of repos) {
|
|
28026
|
-
const repoDir =
|
|
28026
|
+
const repoDir = path39.join(slotPath, repo.path);
|
|
28027
28027
|
if (!existsSync2(repoDir)) {
|
|
28028
28028
|
continue;
|
|
28029
28029
|
}
|
|
@@ -28144,7 +28144,7 @@ ${lines.join("\n")}`;
|
|
|
28144
28144
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
28145
28145
|
*/
|
|
28146
28146
|
async materialize(repo, workspacePath) {
|
|
28147
|
-
const targetDir =
|
|
28147
|
+
const targetDir = path40.join(workspacePath, repo.path);
|
|
28148
28148
|
const sourceUrl = getSourceUrl(repo.source);
|
|
28149
28149
|
const startedAt = Date.now();
|
|
28150
28150
|
if (this.verbose) {
|
|
@@ -28235,7 +28235,7 @@ ${lines.join("\n")}`;
|
|
|
28235
28235
|
async reset(repos, workspacePath, reset) {
|
|
28236
28236
|
const cleanFlag = reset === "strict" ? "-fdx" : "-fd";
|
|
28237
28237
|
for (const repo of repos) {
|
|
28238
|
-
const targetDir =
|
|
28238
|
+
const targetDir = path40.join(workspacePath, repo.path);
|
|
28239
28239
|
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
28240
28240
|
await this.runGit(["clean", cleanFlag], { cwd: targetDir });
|
|
28241
28241
|
}
|
|
@@ -28245,11 +28245,11 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
28245
28245
|
if (!templatePath) {
|
|
28246
28246
|
return void 0;
|
|
28247
28247
|
}
|
|
28248
|
-
const resolved =
|
|
28248
|
+
const resolved = path41.resolve(templatePath);
|
|
28249
28249
|
const stats = await stat6(resolved);
|
|
28250
28250
|
if (stats.isFile()) {
|
|
28251
28251
|
return {
|
|
28252
|
-
dir:
|
|
28252
|
+
dir: path41.dirname(resolved),
|
|
28253
28253
|
workspaceFile: resolved
|
|
28254
28254
|
};
|
|
28255
28255
|
}
|
|
@@ -28261,14 +28261,14 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
28261
28261
|
if (workspaceFiles.length === 1) {
|
|
28262
28262
|
return {
|
|
28263
28263
|
dir: resolved,
|
|
28264
|
-
workspaceFile:
|
|
28264
|
+
workspaceFile: path41.join(resolved, workspaceFiles[0])
|
|
28265
28265
|
};
|
|
28266
28266
|
}
|
|
28267
28267
|
if (workspaceFiles.length > 1) {
|
|
28268
28268
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
28269
28269
|
return {
|
|
28270
28270
|
dir: resolved,
|
|
28271
|
-
workspaceFile: conventionFile ?
|
|
28271
|
+
workspaceFile: conventionFile ? path41.join(resolved, conventionFile) : void 0
|
|
28272
28272
|
};
|
|
28273
28273
|
}
|
|
28274
28274
|
return { dir: resolved };
|
|
@@ -28468,7 +28468,7 @@ async function runEvaluation(options) {
|
|
|
28468
28468
|
];
|
|
28469
28469
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveGraderProvider);
|
|
28470
28470
|
const typeRegistry = createBuiltinRegistry();
|
|
28471
|
-
const discoveryBaseDir = evalFilePath ?
|
|
28471
|
+
const discoveryBaseDir = evalFilePath ? path422.dirname(path422.resolve(evalFilePath)) : process.cwd();
|
|
28472
28472
|
const evalDir = discoveryBaseDir;
|
|
28473
28473
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
28474
28474
|
await discoverGraders(typeRegistry, discoveryBaseDir);
|
|
@@ -28657,7 +28657,7 @@ async function runEvaluation(options) {
|
|
|
28657
28657
|
}
|
|
28658
28658
|
try {
|
|
28659
28659
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
28660
|
-
const copiedWorkspaceFile =
|
|
28660
|
+
const copiedWorkspaceFile = path422.join(sharedWorkspacePath, path422.basename(suiteWorkspaceFile));
|
|
28661
28661
|
try {
|
|
28662
28662
|
await stat7(copiedWorkspaceFile);
|
|
28663
28663
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
@@ -28770,7 +28770,7 @@ async function runEvaluation(options) {
|
|
|
28770
28770
|
dataset: evalCase.dataset,
|
|
28771
28771
|
score: 0,
|
|
28772
28772
|
assertions: [],
|
|
28773
|
-
|
|
28773
|
+
output: [],
|
|
28774
28774
|
target: target.name,
|
|
28775
28775
|
error: `Suite budget exceeded ($${cumulativeBudgetCost.toFixed(4)} / $${totalBudgetUsd.toFixed(4)})`,
|
|
28776
28776
|
budgetExceeded: true,
|
|
@@ -28806,7 +28806,7 @@ async function runEvaluation(options) {
|
|
|
28806
28806
|
dataset: evalCase.dataset,
|
|
28807
28807
|
score: 0,
|
|
28808
28808
|
assertions: [],
|
|
28809
|
-
|
|
28809
|
+
output: [],
|
|
28810
28810
|
target: target.name,
|
|
28811
28811
|
error: errorMsg,
|
|
28812
28812
|
executionStatus: "execution_error",
|
|
@@ -29234,7 +29234,7 @@ async function runEvalCase(options) {
|
|
|
29234
29234
|
);
|
|
29235
29235
|
}
|
|
29236
29236
|
if (caseWorkspaceFile && workspacePath) {
|
|
29237
|
-
const copiedFile =
|
|
29237
|
+
const copiedFile = path422.join(workspacePath, path422.basename(caseWorkspaceFile));
|
|
29238
29238
|
try {
|
|
29239
29239
|
await stat7(copiedFile);
|
|
29240
29240
|
caseWorkspaceFile = copiedFile;
|
|
@@ -29294,10 +29294,10 @@ async function runEvalCase(options) {
|
|
|
29294
29294
|
const files = evalCase.metadata.agent_skills_files;
|
|
29295
29295
|
if (baseDir && files.length > 0) {
|
|
29296
29296
|
for (const relPath of files) {
|
|
29297
|
-
const srcPath =
|
|
29298
|
-
const destPath =
|
|
29297
|
+
const srcPath = path422.resolve(baseDir, relPath);
|
|
29298
|
+
const destPath = path422.resolve(workspacePath, relPath);
|
|
29299
29299
|
try {
|
|
29300
|
-
await mkdir13(
|
|
29300
|
+
await mkdir13(path422.dirname(destPath), { recursive: true });
|
|
29301
29301
|
await copyFile2(srcPath, destPath);
|
|
29302
29302
|
} catch (error) {
|
|
29303
29303
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -29773,7 +29773,6 @@ async function evaluateCandidate(options) {
|
|
|
29773
29773
|
conversationId: evalCase.conversation_id,
|
|
29774
29774
|
score: score.score,
|
|
29775
29775
|
assertions: score.assertions,
|
|
29776
|
-
outputText: candidate,
|
|
29777
29776
|
target: target.name,
|
|
29778
29777
|
tokenUsage,
|
|
29779
29778
|
costUsd,
|
|
@@ -29784,7 +29783,7 @@ async function evaluateCandidate(options) {
|
|
|
29784
29783
|
input,
|
|
29785
29784
|
scores,
|
|
29786
29785
|
trace: trace2,
|
|
29787
|
-
output,
|
|
29786
|
+
output: output ?? [{ role: "assistant", content: candidate }],
|
|
29788
29787
|
fileChanges,
|
|
29789
29788
|
executionStatus: classifyQualityStatus(score.score)
|
|
29790
29789
|
};
|
|
@@ -29918,7 +29917,7 @@ async function runEvaluatorList(options) {
|
|
|
29918
29917
|
fileChanges,
|
|
29919
29918
|
workspacePath
|
|
29920
29919
|
};
|
|
29921
|
-
const evalFileDir = evalCase.guideline_paths[0] ?
|
|
29920
|
+
const evalFileDir = evalCase.guideline_paths[0] ? path422.dirname(evalCase.guideline_paths[0]) : process.cwd();
|
|
29922
29921
|
const dispatchContext = {
|
|
29923
29922
|
graderProvider,
|
|
29924
29923
|
targetResolver,
|
|
@@ -29949,7 +29948,7 @@ async function runEvaluatorList(options) {
|
|
|
29949
29948
|
weight,
|
|
29950
29949
|
verdict: score2.verdict,
|
|
29951
29950
|
assertions: score2.assertions,
|
|
29952
|
-
|
|
29951
|
+
input: score2.evaluatorRawRequest,
|
|
29953
29952
|
details: score2.details,
|
|
29954
29953
|
scores: mapChildResults(score2.scores),
|
|
29955
29954
|
tokenUsage: score2.tokenUsage,
|
|
@@ -30129,7 +30128,7 @@ function buildErrorResult(evalCase, targetName, timestamp, error, promptInputs,
|
|
|
30129
30128
|
conversationId: evalCase.conversation_id,
|
|
30130
30129
|
score: 0,
|
|
30131
30130
|
assertions: [{ text: `Error: ${message}`, passed: false }],
|
|
30132
|
-
|
|
30131
|
+
output: [{ role: "assistant", content: `Error occurred: ${message}` }],
|
|
30133
30132
|
target: targetName,
|
|
30134
30133
|
requests,
|
|
30135
30134
|
input,
|
|
@@ -30173,7 +30172,7 @@ function buildResultInput(promptInputs) {
|
|
|
30173
30172
|
content: message.content
|
|
30174
30173
|
}));
|
|
30175
30174
|
}
|
|
30176
|
-
return promptInputs.question;
|
|
30175
|
+
return [{ role: "user", content: promptInputs.question }];
|
|
30177
30176
|
}
|
|
30178
30177
|
function aggregateEvaluatorTokenUsage(scores) {
|
|
30179
30178
|
if (!scores || scores.length === 0) return void 0;
|
|
@@ -30239,7 +30238,7 @@ function mapChildResults(children) {
|
|
|
30239
30238
|
weight: child.weight,
|
|
30240
30239
|
verdict: child.verdict,
|
|
30241
30240
|
assertions: child.assertions,
|
|
30242
|
-
|
|
30241
|
+
input: child.evaluatorRawRequest,
|
|
30243
30242
|
scores: mapChildResults(child.scores),
|
|
30244
30243
|
details: child.details,
|
|
30245
30244
|
tokenUsage: child.tokenUsage
|
|
@@ -30287,7 +30286,7 @@ async function evaluate(config) {
|
|
|
30287
30286
|
}
|
|
30288
30287
|
const gitRoot = await findGitRoot(process.cwd());
|
|
30289
30288
|
const repoRoot = gitRoot ?? process.cwd();
|
|
30290
|
-
const testFilePath = config.specFile ?
|
|
30289
|
+
const testFilePath = config.specFile ? path43.resolve(config.specFile) : path43.join(process.cwd(), "__programmatic__.yaml");
|
|
30291
30290
|
await loadEnvHierarchy(repoRoot, testFilePath);
|
|
30292
30291
|
let resolvedTarget;
|
|
30293
30292
|
let taskProvider;
|
|
@@ -30416,10 +30415,10 @@ function computeSummary(results, durationMs) {
|
|
|
30416
30415
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
30417
30416
|
async function discoverDefaultTarget(repoRoot) {
|
|
30418
30417
|
const cwd = process.cwd();
|
|
30419
|
-
const chain = buildDirectoryChain(
|
|
30418
|
+
const chain = buildDirectoryChain(path43.join(cwd, "_placeholder"), repoRoot);
|
|
30420
30419
|
for (const dir of chain) {
|
|
30421
30420
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
30422
|
-
const targetsPath =
|
|
30421
|
+
const targetsPath = path43.join(dir, candidate);
|
|
30423
30422
|
if (!existsSync4(targetsPath)) continue;
|
|
30424
30423
|
try {
|
|
30425
30424
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -30436,7 +30435,7 @@ async function loadEnvHierarchy(repoRoot, startPath) {
|
|
|
30436
30435
|
const chain = buildDirectoryChain(startPath, repoRoot);
|
|
30437
30436
|
const envFiles = [];
|
|
30438
30437
|
for (const dir of chain) {
|
|
30439
|
-
const envPath =
|
|
30438
|
+
const envPath = path43.join(dir, ".env");
|
|
30440
30439
|
if (existsSync4(envPath)) envFiles.push(envPath);
|
|
30441
30440
|
}
|
|
30442
30441
|
for (let i = 0; i < envFiles.length; i++) {
|
|
@@ -30617,7 +30616,7 @@ var ResponseCache = class {
|
|
|
30617
30616
|
async get(key) {
|
|
30618
30617
|
const filePath = this.keyToPath(key);
|
|
30619
30618
|
try {
|
|
30620
|
-
const data = await
|
|
30619
|
+
const data = await readFile12(filePath, "utf8");
|
|
30621
30620
|
return JSON.parse(data);
|
|
30622
30621
|
} catch {
|
|
30623
30622
|
return void 0;
|
|
@@ -30625,13 +30624,13 @@ var ResponseCache = class {
|
|
|
30625
30624
|
}
|
|
30626
30625
|
async set(key, value) {
|
|
30627
30626
|
const filePath = this.keyToPath(key);
|
|
30628
|
-
const dir =
|
|
30627
|
+
const dir = path44.dirname(filePath);
|
|
30629
30628
|
await mkdir14(dir, { recursive: true });
|
|
30630
30629
|
await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
30631
30630
|
}
|
|
30632
30631
|
keyToPath(key) {
|
|
30633
30632
|
const prefix = key.slice(0, 2);
|
|
30634
|
-
return
|
|
30633
|
+
return path44.join(this.cachePath, prefix, `${key}.json`);
|
|
30635
30634
|
}
|
|
30636
30635
|
};
|
|
30637
30636
|
function shouldEnableCache(params) {
|
|
@@ -30646,7 +30645,6 @@ function shouldSkipCacheForTemperature(targetConfig) {
|
|
|
30646
30645
|
return false;
|
|
30647
30646
|
}
|
|
30648
30647
|
var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
30649
|
-
"outputText",
|
|
30650
30648
|
"requests",
|
|
30651
30649
|
"trace",
|
|
30652
30650
|
"workspacePath",
|
|
@@ -30663,7 +30661,7 @@ var STRIPPED_TOP_LEVEL_FIELDS = /* @__PURE__ */ new Set([
|
|
|
30663
30661
|
"startTime",
|
|
30664
30662
|
"endTime"
|
|
30665
30663
|
]);
|
|
30666
|
-
var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "
|
|
30664
|
+
var STRIPPED_EVALUATOR_FIELDS = /* @__PURE__ */ new Set(["rawRequest", "input"]);
|
|
30667
30665
|
function trimEvaluatorResult(result) {
|
|
30668
30666
|
const trimmed = {};
|
|
30669
30667
|
for (const [key, value] of Object.entries(result)) {
|
|
@@ -30818,7 +30816,11 @@ var OtelTraceExporter = class {
|
|
|
30818
30816
|
rootSpan.setAttribute("agentv.target", result.target);
|
|
30819
30817
|
if (result.dataset) rootSpan.setAttribute("agentv.dataset", result.dataset);
|
|
30820
30818
|
rootSpan.setAttribute("agentv.score", result.score);
|
|
30821
|
-
if (captureContent
|
|
30819
|
+
if (captureContent && result.output.length > 0) {
|
|
30820
|
+
const lastMsg = result.output[result.output.length - 1];
|
|
30821
|
+
const text2 = typeof lastMsg.content === "string" ? lastMsg.content : JSON.stringify(lastMsg.content);
|
|
30822
|
+
rootSpan.setAttribute("agentv.output_text", text2);
|
|
30823
|
+
}
|
|
30822
30824
|
if (result.durationMs != null)
|
|
30823
30825
|
rootSpan.setAttribute("agentv.trace.duration_ms", result.durationMs);
|
|
30824
30826
|
if (result.costUsd != null) rootSpan.setAttribute("agentv.trace.cost_usd", result.costUsd);
|
|
@@ -31119,6 +31121,8 @@ export {
|
|
|
31119
31121
|
resolveFileReference,
|
|
31120
31122
|
CLI_PLACEHOLDERS,
|
|
31121
31123
|
resolveTargetDefinition,
|
|
31124
|
+
interpolateEnv,
|
|
31125
|
+
loadCasesFromFile,
|
|
31122
31126
|
KNOWN_PROVIDERS,
|
|
31123
31127
|
PROVIDER_ALIASES,
|
|
31124
31128
|
computeTraceSummary,
|
|
@@ -31248,4 +31252,4 @@ export {
|
|
|
31248
31252
|
OtelStreamingObserver,
|
|
31249
31253
|
createAgentKernel
|
|
31250
31254
|
};
|
|
31251
|
-
//# sourceMappingURL=chunk-
|
|
31255
|
+
//# sourceMappingURL=chunk-F4UDJ7LG.js.map
|