agentv 4.9.1 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-XOSNETAV.js → chunk-BAUNAXHT.js} +1 -1
- package/dist/chunk-BPGJ4HBU.js +183 -0
- package/dist/chunk-BPGJ4HBU.js.map +1 -0
- package/dist/{chunk-2IKIOZ4Z.js → chunk-FH24D7XW.js} +1090 -303
- package/dist/chunk-FH24D7XW.js.map +1 -0
- package/dist/{chunk-RHAXSXIY.js → chunk-FQGY6QXQ.js} +1360 -653
- package/dist/chunk-FQGY6QXQ.js.map +1 -0
- package/dist/chunk-NPVGBFF6.js +151 -0
- package/dist/chunk-NPVGBFF6.js.map +1 -0
- package/dist/{chunk-2JW4HVCX.js → chunk-QRYAMYT7.js} +1120 -731
- package/dist/chunk-QRYAMYT7.js.map +1 -0
- package/dist/cli.js +6 -4
- package/dist/cli.js.map +1 -1
- package/dist/{dist-DDFE3W2A.js → dist-HNSXNRVK.js} +36 -3
- package/dist/docker-workspace-RPPXBT27-B4AQHVWA.js +11 -0
- package/dist/{esm-CZAWIY6F.js → esm-UYZ3HJBU.js} +2 -2
- package/dist/esm-UYZ3HJBU.js.map +1 -0
- package/dist/exec-AR6JUUN5-6MBPURPR.js +11 -0
- package/dist/exec-AR6JUUN5-6MBPURPR.js.map +1 -0
- package/dist/index.js +6 -4
- package/dist/{interactive-VMDBXBRL.js → interactive-SIOZB665.js} +6 -4
- package/dist/{interactive-VMDBXBRL.js.map → interactive-SIOZB665.js.map} +1 -1
- package/dist/{src-ML4D2MC2.js → src-PXDA7QIS.js} +2 -2
- package/dist/studio/assets/index-Bi-KHfNm.js +65 -0
- package/dist/studio/assets/index-D_j-w4UO.css +1 -0
- package/dist/studio/assets/{index-DcwjOyrk.js → index-VyDFrnoK.js} +1 -1
- package/dist/studio/index.html +2 -2
- package/package.json +1 -1
- package/dist/chunk-2IKIOZ4Z.js.map +0 -1
- package/dist/chunk-2JW4HVCX.js.map +0 -1
- package/dist/chunk-RHAXSXIY.js.map +0 -1
- package/dist/studio/assets/index-DHxVz6M9.css +0 -1
- package/dist/studio/assets/index-Y5InSvcS.js +0 -65
- /package/dist/{chunk-XOSNETAV.js.map → chunk-BAUNAXHT.js.map} +0 -0
- /package/dist/{dist-DDFE3W2A.js.map → dist-HNSXNRVK.js.map} +0 -0
- /package/dist/{esm-CZAWIY6F.js.map → docker-workspace-RPPXBT27-B4AQHVWA.js.map} +0 -0
- /package/dist/{src-ML4D2MC2.js.map → src-PXDA7QIS.js.map} +0 -0
|
@@ -1,7 +1,17 @@
|
|
|
1
1
|
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
2
|
+
import {
|
|
3
|
+
execFileWithStdin,
|
|
4
|
+
execShellWithStdin
|
|
5
|
+
} from "./chunk-NPVGBFF6.js";
|
|
2
6
|
import {
|
|
3
7
|
require_token_error
|
|
4
8
|
} from "./chunk-HQDCIXVH.js";
|
|
9
|
+
import {
|
|
10
|
+
SpanStatusCode,
|
|
11
|
+
context,
|
|
12
|
+
init_esm,
|
|
13
|
+
trace
|
|
14
|
+
} from "./chunk-LRULMAAA.js";
|
|
5
15
|
import {
|
|
6
16
|
AISDKError,
|
|
7
17
|
APICallError,
|
|
@@ -150,12 +160,6 @@ import {
|
|
|
150
160
|
withoutTrailingSlash,
|
|
151
161
|
zodSchema
|
|
152
162
|
} from "./chunk-ZKO2LGRR.js";
|
|
153
|
-
import {
|
|
154
|
-
SpanStatusCode,
|
|
155
|
-
context,
|
|
156
|
-
init_esm,
|
|
157
|
-
trace
|
|
158
|
-
} from "./chunk-LRULMAAA.js";
|
|
159
163
|
import {
|
|
160
164
|
__commonJS,
|
|
161
165
|
__export,
|
|
@@ -301,7 +305,7 @@ var require_dist = __commonJS({
|
|
|
301
305
|
}
|
|
302
306
|
});
|
|
303
307
|
|
|
304
|
-
// ../../packages/core/dist/chunk-
|
|
308
|
+
// ../../packages/core/dist/chunk-5POFMJJ7.js
|
|
305
309
|
import { constants } from "node:fs";
|
|
306
310
|
import { access, readFile } from "node:fs/promises";
|
|
307
311
|
import path from "node:path";
|
|
@@ -419,7 +423,7 @@ __export(external_exports2, {
|
|
|
419
423
|
void: () => voidType
|
|
420
424
|
});
|
|
421
425
|
|
|
422
|
-
// ../../packages/core/dist/chunk-
|
|
426
|
+
// ../../packages/core/dist/chunk-5POFMJJ7.js
|
|
423
427
|
import { readFile as readFile2 } from "node:fs/promises";
|
|
424
428
|
import path3 from "node:path";
|
|
425
429
|
import fg from "fast-glob";
|
|
@@ -2308,10 +2312,10 @@ async function expandFileReferences(tests, evalFileDir) {
|
|
|
2308
2312
|
}
|
|
2309
2313
|
|
|
2310
2314
|
// ../../packages/core/dist/index.js
|
|
2311
|
-
import { readFile as
|
|
2312
|
-
import
|
|
2315
|
+
import { readFile as readFile8 } from "node:fs/promises";
|
|
2316
|
+
import path8 from "node:path";
|
|
2313
2317
|
import micromatch2 from "micromatch";
|
|
2314
|
-
import { parse as
|
|
2318
|
+
import { parse as parse3 } from "yaml";
|
|
2315
2319
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
2316
2320
|
import path4 from "node:path";
|
|
2317
2321
|
import { readFile as readFile22 } from "node:fs/promises";
|
|
@@ -2321,17 +2325,22 @@ import { constants as constants2 } from "node:fs";
|
|
|
2321
2325
|
import { access as access2 } from "node:fs/promises";
|
|
2322
2326
|
import path22 from "node:path";
|
|
2323
2327
|
import { fileURLToPath } from "node:url";
|
|
2324
|
-
import path42 from "node:path";
|
|
2325
|
-
import { readFile as readFile32 } from "node:fs/promises";
|
|
2326
2328
|
import { readFile as readFile5 } from "node:fs/promises";
|
|
2327
|
-
import
|
|
2329
|
+
import path5 from "node:path";
|
|
2330
|
+
import { parse as parse2 } from "yaml";
|
|
2331
|
+
import { readFile as readFile32 } from "node:fs/promises";
|
|
2332
|
+
import path42 from "node:path";
|
|
2333
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
2334
|
+
import { readFile as readFile4 } from "node:fs/promises";
|
|
2335
|
+
import { readFile as readFile7 } from "node:fs/promises";
|
|
2336
|
+
import path7 from "node:path";
|
|
2328
2337
|
import micromatch from "micromatch";
|
|
2329
2338
|
import { parse as parseYaml2 } from "yaml";
|
|
2330
|
-
import { readFile as
|
|
2331
|
-
import
|
|
2339
|
+
import { readFile as readFile6 } from "node:fs/promises";
|
|
2340
|
+
import path6 from "node:path";
|
|
2332
2341
|
import { readFileSync } from "node:fs";
|
|
2333
|
-
import
|
|
2334
|
-
import { parse as
|
|
2342
|
+
import path9 from "node:path";
|
|
2343
|
+
import { parse as parse4 } from "yaml";
|
|
2335
2344
|
import { createOpenAI } from "@ai-sdk/openai";
|
|
2336
2345
|
|
|
2337
2346
|
// ../../node_modules/.bun/@openrouter+ai-sdk-provider@2.3.3+3ab978b6804fd9e7/node_modules/@openrouter/ai-sdk-provider/dist/index.mjs
|
|
@@ -6941,7 +6950,7 @@ function createOpenRouter(options = {}) {
|
|
|
6941
6950
|
);
|
|
6942
6951
|
const createChatModel = (modelId, settings = {}) => new OpenRouterChatLanguageModel(modelId, settings, {
|
|
6943
6952
|
provider: "openrouter.chat",
|
|
6944
|
-
url: ({ path:
|
|
6953
|
+
url: ({ path: path53 }) => `${baseURL}${path53}`,
|
|
6945
6954
|
headers: getHeaders,
|
|
6946
6955
|
compatibility,
|
|
6947
6956
|
fetch: options.fetch,
|
|
@@ -6949,7 +6958,7 @@ function createOpenRouter(options = {}) {
|
|
|
6949
6958
|
});
|
|
6950
6959
|
const createCompletionModel = (modelId, settings = {}) => new OpenRouterCompletionLanguageModel(modelId, settings, {
|
|
6951
6960
|
provider: "openrouter.completion",
|
|
6952
|
-
url: ({ path:
|
|
6961
|
+
url: ({ path: path53 }) => `${baseURL}${path53}`,
|
|
6953
6962
|
headers: getHeaders,
|
|
6954
6963
|
compatibility,
|
|
6955
6964
|
fetch: options.fetch,
|
|
@@ -6957,14 +6966,14 @@ function createOpenRouter(options = {}) {
|
|
|
6957
6966
|
});
|
|
6958
6967
|
const createEmbeddingModel = (modelId, settings = {}) => new OpenRouterEmbeddingModel(modelId, settings, {
|
|
6959
6968
|
provider: "openrouter.embedding",
|
|
6960
|
-
url: ({ path:
|
|
6969
|
+
url: ({ path: path53 }) => `${baseURL}${path53}`,
|
|
6961
6970
|
headers: getHeaders,
|
|
6962
6971
|
fetch: options.fetch,
|
|
6963
6972
|
extraBody: options.extraBody
|
|
6964
6973
|
});
|
|
6965
6974
|
const createImageModel = (modelId, settings = {}) => new OpenRouterImageModel(modelId, settings, {
|
|
6966
6975
|
provider: "openrouter.image",
|
|
6967
|
-
url: ({ path:
|
|
6976
|
+
url: ({ path: path53 }) => `${baseURL}${path53}`,
|
|
6968
6977
|
headers: getHeaders,
|
|
6969
6978
|
fetch: options.fetch,
|
|
6970
6979
|
extraBody: options.extraBody
|
|
@@ -12960,24 +12969,24 @@ import { spawn } from "node:child_process";
|
|
|
12960
12969
|
import { randomUUID } from "node:crypto";
|
|
12961
12970
|
import { createWriteStream } from "node:fs";
|
|
12962
12971
|
import { mkdir } from "node:fs/promises";
|
|
12972
|
+
import path11 from "node:path";
|
|
12963
12973
|
import path10 from "node:path";
|
|
12964
|
-
import path9 from "node:path";
|
|
12965
12974
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
12966
12975
|
import { createWriteStream as createWriteStream2 } from "node:fs";
|
|
12967
12976
|
import { mkdir as mkdir2 } from "node:fs/promises";
|
|
12968
|
-
import
|
|
12977
|
+
import path12 from "node:path";
|
|
12969
12978
|
import { exec as execWithCallback } from "node:child_process";
|
|
12970
12979
|
import fs from "node:fs/promises";
|
|
12971
12980
|
import os from "node:os";
|
|
12972
|
-
import
|
|
12981
|
+
import path13 from "node:path";
|
|
12973
12982
|
import { promisify } from "node:util";
|
|
12974
12983
|
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
12975
12984
|
import { createWriteStream as createWriteStream3 } from "node:fs";
|
|
12976
12985
|
import { mkdir as mkdir3 } from "node:fs/promises";
|
|
12977
|
-
import
|
|
12986
|
+
import path14 from "node:path";
|
|
12978
12987
|
import { randomUUID as randomUUID5 } from "node:crypto";
|
|
12979
12988
|
import { mkdir as mkdir4 } from "node:fs/promises";
|
|
12980
|
-
import
|
|
12989
|
+
import path16 from "node:path";
|
|
12981
12990
|
import { Readable, Writable } from "node:stream";
|
|
12982
12991
|
import { spawn as spawn2 } from "node:child_process";
|
|
12983
12992
|
|
|
@@ -14462,63 +14471,63 @@ var RequestError = class _RequestError extends Error {
|
|
|
14462
14471
|
import { randomUUID as randomUUID4 } from "node:crypto";
|
|
14463
14472
|
import { createWriteStream as createWriteStream4, existsSync, readdirSync } from "node:fs";
|
|
14464
14473
|
import { arch, platform } from "node:os";
|
|
14465
|
-
import
|
|
14466
|
-
import { fileURLToPath as
|
|
14467
|
-
import { readFile as
|
|
14474
|
+
import path15 from "node:path";
|
|
14475
|
+
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
14476
|
+
import { readFile as readFile10 } from "node:fs/promises";
|
|
14468
14477
|
import { homedir as homedir2 } from "node:os";
|
|
14469
|
-
import
|
|
14470
|
-
import { readFile as
|
|
14478
|
+
import path18 from "node:path";
|
|
14479
|
+
import { readFile as readFile9, readdir, stat } from "node:fs/promises";
|
|
14471
14480
|
import { homedir } from "node:os";
|
|
14472
|
-
import
|
|
14481
|
+
import path17 from "node:path";
|
|
14473
14482
|
import { parse as parseYaml22 } from "yaml";
|
|
14474
14483
|
import { randomUUID as randomUUID6 } from "node:crypto";
|
|
14475
14484
|
import { existsSync as existsSync2 } from "node:fs";
|
|
14476
14485
|
import { mkdir as mkdir5 } from "node:fs/promises";
|
|
14477
|
-
import
|
|
14486
|
+
import path19 from "node:path";
|
|
14478
14487
|
import { execSync, spawn as spawn3 } from "node:child_process";
|
|
14479
14488
|
import { randomUUID as randomUUID7 } from "node:crypto";
|
|
14480
14489
|
import { accessSync, createWriteStream as createWriteStream5, readFileSync as readFileSync2 } from "node:fs";
|
|
14481
14490
|
import { mkdir as mkdir6, mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
14482
14491
|
import { tmpdir } from "node:os";
|
|
14483
|
-
import
|
|
14492
|
+
import path20 from "node:path";
|
|
14484
14493
|
import { execSync as execSync2 } from "node:child_process";
|
|
14485
14494
|
import { randomUUID as randomUUID8 } from "node:crypto";
|
|
14486
14495
|
import { accessSync as accessSync2, createWriteStream as createWriteStream6, mkdirSync } from "node:fs";
|
|
14487
14496
|
import { mkdir as mkdir7 } from "node:fs/promises";
|
|
14488
|
-
import
|
|
14497
|
+
import path222 from "node:path";
|
|
14489
14498
|
import { createInterface } from "node:readline";
|
|
14490
|
-
import { fileURLToPath as
|
|
14499
|
+
import { fileURLToPath as fileURLToPath4, pathToFileURL } from "node:url";
|
|
14491
14500
|
import os2 from "node:os";
|
|
14492
|
-
import
|
|
14501
|
+
import path21 from "node:path";
|
|
14493
14502
|
import { exec as exec2 } from "node:child_process";
|
|
14494
14503
|
import { constants as constants3, access as access3, stat as stat5 } from "node:fs/promises";
|
|
14495
|
-
import
|
|
14504
|
+
import path33 from "node:path";
|
|
14496
14505
|
import { promisify as promisify3 } from "node:util";
|
|
14497
14506
|
import { stat as stat4, writeFile as writeFile4 } from "node:fs/promises";
|
|
14498
|
-
import
|
|
14507
|
+
import path31 from "node:path";
|
|
14499
14508
|
import { constants as constants22 } from "node:fs";
|
|
14500
14509
|
import { access as access22, mkdir as mkdir8, readdir as readdir2, rm as rm2, stat as stat2 } from "node:fs/promises";
|
|
14501
|
-
import path222 from "node:path";
|
|
14502
14510
|
import path23 from "node:path";
|
|
14503
14511
|
import path24 from "node:path";
|
|
14504
|
-
import { readFile as readFile9 } from "node:fs/promises";
|
|
14505
14512
|
import path25 from "node:path";
|
|
14513
|
+
import { readFile as readFile11 } from "node:fs/promises";
|
|
14514
|
+
import path26 from "node:path";
|
|
14506
14515
|
import { exec, spawn as spawn4 } from "node:child_process";
|
|
14507
14516
|
import { mkdir as mkdir9, writeFile as writeFile2 } from "node:fs/promises";
|
|
14508
|
-
import
|
|
14517
|
+
import path28 from "node:path";
|
|
14509
14518
|
import { promisify as promisify2 } from "node:util";
|
|
14510
|
-
import
|
|
14511
|
-
import { copyFile, mkdir as mkdir10, readFile as
|
|
14519
|
+
import path27 from "node:path";
|
|
14520
|
+
import { copyFile, mkdir as mkdir10, readFile as readFile12, readdir as readdir3, stat as stat3, writeFile as writeFile3 } from "node:fs/promises";
|
|
14521
|
+
import path30 from "node:path";
|
|
14512
14522
|
import path29 from "node:path";
|
|
14513
|
-
import path28 from "node:path";
|
|
14514
14523
|
import JSON5 from "json5";
|
|
14515
14524
|
import { writeFile as writeFile5 } from "node:fs/promises";
|
|
14516
|
-
import
|
|
14525
|
+
import path322 from "node:path";
|
|
14517
14526
|
import { constants as constants4 } from "node:fs";
|
|
14518
|
-
import { access as access4, readFile as
|
|
14519
|
-
import path33 from "node:path";
|
|
14520
|
-
import { parse as parse4 } from "yaml";
|
|
14527
|
+
import { access as access4, readFile as readFile13 } from "node:fs/promises";
|
|
14521
14528
|
import path34 from "node:path";
|
|
14529
|
+
import { parse as parse5 } from "yaml";
|
|
14530
|
+
import path35 from "node:path";
|
|
14522
14531
|
import fg2 from "fast-glob";
|
|
14523
14532
|
import { mkdtemp as mkdtemp2, rm as rm3, writeFile as writeFile6 } from "node:fs/promises";
|
|
14524
14533
|
import { tmpdir as tmpdir2 } from "node:os";
|
|
@@ -14526,52 +14535,58 @@ import { dirname, join } from "node:path";
|
|
|
14526
14535
|
import { randomBytes } from "node:crypto";
|
|
14527
14536
|
import { createServer } from "node:http";
|
|
14528
14537
|
import fs2 from "node:fs/promises";
|
|
14529
|
-
import
|
|
14538
|
+
import path36 from "node:path";
|
|
14530
14539
|
import { createHash as createHash2, randomUUID as randomUUID9 } from "node:crypto";
|
|
14531
14540
|
import { existsSync as existsSync5 } from "node:fs";
|
|
14532
14541
|
import { copyFile as copyFile2, mkdir as mkdir14, readdir as readdir7, stat as stat8 } from "node:fs/promises";
|
|
14533
|
-
import
|
|
14542
|
+
import path45 from "node:path";
|
|
14534
14543
|
import micromatch3 from "micromatch";
|
|
14535
|
-
import path36 from "node:path";
|
|
14536
14544
|
import path37 from "node:path";
|
|
14537
|
-
import fg22 from "fast-glob";
|
|
14538
14545
|
import path38 from "node:path";
|
|
14546
|
+
import fg22 from "fast-glob";
|
|
14547
|
+
import path39 from "node:path";
|
|
14539
14548
|
import fg3 from "fast-glob";
|
|
14540
14549
|
import { exec as execCallback } from "node:child_process";
|
|
14541
14550
|
import { readdirSync as readdirSync2, statSync } from "node:fs";
|
|
14542
|
-
import
|
|
14551
|
+
import path40 from "node:path";
|
|
14543
14552
|
import { promisify as promisify4 } from "node:util";
|
|
14544
14553
|
import { cp, mkdir as mkdir12, readdir as readdir4, rm as rm4, stat as stat6 } from "node:fs/promises";
|
|
14545
|
-
import
|
|
14554
|
+
import path41 from "node:path";
|
|
14546
14555
|
import { execFile } from "node:child_process";
|
|
14547
14556
|
import { createHash } from "node:crypto";
|
|
14548
14557
|
import { existsSync as existsSync3 } from "node:fs";
|
|
14549
|
-
import { cp as cp2, mkdir as mkdir13, readFile as
|
|
14550
|
-
import
|
|
14558
|
+
import { cp as cp2, mkdir as mkdir13, readFile as readFile14, readdir as readdir5, rm as rm5, unlink, writeFile as writeFile7 } from "node:fs/promises";
|
|
14559
|
+
import path422 from "node:path";
|
|
14551
14560
|
import { promisify as promisify5 } from "node:util";
|
|
14552
14561
|
import { execFile as execFile2 } from "node:child_process";
|
|
14553
14562
|
import { existsSync as existsSync4 } from "node:fs";
|
|
14554
|
-
import
|
|
14563
|
+
import path43 from "node:path";
|
|
14555
14564
|
import { promisify as promisify6 } from "node:util";
|
|
14556
14565
|
import { readdir as readdir6, stat as stat7 } from "node:fs/promises";
|
|
14557
|
-
import
|
|
14566
|
+
import path44 from "node:path";
|
|
14558
14567
|
import { existsSync as existsSync6 } from "node:fs";
|
|
14559
|
-
import path45 from "node:path";
|
|
14560
|
-
import { readFile as readFile13 } from "node:fs/promises";
|
|
14561
14568
|
import path46 from "node:path";
|
|
14562
|
-
import {
|
|
14563
|
-
import { mkdir as mkdir15, readFile as readFile14, writeFile as writeFile8 } from "node:fs/promises";
|
|
14569
|
+
import { readFile as readFile15 } from "node:fs/promises";
|
|
14564
14570
|
import path47 from "node:path";
|
|
14565
|
-
import {
|
|
14571
|
+
import { parse as parse6 } from "yaml";
|
|
14572
|
+
import { mkdir as mkdir15, readFile as readFile16, writeFile as writeFile8 } from "node:fs/promises";
|
|
14566
14573
|
import path48 from "node:path";
|
|
14567
|
-
import {
|
|
14568
|
-
import {
|
|
14569
|
-
import {
|
|
14574
|
+
import { execFile as execFile3 } from "node:child_process";
|
|
14575
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync2, readFileSync as readFileSync3, rmSync, writeFileSync } from "node:fs";
|
|
14576
|
+
import { cp as cp3, mkdtemp as mkdtemp3, readdir as readdir8, rm as rm6, stat as stat9 } from "node:fs/promises";
|
|
14577
|
+
import os3 from "node:os";
|
|
14570
14578
|
import path49 from "node:path";
|
|
14579
|
+
import { promisify as promisify7 } from "node:util";
|
|
14580
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync3, readFileSync as readFileSync4, readdirSync as readdirSync3, statSync as statSync2, writeFileSync as writeFileSync2 } from "node:fs";
|
|
14581
|
+
import path50 from "node:path";
|
|
14582
|
+
import { parse as parseYaml3, stringify as stringifyYaml } from "yaml";
|
|
14571
14583
|
import { readdir as readdir9, stat as stat10 } from "node:fs/promises";
|
|
14584
|
+
import { homedir as homedir3 } from "node:os";
|
|
14585
|
+
import path51 from "node:path";
|
|
14586
|
+
import { readdir as readdir10, stat as stat11 } from "node:fs/promises";
|
|
14572
14587
|
import { homedir as homedir4 } from "node:os";
|
|
14573
|
-
import
|
|
14574
|
-
import { readFile as
|
|
14588
|
+
import path52 from "node:path";
|
|
14589
|
+
import { readFile as readFile17 } from "node:fs/promises";
|
|
14575
14590
|
function computeTraceSummary(messages) {
|
|
14576
14591
|
const toolCallCounts = {};
|
|
14577
14592
|
const toolDurations = {};
|
|
@@ -14954,10 +14969,12 @@ async function loadConfig(evalFilePath, repoRoot) {
|
|
|
14954
14969
|
parsed.execution,
|
|
14955
14970
|
configPath
|
|
14956
14971
|
);
|
|
14972
|
+
const results = parseResultsConfig(parsed.results, configPath);
|
|
14957
14973
|
return {
|
|
14958
14974
|
required_version: requiredVersion,
|
|
14959
14975
|
eval_patterns: evalPatterns,
|
|
14960
|
-
execution: executionDefaults
|
|
14976
|
+
execution: executionDefaults,
|
|
14977
|
+
results
|
|
14961
14978
|
};
|
|
14962
14979
|
} catch (error) {
|
|
14963
14980
|
logWarning(
|
|
@@ -15192,9 +15209,221 @@ function parseExecutionDefaults(raw, configPath) {
|
|
|
15192
15209
|
}
|
|
15193
15210
|
return Object.keys(result).length > 0 ? result : void 0;
|
|
15194
15211
|
}
|
|
15212
|
+
function parseResultsConfig(raw, configPath) {
|
|
15213
|
+
if (raw === void 0 || raw === null) {
|
|
15214
|
+
return void 0;
|
|
15215
|
+
}
|
|
15216
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
15217
|
+
logWarning(`Invalid results in ${configPath}, expected object`);
|
|
15218
|
+
return void 0;
|
|
15219
|
+
}
|
|
15220
|
+
const obj = raw;
|
|
15221
|
+
const exportConfig = parseResultsExportConfig(obj.export, configPath);
|
|
15222
|
+
if (!exportConfig) {
|
|
15223
|
+
return void 0;
|
|
15224
|
+
}
|
|
15225
|
+
return { export: exportConfig };
|
|
15226
|
+
}
|
|
15227
|
+
function parseResultsExportConfig(raw, configPath) {
|
|
15228
|
+
if (raw === void 0 || raw === null) {
|
|
15229
|
+
return void 0;
|
|
15230
|
+
}
|
|
15231
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
15232
|
+
logWarning(`Invalid results.export in ${configPath}, expected object`);
|
|
15233
|
+
return void 0;
|
|
15234
|
+
}
|
|
15235
|
+
const obj = raw;
|
|
15236
|
+
const repo = typeof obj.repo === "string" ? obj.repo.trim() : "";
|
|
15237
|
+
const exportPath = typeof obj.path === "string" ? obj.path.trim() : "";
|
|
15238
|
+
if (!repo) {
|
|
15239
|
+
logWarning(`Invalid results.export.repo in ${configPath}, expected non-empty string`);
|
|
15240
|
+
return void 0;
|
|
15241
|
+
}
|
|
15242
|
+
if (!exportPath) {
|
|
15243
|
+
logWarning(`Invalid results.export.path in ${configPath}, expected non-empty string`);
|
|
15244
|
+
return void 0;
|
|
15245
|
+
}
|
|
15246
|
+
if (obj.auto_push !== void 0 && typeof obj.auto_push !== "boolean") {
|
|
15247
|
+
logWarning(`Invalid results.export.auto_push in ${configPath}, expected boolean`);
|
|
15248
|
+
return void 0;
|
|
15249
|
+
}
|
|
15250
|
+
let branchPrefix;
|
|
15251
|
+
if (obj.branch_prefix !== void 0) {
|
|
15252
|
+
if (typeof obj.branch_prefix !== "string" || obj.branch_prefix.trim().length === 0) {
|
|
15253
|
+
logWarning(
|
|
15254
|
+
`Invalid results.export.branch_prefix in ${configPath}, expected non-empty string`
|
|
15255
|
+
);
|
|
15256
|
+
return void 0;
|
|
15257
|
+
}
|
|
15258
|
+
branchPrefix = obj.branch_prefix.trim();
|
|
15259
|
+
}
|
|
15260
|
+
return {
|
|
15261
|
+
repo,
|
|
15262
|
+
path: exportPath,
|
|
15263
|
+
...typeof obj.auto_push === "boolean" && { auto_push: obj.auto_push },
|
|
15264
|
+
...branchPrefix && { branch_prefix: branchPrefix }
|
|
15265
|
+
};
|
|
15266
|
+
}
|
|
15195
15267
|
function logWarning(message) {
|
|
15196
15268
|
console.warn(`${ANSI_YELLOW2}Warning: ${message}${ANSI_RESET22}`);
|
|
15197
15269
|
}
|
|
15270
|
+
var MIME_TYPE_ALIASES = {
|
|
15271
|
+
csv: "text/csv",
|
|
15272
|
+
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
15273
|
+
htm: "text/html",
|
|
15274
|
+
html: "text/html",
|
|
15275
|
+
json: "application/json",
|
|
15276
|
+
markdown: "text/markdown",
|
|
15277
|
+
md: "text/markdown",
|
|
15278
|
+
pdf: "application/pdf",
|
|
15279
|
+
sql: "application/sql",
|
|
15280
|
+
txt: "text/plain",
|
|
15281
|
+
xhtml: "application/xhtml+xml",
|
|
15282
|
+
xls: "application/vnd.ms-excel",
|
|
15283
|
+
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
15284
|
+
xml: "application/xml",
|
|
15285
|
+
yaml: "application/yaml",
|
|
15286
|
+
yml: "application/yaml"
|
|
15287
|
+
};
|
|
15288
|
+
var REPLACEMENT_CHAR = "\uFFFD";
|
|
15289
|
+
async function extractTextWithPreprocessors(content, preprocessors, options = {}) {
|
|
15290
|
+
if (typeof content === "string") {
|
|
15291
|
+
return { text: content, warnings: [] };
|
|
15292
|
+
}
|
|
15293
|
+
if (!content || content.length === 0) {
|
|
15294
|
+
return { text: "", warnings: [] };
|
|
15295
|
+
}
|
|
15296
|
+
const parts = [];
|
|
15297
|
+
const warnings = [];
|
|
15298
|
+
for (const block of content) {
|
|
15299
|
+
if (block.type === "text") {
|
|
15300
|
+
parts.push(block.text);
|
|
15301
|
+
continue;
|
|
15302
|
+
}
|
|
15303
|
+
if (block.type !== "file") {
|
|
15304
|
+
continue;
|
|
15305
|
+
}
|
|
15306
|
+
const result = await preprocessContentFile(block, preprocessors, options.basePath);
|
|
15307
|
+
if (result.text) {
|
|
15308
|
+
parts.push(result.text);
|
|
15309
|
+
}
|
|
15310
|
+
warnings.push(...result.warnings);
|
|
15311
|
+
}
|
|
15312
|
+
return { text: parts.join("\n"), warnings };
|
|
15313
|
+
}
|
|
15314
|
+
async function preprocessContentFile(block, preprocessors, basePath) {
|
|
15315
|
+
const mediaType = normalizePreprocessorType(block.media_type);
|
|
15316
|
+
const resolvedPath = resolveLocalFilePath(block.path, basePath);
|
|
15317
|
+
if (!resolvedPath) {
|
|
15318
|
+
return {
|
|
15319
|
+
text: "",
|
|
15320
|
+
warnings: [
|
|
15321
|
+
{
|
|
15322
|
+
file: block.path,
|
|
15323
|
+
mediaType: block.media_type,
|
|
15324
|
+
reason: "remote file paths are not supported for preprocessing"
|
|
15325
|
+
}
|
|
15326
|
+
]
|
|
15327
|
+
};
|
|
15328
|
+
}
|
|
15329
|
+
const preprocessor = preprocessors?.find(
|
|
15330
|
+
(entry) => normalizePreprocessorType(entry.type) === mediaType
|
|
15331
|
+
);
|
|
15332
|
+
if (preprocessor) {
|
|
15333
|
+
return runContentPreprocessor(block, resolvedPath, preprocessor);
|
|
15334
|
+
}
|
|
15335
|
+
try {
|
|
15336
|
+
const buffer = await readFile32(resolvedPath);
|
|
15337
|
+
const text2 = buffer.toString("utf8").replace(/\r\n/g, "\n");
|
|
15338
|
+
if (buffer.includes(0) || text2.includes(REPLACEMENT_CHAR)) {
|
|
15339
|
+
return {
|
|
15340
|
+
text: "",
|
|
15341
|
+
warnings: [
|
|
15342
|
+
{
|
|
15343
|
+
file: block.path,
|
|
15344
|
+
mediaType: block.media_type,
|
|
15345
|
+
reason: "default UTF-8 read produced binary or invalid text; configure a preprocessor"
|
|
15346
|
+
}
|
|
15347
|
+
]
|
|
15348
|
+
};
|
|
15349
|
+
}
|
|
15350
|
+
return { text: formatFileText(block.path, text2), warnings: [] };
|
|
15351
|
+
} catch (error) {
|
|
15352
|
+
return {
|
|
15353
|
+
text: "",
|
|
15354
|
+
warnings: [
|
|
15355
|
+
{
|
|
15356
|
+
file: block.path,
|
|
15357
|
+
mediaType: block.media_type,
|
|
15358
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
15359
|
+
}
|
|
15360
|
+
]
|
|
15361
|
+
};
|
|
15362
|
+
}
|
|
15363
|
+
}
|
|
15364
|
+
async function runContentPreprocessor(block, resolvedPath, preprocessor) {
|
|
15365
|
+
try {
|
|
15366
|
+
const argv = preprocessor.resolvedCommand ?? preprocessor.command;
|
|
15367
|
+
const { stdout, stderr, exitCode } = await execFileWithStdin(
|
|
15368
|
+
argv,
|
|
15369
|
+
JSON.stringify({
|
|
15370
|
+
path: resolvedPath,
|
|
15371
|
+
original_path: block.path,
|
|
15372
|
+
media_type: block.media_type
|
|
15373
|
+
})
|
|
15374
|
+
);
|
|
15375
|
+
if (exitCode !== 0) {
|
|
15376
|
+
return {
|
|
15377
|
+
text: "",
|
|
15378
|
+
warnings: [
|
|
15379
|
+
{
|
|
15380
|
+
file: block.path,
|
|
15381
|
+
mediaType: block.media_type,
|
|
15382
|
+
reason: stderr.trim() || `preprocessor exited with code ${exitCode}`
|
|
15383
|
+
}
|
|
15384
|
+
]
|
|
15385
|
+
};
|
|
15386
|
+
}
|
|
15387
|
+
return { text: formatFileText(block.path, stdout.trim()), warnings: [] };
|
|
15388
|
+
} catch (error) {
|
|
15389
|
+
return {
|
|
15390
|
+
text: "",
|
|
15391
|
+
warnings: [
|
|
15392
|
+
{
|
|
15393
|
+
file: block.path,
|
|
15394
|
+
mediaType: block.media_type,
|
|
15395
|
+
reason: error instanceof Error ? error.message : String(error)
|
|
15396
|
+
}
|
|
15397
|
+
]
|
|
15398
|
+
};
|
|
15399
|
+
}
|
|
15400
|
+
}
|
|
15401
|
+
function appendPreprocessingWarnings(text2, warnings) {
|
|
15402
|
+
if (warnings.length === 0) {
|
|
15403
|
+
return text2;
|
|
15404
|
+
}
|
|
15405
|
+
const notes = warnings.map(
|
|
15406
|
+
(warning) => `[file preprocessing warning] ${warning.file} (${warning.mediaType}): ${warning.reason}`
|
|
15407
|
+
);
|
|
15408
|
+
return [text2, ...notes].filter((part) => part.length > 0).join("\n");
|
|
15409
|
+
}
|
|
15410
|
+
function normalizePreprocessorType(value) {
|
|
15411
|
+
const normalized = value.trim().toLowerCase();
|
|
15412
|
+
return MIME_TYPE_ALIASES[normalized] ?? normalized;
|
|
15413
|
+
}
|
|
15414
|
+
function resolveLocalFilePath(value, basePath) {
|
|
15415
|
+
if (value.startsWith("file://")) {
|
|
15416
|
+
return fileURLToPath2(value);
|
|
15417
|
+
}
|
|
15418
|
+
if (/^[a-z]+:\/\//i.test(value)) {
|
|
15419
|
+
return void 0;
|
|
15420
|
+
}
|
|
15421
|
+
return basePath ? path42.resolve(basePath, value) : path42.resolve(value);
|
|
15422
|
+
}
|
|
15423
|
+
function formatFileText(filePath, text2) {
|
|
15424
|
+
return `[[ file: ${filePath} ]]
|
|
15425
|
+
${text2}`;
|
|
15426
|
+
}
|
|
15198
15427
|
var TEMPLATE_VARIABLES = {
|
|
15199
15428
|
EXPECTED_OUTPUT: "expected_output",
|
|
15200
15429
|
CRITERIA: "criteria",
|
|
@@ -15221,7 +15450,7 @@ var DEPRECATED_TEMPLATE_VARIABLES = /* @__PURE__ */ new Map([
|
|
|
15221
15450
|
var ANSI_YELLOW22 = "\x1B[33m";
|
|
15222
15451
|
var ANSI_RESET3 = "\x1B[0m";
|
|
15223
15452
|
async function validateCustomPromptContent(promptPath) {
|
|
15224
|
-
const content = await
|
|
15453
|
+
const content = await readFile4(promptPath, "utf8");
|
|
15225
15454
|
validateTemplateVariables(content, promptPath);
|
|
15226
15455
|
}
|
|
15227
15456
|
function validateTemplateVariables(content, source) {
|
|
@@ -15269,6 +15498,7 @@ function validateTemplateVariables(content, source) {
|
|
|
15269
15498
|
}
|
|
15270
15499
|
var ANSI_YELLOW3 = "\x1B[33m";
|
|
15271
15500
|
var ANSI_RESET4 = "\x1B[0m";
|
|
15501
|
+
var MAX_ASSERTION_INCLUDE_DEPTH = 3;
|
|
15272
15502
|
var PROMPT_FILE_PREFIX = "file://";
|
|
15273
15503
|
function normalizeEvaluatorType(type) {
|
|
15274
15504
|
return type.replace(/_/g, "-");
|
|
@@ -15276,22 +15506,104 @@ function normalizeEvaluatorType(type) {
|
|
|
15276
15506
|
function isDeprecatedJudgeType(type) {
|
|
15277
15507
|
return type === "code-judge" || type === "llm-judge";
|
|
15278
15508
|
}
|
|
15279
|
-
async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId) {
|
|
15509
|
+
async function parseEvaluators(rawEvalCase, globalExecution, searchRoots, evalId, defaultPreprocessors) {
|
|
15280
15510
|
const execution = rawEvalCase.execution;
|
|
15281
15511
|
const executionObject = isJsonObject2(execution) ? execution : void 0;
|
|
15282
15512
|
const caseEvaluators = rawEvalCase.assertions ?? rawEvalCase.assert ?? (executionObject ? executionObject.evaluators : void 0) ?? // deprecated: use assertions
|
|
15283
15513
|
rawEvalCase.evaluators;
|
|
15284
15514
|
const skipDefaults = executionObject?.skip_defaults === true;
|
|
15285
15515
|
const rootEvaluators = skipDefaults ? void 0 : globalExecution?.assertions ?? globalExecution?.assert ?? globalExecution?.evaluators;
|
|
15286
|
-
const parsedCase = await parseEvaluatorList(
|
|
15287
|
-
|
|
15516
|
+
const parsedCase = await parseEvaluatorList(
|
|
15517
|
+
caseEvaluators,
|
|
15518
|
+
searchRoots,
|
|
15519
|
+
evalId,
|
|
15520
|
+
defaultPreprocessors
|
|
15521
|
+
);
|
|
15522
|
+
const parsedRoot = await parseEvaluatorList(
|
|
15523
|
+
rootEvaluators,
|
|
15524
|
+
searchRoots,
|
|
15525
|
+
evalId,
|
|
15526
|
+
defaultPreprocessors
|
|
15527
|
+
);
|
|
15288
15528
|
if (!parsedCase && !parsedRoot) {
|
|
15289
15529
|
return void 0;
|
|
15290
15530
|
}
|
|
15291
15531
|
const evaluators = [...parsedCase ?? [], ...parsedRoot ?? []];
|
|
15292
15532
|
return evaluators.length > 0 ? evaluators : void 0;
|
|
15293
15533
|
}
|
|
15294
|
-
|
|
15534
|
+
function isIncludeEntry(value) {
|
|
15535
|
+
return isJsonObject2(value) && typeof value.include === "string" && Object.keys(value).length === 1;
|
|
15536
|
+
}
|
|
15537
|
+
function isTemplateReference(value) {
|
|
15538
|
+
return !value.startsWith(".") && !value.includes("/") && !value.includes("\\");
|
|
15539
|
+
}
|
|
15540
|
+
async function resolveAssertionTemplateReference(include, searchRoots) {
|
|
15541
|
+
const templateCandidates = isTemplateReference(include) ? [
|
|
15542
|
+
path5.join(".agentv", "templates", `${include}.yaml`),
|
|
15543
|
+
path5.join(".agentv", "templates", `${include}.yml`)
|
|
15544
|
+
] : [include];
|
|
15545
|
+
const attempted = [];
|
|
15546
|
+
for (const candidate of templateCandidates) {
|
|
15547
|
+
const resolved = await resolveFileReference22(candidate, searchRoots);
|
|
15548
|
+
attempted.push(...resolved.attempted);
|
|
15549
|
+
if (resolved.resolvedPath) {
|
|
15550
|
+
return {
|
|
15551
|
+
displayPath: resolved.displayPath,
|
|
15552
|
+
resolvedPath: resolved.resolvedPath,
|
|
15553
|
+
attempted
|
|
15554
|
+
};
|
|
15555
|
+
}
|
|
15556
|
+
}
|
|
15557
|
+
return {
|
|
15558
|
+
displayPath: templateCandidates[0] ?? include,
|
|
15559
|
+
resolvedPath: "",
|
|
15560
|
+
attempted
|
|
15561
|
+
};
|
|
15562
|
+
}
|
|
15563
|
+
async function loadAssertionTemplateEntries(include, searchRoots, evalId, includeContext) {
|
|
15564
|
+
const nextDepth = includeContext.depth + 1;
|
|
15565
|
+
if (nextDepth > MAX_ASSERTION_INCLUDE_DEPTH) {
|
|
15566
|
+
const chain = [...includeContext.chain, include].join(" -> ");
|
|
15567
|
+
throw new Error(
|
|
15568
|
+
`Assertion template include depth exceeded ${MAX_ASSERTION_INCLUDE_DEPTH} in '${evalId}'. Include chain: ${chain}`
|
|
15569
|
+
);
|
|
15570
|
+
}
|
|
15571
|
+
const resolved = await resolveAssertionTemplateReference(include, searchRoots);
|
|
15572
|
+
if (!resolved.resolvedPath) {
|
|
15573
|
+
const attempted = resolved.attempted.length > 0 ? `
|
|
15574
|
+
${resolved.attempted.map((attempt) => ` Tried: ${attempt}`).join("\n")}` : "";
|
|
15575
|
+
throw new Error(
|
|
15576
|
+
`Assertion template not found in '${evalId}': ${resolved.displayPath}${attempted}`
|
|
15577
|
+
);
|
|
15578
|
+
}
|
|
15579
|
+
if (includeContext.chain.includes(resolved.resolvedPath)) {
|
|
15580
|
+
const cycle = [...includeContext.chain, resolved.resolvedPath].join(" -> ");
|
|
15581
|
+
throw new Error(`Assertion template cycle detected in '${evalId}': ${cycle}`);
|
|
15582
|
+
}
|
|
15583
|
+
const content = await readFile5(resolved.resolvedPath, "utf8");
|
|
15584
|
+
const parsed = interpolateEnv(parse2(content), process.env);
|
|
15585
|
+
if (!isJsonObject2(parsed)) {
|
|
15586
|
+
throw new Error(
|
|
15587
|
+
`Invalid assertion template file in '${evalId}': ${resolved.resolvedPath} (expected a YAML object with an assertions array)`
|
|
15588
|
+
);
|
|
15589
|
+
}
|
|
15590
|
+
const assertions = parsed.assertions;
|
|
15591
|
+
if (!Array.isArray(assertions)) {
|
|
15592
|
+
throw new Error(
|
|
15593
|
+
`Invalid assertion template file in '${evalId}': ${resolved.resolvedPath} is missing a top-level assertions array`
|
|
15594
|
+
);
|
|
15595
|
+
}
|
|
15596
|
+
const templateDir = path5.dirname(resolved.resolvedPath);
|
|
15597
|
+
const nestedSearchRoots = [
|
|
15598
|
+
templateDir,
|
|
15599
|
+
...searchRoots.filter((root) => path5.resolve(root) !== templateDir)
|
|
15600
|
+
];
|
|
15601
|
+
return await expandEvaluatorEntries(assertions, nestedSearchRoots, evalId, {
|
|
15602
|
+
depth: nextDepth,
|
|
15603
|
+
chain: [...includeContext.chain, resolved.resolvedPath]
|
|
15604
|
+
}) ?? [];
|
|
15605
|
+
}
|
|
15606
|
+
async function expandEvaluatorEntries(candidateEvaluators, searchRoots, evalId, includeContext = { depth: 0, chain: [] }) {
|
|
15295
15607
|
if (candidateEvaluators === void 0) {
|
|
15296
15608
|
return void 0;
|
|
15297
15609
|
}
|
|
@@ -15299,13 +15611,34 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15299
15611
|
logWarning2(`Skipping evaluators for '${evalId}': expected array`);
|
|
15300
15612
|
return void 0;
|
|
15301
15613
|
}
|
|
15302
|
-
const
|
|
15303
|
-
const
|
|
15614
|
+
const expanded = [];
|
|
15615
|
+
for (const rawEvaluator of candidateEvaluators) {
|
|
15616
|
+
if (isIncludeEntry(rawEvaluator)) {
|
|
15617
|
+
const included = await loadAssertionTemplateEntries(
|
|
15618
|
+
rawEvaluator.include,
|
|
15619
|
+
searchRoots,
|
|
15620
|
+
evalId,
|
|
15621
|
+
includeContext
|
|
15622
|
+
);
|
|
15623
|
+
expanded.push(...included);
|
|
15624
|
+
continue;
|
|
15625
|
+
}
|
|
15626
|
+
expanded.push(rawEvaluator);
|
|
15627
|
+
}
|
|
15628
|
+
return expanded;
|
|
15629
|
+
}
|
|
15630
|
+
async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId, defaultPreprocessors) {
|
|
15631
|
+
const expandedEvaluators = await expandEvaluatorEntries(candidateEvaluators, searchRoots, evalId);
|
|
15632
|
+
if (!expandedEvaluators) {
|
|
15633
|
+
return void 0;
|
|
15634
|
+
}
|
|
15635
|
+
const firstStringIndex = expandedEvaluators.findIndex((e) => typeof e === "string");
|
|
15636
|
+
const processedEvaluators = firstStringIndex === -1 ? [...expandedEvaluators] : (() => {
|
|
15304
15637
|
const PLACEHOLDER = Symbol("rubric-placeholder");
|
|
15305
15638
|
const strings = [];
|
|
15306
15639
|
const result = [];
|
|
15307
15640
|
let rubricInserted = false;
|
|
15308
|
-
for (const item of
|
|
15641
|
+
for (const item of expandedEvaluators) {
|
|
15309
15642
|
if (typeof item === "string") {
|
|
15310
15643
|
const trimmed = item.trim();
|
|
15311
15644
|
if (trimmed.length === 0) {
|
|
@@ -15356,6 +15689,13 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15356
15689
|
continue;
|
|
15357
15690
|
}
|
|
15358
15691
|
const negate = rawEvaluator.negate === true ? true : void 0;
|
|
15692
|
+
const mergedPreprocessors = await parseMergedPreprocessors(
|
|
15693
|
+
rawEvaluator.preprocessors,
|
|
15694
|
+
defaultPreprocessors,
|
|
15695
|
+
searchRoots,
|
|
15696
|
+
name21,
|
|
15697
|
+
evalId
|
|
15698
|
+
);
|
|
15359
15699
|
if (isCustomType) {
|
|
15360
15700
|
const weight2 = validateWeight(rawEvaluator.weight, name21, evalId);
|
|
15361
15701
|
const { required: required2, min_score: min_score2 } = parseRequiredAndMinScore(
|
|
@@ -15414,7 +15754,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15414
15754
|
if (cwd) {
|
|
15415
15755
|
const resolved = await resolveFileReference22(cwd, searchRoots);
|
|
15416
15756
|
if (resolved.resolvedPath) {
|
|
15417
|
-
resolvedCwd =
|
|
15757
|
+
resolvedCwd = path5.resolve(resolved.resolvedPath);
|
|
15418
15758
|
} else {
|
|
15419
15759
|
logWarning2(
|
|
15420
15760
|
`Code-grader evaluator '${name21}' in '${evalId}': cwd not found (${resolved.displayPath})`,
|
|
@@ -15460,6 +15800,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15460
15800
|
"cwd",
|
|
15461
15801
|
"weight",
|
|
15462
15802
|
"target",
|
|
15803
|
+
"preprocessors",
|
|
15463
15804
|
"required",
|
|
15464
15805
|
"negate"
|
|
15465
15806
|
]);
|
|
@@ -15480,6 +15821,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15480
15821
|
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
15481
15822
|
...negate !== void 0 ? { negate } : {},
|
|
15482
15823
|
...Object.keys(config2).length > 0 ? { config: config2 } : {},
|
|
15824
|
+
...mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {},
|
|
15483
15825
|
...targetConfig !== void 0 ? { target: targetConfig } : {}
|
|
15484
15826
|
});
|
|
15485
15827
|
continue;
|
|
@@ -15511,8 +15853,16 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15511
15853
|
);
|
|
15512
15854
|
continue;
|
|
15513
15855
|
}
|
|
15856
|
+
const expandedMembers = await expandEvaluatorEntries(
|
|
15857
|
+
rawMembers,
|
|
15858
|
+
searchRoots,
|
|
15859
|
+
`${evalId}:${name21}`
|
|
15860
|
+
);
|
|
15861
|
+
if (!expandedMembers) {
|
|
15862
|
+
continue;
|
|
15863
|
+
}
|
|
15514
15864
|
const memberEvaluators = [];
|
|
15515
|
-
for (const rawMember of
|
|
15865
|
+
for (const rawMember of expandedMembers) {
|
|
15516
15866
|
if (!isJsonObject2(rawMember)) {
|
|
15517
15867
|
logWarning2(`Skipping invalid member evaluator in composite '${name21}' (expected object)`);
|
|
15518
15868
|
continue;
|
|
@@ -15589,7 +15939,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
15589
15939
|
aggregatorPrompt = fileRef;
|
|
15590
15940
|
const resolved = await resolveFileReference22(fileRef, searchRoots);
|
|
15591
15941
|
if (resolved.resolvedPath) {
|
|
15592
|
-
promptPath2 =
|
|
15942
|
+
promptPath2 = path5.resolve(resolved.resolvedPath);
|
|
15593
15943
|
} else {
|
|
15594
15944
|
throw new Error(
|
|
15595
15945
|
`Composite aggregator in '${evalId}': prompt file not found: ${resolved.displayPath}`
|
|
@@ -16243,7 +16593,8 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16243
16593
|
...weight2 !== void 0 ? { weight: weight2 } : {},
|
|
16244
16594
|
...required2 !== void 0 ? { required: required2 } : {},
|
|
16245
16595
|
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
16246
|
-
...negate !== void 0 ? { negate } : {}
|
|
16596
|
+
...negate !== void 0 ? { negate } : {},
|
|
16597
|
+
...mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}
|
|
16247
16598
|
});
|
|
16248
16599
|
continue;
|
|
16249
16600
|
}
|
|
@@ -16268,7 +16619,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16268
16619
|
const commandPath = commandArray[commandArray.length - 1];
|
|
16269
16620
|
const resolved = await resolveFileReference22(commandPath, searchRoots);
|
|
16270
16621
|
if (resolved.resolvedPath) {
|
|
16271
|
-
resolvedPromptScript = [...commandArray.slice(0, -1),
|
|
16622
|
+
resolvedPromptScript = [...commandArray.slice(0, -1), path5.resolve(resolved.resolvedPath)];
|
|
16272
16623
|
} else {
|
|
16273
16624
|
throw new Error(
|
|
16274
16625
|
`Evaluator '${name21}' in '${evalId}': prompt command file not found: ${resolved.displayPath}`
|
|
@@ -16283,7 +16634,7 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16283
16634
|
prompt = fileRef;
|
|
16284
16635
|
const resolved = await resolveFileReference22(fileRef, searchRoots);
|
|
16285
16636
|
if (resolved.resolvedPath) {
|
|
16286
|
-
promptPath =
|
|
16637
|
+
promptPath = path5.resolve(resolved.resolvedPath);
|
|
16287
16638
|
try {
|
|
16288
16639
|
await validateCustomPromptContent(promptPath);
|
|
16289
16640
|
} catch (error) {
|
|
@@ -16326,7 +16677,8 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16326
16677
|
...weight2 !== void 0 ? { weight: weight2 } : {},
|
|
16327
16678
|
...required2 !== void 0 ? { required: required2 } : {},
|
|
16328
16679
|
...min_score2 !== void 0 ? { min_score: min_score2 } : {},
|
|
16329
|
-
...negate !== void 0 ? { negate } : {}
|
|
16680
|
+
...negate !== void 0 ? { negate } : {},
|
|
16681
|
+
...mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}
|
|
16330
16682
|
});
|
|
16331
16683
|
continue;
|
|
16332
16684
|
}
|
|
@@ -16351,7 +16703,8 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16351
16703
|
"negate",
|
|
16352
16704
|
"max_steps",
|
|
16353
16705
|
"maxSteps",
|
|
16354
|
-
"temperature"
|
|
16706
|
+
"temperature",
|
|
16707
|
+
"preprocessors"
|
|
16355
16708
|
]);
|
|
16356
16709
|
const config = {};
|
|
16357
16710
|
for (const [key, value] of Object.entries(rawEvaluator)) {
|
|
@@ -16381,30 +16734,70 @@ async function parseEvaluatorList(candidateEvaluators, searchRoots, evalId) {
|
|
|
16381
16734
|
...negate !== void 0 ? { negate } : {},
|
|
16382
16735
|
...finalConfig ? { config: finalConfig } : {},
|
|
16383
16736
|
...llmMaxSteps !== void 0 ? { max_steps: llmMaxSteps } : {},
|
|
16384
|
-
...llmTemperature !== void 0 ? { temperature: llmTemperature } : {}
|
|
16737
|
+
...llmTemperature !== void 0 ? { temperature: llmTemperature } : {},
|
|
16738
|
+
...mergedPreprocessors ? { preprocessors: mergedPreprocessors } : {}
|
|
16385
16739
|
});
|
|
16386
16740
|
}
|
|
16387
16741
|
return evaluators.length > 0 ? evaluators : void 0;
|
|
16388
16742
|
}
|
|
16389
|
-
|
|
16390
|
-
|
|
16391
|
-
|
|
16392
|
-
|
|
16393
|
-
|
|
16394
|
-
|
|
16395
|
-
|
|
16396
|
-
|
|
16397
|
-
|
|
16398
|
-
|
|
16399
|
-
|
|
16400
|
-
|
|
16401
|
-
|
|
16402
|
-
|
|
16403
|
-
|
|
16404
|
-
function
|
|
16405
|
-
if (
|
|
16743
|
+
async function parseMergedPreprocessors(rawValue, defaultPreprocessors, searchRoots, evaluatorName, evalId) {
|
|
16744
|
+
const parsedDefaults = defaultPreprocessors ?? [];
|
|
16745
|
+
const parsedOverrides = await parsePreprocessors(rawValue, searchRoots, evaluatorName, evalId);
|
|
16746
|
+
if (parsedDefaults.length === 0 && (!parsedOverrides || parsedOverrides.length === 0)) {
|
|
16747
|
+
return void 0;
|
|
16748
|
+
}
|
|
16749
|
+
const merged = /* @__PURE__ */ new Map();
|
|
16750
|
+
for (const entry of parsedDefaults) {
|
|
16751
|
+
merged.set(normalizePreprocessorType(entry.type), entry);
|
|
16752
|
+
}
|
|
16753
|
+
for (const entry of parsedOverrides ?? []) {
|
|
16754
|
+
merged.set(normalizePreprocessorType(entry.type), entry);
|
|
16755
|
+
}
|
|
16756
|
+
return [...merged.values()];
|
|
16757
|
+
}
|
|
16758
|
+
async function parsePreprocessors(rawValue, searchRoots, evaluatorName, evalId) {
|
|
16759
|
+
if (rawValue === void 0) {
|
|
16406
16760
|
return void 0;
|
|
16407
16761
|
}
|
|
16762
|
+
if (!Array.isArray(rawValue)) {
|
|
16763
|
+
throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessors must be an array`);
|
|
16764
|
+
}
|
|
16765
|
+
const preprocessors = [];
|
|
16766
|
+
for (const rawEntry of rawValue) {
|
|
16767
|
+
if (!isJsonObject2(rawEntry)) {
|
|
16768
|
+
throw new Error(
|
|
16769
|
+
`Evaluator '${evaluatorName}' in '${evalId}': each preprocessor must be an object`
|
|
16770
|
+
);
|
|
16771
|
+
}
|
|
16772
|
+
const type = asString(rawEntry.type)?.trim();
|
|
16773
|
+
if (!type) {
|
|
16774
|
+
throw new Error(`Evaluator '${evaluatorName}' in '${evalId}': preprocessor.type is required`);
|
|
16775
|
+
}
|
|
16776
|
+
const command = asStringArray(
|
|
16777
|
+
rawEntry.command,
|
|
16778
|
+
`preprocessor command for evaluator '${evaluatorName}' in '${evalId}'`
|
|
16779
|
+
);
|
|
16780
|
+
if (!command || command.length === 0) {
|
|
16781
|
+
throw new Error(
|
|
16782
|
+
`Evaluator '${evaluatorName}' in '${evalId}': preprocessor '${type}' requires command`
|
|
16783
|
+
);
|
|
16784
|
+
}
|
|
16785
|
+
const commandPath = command[command.length - 1];
|
|
16786
|
+
const resolved = await resolveFileReference22(commandPath, searchRoots);
|
|
16787
|
+
if (!resolved.resolvedPath) {
|
|
16788
|
+
throw new Error(
|
|
16789
|
+
`Evaluator '${evaluatorName}' in '${evalId}': preprocessor command file not found: ${resolved.displayPath}`
|
|
16790
|
+
);
|
|
16791
|
+
}
|
|
16792
|
+
preprocessors.push({
|
|
16793
|
+
type,
|
|
16794
|
+
command,
|
|
16795
|
+
resolvedCommand: [...command.slice(0, -1), path5.resolve(resolved.resolvedPath)]
|
|
16796
|
+
});
|
|
16797
|
+
}
|
|
16798
|
+
return preprocessors;
|
|
16799
|
+
}
|
|
16800
|
+
function generateAssertionName(typeValue, rawEvaluator) {
|
|
16408
16801
|
const value = asString(rawEvaluator.value);
|
|
16409
16802
|
const arrayValue = Array.isArray(rawEvaluator.value) ? rawEvaluator.value : void 0;
|
|
16410
16803
|
switch (typeValue) {
|
|
@@ -16437,7 +16830,7 @@ function generateAssertionName(typeValue, rawEvaluator) {
|
|
|
16437
16830
|
case "rubrics":
|
|
16438
16831
|
return "rubrics";
|
|
16439
16832
|
default:
|
|
16440
|
-
return
|
|
16833
|
+
return typeValue;
|
|
16441
16834
|
}
|
|
16442
16835
|
}
|
|
16443
16836
|
function coerceEvaluator(candidate, contextId) {
|
|
@@ -16854,7 +17247,7 @@ var IMAGE_MEDIA_TYPES = {
|
|
|
16854
17247
|
".bmp": "image/bmp"
|
|
16855
17248
|
};
|
|
16856
17249
|
function detectImageMediaType(filePath) {
|
|
16857
|
-
const ext =
|
|
17250
|
+
const ext = path6.extname(filePath).toLowerCase();
|
|
16858
17251
|
return IMAGE_MEDIA_TYPES[ext];
|
|
16859
17252
|
}
|
|
16860
17253
|
var ANSI_YELLOW4 = "\x1B[33m";
|
|
@@ -16904,12 +17297,12 @@ async function processMessages(options) {
|
|
|
16904
17297
|
continue;
|
|
16905
17298
|
}
|
|
16906
17299
|
try {
|
|
16907
|
-
const fileContent = (await
|
|
17300
|
+
const fileContent = (await readFile6(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
16908
17301
|
processedContent.push({
|
|
16909
17302
|
...cloneJsonObject(rawSegment),
|
|
16910
17303
|
path: displayPath,
|
|
16911
17304
|
text: fileContent,
|
|
16912
|
-
resolvedPath:
|
|
17305
|
+
resolvedPath: path6.resolve(resolvedPath)
|
|
16913
17306
|
});
|
|
16914
17307
|
if (verbose) {
|
|
16915
17308
|
const label = messageType === "input" ? "[File]" : "[Expected Output File]";
|
|
@@ -16945,7 +17338,7 @@ async function processMessages(options) {
|
|
|
16945
17338
|
continue;
|
|
16946
17339
|
}
|
|
16947
17340
|
try {
|
|
16948
|
-
const imageBuffer = await
|
|
17341
|
+
const imageBuffer = await readFile6(resolvedPath);
|
|
16949
17342
|
const base64 = imageBuffer.toString("base64");
|
|
16950
17343
|
processedContent.push({
|
|
16951
17344
|
type: "image",
|
|
@@ -17022,12 +17415,12 @@ async function processExpectedMessages(options) {
|
|
|
17022
17415
|
continue;
|
|
17023
17416
|
}
|
|
17024
17417
|
try {
|
|
17025
|
-
const fileContent = (await
|
|
17418
|
+
const fileContent = (await readFile6(resolvedPath, "utf8")).replace(/\r\n/g, "\n");
|
|
17026
17419
|
processedContent.push({
|
|
17027
17420
|
type: "file",
|
|
17028
17421
|
path: displayPath,
|
|
17029
17422
|
text: fileContent,
|
|
17030
|
-
resolvedPath:
|
|
17423
|
+
resolvedPath: path6.resolve(resolvedPath)
|
|
17031
17424
|
});
|
|
17032
17425
|
if (verbose) {
|
|
17033
17426
|
console.log(` [Expected Output File] Found: ${displayPath}`);
|
|
@@ -17062,7 +17455,7 @@ async function processExpectedMessages(options) {
|
|
|
17062
17455
|
continue;
|
|
17063
17456
|
}
|
|
17064
17457
|
try {
|
|
17065
|
-
const imageBuffer = await
|
|
17458
|
+
const imageBuffer = await readFile6(resolvedPath);
|
|
17066
17459
|
const base64 = imageBuffer.toString("base64");
|
|
17067
17460
|
processedContent.push({
|
|
17068
17461
|
type: "image",
|
|
@@ -17167,7 +17560,7 @@ function matchesFilter(id, filter2) {
|
|
|
17167
17560
|
return typeof filter2 === "string" ? micromatch.isMatch(id, filter2) : filter2.some((pattern) => micromatch.isMatch(id, pattern));
|
|
17168
17561
|
}
|
|
17169
17562
|
function detectFormat(filePath) {
|
|
17170
|
-
const ext =
|
|
17563
|
+
const ext = path7.extname(filePath).toLowerCase();
|
|
17171
17564
|
if (ext === ".jsonl") return "jsonl";
|
|
17172
17565
|
if (ext === ".yaml" || ext === ".yml") return "yaml";
|
|
17173
17566
|
if (ext === ".json") return "agent-skills-json";
|
|
@@ -17176,9 +17569,9 @@ function detectFormat(filePath) {
|
|
|
17176
17569
|
);
|
|
17177
17570
|
}
|
|
17178
17571
|
async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
17179
|
-
const dir =
|
|
17180
|
-
const base =
|
|
17181
|
-
const sidecarPath =
|
|
17572
|
+
const dir = path7.dirname(jsonlPath);
|
|
17573
|
+
const base = path7.basename(jsonlPath, ".jsonl");
|
|
17574
|
+
const sidecarPath = path7.join(dir, `${base}.yaml`);
|
|
17182
17575
|
if (!await fileExists2(sidecarPath)) {
|
|
17183
17576
|
if (verbose) {
|
|
17184
17577
|
logWarning4(`Sidecar metadata file not found: ${sidecarPath} (using defaults)`);
|
|
@@ -17186,7 +17579,7 @@ async function loadSidecarMetadata(jsonlPath, verbose) {
|
|
|
17186
17579
|
return {};
|
|
17187
17580
|
}
|
|
17188
17581
|
try {
|
|
17189
|
-
const content = await
|
|
17582
|
+
const content = await readFile7(sidecarPath, "utf8");
|
|
17190
17583
|
const parsed = interpolateEnv(parseYaml2(content), process.env);
|
|
17191
17584
|
if (!isJsonObject(parsed)) {
|
|
17192
17585
|
logWarning4(`Invalid sidecar metadata format in ${sidecarPath}`);
|
|
@@ -17227,13 +17620,13 @@ function parseJsonlContent(content, filePath) {
|
|
|
17227
17620
|
async function loadTestsFromJsonl(evalFilePath, repoRoot, options) {
|
|
17228
17621
|
const verbose = options?.verbose ?? false;
|
|
17229
17622
|
const filterPattern = options?.filter;
|
|
17230
|
-
const absoluteTestPath =
|
|
17623
|
+
const absoluteTestPath = path7.resolve(evalFilePath);
|
|
17231
17624
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
17232
17625
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
17233
17626
|
const sidecar = await loadSidecarMetadata(absoluteTestPath, verbose);
|
|
17234
|
-
const rawFile = await
|
|
17627
|
+
const rawFile = await readFile7(absoluteTestPath, "utf8");
|
|
17235
17628
|
const rawCases = parseJsonlContent(rawFile, evalFilePath);
|
|
17236
|
-
const fallbackSuiteName =
|
|
17629
|
+
const fallbackSuiteName = path7.basename(absoluteTestPath, ".jsonl") || "eval";
|
|
17237
17630
|
const suiteName = sidecar.name && sidecar.name.trim().length > 0 ? sidecar.name : fallbackSuiteName;
|
|
17238
17631
|
const globalEvaluator = coerceEvaluator(sidecar.evaluator, "sidecar") ?? "llm-grader";
|
|
17239
17632
|
const globalExecution = sidecar.execution;
|
|
@@ -17408,11 +17801,13 @@ function parseRepoCheckout(raw) {
|
|
|
17408
17801
|
if (!isJsonObject(raw)) return void 0;
|
|
17409
17802
|
const obj = raw;
|
|
17410
17803
|
const ref = typeof obj.ref === "string" ? obj.ref : void 0;
|
|
17804
|
+
const baseCommit = typeof obj.base_commit === "string" ? obj.base_commit : void 0;
|
|
17411
17805
|
const resolve2 = obj.resolve === "remote" || obj.resolve === "local" ? obj.resolve : void 0;
|
|
17412
17806
|
const ancestor = typeof obj.ancestor === "number" ? obj.ancestor : void 0;
|
|
17413
|
-
if (!ref && !resolve2 && ancestor === void 0) return void 0;
|
|
17807
|
+
if (!ref && !baseCommit && !resolve2 && ancestor === void 0) return void 0;
|
|
17414
17808
|
return {
|
|
17415
17809
|
...ref !== void 0 && { ref },
|
|
17810
|
+
...baseCommit !== void 0 && { base_commit: baseCommit },
|
|
17416
17811
|
...resolve2 !== void 0 && { resolve: resolve2 },
|
|
17417
17812
|
...ancestor !== void 0 && { ancestor }
|
|
17418
17813
|
};
|
|
@@ -17435,12 +17830,12 @@ function parseRepoConfig(raw) {
|
|
|
17435
17830
|
const obj = raw;
|
|
17436
17831
|
const repoPath = typeof obj.path === "string" ? obj.path : void 0;
|
|
17437
17832
|
const source = parseRepoSource(obj.source);
|
|
17438
|
-
if (!repoPath || !source) return void 0;
|
|
17439
17833
|
const checkout = parseRepoCheckout(obj.checkout);
|
|
17440
17834
|
const clone = parseRepoClone(obj.clone);
|
|
17835
|
+
if (!repoPath && !source && !checkout && !clone) return void 0;
|
|
17441
17836
|
return {
|
|
17442
|
-
path: repoPath,
|
|
17443
|
-
source,
|
|
17837
|
+
...repoPath !== void 0 && { path: repoPath },
|
|
17838
|
+
...source !== void 0 && { source },
|
|
17444
17839
|
...checkout !== void 0 && { checkout },
|
|
17445
17840
|
...clone !== void 0 && { clone }
|
|
17446
17841
|
};
|
|
@@ -17489,7 +17884,8 @@ ${messageContent}`);
|
|
|
17489
17884
|
segmentsByMessage,
|
|
17490
17885
|
mode
|
|
17491
17886
|
}) : void 0;
|
|
17492
|
-
|
|
17887
|
+
const systemMessage = extractSystemMessage(testCase.input, segmentsByMessage, mode);
|
|
17888
|
+
return { question, chatPrompt, systemMessage };
|
|
17493
17889
|
}
|
|
17494
17890
|
function needsRoleMarkers(messages, processedSegmentsByMessage) {
|
|
17495
17891
|
if (messages.some((msg) => msg.role === "assistant" || msg.role === "tool")) {
|
|
@@ -17503,6 +17899,26 @@ function needsRoleMarkers(messages, processedSegmentsByMessage) {
|
|
|
17503
17899
|
}
|
|
17504
17900
|
return messagesWithContent > 1;
|
|
17505
17901
|
}
|
|
17902
|
+
function extractSystemMessage(messages, segmentsByMessage, mode) {
|
|
17903
|
+
const systemParts = [];
|
|
17904
|
+
for (let i = 0; i < messages.length; i++) {
|
|
17905
|
+
if (messages[i].role !== "system") {
|
|
17906
|
+
break;
|
|
17907
|
+
}
|
|
17908
|
+
const segments = segmentsByMessage[i];
|
|
17909
|
+
const contentParts = [];
|
|
17910
|
+
for (const segment of segments) {
|
|
17911
|
+
const formatted = formatSegment(segment, mode);
|
|
17912
|
+
if (formatted) {
|
|
17913
|
+
contentParts.push(formatted);
|
|
17914
|
+
}
|
|
17915
|
+
}
|
|
17916
|
+
if (contentParts.length > 0) {
|
|
17917
|
+
systemParts.push(contentParts.join("\n"));
|
|
17918
|
+
}
|
|
17919
|
+
}
|
|
17920
|
+
return systemParts.length > 0 ? systemParts.join("\n\n") : void 0;
|
|
17921
|
+
}
|
|
17506
17922
|
function buildChatPromptFromSegments(options) {
|
|
17507
17923
|
const { messages, segmentsByMessage, systemPrompt, mode = "lm" } = options;
|
|
17508
17924
|
if (messages.length === 0) {
|
|
@@ -17583,9 +17999,9 @@ function resolveTests(suite) {
|
|
|
17583
17999
|
}
|
|
17584
18000
|
async function readTestSuiteMetadata(testFilePath) {
|
|
17585
18001
|
try {
|
|
17586
|
-
const absolutePath =
|
|
17587
|
-
const content = await
|
|
17588
|
-
const parsed = interpolateEnv(
|
|
18002
|
+
const absolutePath = path8.resolve(testFilePath);
|
|
18003
|
+
const content = await readFile8(absolutePath, "utf8");
|
|
18004
|
+
const parsed = interpolateEnv(parse3(content), process.env);
|
|
17589
18005
|
if (!isJsonObject(parsed)) {
|
|
17590
18006
|
return {};
|
|
17591
18007
|
}
|
|
@@ -17638,25 +18054,31 @@ var loadEvalCases = loadTests;
|
|
|
17638
18054
|
async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
17639
18055
|
const verbose = options?.verbose ?? false;
|
|
17640
18056
|
const filterPattern = options?.filter;
|
|
17641
|
-
const absoluteTestPath =
|
|
18057
|
+
const absoluteTestPath = path8.resolve(evalFilePath);
|
|
17642
18058
|
const repoRootPath = resolveToAbsolutePath(repoRoot);
|
|
17643
18059
|
const searchRoots = buildSearchRoots2(absoluteTestPath, repoRootPath);
|
|
17644
18060
|
const config = await loadConfig(absoluteTestPath, repoRootPath);
|
|
17645
|
-
const rawFile = await
|
|
17646
|
-
const interpolated = interpolateEnv(
|
|
18061
|
+
const rawFile = await readFile8(absoluteTestPath, "utf8");
|
|
18062
|
+
const interpolated = interpolateEnv(parse3(rawFile), process.env);
|
|
17647
18063
|
if (!isJsonObject(interpolated)) {
|
|
17648
18064
|
throw new Error(`Invalid test file format: ${evalFilePath}`);
|
|
17649
18065
|
}
|
|
17650
18066
|
const suite = interpolated;
|
|
17651
18067
|
const suiteNameFromFile = asString5(suite.name)?.trim();
|
|
17652
|
-
const fallbackSuiteName =
|
|
18068
|
+
const fallbackSuiteName = path8.basename(absoluteTestPath).replace(/\.eval\.ya?ml$/i, "").replace(/\.ya?ml$/i, "") || "eval";
|
|
17653
18069
|
const suiteName = suiteNameFromFile && suiteNameFromFile.length > 0 ? suiteNameFromFile : fallbackSuiteName;
|
|
17654
18070
|
const rawTestCases = resolveTests(suite);
|
|
17655
18071
|
const globalEvaluator = coerceEvaluator(suite.evaluator, "global") ?? "llm-grader";
|
|
17656
|
-
const
|
|
18072
|
+
const suitePreprocessors = await parsePreprocessors(
|
|
18073
|
+
suite.preprocessors,
|
|
18074
|
+
searchRoots,
|
|
18075
|
+
"<suite>",
|
|
18076
|
+
absoluteTestPath
|
|
18077
|
+
);
|
|
18078
|
+
const evalFileDir = path8.dirname(absoluteTestPath);
|
|
17657
18079
|
let expandedTestCases;
|
|
17658
18080
|
if (typeof rawTestCases === "string") {
|
|
17659
|
-
const externalPath =
|
|
18081
|
+
const externalPath = path8.resolve(evalFileDir, rawTestCases);
|
|
17660
18082
|
expandedTestCases = await loadCasesFromFile(externalPath);
|
|
17661
18083
|
} else if (Array.isArray(rawTestCases)) {
|
|
17662
18084
|
expandedTestCases = await expandFileReferences(rawTestCases, evalFileDir);
|
|
@@ -17754,7 +18176,8 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17754
18176
|
testCaseConfig,
|
|
17755
18177
|
globalExecution,
|
|
17756
18178
|
searchRoots,
|
|
17757
|
-
id ?? "unknown"
|
|
18179
|
+
id ?? "unknown",
|
|
18180
|
+
suitePreprocessors
|
|
17758
18181
|
);
|
|
17759
18182
|
} catch (error) {
|
|
17760
18183
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -17777,7 +18200,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17777
18200
|
const testCase = {
|
|
17778
18201
|
id,
|
|
17779
18202
|
suite: suiteName,
|
|
17780
|
-
category: options?.category,
|
|
18203
|
+
category: suite.category ?? options?.category,
|
|
17781
18204
|
conversation_id: conversationId,
|
|
17782
18205
|
question,
|
|
17783
18206
|
input: inputMessages,
|
|
@@ -17787,6 +18210,7 @@ async function loadTestsFromYaml(evalFilePath, repoRoot, options) {
|
|
|
17787
18210
|
criteria: outcome ?? "",
|
|
17788
18211
|
evaluator: testCaseEvaluatorKind,
|
|
17789
18212
|
assertions: evaluators,
|
|
18213
|
+
...suitePreprocessors ? { preprocessors: suitePreprocessors } : {},
|
|
17790
18214
|
workspace: mergedWorkspace,
|
|
17791
18215
|
metadata,
|
|
17792
18216
|
targets: caseTargets,
|
|
@@ -17827,8 +18251,8 @@ function parseWorkspaceScriptConfig(raw, evalFileDir) {
|
|
|
17827
18251
|
if (!command) return void 0;
|
|
17828
18252
|
const timeoutMs = typeof obj.timeout_ms === "number" ? obj.timeout_ms : void 0;
|
|
17829
18253
|
let cwd = typeof obj.cwd === "string" ? obj.cwd : void 0;
|
|
17830
|
-
if (cwd && !
|
|
17831
|
-
cwd =
|
|
18254
|
+
if (cwd && !path8.isAbsolute(cwd)) {
|
|
18255
|
+
cwd = path8.resolve(evalFileDir, cwd);
|
|
17832
18256
|
}
|
|
17833
18257
|
const config = { command };
|
|
17834
18258
|
if (timeoutMs !== void 0) {
|
|
@@ -17866,20 +18290,20 @@ function parseWorkspaceHooksConfig(raw, evalFileDir) {
|
|
|
17866
18290
|
}
|
|
17867
18291
|
async function resolveWorkspaceConfig(raw, evalFileDir) {
|
|
17868
18292
|
if (typeof raw === "string") {
|
|
17869
|
-
const workspaceFilePath =
|
|
18293
|
+
const workspaceFilePath = path8.resolve(evalFileDir, raw);
|
|
17870
18294
|
let content;
|
|
17871
18295
|
try {
|
|
17872
|
-
content = await
|
|
18296
|
+
content = await readFile8(workspaceFilePath, "utf8");
|
|
17873
18297
|
} catch {
|
|
17874
18298
|
throw new Error(`Workspace file not found: ${raw} (resolved to ${workspaceFilePath})`);
|
|
17875
18299
|
}
|
|
17876
|
-
const parsed = interpolateEnv(
|
|
18300
|
+
const parsed = interpolateEnv(parse3(content), process.env);
|
|
17877
18301
|
if (!isJsonObject(parsed)) {
|
|
17878
18302
|
throw new Error(
|
|
17879
18303
|
`Invalid workspace file format: ${workspaceFilePath} (expected a YAML object)`
|
|
17880
18304
|
);
|
|
17881
18305
|
}
|
|
17882
|
-
const workspaceFileDir =
|
|
18306
|
+
const workspaceFileDir = path8.dirname(workspaceFilePath);
|
|
17883
18307
|
return parseWorkspaceConfig(parsed, workspaceFileDir);
|
|
17884
18308
|
}
|
|
17885
18309
|
return parseWorkspaceConfig(raw, evalFileDir);
|
|
@@ -17899,8 +18323,8 @@ function parseWorkspaceConfig(raw, evalFileDir) {
|
|
|
17899
18323
|
throw new Error("workspace.static has been removed. Use workspace.mode='static'.");
|
|
17900
18324
|
}
|
|
17901
18325
|
let template = typeof obj.template === "string" ? obj.template : void 0;
|
|
17902
|
-
if (template && !
|
|
17903
|
-
template =
|
|
18326
|
+
if (template && !path8.isAbsolute(template)) {
|
|
18327
|
+
template = path8.resolve(evalFileDir, template);
|
|
17904
18328
|
}
|
|
17905
18329
|
const isolation = obj.isolation === "shared" || obj.isolation === "per_test" ? obj.isolation : void 0;
|
|
17906
18330
|
const repos = Array.isArray(obj.repos) ? obj.repos.map(parseRepoConfig).filter(Boolean) : void 0;
|
|
@@ -17908,14 +18332,28 @@ function parseWorkspaceConfig(raw, evalFileDir) {
|
|
|
17908
18332
|
const explicitMode = obj.mode === "pooled" || obj.mode === "temp" || obj.mode === "static" ? obj.mode : void 0;
|
|
17909
18333
|
const workspacePath = typeof obj.path === "string" ? obj.path : void 0;
|
|
17910
18334
|
const mode = explicitMode ?? (workspacePath ? "static" : void 0);
|
|
17911
|
-
|
|
18335
|
+
const docker = parseDockerWorkspaceConfig(obj.docker);
|
|
18336
|
+
if (!template && !isolation && !repos && !hooks && !mode && !workspacePath && !docker)
|
|
18337
|
+
return void 0;
|
|
17912
18338
|
return {
|
|
17913
18339
|
...template !== void 0 && { template },
|
|
17914
18340
|
...isolation !== void 0 && { isolation },
|
|
17915
18341
|
...repos !== void 0 && { repos },
|
|
17916
18342
|
...hooks !== void 0 && { hooks },
|
|
17917
18343
|
...mode !== void 0 && { mode },
|
|
17918
|
-
...workspacePath !== void 0 && { path: workspacePath }
|
|
18344
|
+
...workspacePath !== void 0 && { path: workspacePath },
|
|
18345
|
+
...docker !== void 0 && { docker }
|
|
18346
|
+
};
|
|
18347
|
+
}
|
|
18348
|
+
function parseDockerWorkspaceConfig(raw) {
|
|
18349
|
+
if (!isJsonObject(raw)) return void 0;
|
|
18350
|
+
const obj = raw;
|
|
18351
|
+
if (typeof obj.image !== "string") return void 0;
|
|
18352
|
+
return {
|
|
18353
|
+
image: obj.image,
|
|
18354
|
+
...typeof obj.timeout === "number" && { timeout: obj.timeout },
|
|
18355
|
+
...typeof obj.memory === "string" && { memory: obj.memory },
|
|
18356
|
+
...typeof obj.cpus === "number" && { cpus: obj.cpus }
|
|
17919
18357
|
};
|
|
17920
18358
|
}
|
|
17921
18359
|
function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
|
|
@@ -17944,7 +18382,8 @@ function mergeWorkspaceConfigs(suiteLevel, caseLevel) {
|
|
|
17944
18382
|
repos: caseLevel.repos ?? suiteLevel.repos,
|
|
17945
18383
|
...hasHooks && { hooks: mergedHooks },
|
|
17946
18384
|
mode: caseLevel.mode ?? suiteLevel.mode,
|
|
17947
|
-
path: caseLevel.path ?? suiteLevel.path
|
|
18385
|
+
path: caseLevel.path ?? suiteLevel.path,
|
|
18386
|
+
docker: caseLevel.docker ?? suiteLevel.docker
|
|
17948
18387
|
};
|
|
17949
18388
|
}
|
|
17950
18389
|
function asString5(value) {
|
|
@@ -18206,8 +18645,8 @@ function transpileEvalYaml(suite, source = "EVAL.yaml") {
|
|
|
18206
18645
|
}
|
|
18207
18646
|
function transpileEvalYamlFile(evalYamlPath) {
|
|
18208
18647
|
const content = readFileSync(evalYamlPath, "utf8");
|
|
18209
|
-
const parsed =
|
|
18210
|
-
return transpileEvalYaml(parsed,
|
|
18648
|
+
const parsed = parse4(content);
|
|
18649
|
+
return transpileEvalYaml(parsed, path9.basename(evalYamlPath));
|
|
18211
18650
|
}
|
|
18212
18651
|
function getOutputFilenames(result) {
|
|
18213
18652
|
const names = /* @__PURE__ */ new Map();
|
|
@@ -18739,7 +19178,7 @@ function normalizeInputFiles(inputFiles) {
|
|
|
18739
19178
|
}
|
|
18740
19179
|
const deduped = /* @__PURE__ */ new Map();
|
|
18741
19180
|
for (const inputFile of inputFiles) {
|
|
18742
|
-
const absolutePath =
|
|
19181
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18743
19182
|
if (!deduped.has(absolutePath)) {
|
|
18744
19183
|
deduped.set(absolutePath, absolutePath);
|
|
18745
19184
|
}
|
|
@@ -18752,7 +19191,7 @@ function collectInputFiles(inputFiles) {
|
|
|
18752
19191
|
}
|
|
18753
19192
|
const unique = /* @__PURE__ */ new Map();
|
|
18754
19193
|
for (const inputFile of inputFiles) {
|
|
18755
|
-
const absolutePath =
|
|
19194
|
+
const absolutePath = path10.resolve(inputFile);
|
|
18756
19195
|
if (!unique.has(absolutePath)) {
|
|
18757
19196
|
unique.set(absolutePath, absolutePath);
|
|
18758
19197
|
}
|
|
@@ -18764,7 +19203,7 @@ function buildMandatoryPrereadBlock(inputFiles) {
|
|
|
18764
19203
|
return "";
|
|
18765
19204
|
}
|
|
18766
19205
|
const buildList = (files) => files.map((absolutePath) => {
|
|
18767
|
-
const fileName =
|
|
19206
|
+
const fileName = path10.basename(absolutePath);
|
|
18768
19207
|
const fileUri = pathToFileUri(absolutePath);
|
|
18769
19208
|
return `* [${fileName}](${fileUri})`;
|
|
18770
19209
|
});
|
|
@@ -18780,7 +19219,7 @@ ${buildList(inputFiles).join("\n")}.`);
|
|
|
18780
19219
|
return sections.join("\n");
|
|
18781
19220
|
}
|
|
18782
19221
|
function pathToFileUri(filePath) {
|
|
18783
|
-
const absolutePath =
|
|
19222
|
+
const absolutePath = path10.isAbsolute(filePath) ? filePath : path10.resolve(filePath);
|
|
18784
19223
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
18785
19224
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
18786
19225
|
return `file:///${normalizedPath}`;
|
|
@@ -18926,10 +19365,10 @@ var ClaudeCliProvider = class {
|
|
|
18926
19365
|
}
|
|
18927
19366
|
resolveCwd(cwdOverride) {
|
|
18928
19367
|
if (cwdOverride) {
|
|
18929
|
-
return
|
|
19368
|
+
return path11.resolve(cwdOverride);
|
|
18930
19369
|
}
|
|
18931
19370
|
if (this.config.cwd) {
|
|
18932
|
-
return
|
|
19371
|
+
return path11.resolve(this.config.cwd);
|
|
18933
19372
|
}
|
|
18934
19373
|
return void 0;
|
|
18935
19374
|
}
|
|
@@ -18939,9 +19378,9 @@ var ClaudeCliProvider = class {
|
|
|
18939
19378
|
return void 0;
|
|
18940
19379
|
}
|
|
18941
19380
|
if (this.config.logDir) {
|
|
18942
|
-
return
|
|
19381
|
+
return path11.resolve(this.config.logDir);
|
|
18943
19382
|
}
|
|
18944
|
-
return
|
|
19383
|
+
return path11.join(process.cwd(), ".agentv", "logs", "claude-cli");
|
|
18945
19384
|
}
|
|
18946
19385
|
async createStreamLogger(request) {
|
|
18947
19386
|
const logDir = this.resolveLogDirectory();
|
|
@@ -18955,7 +19394,7 @@ var ClaudeCliProvider = class {
|
|
|
18955
19394
|
console.warn(`Skipping Claude CLI stream logging (could not create ${logDir}): ${message}`);
|
|
18956
19395
|
return void 0;
|
|
18957
19396
|
}
|
|
18958
|
-
const filePath =
|
|
19397
|
+
const filePath = path11.join(logDir, buildLogFilename(request, this.targetName));
|
|
18959
19398
|
try {
|
|
18960
19399
|
const logger = await ClaudeCliStreamLogger.create({
|
|
18961
19400
|
filePath,
|
|
@@ -19412,10 +19851,10 @@ var ClaudeSdkProvider = class {
|
|
|
19412
19851
|
}
|
|
19413
19852
|
resolveCwd(cwdOverride) {
|
|
19414
19853
|
if (cwdOverride) {
|
|
19415
|
-
return
|
|
19854
|
+
return path12.resolve(cwdOverride);
|
|
19416
19855
|
}
|
|
19417
19856
|
if (this.config.cwd) {
|
|
19418
|
-
return
|
|
19857
|
+
return path12.resolve(this.config.cwd);
|
|
19419
19858
|
}
|
|
19420
19859
|
return void 0;
|
|
19421
19860
|
}
|
|
@@ -19425,9 +19864,9 @@ var ClaudeSdkProvider = class {
|
|
|
19425
19864
|
return void 0;
|
|
19426
19865
|
}
|
|
19427
19866
|
if (this.config.logDir) {
|
|
19428
|
-
return
|
|
19867
|
+
return path12.resolve(this.config.logDir);
|
|
19429
19868
|
}
|
|
19430
|
-
return
|
|
19869
|
+
return path12.join(process.cwd(), ".agentv", "logs", "claude");
|
|
19431
19870
|
}
|
|
19432
19871
|
async createStreamLogger(request) {
|
|
19433
19872
|
const logDir = this.resolveLogDirectory();
|
|
@@ -19441,7 +19880,7 @@ var ClaudeSdkProvider = class {
|
|
|
19441
19880
|
console.warn(`Skipping Claude stream logging (could not create ${logDir}): ${message}`);
|
|
19442
19881
|
return void 0;
|
|
19443
19882
|
}
|
|
19444
|
-
const filePath =
|
|
19883
|
+
const filePath = path12.join(logDir, buildLogFilename2(request, this.targetName));
|
|
19445
19884
|
try {
|
|
19446
19885
|
const logger = await ClaudeStreamLogger.create({
|
|
19447
19886
|
filePath,
|
|
@@ -20124,7 +20563,7 @@ function normalizeInputFiles2(inputFiles) {
|
|
|
20124
20563
|
}
|
|
20125
20564
|
const unique = /* @__PURE__ */ new Map();
|
|
20126
20565
|
for (const inputFile of inputFiles) {
|
|
20127
|
-
const absolutePath =
|
|
20566
|
+
const absolutePath = path13.resolve(inputFile);
|
|
20128
20567
|
if (!unique.has(absolutePath)) {
|
|
20129
20568
|
unique.set(absolutePath, absolutePath);
|
|
20130
20569
|
}
|
|
@@ -20138,7 +20577,7 @@ function formatFileList(files, template) {
|
|
|
20138
20577
|
const formatter = template ?? "{path}";
|
|
20139
20578
|
return files.map((filePath) => {
|
|
20140
20579
|
const escapedPath = shellEscape(filePath);
|
|
20141
|
-
const escapedName = shellEscape(
|
|
20580
|
+
const escapedName = shellEscape(path13.basename(filePath));
|
|
20142
20581
|
return formatter.replaceAll("{path}", escapedPath).replaceAll("{basename}", escapedName);
|
|
20143
20582
|
}).join(" ");
|
|
20144
20583
|
}
|
|
@@ -20162,7 +20601,7 @@ function generateOutputFilePath(evalCaseId, extension = ".json") {
|
|
|
20162
20601
|
const safeEvalId = evalCaseId || "unknown";
|
|
20163
20602
|
const timestamp = Date.now();
|
|
20164
20603
|
const random = Math.random().toString(36).substring(2, 9);
|
|
20165
|
-
return
|
|
20604
|
+
return path13.join(os.tmpdir(), `agentv-${safeEvalId}-${timestamp}-${random}${extension}`);
|
|
20166
20605
|
}
|
|
20167
20606
|
function formatTimeoutSuffix2(timeoutMs) {
|
|
20168
20607
|
if (!timeoutMs || timeoutMs <= 0) {
|
|
@@ -20401,10 +20840,10 @@ ${basePrompt}` : basePrompt;
|
|
|
20401
20840
|
}
|
|
20402
20841
|
resolveCwd(cwdOverride) {
|
|
20403
20842
|
if (cwdOverride) {
|
|
20404
|
-
return
|
|
20843
|
+
return path14.resolve(cwdOverride);
|
|
20405
20844
|
}
|
|
20406
20845
|
if (this.config.cwd) {
|
|
20407
|
-
return
|
|
20846
|
+
return path14.resolve(this.config.cwd);
|
|
20408
20847
|
}
|
|
20409
20848
|
return void 0;
|
|
20410
20849
|
}
|
|
@@ -20414,9 +20853,9 @@ ${basePrompt}` : basePrompt;
|
|
|
20414
20853
|
return void 0;
|
|
20415
20854
|
}
|
|
20416
20855
|
if (this.config.logDir) {
|
|
20417
|
-
return
|
|
20856
|
+
return path14.resolve(this.config.logDir);
|
|
20418
20857
|
}
|
|
20419
|
-
return
|
|
20858
|
+
return path14.join(process.cwd(), ".agentv", "logs", "codex");
|
|
20420
20859
|
}
|
|
20421
20860
|
async createStreamLogger(request) {
|
|
20422
20861
|
const logDir = this.resolveLogDirectory();
|
|
@@ -20430,7 +20869,7 @@ ${basePrompt}` : basePrompt;
|
|
|
20430
20869
|
console.warn(`Skipping Codex SDK stream logging (could not create ${logDir}): ${message}`);
|
|
20431
20870
|
return void 0;
|
|
20432
20871
|
}
|
|
20433
|
-
const filePath =
|
|
20872
|
+
const filePath = path14.join(logDir, buildLogFilename3(request, this.targetName));
|
|
20434
20873
|
try {
|
|
20435
20874
|
const logger = await CodexSdkStreamLogger.create({
|
|
20436
20875
|
filePath,
|
|
@@ -20622,7 +21061,7 @@ function subscribeToCopilotCliLogEntries(listener) {
|
|
|
20622
21061
|
};
|
|
20623
21062
|
}
|
|
20624
21063
|
function resolvePlatformCliPath() {
|
|
20625
|
-
const
|
|
21064
|
+
const os4 = platform();
|
|
20626
21065
|
const cpu = arch();
|
|
20627
21066
|
const platformMap = {
|
|
20628
21067
|
linux: "linux",
|
|
@@ -20633,17 +21072,17 @@ function resolvePlatformCliPath() {
|
|
|
20633
21072
|
x64: "x64",
|
|
20634
21073
|
arm64: "arm64"
|
|
20635
21074
|
};
|
|
20636
|
-
const osPart = platformMap[
|
|
21075
|
+
const osPart = platformMap[os4];
|
|
20637
21076
|
const archPart = archMap[cpu];
|
|
20638
21077
|
if (!osPart || !archPart) {
|
|
20639
21078
|
return void 0;
|
|
20640
21079
|
}
|
|
20641
21080
|
const packageName = `@github/copilot-${osPart}-${archPart}`;
|
|
20642
|
-
const binaryName =
|
|
21081
|
+
const binaryName = os4 === "win32" ? "copilot.exe" : "copilot";
|
|
20643
21082
|
try {
|
|
20644
21083
|
const resolved = import.meta.resolve(`${packageName}/package.json`);
|
|
20645
|
-
const packageJsonPath = resolved.startsWith("file:") ?
|
|
20646
|
-
const binaryPath =
|
|
21084
|
+
const packageJsonPath = resolved.startsWith("file:") ? fileURLToPath3(resolved) : resolved;
|
|
21085
|
+
const binaryPath = path15.join(path15.dirname(packageJsonPath), binaryName);
|
|
20647
21086
|
if (existsSync(binaryPath)) {
|
|
20648
21087
|
return binaryPath;
|
|
20649
21088
|
}
|
|
@@ -20651,7 +21090,7 @@ function resolvePlatformCliPath() {
|
|
|
20651
21090
|
}
|
|
20652
21091
|
let searchDir = process.cwd();
|
|
20653
21092
|
for (let i = 0; i < 10; i++) {
|
|
20654
|
-
const standardPath =
|
|
21093
|
+
const standardPath = path15.join(
|
|
20655
21094
|
searchDir,
|
|
20656
21095
|
"node_modules",
|
|
20657
21096
|
...packageName.split("/"),
|
|
@@ -20660,13 +21099,13 @@ function resolvePlatformCliPath() {
|
|
|
20660
21099
|
if (existsSync(standardPath)) {
|
|
20661
21100
|
return standardPath;
|
|
20662
21101
|
}
|
|
20663
|
-
const bunDir =
|
|
21102
|
+
const bunDir = path15.join(searchDir, "node_modules", ".bun");
|
|
20664
21103
|
const prefix = `@github+copilot-${osPart}-${archPart}@`;
|
|
20665
21104
|
try {
|
|
20666
21105
|
const entries = readdirSync(bunDir);
|
|
20667
21106
|
for (const entry of entries) {
|
|
20668
21107
|
if (entry.startsWith(prefix)) {
|
|
20669
|
-
const candidate =
|
|
21108
|
+
const candidate = path15.join(
|
|
20670
21109
|
bunDir,
|
|
20671
21110
|
entry,
|
|
20672
21111
|
"node_modules",
|
|
@@ -20681,7 +21120,7 @@ function resolvePlatformCliPath() {
|
|
|
20681
21120
|
}
|
|
20682
21121
|
} catch {
|
|
20683
21122
|
}
|
|
20684
|
-
const parent =
|
|
21123
|
+
const parent = path15.dirname(searchDir);
|
|
20685
21124
|
if (parent === searchDir) break;
|
|
20686
21125
|
searchDir = parent;
|
|
20687
21126
|
}
|
|
@@ -21023,10 +21462,10 @@ var CopilotCliProvider = class {
|
|
|
21023
21462
|
}
|
|
21024
21463
|
resolveCwd(cwdOverride) {
|
|
21025
21464
|
if (cwdOverride) {
|
|
21026
|
-
return
|
|
21465
|
+
return path16.resolve(cwdOverride);
|
|
21027
21466
|
}
|
|
21028
21467
|
if (this.config.cwd) {
|
|
21029
|
-
return
|
|
21468
|
+
return path16.resolve(this.config.cwd);
|
|
21030
21469
|
}
|
|
21031
21470
|
return void 0;
|
|
21032
21471
|
}
|
|
@@ -21045,9 +21484,9 @@ var CopilotCliProvider = class {
|
|
|
21045
21484
|
return void 0;
|
|
21046
21485
|
}
|
|
21047
21486
|
if (this.config.logDir) {
|
|
21048
|
-
return
|
|
21487
|
+
return path16.resolve(this.config.logDir);
|
|
21049
21488
|
}
|
|
21050
|
-
return
|
|
21489
|
+
return path16.join(process.cwd(), ".agentv", "logs", "copilot-cli");
|
|
21051
21490
|
}
|
|
21052
21491
|
async createStreamLogger(request) {
|
|
21053
21492
|
const logDir = this.resolveLogDirectory();
|
|
@@ -21061,7 +21500,7 @@ var CopilotCliProvider = class {
|
|
|
21061
21500
|
console.warn(`Skipping Copilot CLI stream logging (could not create ${logDir}): ${message}`);
|
|
21062
21501
|
return void 0;
|
|
21063
21502
|
}
|
|
21064
|
-
const filePath =
|
|
21503
|
+
const filePath = path16.join(logDir, buildLogFilename4(request, this.targetName, "copilot-cli"));
|
|
21065
21504
|
try {
|
|
21066
21505
|
const logger = await CopilotStreamLogger.create(
|
|
21067
21506
|
{
|
|
@@ -21279,7 +21718,7 @@ function parseCopilotEvents(eventsJsonl) {
|
|
|
21279
21718
|
durationMs
|
|
21280
21719
|
};
|
|
21281
21720
|
}
|
|
21282
|
-
var DEFAULT_SESSION_STATE_DIR = () =>
|
|
21721
|
+
var DEFAULT_SESSION_STATE_DIR = () => path17.join(homedir(), ".copilot", "session-state");
|
|
21283
21722
|
async function discoverCopilotSessions(opts) {
|
|
21284
21723
|
const sessionStateDir = opts?.sessionStateDir ?? DEFAULT_SESSION_STATE_DIR();
|
|
21285
21724
|
const limit = opts?.limit ?? 10;
|
|
@@ -21291,11 +21730,11 @@ async function discoverCopilotSessions(opts) {
|
|
|
21291
21730
|
}
|
|
21292
21731
|
const sessions = [];
|
|
21293
21732
|
for (const entry of entries) {
|
|
21294
|
-
const sessionDir =
|
|
21295
|
-
const workspacePath =
|
|
21296
|
-
const eventsPath =
|
|
21733
|
+
const sessionDir = path17.join(sessionStateDir, entry);
|
|
21734
|
+
const workspacePath = path17.join(sessionDir, "workspace.yaml");
|
|
21735
|
+
const eventsPath = path17.join(sessionDir, "events.jsonl");
|
|
21297
21736
|
try {
|
|
21298
|
-
const workspaceContent = await
|
|
21737
|
+
const workspaceContent = await readFile9(workspacePath, "utf8");
|
|
21299
21738
|
const workspace = parseYaml22(workspaceContent) ?? {};
|
|
21300
21739
|
const cwd = String(workspace.cwd ?? "");
|
|
21301
21740
|
let updatedAt;
|
|
@@ -21352,10 +21791,10 @@ var CopilotLogProvider = class {
|
|
|
21352
21791
|
}
|
|
21353
21792
|
async invoke(_request) {
|
|
21354
21793
|
const sessionDir = await this.resolveSessionDir();
|
|
21355
|
-
const eventsPath =
|
|
21794
|
+
const eventsPath = path18.join(sessionDir, "events.jsonl");
|
|
21356
21795
|
let eventsContent;
|
|
21357
21796
|
try {
|
|
21358
|
-
eventsContent = await
|
|
21797
|
+
eventsContent = await readFile10(eventsPath, "utf8");
|
|
21359
21798
|
} catch (err) {
|
|
21360
21799
|
throw new Error(
|
|
21361
21800
|
`Failed to read Copilot session transcript at ${eventsPath}: ${err instanceof Error ? err.message : String(err)}`
|
|
@@ -21374,8 +21813,8 @@ var CopilotLogProvider = class {
|
|
|
21374
21813
|
return this.config.sessionDir;
|
|
21375
21814
|
}
|
|
21376
21815
|
if (this.config.sessionId) {
|
|
21377
|
-
const stateDir = this.config.sessionStateDir ??
|
|
21378
|
-
return
|
|
21816
|
+
const stateDir = this.config.sessionStateDir ?? path18.join(homedir2(), ".copilot", "session-state");
|
|
21817
|
+
return path18.join(stateDir, this.config.sessionId);
|
|
21379
21818
|
}
|
|
21380
21819
|
if (this.config.discover === "latest") {
|
|
21381
21820
|
const sessions = await discoverCopilotSessions({
|
|
@@ -21700,10 +22139,10 @@ var CopilotSdkProvider = class {
|
|
|
21700
22139
|
}
|
|
21701
22140
|
resolveCwd(cwdOverride) {
|
|
21702
22141
|
if (cwdOverride) {
|
|
21703
|
-
return
|
|
22142
|
+
return path19.resolve(cwdOverride);
|
|
21704
22143
|
}
|
|
21705
22144
|
if (this.config.cwd) {
|
|
21706
|
-
return
|
|
22145
|
+
return path19.resolve(this.config.cwd);
|
|
21707
22146
|
}
|
|
21708
22147
|
return void 0;
|
|
21709
22148
|
}
|
|
@@ -21712,9 +22151,9 @@ var CopilotSdkProvider = class {
|
|
|
21712
22151
|
return void 0;
|
|
21713
22152
|
}
|
|
21714
22153
|
if (this.config.logDir) {
|
|
21715
|
-
return
|
|
22154
|
+
return path19.resolve(this.config.logDir);
|
|
21716
22155
|
}
|
|
21717
|
-
return
|
|
22156
|
+
return path19.join(process.cwd(), ".agentv", "logs", "copilot-sdk");
|
|
21718
22157
|
}
|
|
21719
22158
|
async createStreamLogger(request) {
|
|
21720
22159
|
const logDir = this.resolveLogDirectory();
|
|
@@ -21728,7 +22167,7 @@ var CopilotSdkProvider = class {
|
|
|
21728
22167
|
console.warn(`Skipping Copilot SDK stream logging (could not create ${logDir}): ${message}`);
|
|
21729
22168
|
return void 0;
|
|
21730
22169
|
}
|
|
21731
|
-
const filePath =
|
|
22170
|
+
const filePath = path19.join(logDir, buildLogFilename4(request, this.targetName, "copilot-sdk"));
|
|
21732
22171
|
try {
|
|
21733
22172
|
const logger = await CopilotStreamLogger.create(
|
|
21734
22173
|
{
|
|
@@ -21757,9 +22196,9 @@ var CopilotSdkProvider = class {
|
|
|
21757
22196
|
};
|
|
21758
22197
|
function resolveSkillDirectories(cwd) {
|
|
21759
22198
|
const candidates = [
|
|
21760
|
-
|
|
21761
|
-
|
|
21762
|
-
|
|
22199
|
+
path19.join(cwd, ".claude", "skills"),
|
|
22200
|
+
path19.join(cwd, ".agents", "skills"),
|
|
22201
|
+
path19.join(cwd, ".codex", "skills")
|
|
21763
22202
|
];
|
|
21764
22203
|
return candidates.filter((dir) => existsSync2(dir));
|
|
21765
22204
|
}
|
|
@@ -22031,7 +22470,7 @@ var PiCliProvider = class {
|
|
|
22031
22470
|
const cwd = this.resolveCwd(workspaceRoot, request.cwd);
|
|
22032
22471
|
const logger = await this.createStreamLogger(request).catch(() => void 0);
|
|
22033
22472
|
try {
|
|
22034
|
-
const promptFile =
|
|
22473
|
+
const promptFile = path20.join(cwd, PROMPT_FILENAME);
|
|
22035
22474
|
await writeFile(promptFile, request.question, "utf8");
|
|
22036
22475
|
const args = this.buildPiArgs(request.question, inputFiles);
|
|
22037
22476
|
const result = await this.executePi(args, cwd, request.signal, logger);
|
|
@@ -22094,10 +22533,10 @@ var PiCliProvider = class {
|
|
|
22094
22533
|
}
|
|
22095
22534
|
resolveCwd(workspaceRoot, cwdOverride) {
|
|
22096
22535
|
if (cwdOverride) {
|
|
22097
|
-
return
|
|
22536
|
+
return path20.resolve(cwdOverride);
|
|
22098
22537
|
}
|
|
22099
22538
|
if (this.config.cwd) {
|
|
22100
|
-
return
|
|
22539
|
+
return path20.resolve(this.config.cwd);
|
|
22101
22540
|
}
|
|
22102
22541
|
if (workspaceRoot) {
|
|
22103
22542
|
return workspaceRoot;
|
|
@@ -22203,7 +22642,7 @@ ${prompt}` : prompt;
|
|
|
22203
22642
|
return env;
|
|
22204
22643
|
}
|
|
22205
22644
|
async createWorkspace() {
|
|
22206
|
-
return await mkdtemp(
|
|
22645
|
+
return await mkdtemp(path20.join(tmpdir(), WORKSPACE_PREFIX));
|
|
22207
22646
|
}
|
|
22208
22647
|
async cleanupWorkspace(workspaceRoot) {
|
|
22209
22648
|
try {
|
|
@@ -22213,9 +22652,9 @@ ${prompt}` : prompt;
|
|
|
22213
22652
|
}
|
|
22214
22653
|
resolveLogDirectory() {
|
|
22215
22654
|
if (this.config.logDir) {
|
|
22216
|
-
return
|
|
22655
|
+
return path20.resolve(this.config.logDir);
|
|
22217
22656
|
}
|
|
22218
|
-
return
|
|
22657
|
+
return path20.join(process.cwd(), ".agentv", "logs", "pi-cli");
|
|
22219
22658
|
}
|
|
22220
22659
|
async createStreamLogger(request) {
|
|
22221
22660
|
const logDir = this.resolveLogDirectory();
|
|
@@ -22229,7 +22668,7 @@ ${prompt}` : prompt;
|
|
|
22229
22668
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
22230
22669
|
return void 0;
|
|
22231
22670
|
}
|
|
22232
|
-
const filePath =
|
|
22671
|
+
const filePath = path20.join(logDir, buildLogFilename5(request, this.targetName));
|
|
22233
22672
|
try {
|
|
22234
22673
|
const logger = await PiStreamLogger.create({
|
|
22235
22674
|
filePath,
|
|
@@ -22700,8 +23139,8 @@ function resolveWindowsCmd(executable) {
|
|
|
22700
23139
|
const content = readFileSync2(cmdPath, "utf-8");
|
|
22701
23140
|
const match = content.match(/"?%_prog%"?\s+"([^"]+\.js)"/);
|
|
22702
23141
|
if (match) {
|
|
22703
|
-
const dp0 =
|
|
22704
|
-
const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${
|
|
23142
|
+
const dp0 = path20.dirname(path20.resolve(cmdPath));
|
|
23143
|
+
const scriptPath = match[1].replace(/%dp0%[/\\]?/gi, `${dp0}${path20.sep}`);
|
|
22705
23144
|
try {
|
|
22706
23145
|
accessSync(scriptPath);
|
|
22707
23146
|
return ["node", [scriptPath]];
|
|
@@ -22784,19 +23223,19 @@ function getAgentvHome() {
|
|
|
22784
23223
|
}
|
|
22785
23224
|
return envHome;
|
|
22786
23225
|
}
|
|
22787
|
-
return
|
|
23226
|
+
return path21.join(os2.homedir(), ".agentv");
|
|
22788
23227
|
}
|
|
22789
23228
|
function getWorkspacesRoot() {
|
|
22790
|
-
return
|
|
23229
|
+
return path21.join(getAgentvHome(), "workspaces");
|
|
22791
23230
|
}
|
|
22792
23231
|
function getSubagentsRoot() {
|
|
22793
|
-
return
|
|
23232
|
+
return path21.join(getAgentvHome(), "subagents");
|
|
22794
23233
|
}
|
|
22795
23234
|
function getTraceStateRoot() {
|
|
22796
|
-
return
|
|
23235
|
+
return path21.join(getAgentvHome(), "trace-state");
|
|
22797
23236
|
}
|
|
22798
23237
|
function getWorkspacePoolRoot() {
|
|
22799
|
-
return
|
|
23238
|
+
return path21.join(getAgentvHome(), "workspace-pool");
|
|
22800
23239
|
}
|
|
22801
23240
|
var piCodingAgentModule = null;
|
|
22802
23241
|
var piAiModule = null;
|
|
@@ -22816,7 +23255,7 @@ async function promptInstall() {
|
|
|
22816
23255
|
}
|
|
22817
23256
|
}
|
|
22818
23257
|
function findManagedSdkInstallRoot() {
|
|
22819
|
-
return
|
|
23258
|
+
return path222.join(getAgentvHome(), "deps", "pi-sdk");
|
|
22820
23259
|
}
|
|
22821
23260
|
function resolveGlobalNpmRoot() {
|
|
22822
23261
|
try {
|
|
@@ -22830,7 +23269,7 @@ function resolveGlobalNpmRoot() {
|
|
|
22830
23269
|
}
|
|
22831
23270
|
}
|
|
22832
23271
|
function buildGlobalModuleEntry(moduleName, globalNpmRoot) {
|
|
22833
|
-
return
|
|
23272
|
+
return path222.join(globalNpmRoot, ...moduleName.split("/"), "dist", "index.js");
|
|
22834
23273
|
}
|
|
22835
23274
|
function findAccessiblePath(paths) {
|
|
22836
23275
|
for (const candidate of paths) {
|
|
@@ -22856,11 +23295,11 @@ async function tryImportLocalSdkModules() {
|
|
|
22856
23295
|
async function tryImportManagedSdkModules() {
|
|
22857
23296
|
const managedRoot = findManagedSdkInstallRoot();
|
|
22858
23297
|
const piCodingAgentEntry = findAccessiblePath([
|
|
22859
|
-
|
|
23298
|
+
path222.join(managedRoot, "node_modules", "@mariozechner", "pi-coding-agent", "dist", "index.js")
|
|
22860
23299
|
]);
|
|
22861
23300
|
const piAiEntry = findAccessiblePath([
|
|
22862
|
-
|
|
22863
|
-
|
|
23301
|
+
path222.join(managedRoot, "node_modules", "@mariozechner", "pi-ai", "dist", "index.js"),
|
|
23302
|
+
path222.join(
|
|
22864
23303
|
managedRoot,
|
|
22865
23304
|
"node_modules",
|
|
22866
23305
|
"@mariozechner",
|
|
@@ -22891,7 +23330,7 @@ async function tryImportGlobalSdkModules() {
|
|
|
22891
23330
|
]);
|
|
22892
23331
|
const piAiEntry = findAccessiblePath([
|
|
22893
23332
|
buildGlobalModuleEntry("@mariozechner/pi-ai", globalNpmRoot),
|
|
22894
|
-
|
|
23333
|
+
path222.join(
|
|
22895
23334
|
globalNpmRoot,
|
|
22896
23335
|
"@mariozechner",
|
|
22897
23336
|
"pi-coding-agent",
|
|
@@ -23192,10 +23631,10 @@ ${fileList}`;
|
|
|
23192
23631
|
}
|
|
23193
23632
|
resolveCwd(cwdOverride) {
|
|
23194
23633
|
if (cwdOverride) {
|
|
23195
|
-
return
|
|
23634
|
+
return path222.resolve(cwdOverride);
|
|
23196
23635
|
}
|
|
23197
23636
|
if (this.config.cwd) {
|
|
23198
|
-
return
|
|
23637
|
+
return path222.resolve(this.config.cwd);
|
|
23199
23638
|
}
|
|
23200
23639
|
return process.cwd();
|
|
23201
23640
|
}
|
|
@@ -23214,9 +23653,9 @@ ${fileList}`;
|
|
|
23214
23653
|
}
|
|
23215
23654
|
resolveLogDirectory() {
|
|
23216
23655
|
if (this.config.logDir) {
|
|
23217
|
-
return
|
|
23656
|
+
return path222.resolve(this.config.logDir);
|
|
23218
23657
|
}
|
|
23219
|
-
return
|
|
23658
|
+
return path222.join(process.cwd(), ".agentv", "logs", "pi-coding-agent");
|
|
23220
23659
|
}
|
|
23221
23660
|
async createStreamLogger(request) {
|
|
23222
23661
|
const logDir = this.resolveLogDirectory();
|
|
@@ -23230,7 +23669,7 @@ ${fileList}`;
|
|
|
23230
23669
|
console.warn(`Skipping Pi stream logging (could not create ${logDir}): ${message}`);
|
|
23231
23670
|
return void 0;
|
|
23232
23671
|
}
|
|
23233
|
-
const filePath =
|
|
23672
|
+
const filePath = path222.join(logDir, buildLogFilename6(request, this.targetName));
|
|
23234
23673
|
try {
|
|
23235
23674
|
const logger = await PiStreamLogger2.create({
|
|
23236
23675
|
filePath,
|
|
@@ -23454,7 +23893,7 @@ async function readDirEntries(target) {
|
|
|
23454
23893
|
const entries = await readdir2(target, { withFileTypes: true });
|
|
23455
23894
|
return entries.map((entry) => ({
|
|
23456
23895
|
name: entry.name,
|
|
23457
|
-
absolutePath:
|
|
23896
|
+
absolutePath: path23.join(target, entry.name),
|
|
23458
23897
|
isDirectory: entry.isDirectory()
|
|
23459
23898
|
}));
|
|
23460
23899
|
}
|
|
@@ -23468,7 +23907,7 @@ async function removeIfExists(target) {
|
|
|
23468
23907
|
}
|
|
23469
23908
|
}
|
|
23470
23909
|
function pathToFileUri2(filePath) {
|
|
23471
|
-
const absolutePath =
|
|
23910
|
+
const absolutePath = path24.isAbsolute(filePath) ? filePath : path24.resolve(filePath);
|
|
23472
23911
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
23473
23912
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
23474
23913
|
return `file:///${normalizedPath}`;
|
|
@@ -23560,8 +23999,8 @@ function createBatchRequestPrompt(userQuery, responseFileTmp, responseFileFinal,
|
|
|
23560
23999
|
});
|
|
23561
24000
|
}
|
|
23562
24001
|
function createBatchOrchestratorPrompt(requestFiles, responseFiles, templateContent) {
|
|
23563
|
-
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${
|
|
23564
|
-
const responseList = responseFiles.map((file) => `"${
|
|
24002
|
+
const requestLines = requestFiles.map((file, index) => `${index + 1}. messages/${path25.basename(file)}`).join("\n");
|
|
24003
|
+
const responseList = responseFiles.map((file) => `"${path25.basename(file)}"`).join(", ");
|
|
23565
24004
|
return renderTemplate2(templateContent, {
|
|
23566
24005
|
requestFiles: requestLines,
|
|
23567
24006
|
responseList
|
|
@@ -23600,7 +24039,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
23600
24039
|
const maxAttempts = 10;
|
|
23601
24040
|
while (attempts < maxAttempts) {
|
|
23602
24041
|
try {
|
|
23603
|
-
const content = await
|
|
24042
|
+
const content = await readFile11(responseFileFinal, { encoding: "utf8" });
|
|
23604
24043
|
if (!silent) {
|
|
23605
24044
|
process.stdout.write(`${content}
|
|
23606
24045
|
`);
|
|
@@ -23621,7 +24060,7 @@ async function waitForResponseOutput(responseFileFinal, pollInterval = 1e3, sile
|
|
|
23621
24060
|
}
|
|
23622
24061
|
async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, silent = false, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
23623
24062
|
if (!silent) {
|
|
23624
|
-
const fileList = responseFilesFinal.map((file) =>
|
|
24063
|
+
const fileList = responseFilesFinal.map((file) => path26.basename(file)).join(", ");
|
|
23625
24064
|
console.error(`waiting for ${responseFilesFinal.length} batch response(s): ${fileList}`);
|
|
23626
24065
|
}
|
|
23627
24066
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -23630,7 +24069,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
23630
24069
|
while (pending.size > 0) {
|
|
23631
24070
|
if (Date.now() >= deadline) {
|
|
23632
24071
|
if (!silent) {
|
|
23633
|
-
const remaining = [...pending].map((f) =>
|
|
24072
|
+
const remaining = [...pending].map((f) => path26.basename(f)).join(", ");
|
|
23634
24073
|
console.error(
|
|
23635
24074
|
`error: timed out after ${Math.round(timeoutMs / 1e3)}s waiting for batch responses. Still pending: ${remaining}`
|
|
23636
24075
|
);
|
|
@@ -23657,7 +24096,7 @@ async function waitForBatchResponses(responseFilesFinal, pollInterval = 1e3, sil
|
|
|
23657
24096
|
const maxAttempts = 10;
|
|
23658
24097
|
while (attempts < maxAttempts) {
|
|
23659
24098
|
try {
|
|
23660
|
-
const content = await
|
|
24099
|
+
const content = await readFile11(file, { encoding: "utf8" });
|
|
23661
24100
|
if (!silent) {
|
|
23662
24101
|
process.stdout.write(`${content}
|
|
23663
24102
|
`);
|
|
@@ -23681,7 +24120,7 @@ var DEFAULT_LOCK_NAME = "subagent.lock";
|
|
|
23681
24120
|
var DEFAULT_ALIVE_FILENAME = ".alive";
|
|
23682
24121
|
function getDefaultSubagentRoot(vscodeCmd = "code") {
|
|
23683
24122
|
const folder = vscodeCmd === "code-insiders" ? "vscode-insiders-agents" : "vscode-agents";
|
|
23684
|
-
return
|
|
24123
|
+
return path27.join(getSubagentsRoot(), folder);
|
|
23685
24124
|
}
|
|
23686
24125
|
var DEFAULT_SUBAGENT_ROOT = getDefaultSubagentRoot();
|
|
23687
24126
|
var execAsync2 = promisify2(exec);
|
|
@@ -23746,11 +24185,11 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
23746
24185
|
await raceSpawnError(child);
|
|
23747
24186
|
return true;
|
|
23748
24187
|
}
|
|
23749
|
-
const aliveFile =
|
|
24188
|
+
const aliveFile = path28.join(subagentDir, DEFAULT_ALIVE_FILENAME);
|
|
23750
24189
|
await removeIfExists(aliveFile);
|
|
23751
|
-
const githubAgentsDir =
|
|
24190
|
+
const githubAgentsDir = path28.join(subagentDir, ".github", "agents");
|
|
23752
24191
|
await mkdir9(githubAgentsDir, { recursive: true });
|
|
23753
|
-
const wakeupDst =
|
|
24192
|
+
const wakeupDst = path28.join(githubAgentsDir, "wakeup.md");
|
|
23754
24193
|
await writeFile2(wakeupDst, DEFAULT_WAKEUP_CONTENT, "utf8");
|
|
23755
24194
|
const workspaceChild = spawnVsCode(vscodeCmd, [workspacePath], {
|
|
23756
24195
|
label: "open-workspace"
|
|
@@ -23763,7 +24202,7 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
23763
24202
|
"chat",
|
|
23764
24203
|
"-m",
|
|
23765
24204
|
wakeupChatId,
|
|
23766
|
-
`create a file named .alive in the ${
|
|
24205
|
+
`create a file named .alive in the ${path28.basename(subagentDir)} folder`
|
|
23767
24206
|
];
|
|
23768
24207
|
const wakeupChild = spawnVsCode(vscodeCmd, chatArgs, { label: "send-wakeup-chat" });
|
|
23769
24208
|
await raceSpawnError(wakeupChild);
|
|
@@ -23778,10 +24217,10 @@ async function ensureWorkspaceFocused(workspacePath, workspaceName, subagentDir,
|
|
|
23778
24217
|
return true;
|
|
23779
24218
|
}
|
|
23780
24219
|
async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, requestInstructions, timestamp, vscodeCmd) {
|
|
23781
|
-
const workspacePath =
|
|
23782
|
-
const messagesDir =
|
|
24220
|
+
const workspacePath = path28.join(subagentDir, `${path28.basename(subagentDir)}.code-workspace`);
|
|
24221
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
23783
24222
|
await mkdir9(messagesDir, { recursive: true });
|
|
23784
|
-
const reqFile =
|
|
24223
|
+
const reqFile = path28.join(messagesDir, `${timestamp}_req.md`);
|
|
23785
24224
|
await writeFile2(reqFile, requestInstructions, { encoding: "utf8" });
|
|
23786
24225
|
const reqUri = pathToFileUri2(reqFile);
|
|
23787
24226
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
@@ -23789,16 +24228,16 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
23789
24228
|
chatArgs.push("-a", attachment);
|
|
23790
24229
|
}
|
|
23791
24230
|
chatArgs.push("-a", reqFile);
|
|
23792
|
-
chatArgs.push(`Follow instructions in [${
|
|
24231
|
+
chatArgs.push(`Follow instructions in [${path28.basename(reqFile)}](${reqUri})`);
|
|
23793
24232
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
23794
24233
|
workspacePath,
|
|
23795
|
-
|
|
24234
|
+
path28.basename(subagentDir),
|
|
23796
24235
|
subagentDir,
|
|
23797
24236
|
vscodeCmd
|
|
23798
24237
|
);
|
|
23799
24238
|
if (!workspaceReady) {
|
|
23800
24239
|
throw new Error(
|
|
23801
|
-
`VS Code workspace '${
|
|
24240
|
+
`VS Code workspace '${path28.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
23802
24241
|
);
|
|
23803
24242
|
}
|
|
23804
24243
|
await sleep2(500);
|
|
@@ -23806,8 +24245,8 @@ async function launchVsCodeWithChat(subagentDir, chatId, attachmentPaths, reques
|
|
|
23806
24245
|
await raceSpawnError(child);
|
|
23807
24246
|
}
|
|
23808
24247
|
async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, chatInstruction, vscodeCmd) {
|
|
23809
|
-
const workspacePath =
|
|
23810
|
-
const messagesDir =
|
|
24248
|
+
const workspacePath = path28.join(subagentDir, `${path28.basename(subagentDir)}.code-workspace`);
|
|
24249
|
+
const messagesDir = path28.join(subagentDir, "messages");
|
|
23811
24250
|
await mkdir9(messagesDir, { recursive: true });
|
|
23812
24251
|
const chatArgs = ["-r", "chat", "-m", chatId];
|
|
23813
24252
|
for (const attachment of attachmentPaths) {
|
|
@@ -23816,13 +24255,13 @@ async function launchVsCodeWithBatchChat(subagentDir, chatId, attachmentPaths, c
|
|
|
23816
24255
|
chatArgs.push(chatInstruction);
|
|
23817
24256
|
const workspaceReady = await ensureWorkspaceFocused(
|
|
23818
24257
|
workspacePath,
|
|
23819
|
-
|
|
24258
|
+
path28.basename(subagentDir),
|
|
23820
24259
|
subagentDir,
|
|
23821
24260
|
vscodeCmd
|
|
23822
24261
|
);
|
|
23823
24262
|
if (!workspaceReady) {
|
|
23824
24263
|
throw new Error(
|
|
23825
|
-
`VS Code workspace '${
|
|
24264
|
+
`VS Code workspace '${path28.basename(subagentDir)}' failed to become ready within the timeout. Check that '${vscodeCmd}' can open workspaces.`
|
|
23826
24265
|
);
|
|
23827
24266
|
}
|
|
23828
24267
|
await sleep2(500);
|
|
@@ -23844,10 +24283,10 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
23844
24283
|
}
|
|
23845
24284
|
const transformedFolders = workspace.folders.map((folder) => {
|
|
23846
24285
|
const folderPath = folder.path;
|
|
23847
|
-
if (
|
|
24286
|
+
if (path29.isAbsolute(folderPath)) {
|
|
23848
24287
|
return folder;
|
|
23849
24288
|
}
|
|
23850
|
-
const absolutePath =
|
|
24289
|
+
const absolutePath = path29.resolve(templateDir, folderPath);
|
|
23851
24290
|
return {
|
|
23852
24291
|
...folder,
|
|
23853
24292
|
path: absolutePath
|
|
@@ -23869,19 +24308,19 @@ function transformWorkspacePaths(workspaceContent, templateDir) {
|
|
|
23869
24308
|
if (locationMap && typeof locationMap === "object") {
|
|
23870
24309
|
const transformedMap = {};
|
|
23871
24310
|
for (const [locationPath, value] of Object.entries(locationMap)) {
|
|
23872
|
-
const isAbsolute =
|
|
24311
|
+
const isAbsolute = path29.isAbsolute(locationPath);
|
|
23873
24312
|
if (isAbsolute) {
|
|
23874
24313
|
transformedMap[locationPath] = value;
|
|
23875
24314
|
} else {
|
|
23876
24315
|
const firstGlobIndex = locationPath.search(/[*]/);
|
|
23877
24316
|
if (firstGlobIndex === -1) {
|
|
23878
|
-
const resolvedPath =
|
|
24317
|
+
const resolvedPath = path29.resolve(templateDir, locationPath).replace(/\\/g, "/");
|
|
23879
24318
|
transformedMap[resolvedPath] = value;
|
|
23880
24319
|
} else {
|
|
23881
24320
|
const basePathEnd = locationPath.lastIndexOf("/", firstGlobIndex);
|
|
23882
24321
|
const basePath = basePathEnd !== -1 ? locationPath.substring(0, basePathEnd) : ".";
|
|
23883
24322
|
const patternPath = locationPath.substring(basePathEnd !== -1 ? basePathEnd : 0);
|
|
23884
|
-
const resolvedPath = (
|
|
24323
|
+
const resolvedPath = (path29.resolve(templateDir, basePath) + patternPath).replace(
|
|
23885
24324
|
/\\/g,
|
|
23886
24325
|
"/"
|
|
23887
24326
|
);
|
|
@@ -23920,7 +24359,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
23920
24359
|
number: Number.parseInt(entry.name.split("-")[1] ?? "", 10)
|
|
23921
24360
|
})).filter((entry) => Number.isInteger(entry.number)).sort((a, b) => a.number - b.number);
|
|
23922
24361
|
for (const subagent of subagents) {
|
|
23923
|
-
const lockFile =
|
|
24362
|
+
const lockFile = path30.join(subagent.absolutePath, DEFAULT_LOCK_NAME);
|
|
23924
24363
|
if (!await pathExists(lockFile)) {
|
|
23925
24364
|
return subagent.absolutePath;
|
|
23926
24365
|
}
|
|
@@ -23930,7 +24369,7 @@ async function findUnlockedSubagent(subagentRoot) {
|
|
|
23930
24369
|
async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
23931
24370
|
let workspaceContent;
|
|
23932
24371
|
if (workspaceTemplate) {
|
|
23933
|
-
const workspaceSrc =
|
|
24372
|
+
const workspaceSrc = path30.resolve(workspaceTemplate);
|
|
23934
24373
|
if (!await pathExists(workspaceSrc)) {
|
|
23935
24374
|
throw new Error(`workspace template not found: ${workspaceSrc}`);
|
|
23936
24375
|
}
|
|
@@ -23938,18 +24377,18 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
23938
24377
|
if (!stats.isFile()) {
|
|
23939
24378
|
throw new Error(`workspace template must be a file, not a directory: ${workspaceSrc}`);
|
|
23940
24379
|
}
|
|
23941
|
-
const templateText = await
|
|
24380
|
+
const templateText = await readFile12(workspaceSrc, "utf8");
|
|
23942
24381
|
workspaceContent = JSON.parse(templateText);
|
|
23943
24382
|
} else {
|
|
23944
24383
|
workspaceContent = DEFAULT_WORKSPACE_TEMPLATE;
|
|
23945
24384
|
}
|
|
23946
|
-
const workspaceName = `${
|
|
23947
|
-
const workspaceDst =
|
|
23948
|
-
const templateDir = workspaceTemplate ?
|
|
24385
|
+
const workspaceName = `${path30.basename(subagentDir)}.code-workspace`;
|
|
24386
|
+
const workspaceDst = path30.join(subagentDir, workspaceName);
|
|
24387
|
+
const templateDir = workspaceTemplate ? path30.dirname(path30.resolve(workspaceTemplate)) : subagentDir;
|
|
23949
24388
|
const workspaceJson = JSON.stringify(workspaceContent, null, 2);
|
|
23950
24389
|
let transformedContent = transformWorkspacePaths(workspaceJson, templateDir);
|
|
23951
24390
|
if (cwd) {
|
|
23952
|
-
const absCwd =
|
|
24391
|
+
const absCwd = path30.resolve(cwd);
|
|
23953
24392
|
const parsed = JSON.parse(transformedContent);
|
|
23954
24393
|
const alreadyPresent = parsed.folders.some((f) => f.path === absCwd);
|
|
23955
24394
|
if (!alreadyPresent) {
|
|
@@ -23958,35 +24397,35 @@ async function copyAgentConfig(subagentDir, workspaceTemplate, cwd) {
|
|
|
23958
24397
|
}
|
|
23959
24398
|
}
|
|
23960
24399
|
await writeFile3(workspaceDst, transformedContent, "utf8");
|
|
23961
|
-
const messagesDir =
|
|
24400
|
+
const messagesDir = path30.join(subagentDir, "messages");
|
|
23962
24401
|
await mkdir10(messagesDir, { recursive: true });
|
|
23963
24402
|
return { workspace: workspaceDst, messagesDir };
|
|
23964
24403
|
}
|
|
23965
24404
|
async function createSubagentLock(subagentDir) {
|
|
23966
|
-
const messagesDir =
|
|
24405
|
+
const messagesDir = path30.join(subagentDir, "messages");
|
|
23967
24406
|
if (await pathExists(messagesDir)) {
|
|
23968
24407
|
const files = await readdir3(messagesDir);
|
|
23969
24408
|
await Promise.all(
|
|
23970
24409
|
files.map(async (file) => {
|
|
23971
|
-
const target =
|
|
24410
|
+
const target = path30.join(messagesDir, file);
|
|
23972
24411
|
await removeIfExists(target);
|
|
23973
24412
|
})
|
|
23974
24413
|
);
|
|
23975
24414
|
}
|
|
23976
|
-
const githubAgentsDir =
|
|
24415
|
+
const githubAgentsDir = path30.join(subagentDir, ".github", "agents");
|
|
23977
24416
|
if (await pathExists(githubAgentsDir)) {
|
|
23978
24417
|
const agentFiles = await readdir3(githubAgentsDir);
|
|
23979
24418
|
const preservedFiles = /* @__PURE__ */ new Set(["wakeup.md", "subagent.md"]);
|
|
23980
24419
|
await Promise.all(
|
|
23981
|
-
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(
|
|
24420
|
+
agentFiles.filter((file) => file.endsWith(".md") && !preservedFiles.has(file)).map((file) => removeIfExists(path30.join(githubAgentsDir, file)))
|
|
23982
24421
|
);
|
|
23983
24422
|
}
|
|
23984
|
-
const lockFile =
|
|
24423
|
+
const lockFile = path30.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
23985
24424
|
await writeFile3(lockFile, "", { encoding: "utf8" });
|
|
23986
24425
|
return lockFile;
|
|
23987
24426
|
}
|
|
23988
24427
|
async function removeSubagentLock(subagentDir) {
|
|
23989
|
-
const lockFile =
|
|
24428
|
+
const lockFile = path30.join(subagentDir, DEFAULT_LOCK_NAME);
|
|
23990
24429
|
await removeIfExists(lockFile);
|
|
23991
24430
|
}
|
|
23992
24431
|
async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspaceTemplate, dryRun, cwd) {
|
|
@@ -24006,9 +24445,9 @@ async function prepareSubagentDirectory(subagentDir, promptFile, chatId, workspa
|
|
|
24006
24445
|
return 1;
|
|
24007
24446
|
}
|
|
24008
24447
|
if (promptFile) {
|
|
24009
|
-
const githubAgentsDir =
|
|
24448
|
+
const githubAgentsDir = path30.join(subagentDir, ".github", "agents");
|
|
24010
24449
|
await mkdir10(githubAgentsDir, { recursive: true });
|
|
24011
|
-
const agentFile =
|
|
24450
|
+
const agentFile = path30.join(githubAgentsDir, `${chatId}.md`);
|
|
24012
24451
|
try {
|
|
24013
24452
|
await copyFile(promptFile, agentFile);
|
|
24014
24453
|
} catch (error) {
|
|
@@ -24025,7 +24464,7 @@ async function resolvePromptFile(promptFile) {
|
|
|
24025
24464
|
if (!promptFile) {
|
|
24026
24465
|
return void 0;
|
|
24027
24466
|
}
|
|
24028
|
-
const resolvedPrompt =
|
|
24467
|
+
const resolvedPrompt = path31.resolve(promptFile);
|
|
24029
24468
|
if (!await pathExists(resolvedPrompt)) {
|
|
24030
24469
|
throw new Error(`Prompt file not found: ${resolvedPrompt}`);
|
|
24031
24470
|
}
|
|
@@ -24041,7 +24480,7 @@ async function resolveAttachments(extraAttachments) {
|
|
|
24041
24480
|
}
|
|
24042
24481
|
const resolved = [];
|
|
24043
24482
|
for (const attachment of extraAttachments) {
|
|
24044
|
-
const resolvedPath =
|
|
24483
|
+
const resolvedPath = path31.resolve(attachment);
|
|
24045
24484
|
if (!await pathExists(resolvedPath)) {
|
|
24046
24485
|
throw new Error(`Attachment not found: ${resolvedPath}`);
|
|
24047
24486
|
}
|
|
@@ -24083,7 +24522,7 @@ async function dispatchAgentSession(options) {
|
|
|
24083
24522
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
24084
24523
|
};
|
|
24085
24524
|
}
|
|
24086
|
-
const subagentName =
|
|
24525
|
+
const subagentName = path31.basename(subagentDir);
|
|
24087
24526
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
24088
24527
|
const preparationResult = await prepareSubagentDirectory(
|
|
24089
24528
|
subagentDir,
|
|
@@ -24111,9 +24550,9 @@ async function dispatchAgentSession(options) {
|
|
|
24111
24550
|
};
|
|
24112
24551
|
}
|
|
24113
24552
|
const timestamp = generateTimestamp();
|
|
24114
|
-
const messagesDir =
|
|
24115
|
-
const responseFileTmp =
|
|
24116
|
-
const responseFileFinal =
|
|
24553
|
+
const messagesDir = path31.join(subagentDir, "messages");
|
|
24554
|
+
const responseFileTmp = path31.join(messagesDir, `${timestamp}_res.tmp.md`);
|
|
24555
|
+
const responseFileFinal = path31.join(messagesDir, `${timestamp}_res.md`);
|
|
24117
24556
|
const requestInstructions = createRequestPrompt(
|
|
24118
24557
|
userQuery,
|
|
24119
24558
|
responseFileTmp,
|
|
@@ -24218,7 +24657,7 @@ async function dispatchBatchAgent(options) {
|
|
|
24218
24657
|
error: "No unlocked subagents available. Provision additional subagents with: subagent code provision --subagents <desired_total>"
|
|
24219
24658
|
};
|
|
24220
24659
|
}
|
|
24221
|
-
subagentName =
|
|
24660
|
+
subagentName = path31.basename(subagentDir);
|
|
24222
24661
|
const chatId = Math.random().toString(16).slice(2, 10);
|
|
24223
24662
|
const preparationResult = await prepareSubagentDirectory(
|
|
24224
24663
|
subagentDir,
|
|
@@ -24249,17 +24688,17 @@ async function dispatchBatchAgent(options) {
|
|
|
24249
24688
|
};
|
|
24250
24689
|
}
|
|
24251
24690
|
const timestamp = generateTimestamp();
|
|
24252
|
-
const messagesDir =
|
|
24691
|
+
const messagesDir = path31.join(subagentDir, "messages");
|
|
24253
24692
|
requestFiles = userQueries.map(
|
|
24254
|
-
(_, index) =>
|
|
24693
|
+
(_, index) => path31.join(messagesDir, `${timestamp}_${index}_req.md`)
|
|
24255
24694
|
);
|
|
24256
24695
|
const responseTmpFiles = userQueries.map(
|
|
24257
|
-
(_, index) =>
|
|
24696
|
+
(_, index) => path31.join(messagesDir, `${timestamp}_${index}_res.tmp.md`)
|
|
24258
24697
|
);
|
|
24259
24698
|
responseFilesFinal = userQueries.map(
|
|
24260
|
-
(_, index) =>
|
|
24699
|
+
(_, index) => path31.join(messagesDir, `${timestamp}_${index}_res.md`)
|
|
24261
24700
|
);
|
|
24262
|
-
const orchestratorFile =
|
|
24701
|
+
const orchestratorFile = path31.join(messagesDir, `${timestamp}_orchestrator.md`);
|
|
24263
24702
|
if (!dryRun) {
|
|
24264
24703
|
await Promise.all(
|
|
24265
24704
|
userQueries.map((query, index) => {
|
|
@@ -24372,7 +24811,7 @@ async function provisionSubagents(options) {
|
|
|
24372
24811
|
if (!Number.isInteger(subagents) || subagents < 1) {
|
|
24373
24812
|
throw new Error("subagents must be a positive integer");
|
|
24374
24813
|
}
|
|
24375
|
-
const targetPath =
|
|
24814
|
+
const targetPath = path322.resolve(targetRoot);
|
|
24376
24815
|
if (!dryRun) {
|
|
24377
24816
|
await ensureDir(targetPath);
|
|
24378
24817
|
}
|
|
@@ -24392,7 +24831,7 @@ async function provisionSubagents(options) {
|
|
|
24392
24831
|
continue;
|
|
24393
24832
|
}
|
|
24394
24833
|
highestNumber = Math.max(highestNumber, parsed);
|
|
24395
|
-
const lockFile =
|
|
24834
|
+
const lockFile = path322.join(entry.absolutePath, lockName);
|
|
24396
24835
|
const locked = await pathExists(lockFile);
|
|
24397
24836
|
if (locked) {
|
|
24398
24837
|
lockedSubagents.add(entry.absolutePath);
|
|
@@ -24409,10 +24848,10 @@ async function provisionSubagents(options) {
|
|
|
24409
24848
|
break;
|
|
24410
24849
|
}
|
|
24411
24850
|
const subagentDir = subagent.absolutePath;
|
|
24412
|
-
const githubAgentsDir =
|
|
24413
|
-
const lockFile =
|
|
24414
|
-
const workspaceDst =
|
|
24415
|
-
const wakeupDst =
|
|
24851
|
+
const githubAgentsDir = path322.join(subagentDir, ".github", "agents");
|
|
24852
|
+
const lockFile = path322.join(subagentDir, lockName);
|
|
24853
|
+
const workspaceDst = path322.join(subagentDir, `${path322.basename(subagentDir)}.code-workspace`);
|
|
24854
|
+
const wakeupDst = path322.join(githubAgentsDir, "wakeup.md");
|
|
24416
24855
|
const isLocked = await pathExists(lockFile);
|
|
24417
24856
|
if (isLocked && !force) {
|
|
24418
24857
|
continue;
|
|
@@ -24450,10 +24889,10 @@ async function provisionSubagents(options) {
|
|
|
24450
24889
|
let nextIndex = highestNumber;
|
|
24451
24890
|
while (subagentsProvisioned < subagents) {
|
|
24452
24891
|
nextIndex += 1;
|
|
24453
|
-
const subagentDir =
|
|
24454
|
-
const githubAgentsDir =
|
|
24455
|
-
const workspaceDst =
|
|
24456
|
-
const wakeupDst =
|
|
24892
|
+
const subagentDir = path322.join(targetPath, `subagent-${nextIndex}`);
|
|
24893
|
+
const githubAgentsDir = path322.join(subagentDir, ".github", "agents");
|
|
24894
|
+
const workspaceDst = path322.join(subagentDir, `${path322.basename(subagentDir)}.code-workspace`);
|
|
24895
|
+
const wakeupDst = path322.join(githubAgentsDir, "wakeup.md");
|
|
24457
24896
|
if (!dryRun) {
|
|
24458
24897
|
await ensureDir(subagentDir);
|
|
24459
24898
|
await ensureDir(githubAgentsDir);
|
|
@@ -24639,7 +25078,7 @@ var VSCodeProvider = class {
|
|
|
24639
25078
|
async function locateVSCodeExecutable(candidate) {
|
|
24640
25079
|
const includesPathSeparator = candidate.includes("/") || candidate.includes("\\");
|
|
24641
25080
|
if (includesPathSeparator) {
|
|
24642
|
-
const resolved =
|
|
25081
|
+
const resolved = path33.isAbsolute(candidate) ? candidate : path33.resolve(candidate);
|
|
24643
25082
|
try {
|
|
24644
25083
|
await access3(resolved, constants3.F_OK);
|
|
24645
25084
|
return resolved;
|
|
@@ -24668,7 +25107,7 @@ async function resolveWorkspaceTemplateFile(template) {
|
|
|
24668
25107
|
return void 0;
|
|
24669
25108
|
}
|
|
24670
25109
|
try {
|
|
24671
|
-
const stats = await stat5(
|
|
25110
|
+
const stats = await stat5(path33.resolve(template));
|
|
24672
25111
|
return stats.isFile() ? template : void 0;
|
|
24673
25112
|
} catch {
|
|
24674
25113
|
return template;
|
|
@@ -24692,7 +25131,7 @@ function buildMandatoryPrereadBlock2(attachmentFiles) {
|
|
|
24692
25131
|
return "";
|
|
24693
25132
|
}
|
|
24694
25133
|
const buildList = (files) => files.map((absolutePath) => {
|
|
24695
|
-
const fileName =
|
|
25134
|
+
const fileName = path33.basename(absolutePath);
|
|
24696
25135
|
const fileUri = pathToFileUri3(absolutePath);
|
|
24697
25136
|
return `* [${fileName}](${fileUri})`;
|
|
24698
25137
|
});
|
|
@@ -24713,7 +25152,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
24713
25152
|
}
|
|
24714
25153
|
const unique = /* @__PURE__ */ new Map();
|
|
24715
25154
|
for (const attachment of attachments) {
|
|
24716
|
-
const absolutePath =
|
|
25155
|
+
const absolutePath = path33.resolve(attachment);
|
|
24717
25156
|
if (!unique.has(absolutePath)) {
|
|
24718
25157
|
unique.set(absolutePath, absolutePath);
|
|
24719
25158
|
}
|
|
@@ -24721,7 +25160,7 @@ function collectAttachmentFiles(attachments) {
|
|
|
24721
25160
|
return Array.from(unique.values());
|
|
24722
25161
|
}
|
|
24723
25162
|
function pathToFileUri3(filePath) {
|
|
24724
|
-
const absolutePath =
|
|
25163
|
+
const absolutePath = path33.isAbsolute(filePath) ? filePath : path33.resolve(filePath);
|
|
24725
25164
|
const normalizedPath = absolutePath.replace(/\\/g, "/");
|
|
24726
25165
|
if (/^[a-zA-Z]:\//.test(normalizedPath)) {
|
|
24727
25166
|
return `file:///${normalizedPath}`;
|
|
@@ -24734,7 +25173,7 @@ function normalizeAttachments(attachments) {
|
|
|
24734
25173
|
}
|
|
24735
25174
|
const deduped = /* @__PURE__ */ new Set();
|
|
24736
25175
|
for (const attachment of attachments) {
|
|
24737
|
-
deduped.add(
|
|
25176
|
+
deduped.add(path33.resolve(attachment));
|
|
24738
25177
|
}
|
|
24739
25178
|
return Array.from(deduped);
|
|
24740
25179
|
}
|
|
@@ -24743,7 +25182,7 @@ function mergeAttachments(all) {
|
|
|
24743
25182
|
for (const list of all) {
|
|
24744
25183
|
if (!list) continue;
|
|
24745
25184
|
for (const inputFile of list) {
|
|
24746
|
-
deduped.add(
|
|
25185
|
+
deduped.add(path33.resolve(inputFile));
|
|
24747
25186
|
}
|
|
24748
25187
|
}
|
|
24749
25188
|
return deduped.size > 0 ? Array.from(deduped) : void 0;
|
|
@@ -24826,12 +25265,12 @@ async function fileExists3(filePath) {
|
|
|
24826
25265
|
}
|
|
24827
25266
|
}
|
|
24828
25267
|
async function readTargetDefinitions(filePath) {
|
|
24829
|
-
const absolutePath =
|
|
25268
|
+
const absolutePath = path34.resolve(filePath);
|
|
24830
25269
|
if (!await fileExists3(absolutePath)) {
|
|
24831
25270
|
throw new Error(`targets.yaml not found at ${absolutePath}`);
|
|
24832
25271
|
}
|
|
24833
|
-
const raw = await
|
|
24834
|
-
const parsed =
|
|
25272
|
+
const raw = await readFile13(absolutePath, "utf8");
|
|
25273
|
+
const parsed = parse5(raw);
|
|
24835
25274
|
if (!isRecord(parsed)) {
|
|
24836
25275
|
throw new Error(`targets.yaml at ${absolutePath} must be a YAML object with a 'targets' field`);
|
|
24837
25276
|
}
|
|
@@ -24847,11 +25286,11 @@ function listTargetNames(definitions) {
|
|
|
24847
25286
|
async function discoverProviders(registry, baseDir) {
|
|
24848
25287
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
24849
25288
|
const candidateDirs = [];
|
|
24850
|
-
let dir =
|
|
24851
|
-
const root =
|
|
25289
|
+
let dir = path35.resolve(baseDir);
|
|
25290
|
+
const root = path35.parse(dir).root;
|
|
24852
25291
|
while (dir !== root) {
|
|
24853
|
-
candidateDirs.push(
|
|
24854
|
-
dir =
|
|
25292
|
+
candidateDirs.push(path35.join(dir, ".agentv", "providers"));
|
|
25293
|
+
dir = path35.dirname(dir);
|
|
24855
25294
|
}
|
|
24856
25295
|
let files = [];
|
|
24857
25296
|
for (const providersDir of candidateDirs) {
|
|
@@ -24867,7 +25306,7 @@ async function discoverProviders(registry, baseDir) {
|
|
|
24867
25306
|
}
|
|
24868
25307
|
const discoveredKinds = [];
|
|
24869
25308
|
for (const filePath of files) {
|
|
24870
|
-
const basename =
|
|
25309
|
+
const basename = path35.basename(filePath);
|
|
24871
25310
|
const kindName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
24872
25311
|
if (registry.has(kindName)) {
|
|
24873
25312
|
continue;
|
|
@@ -24983,148 +25422,6 @@ function negateScore(score) {
|
|
|
24983
25422
|
}))
|
|
24984
25423
|
};
|
|
24985
25424
|
}
|
|
24986
|
-
function shellEscapePath(value) {
|
|
24987
|
-
if (process.platform === "win32") {
|
|
24988
|
-
return `"${value.replaceAll('"', '""')}"`;
|
|
24989
|
-
}
|
|
24990
|
-
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
|
24991
|
-
}
|
|
24992
|
-
async function execFileWithStdin(argv, stdinPayload, options = {}) {
|
|
24993
|
-
if (argv.length === 0) {
|
|
24994
|
-
throw new Error("Executable argv must include at least one entry");
|
|
24995
|
-
}
|
|
24996
|
-
if (typeof Bun !== "undefined") {
|
|
24997
|
-
return execFileWithStdinBun(argv, stdinPayload, options);
|
|
24998
|
-
}
|
|
24999
|
-
return execFileWithStdinNode(argv, stdinPayload, options);
|
|
25000
|
-
}
|
|
25001
|
-
async function execFileWithStdinBun(argv, stdinPayload, options) {
|
|
25002
|
-
const command = [...argv];
|
|
25003
|
-
const encoder = new TextEncoder();
|
|
25004
|
-
const proc = Bun.spawn(command, {
|
|
25005
|
-
cwd: options.cwd,
|
|
25006
|
-
stdin: encoder.encode(stdinPayload),
|
|
25007
|
-
stdout: "pipe",
|
|
25008
|
-
stderr: "pipe",
|
|
25009
|
-
// Merge additional env vars with process.env
|
|
25010
|
-
env: options.env ? { ...process.env, ...options.env } : process.env
|
|
25011
|
-
});
|
|
25012
|
-
let timedOut = false;
|
|
25013
|
-
const timeout = options.timeoutMs !== void 0 ? setTimeout(() => {
|
|
25014
|
-
timedOut = true;
|
|
25015
|
-
proc.kill("SIGKILL");
|
|
25016
|
-
}, options.timeoutMs) : void 0;
|
|
25017
|
-
try {
|
|
25018
|
-
const stdoutPromise = proc.stdout ? new Response(proc.stdout).text() : Promise.resolve("");
|
|
25019
|
-
const stderrPromise = proc.stderr ? new Response(proc.stderr).text() : Promise.resolve("");
|
|
25020
|
-
const [stdout, stderr, exitCode] = await Promise.all([
|
|
25021
|
-
stdoutPromise,
|
|
25022
|
-
stderrPromise,
|
|
25023
|
-
proc.exited
|
|
25024
|
-
]);
|
|
25025
|
-
if (timedOut) {
|
|
25026
|
-
throw new Error(`Process timed out after ${options.timeoutMs}ms`);
|
|
25027
|
-
}
|
|
25028
|
-
return {
|
|
25029
|
-
stdout: stdout.replace(/\r\n/g, "\n"),
|
|
25030
|
-
stderr: stderr.replace(/\r\n/g, "\n"),
|
|
25031
|
-
exitCode
|
|
25032
|
-
};
|
|
25033
|
-
} finally {
|
|
25034
|
-
if (timeout !== void 0) {
|
|
25035
|
-
clearTimeout(timeout);
|
|
25036
|
-
}
|
|
25037
|
-
}
|
|
25038
|
-
}
|
|
25039
|
-
async function execFileWithStdinNode(argv, stdinPayload, options) {
|
|
25040
|
-
const { spawn: spawn5 } = await import("node:child_process");
|
|
25041
|
-
return new Promise((resolve2, reject) => {
|
|
25042
|
-
const [cmd, ...args] = argv;
|
|
25043
|
-
const child = spawn5(cmd, args, {
|
|
25044
|
-
cwd: options.cwd,
|
|
25045
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
25046
|
-
// Merge additional env vars with process.env
|
|
25047
|
-
env: options.env ? { ...process.env, ...options.env } : process.env
|
|
25048
|
-
});
|
|
25049
|
-
const stdoutChunks = [];
|
|
25050
|
-
const stderrChunks = [];
|
|
25051
|
-
child.stdout?.on("data", (chunk) => stdoutChunks.push(chunk));
|
|
25052
|
-
child.stderr?.on("data", (chunk) => stderrChunks.push(chunk));
|
|
25053
|
-
let timedOut = false;
|
|
25054
|
-
const timeout = options.timeoutMs !== void 0 ? setTimeout(() => {
|
|
25055
|
-
timedOut = true;
|
|
25056
|
-
child.kill("SIGKILL");
|
|
25057
|
-
}, options.timeoutMs) : void 0;
|
|
25058
|
-
child.on("error", (error) => {
|
|
25059
|
-
if (timeout !== void 0) clearTimeout(timeout);
|
|
25060
|
-
reject(error);
|
|
25061
|
-
});
|
|
25062
|
-
child.on("close", (code) => {
|
|
25063
|
-
if (timeout !== void 0) clearTimeout(timeout);
|
|
25064
|
-
if (timedOut) {
|
|
25065
|
-
reject(new Error(`Process timed out after ${options.timeoutMs}ms`));
|
|
25066
|
-
return;
|
|
25067
|
-
}
|
|
25068
|
-
const stdout = Buffer.concat(stdoutChunks).toString("utf8").replace(/\r\n/g, "\n");
|
|
25069
|
-
const stderr = Buffer.concat(stderrChunks).toString("utf8").replace(/\r\n/g, "\n");
|
|
25070
|
-
resolve2({
|
|
25071
|
-
stdout,
|
|
25072
|
-
stderr,
|
|
25073
|
-
exitCode: code ?? 0
|
|
25074
|
-
});
|
|
25075
|
-
});
|
|
25076
|
-
if (child.stdin) {
|
|
25077
|
-
child.stdin.write(stdinPayload);
|
|
25078
|
-
child.stdin.end();
|
|
25079
|
-
}
|
|
25080
|
-
});
|
|
25081
|
-
}
|
|
25082
|
-
async function execShellWithStdin(command, stdinPayload, options = {}) {
|
|
25083
|
-
const { mkdir: mkdir16, readFile: readFile16, rm: rm6, writeFile: writeFile9 } = await import("node:fs/promises");
|
|
25084
|
-
const { tmpdir: tmpdir3 } = await import("node:os");
|
|
25085
|
-
const path51 = await import("node:path");
|
|
25086
|
-
const { randomUUID: randomUUID10 } = await import("node:crypto");
|
|
25087
|
-
const dir = path51.join(tmpdir3(), `agentv-exec-${randomUUID10()}`);
|
|
25088
|
-
await mkdir16(dir, { recursive: true });
|
|
25089
|
-
const stdinPath = path51.join(dir, "stdin.txt");
|
|
25090
|
-
const stdoutPath = path51.join(dir, "stdout.txt");
|
|
25091
|
-
const stderrPath = path51.join(dir, "stderr.txt");
|
|
25092
|
-
await writeFile9(stdinPath, stdinPayload, "utf8");
|
|
25093
|
-
const wrappedCommand = process.platform === "win32" ? `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}` : `(${command}) < ${shellEscapePath(stdinPath)} > ${shellEscapePath(stdoutPath)} 2> ${shellEscapePath(stderrPath)}`;
|
|
25094
|
-
const { spawn: spawn5 } = await import("node:child_process");
|
|
25095
|
-
try {
|
|
25096
|
-
const exitCode = await new Promise((resolve2, reject) => {
|
|
25097
|
-
const child = spawn5(wrappedCommand, {
|
|
25098
|
-
shell: true,
|
|
25099
|
-
cwd: options.cwd,
|
|
25100
|
-
stdio: ["ignore", "ignore", "ignore"],
|
|
25101
|
-
// Merge additional env vars with process.env
|
|
25102
|
-
env: options.env ? { ...process.env, ...options.env } : process.env
|
|
25103
|
-
});
|
|
25104
|
-
const timeout = options.timeoutMs ? setTimeout(() => {
|
|
25105
|
-
child.kill();
|
|
25106
|
-
reject(new Error(`Process timed out after ${options.timeoutMs}ms`));
|
|
25107
|
-
}, options.timeoutMs) : void 0;
|
|
25108
|
-
child.on("error", (error) => {
|
|
25109
|
-
if (timeout !== void 0) {
|
|
25110
|
-
clearTimeout(timeout);
|
|
25111
|
-
}
|
|
25112
|
-
reject(error);
|
|
25113
|
-
});
|
|
25114
|
-
child.on("exit", (code) => {
|
|
25115
|
-
if (timeout !== void 0) {
|
|
25116
|
-
clearTimeout(timeout);
|
|
25117
|
-
}
|
|
25118
|
-
resolve2(code ?? 0);
|
|
25119
|
-
});
|
|
25120
|
-
});
|
|
25121
|
-
const stdout = (await readFile16(stdoutPath, "utf8")).replace(/\r\n/g, "\n");
|
|
25122
|
-
const stderr = (await readFile16(stderrPath, "utf8")).replace(/\r\n/g, "\n");
|
|
25123
|
-
return { stdout, stderr, exitCode };
|
|
25124
|
-
} finally {
|
|
25125
|
-
await rm6(dir, { recursive: true, force: true });
|
|
25126
|
-
}
|
|
25127
|
-
}
|
|
25128
25425
|
var DEFAULT_MAX_CALLS = 50;
|
|
25129
25426
|
async function createTargetProxy(options) {
|
|
25130
25427
|
const { defaultProvider, targetResolver, availableTargets, maxCalls } = options;
|
|
@@ -25395,6 +25692,16 @@ function toCamelCaseDeep(obj) {
|
|
|
25395
25692
|
}
|
|
25396
25693
|
return obj;
|
|
25397
25694
|
}
|
|
25695
|
+
function getRepoCheckoutRef(checkout) {
|
|
25696
|
+
return checkout?.base_commit ?? checkout?.ref ?? "HEAD";
|
|
25697
|
+
}
|
|
25698
|
+
function getRepoCheckoutTargets(repos) {
|
|
25699
|
+
if (!repos) return [];
|
|
25700
|
+
return repos.filter((repo) => repo.checkout?.base_commit || repo.checkout?.ref).map((repo) => ({
|
|
25701
|
+
path: repo.path,
|
|
25702
|
+
ref: getRepoCheckoutRef(repo.checkout)
|
|
25703
|
+
}));
|
|
25704
|
+
}
|
|
25398
25705
|
var FILE_BACKED_OUTPUT_THRESHOLD = 5e4;
|
|
25399
25706
|
var DATA_URI_RE = /^data:([^;]+);base64,(.+)$/s;
|
|
25400
25707
|
async function materializeContentForGrader(messages, getWorkDir) {
|
|
@@ -25528,13 +25835,31 @@ var CodeEvaluator = class {
|
|
|
25528
25835
|
const workspaceEnv = context2.workspacePath ? { AGENTV_WORKSPACE_PATH: context2.workspacePath } : void 0;
|
|
25529
25836
|
const env = proxyEnv || workspaceEnv ? { ...proxyEnv, ...workspaceEnv } : void 0;
|
|
25530
25837
|
try {
|
|
25531
|
-
|
|
25532
|
-
|
|
25533
|
-
|
|
25534
|
-
|
|
25535
|
-
|
|
25536
|
-
|
|
25537
|
-
|
|
25838
|
+
let stdout;
|
|
25839
|
+
if (context2.dockerConfig) {
|
|
25840
|
+
const { DockerWorkspaceProvider: DockerWorkspaceProvider2 } = await import("./docker-workspace-RPPXBT27-B4AQHVWA.js");
|
|
25841
|
+
const dockerProvider = new DockerWorkspaceProvider2(context2.dockerConfig);
|
|
25842
|
+
const result = await dockerProvider.runGraderInContainer({
|
|
25843
|
+
command: [...this.command],
|
|
25844
|
+
stdin: inputPayload,
|
|
25845
|
+
repoCheckouts: getRepoCheckoutTargets(context2.evalCase.workspace?.repos)
|
|
25846
|
+
});
|
|
25847
|
+
if (result.exitCode !== 0) {
|
|
25848
|
+
const trimmedErr = result.stderr.trim();
|
|
25849
|
+
throw new Error(
|
|
25850
|
+
trimmedErr.length > 0 ? `Code evaluator exited with code ${result.exitCode}: ${trimmedErr}` : `Code evaluator exited with code ${result.exitCode}`
|
|
25851
|
+
);
|
|
25852
|
+
}
|
|
25853
|
+
stdout = result.stdout.trim();
|
|
25854
|
+
} else {
|
|
25855
|
+
stdout = await executeScript(
|
|
25856
|
+
this.command,
|
|
25857
|
+
inputPayload,
|
|
25858
|
+
this.agentTimeoutMs,
|
|
25859
|
+
this.cwd,
|
|
25860
|
+
env
|
|
25861
|
+
);
|
|
25862
|
+
}
|
|
25538
25863
|
const parsed = parseJsonSafe(stdout);
|
|
25539
25864
|
const score = clampScore(typeof parsed?.score === "number" ? parsed.score : 0);
|
|
25540
25865
|
const assertions = Array.isArray(parsed?.assertions) ? parsed.assertions.filter(
|
|
@@ -25702,6 +26027,15 @@ var scoreRangeEvaluationSchema = external_exports2.object({
|
|
|
25702
26027
|
checks: external_exports2.array(scoreRangeCheckResultSchema).describe("Scores for each rubric criterion"),
|
|
25703
26028
|
overall_reasoning: external_exports2.string().describe("Overall assessment summary (1-2 sentences)").optional()
|
|
25704
26029
|
});
|
|
26030
|
+
function resolveContentBasePath(context2) {
|
|
26031
|
+
if (context2.workspacePath) {
|
|
26032
|
+
return context2.workspacePath;
|
|
26033
|
+
}
|
|
26034
|
+
if ("config" in context2.target && context2.target.config && typeof context2.target.config === "object" && "cwd" in context2.target.config && typeof context2.target.config.cwd === "string") {
|
|
26035
|
+
return context2.target.config.cwd;
|
|
26036
|
+
}
|
|
26037
|
+
return void 0;
|
|
26038
|
+
}
|
|
25705
26039
|
var LlmGraderEvaluator = class {
|
|
25706
26040
|
kind = "llm-grader";
|
|
25707
26041
|
resolveGraderProvider;
|
|
@@ -25719,24 +26053,46 @@ var LlmGraderEvaluator = class {
|
|
|
25719
26053
|
this.graderTargetProvider = options.graderTargetProvider ?? options.judgeTargetProvider;
|
|
25720
26054
|
}
|
|
25721
26055
|
async evaluate(context2) {
|
|
26056
|
+
const preparedContext = await this.prepareContext(context2);
|
|
25722
26057
|
if (this.graderTargetProvider) {
|
|
25723
|
-
return this.evaluateWithGraderTarget(
|
|
26058
|
+
return this.evaluateWithGraderTarget(preparedContext);
|
|
25724
26059
|
}
|
|
25725
|
-
const graderProvider = await this.resolveGraderProvider(
|
|
26060
|
+
const graderProvider = await this.resolveGraderProvider(preparedContext);
|
|
25726
26061
|
if (!graderProvider) {
|
|
25727
26062
|
throw new Error("No grader provider available for LLM grading");
|
|
25728
26063
|
}
|
|
25729
26064
|
if (graderProvider.kind === "agentv") {
|
|
25730
|
-
return this.evaluateBuiltIn(
|
|
26065
|
+
return this.evaluateBuiltIn(preparedContext, graderProvider);
|
|
25731
26066
|
}
|
|
25732
26067
|
if (isAgentProvider(graderProvider)) {
|
|
25733
|
-
return this.evaluateWithDelegatedAgent(
|
|
26068
|
+
return this.evaluateWithDelegatedAgent(preparedContext, graderProvider);
|
|
25734
26069
|
}
|
|
25735
|
-
const config =
|
|
26070
|
+
const config = preparedContext.evaluator;
|
|
25736
26071
|
if (config?.type === "llm-grader" && config.rubrics && config.rubrics.length > 0) {
|
|
25737
|
-
return this.evaluateWithRubrics(
|
|
26072
|
+
return this.evaluateWithRubrics(preparedContext, graderProvider, config.rubrics);
|
|
26073
|
+
}
|
|
26074
|
+
return this.evaluateFreeform(preparedContext, graderProvider);
|
|
26075
|
+
}
|
|
26076
|
+
async prepareContext(context2) {
|
|
26077
|
+
const config = context2.evaluator;
|
|
26078
|
+
if (config?.type !== "llm-grader" || !context2.output) {
|
|
26079
|
+
return context2;
|
|
25738
26080
|
}
|
|
25739
|
-
|
|
26081
|
+
const lastAssistant = [...context2.output].reverse().find((message) => message.role === "assistant" && message.content !== void 0);
|
|
26082
|
+
if (!lastAssistant || typeof lastAssistant.content === "string") {
|
|
26083
|
+
return context2;
|
|
26084
|
+
}
|
|
26085
|
+
const extracted = await extractTextWithPreprocessors(
|
|
26086
|
+
lastAssistant.content,
|
|
26087
|
+
config.preprocessors,
|
|
26088
|
+
{
|
|
26089
|
+
basePath: resolveContentBasePath(context2)
|
|
26090
|
+
}
|
|
26091
|
+
);
|
|
26092
|
+
return {
|
|
26093
|
+
...context2,
|
|
26094
|
+
candidate: appendPreprocessingWarnings(extracted.text, extracted.warnings)
|
|
26095
|
+
};
|
|
25740
26096
|
}
|
|
25741
26097
|
// ---------------------------------------------------------------------------
|
|
25742
26098
|
// LLM mode (existing)
|
|
@@ -26621,8 +26977,8 @@ function toAiSdkImageParts(images) {
|
|
|
26621
26977
|
}));
|
|
26622
26978
|
}
|
|
26623
26979
|
function resolveSandboxed(basePath, relativePath) {
|
|
26624
|
-
const resolved =
|
|
26625
|
-
if (!resolved.startsWith(basePath +
|
|
26980
|
+
const resolved = path36.resolve(basePath, relativePath);
|
|
26981
|
+
if (!resolved.startsWith(basePath + path36.sep) && resolved !== basePath) {
|
|
26626
26982
|
throw new Error(`Path '${relativePath}' is outside the workspace`);
|
|
26627
26983
|
}
|
|
26628
26984
|
return resolved;
|
|
@@ -26655,11 +27011,11 @@ function createFilesystemTools(workspacePath) {
|
|
|
26655
27011
|
execute: async (input) => {
|
|
26656
27012
|
try {
|
|
26657
27013
|
const resolved = resolveSandboxed(workspacePath, input.path);
|
|
26658
|
-
const
|
|
26659
|
-
if (
|
|
27014
|
+
const stat12 = await fs2.stat(resolved);
|
|
27015
|
+
if (stat12.isDirectory()) {
|
|
26660
27016
|
return { error: `'${input.path}' is a directory, not a file` };
|
|
26661
27017
|
}
|
|
26662
|
-
const buffer = Buffer.alloc(Math.min(
|
|
27018
|
+
const buffer = Buffer.alloc(Math.min(stat12.size, MAX_FILE_SIZE));
|
|
26663
27019
|
const fd = await fs2.open(resolved, "r");
|
|
26664
27020
|
try {
|
|
26665
27021
|
await fd.read(buffer, 0, buffer.length, 0);
|
|
@@ -26667,8 +27023,8 @@ function createFilesystemTools(workspacePath) {
|
|
|
26667
27023
|
await fd.close();
|
|
26668
27024
|
}
|
|
26669
27025
|
const content = buffer.toString("utf-8");
|
|
26670
|
-
const truncated =
|
|
26671
|
-
return { content, truncated, size:
|
|
27026
|
+
const truncated = stat12.size > MAX_FILE_SIZE;
|
|
27027
|
+
return { content, truncated, size: stat12.size };
|
|
26672
27028
|
} catch (error) {
|
|
26673
27029
|
return { error: error instanceof Error ? error.message : String(error) };
|
|
26674
27030
|
}
|
|
@@ -26712,15 +27068,15 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
26712
27068
|
for (const entry of entries) {
|
|
26713
27069
|
if (matches.length >= MAX_SEARCH_MATCHES) return;
|
|
26714
27070
|
if (SEARCH_SKIP_DIRS.has(entry.name)) continue;
|
|
26715
|
-
const fullPath =
|
|
27071
|
+
const fullPath = path36.join(dirPath, entry.name);
|
|
26716
27072
|
if (entry.isDirectory()) {
|
|
26717
27073
|
await searchDirectory(fullPath, workspacePath, regex, matches);
|
|
26718
27074
|
} else if (entry.isFile()) {
|
|
26719
|
-
const ext =
|
|
27075
|
+
const ext = path36.extname(entry.name).toLowerCase();
|
|
26720
27076
|
if (BINARY_EXTENSIONS.has(ext)) continue;
|
|
26721
27077
|
try {
|
|
26722
|
-
const
|
|
26723
|
-
if (
|
|
27078
|
+
const stat12 = await fs2.stat(fullPath);
|
|
27079
|
+
if (stat12.size > MAX_FILE_SIZE) continue;
|
|
26724
27080
|
const content = await fs2.readFile(fullPath, "utf-8");
|
|
26725
27081
|
const lines = content.split("\n");
|
|
26726
27082
|
for (let i = 0; i < lines.length; i++) {
|
|
@@ -26728,7 +27084,7 @@ async function searchDirectory(dirPath, workspacePath, regex, matches) {
|
|
|
26728
27084
|
regex.lastIndex = 0;
|
|
26729
27085
|
if (regex.test(lines[i])) {
|
|
26730
27086
|
matches.push({
|
|
26731
|
-
file:
|
|
27087
|
+
file: path36.relative(workspacePath, fullPath),
|
|
26732
27088
|
line: i + 1,
|
|
26733
27089
|
text: lines[i].substring(0, 200)
|
|
26734
27090
|
});
|
|
@@ -27353,115 +27709,115 @@ var FieldAccuracyEvaluator = class {
|
|
|
27353
27709
|
* Evaluate a single field against the expected value.
|
|
27354
27710
|
*/
|
|
27355
27711
|
evaluateField(fieldConfig, candidateData, expectedData) {
|
|
27356
|
-
const { path:
|
|
27357
|
-
const candidateValue = resolvePath(candidateData,
|
|
27358
|
-
const expectedValue = resolvePath(expectedData,
|
|
27712
|
+
const { path: path53, match, required = true, weight = 1 } = fieldConfig;
|
|
27713
|
+
const candidateValue = resolvePath(candidateData, path53);
|
|
27714
|
+
const expectedValue = resolvePath(expectedData, path53);
|
|
27359
27715
|
if (expectedValue === void 0) {
|
|
27360
27716
|
return {
|
|
27361
|
-
path:
|
|
27717
|
+
path: path53,
|
|
27362
27718
|
score: 1,
|
|
27363
27719
|
// No expected value means no comparison needed
|
|
27364
27720
|
weight,
|
|
27365
27721
|
hit: true,
|
|
27366
|
-
message: `${
|
|
27722
|
+
message: `${path53}: no expected value`
|
|
27367
27723
|
};
|
|
27368
27724
|
}
|
|
27369
27725
|
if (candidateValue === void 0) {
|
|
27370
27726
|
if (required) {
|
|
27371
27727
|
return {
|
|
27372
|
-
path:
|
|
27728
|
+
path: path53,
|
|
27373
27729
|
score: 0,
|
|
27374
27730
|
weight,
|
|
27375
27731
|
hit: false,
|
|
27376
|
-
message: `${
|
|
27732
|
+
message: `${path53} (required, missing)`
|
|
27377
27733
|
};
|
|
27378
27734
|
}
|
|
27379
27735
|
return {
|
|
27380
|
-
path:
|
|
27736
|
+
path: path53,
|
|
27381
27737
|
score: 1,
|
|
27382
27738
|
// Don't penalize missing optional fields
|
|
27383
27739
|
weight: 0,
|
|
27384
27740
|
// Zero weight means it won't affect the score
|
|
27385
27741
|
hit: true,
|
|
27386
|
-
message: `${
|
|
27742
|
+
message: `${path53}: optional field missing`
|
|
27387
27743
|
};
|
|
27388
27744
|
}
|
|
27389
27745
|
switch (match) {
|
|
27390
27746
|
case "exact":
|
|
27391
|
-
return this.compareExact(
|
|
27747
|
+
return this.compareExact(path53, candidateValue, expectedValue, weight);
|
|
27392
27748
|
case "numeric_tolerance":
|
|
27393
27749
|
return this.compareNumericTolerance(
|
|
27394
|
-
|
|
27750
|
+
path53,
|
|
27395
27751
|
candidateValue,
|
|
27396
27752
|
expectedValue,
|
|
27397
27753
|
fieldConfig,
|
|
27398
27754
|
weight
|
|
27399
27755
|
);
|
|
27400
27756
|
case "date":
|
|
27401
|
-
return this.compareDate(
|
|
27757
|
+
return this.compareDate(path53, candidateValue, expectedValue, fieldConfig, weight);
|
|
27402
27758
|
default:
|
|
27403
27759
|
return {
|
|
27404
|
-
path:
|
|
27760
|
+
path: path53,
|
|
27405
27761
|
score: 0,
|
|
27406
27762
|
weight,
|
|
27407
27763
|
hit: false,
|
|
27408
|
-
message: `${
|
|
27764
|
+
message: `${path53}: unknown match type "${match}"`
|
|
27409
27765
|
};
|
|
27410
27766
|
}
|
|
27411
27767
|
}
|
|
27412
27768
|
/**
|
|
27413
27769
|
* Exact equality comparison.
|
|
27414
27770
|
*/
|
|
27415
|
-
compareExact(
|
|
27771
|
+
compareExact(path53, candidateValue, expectedValue, weight) {
|
|
27416
27772
|
if (deepEqual(candidateValue, expectedValue)) {
|
|
27417
27773
|
return {
|
|
27418
|
-
path:
|
|
27774
|
+
path: path53,
|
|
27419
27775
|
score: 1,
|
|
27420
27776
|
weight,
|
|
27421
27777
|
hit: true,
|
|
27422
|
-
message:
|
|
27778
|
+
message: path53
|
|
27423
27779
|
};
|
|
27424
27780
|
}
|
|
27425
27781
|
if (typeof candidateValue !== typeof expectedValue) {
|
|
27426
27782
|
return {
|
|
27427
|
-
path:
|
|
27783
|
+
path: path53,
|
|
27428
27784
|
score: 0,
|
|
27429
27785
|
weight,
|
|
27430
27786
|
hit: false,
|
|
27431
|
-
message: `${
|
|
27787
|
+
message: `${path53} (type mismatch: got ${typeof candidateValue}, expected ${typeof expectedValue})`
|
|
27432
27788
|
};
|
|
27433
27789
|
}
|
|
27434
27790
|
return {
|
|
27435
|
-
path:
|
|
27791
|
+
path: path53,
|
|
27436
27792
|
score: 0,
|
|
27437
27793
|
weight,
|
|
27438
27794
|
hit: false,
|
|
27439
|
-
message: `${
|
|
27795
|
+
message: `${path53} (value mismatch)`
|
|
27440
27796
|
};
|
|
27441
27797
|
}
|
|
27442
27798
|
/**
|
|
27443
27799
|
* Numeric comparison with absolute or relative tolerance.
|
|
27444
27800
|
*/
|
|
27445
|
-
compareNumericTolerance(
|
|
27801
|
+
compareNumericTolerance(path53, candidateValue, expectedValue, fieldConfig, weight) {
|
|
27446
27802
|
const { tolerance = 0, relative = false } = fieldConfig;
|
|
27447
27803
|
const candidateNum = toNumber(candidateValue);
|
|
27448
27804
|
const expectedNum = toNumber(expectedValue);
|
|
27449
27805
|
if (candidateNum === null || expectedNum === null) {
|
|
27450
27806
|
return {
|
|
27451
|
-
path:
|
|
27807
|
+
path: path53,
|
|
27452
27808
|
score: 0,
|
|
27453
27809
|
weight,
|
|
27454
27810
|
hit: false,
|
|
27455
|
-
message: `${
|
|
27811
|
+
message: `${path53} (non-numeric value)`
|
|
27456
27812
|
};
|
|
27457
27813
|
}
|
|
27458
27814
|
if (!Number.isFinite(candidateNum) || !Number.isFinite(expectedNum)) {
|
|
27459
27815
|
return {
|
|
27460
|
-
path:
|
|
27816
|
+
path: path53,
|
|
27461
27817
|
score: 0,
|
|
27462
27818
|
weight,
|
|
27463
27819
|
hit: false,
|
|
27464
|
-
message: `${
|
|
27820
|
+
message: `${path53} (invalid numeric value)`
|
|
27465
27821
|
};
|
|
27466
27822
|
}
|
|
27467
27823
|
const diff = Math.abs(candidateNum - expectedNum);
|
|
@@ -27474,61 +27830,61 @@ var FieldAccuracyEvaluator = class {
|
|
|
27474
27830
|
}
|
|
27475
27831
|
if (withinTolerance) {
|
|
27476
27832
|
return {
|
|
27477
|
-
path:
|
|
27833
|
+
path: path53,
|
|
27478
27834
|
score: 1,
|
|
27479
27835
|
weight,
|
|
27480
27836
|
hit: true,
|
|
27481
|
-
message: `${
|
|
27837
|
+
message: `${path53} (within tolerance: diff=${diff.toFixed(2)})`
|
|
27482
27838
|
};
|
|
27483
27839
|
}
|
|
27484
27840
|
return {
|
|
27485
|
-
path:
|
|
27841
|
+
path: path53,
|
|
27486
27842
|
score: 0,
|
|
27487
27843
|
weight,
|
|
27488
27844
|
hit: false,
|
|
27489
|
-
message: `${
|
|
27845
|
+
message: `${path53} (outside tolerance: diff=${diff.toFixed(2)}, tolerance=${tolerance})`
|
|
27490
27846
|
};
|
|
27491
27847
|
}
|
|
27492
27848
|
/**
|
|
27493
27849
|
* Date comparison with format normalization.
|
|
27494
27850
|
*/
|
|
27495
|
-
compareDate(
|
|
27851
|
+
compareDate(path53, candidateValue, expectedValue, fieldConfig, weight) {
|
|
27496
27852
|
const formats = fieldConfig.formats ?? DEFAULT_DATE_FORMATS;
|
|
27497
27853
|
const candidateDate = parseDate(String(candidateValue), formats);
|
|
27498
27854
|
const expectedDate = parseDate(String(expectedValue), formats);
|
|
27499
27855
|
if (candidateDate === null) {
|
|
27500
27856
|
return {
|
|
27501
|
-
path:
|
|
27857
|
+
path: path53,
|
|
27502
27858
|
score: 0,
|
|
27503
27859
|
weight,
|
|
27504
27860
|
hit: false,
|
|
27505
|
-
message: `${
|
|
27861
|
+
message: `${path53} (unparseable candidate date)`
|
|
27506
27862
|
};
|
|
27507
27863
|
}
|
|
27508
27864
|
if (expectedDate === null) {
|
|
27509
27865
|
return {
|
|
27510
|
-
path:
|
|
27866
|
+
path: path53,
|
|
27511
27867
|
score: 0,
|
|
27512
27868
|
weight,
|
|
27513
27869
|
hit: false,
|
|
27514
|
-
message: `${
|
|
27870
|
+
message: `${path53} (unparseable expected date)`
|
|
27515
27871
|
};
|
|
27516
27872
|
}
|
|
27517
27873
|
if (candidateDate.getFullYear() === expectedDate.getFullYear() && candidateDate.getMonth() === expectedDate.getMonth() && candidateDate.getDate() === expectedDate.getDate()) {
|
|
27518
27874
|
return {
|
|
27519
|
-
path:
|
|
27875
|
+
path: path53,
|
|
27520
27876
|
score: 1,
|
|
27521
27877
|
weight,
|
|
27522
27878
|
hit: true,
|
|
27523
|
-
message:
|
|
27879
|
+
message: path53
|
|
27524
27880
|
};
|
|
27525
27881
|
}
|
|
27526
27882
|
return {
|
|
27527
|
-
path:
|
|
27883
|
+
path: path53,
|
|
27528
27884
|
score: 0,
|
|
27529
27885
|
weight,
|
|
27530
27886
|
hit: false,
|
|
27531
|
-
message: `${
|
|
27887
|
+
message: `${path53} (date mismatch: got ${formatDateISO(candidateDate)}, expected ${formatDateISO(expectedDate)})`
|
|
27532
27888
|
};
|
|
27533
27889
|
}
|
|
27534
27890
|
/**
|
|
@@ -27561,11 +27917,11 @@ var FieldAccuracyEvaluator = class {
|
|
|
27561
27917
|
};
|
|
27562
27918
|
}
|
|
27563
27919
|
};
|
|
27564
|
-
function resolvePath(obj,
|
|
27565
|
-
if (!
|
|
27920
|
+
function resolvePath(obj, path53) {
|
|
27921
|
+
if (!path53 || !obj) {
|
|
27566
27922
|
return void 0;
|
|
27567
27923
|
}
|
|
27568
|
-
const parts =
|
|
27924
|
+
const parts = path53.split(/\.|\[|\]/).filter((p) => p.length > 0);
|
|
27569
27925
|
let current = obj;
|
|
27570
27926
|
for (const part of parts) {
|
|
27571
27927
|
if (current === null || current === void 0) {
|
|
@@ -28047,8 +28403,8 @@ var TokenUsageEvaluator = class {
|
|
|
28047
28403
|
};
|
|
28048
28404
|
}
|
|
28049
28405
|
};
|
|
28050
|
-
function getNestedValue(obj,
|
|
28051
|
-
const parts =
|
|
28406
|
+
function getNestedValue(obj, path53) {
|
|
28407
|
+
const parts = path53.split(".");
|
|
28052
28408
|
let current = obj;
|
|
28053
28409
|
for (const part of parts) {
|
|
28054
28410
|
if (current === null || current === void 0 || typeof current !== "object") {
|
|
@@ -28894,6 +29250,15 @@ async function resolveCustomPrompt(promptConfig, context2, timeoutMs) {
|
|
|
28894
29250
|
}
|
|
28895
29251
|
return void 0;
|
|
28896
29252
|
}
|
|
29253
|
+
function containsTemplateVariables(text2) {
|
|
29254
|
+
const variablePattern = /\{\{\s*([a-zA-Z0-9_]+)\s*\}\}/g;
|
|
29255
|
+
for (const match of text2.matchAll(variablePattern)) {
|
|
29256
|
+
if (VALID_TEMPLATE_VARIABLES.has(match[1])) {
|
|
29257
|
+
return true;
|
|
29258
|
+
}
|
|
29259
|
+
}
|
|
29260
|
+
return false;
|
|
29261
|
+
}
|
|
28897
29262
|
async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
28898
29263
|
const payload = {
|
|
28899
29264
|
criteria: context2.evalCase.criteria,
|
|
@@ -28908,7 +29273,7 @@ async function executePromptTemplate(script, context2, config, timeoutMs) {
|
|
|
28908
29273
|
};
|
|
28909
29274
|
const inputJson = JSON.stringify(toSnakeCaseDeep(payload), null, 2);
|
|
28910
29275
|
const scriptPath = script[script.length - 1];
|
|
28911
|
-
const cwd =
|
|
29276
|
+
const cwd = path37.dirname(scriptPath);
|
|
28912
29277
|
try {
|
|
28913
29278
|
const stdout = await executeScript(script, inputJson, timeoutMs, cwd);
|
|
28914
29279
|
const prompt = stdout.trim();
|
|
@@ -28964,9 +29329,20 @@ var llmGraderFactory = (config, context2) => {
|
|
|
28964
29329
|
},
|
|
28965
29330
|
agentTimeoutMs
|
|
28966
29331
|
);
|
|
29332
|
+
const isFromInlinePrompt = !c.resolvedPromptScript?.length && !c.resolvedPromptPath && !c.promptPath;
|
|
29333
|
+
let evaluatorTemplateOverride;
|
|
29334
|
+
let evalCase = evalContext.evalCase;
|
|
29335
|
+
if (customPrompt) {
|
|
29336
|
+
if (!isFromInlinePrompt || containsTemplateVariables(customPrompt)) {
|
|
29337
|
+
evaluatorTemplateOverride = customPrompt;
|
|
29338
|
+
} else {
|
|
29339
|
+
evalCase = { ...evalCase, criteria: customPrompt };
|
|
29340
|
+
}
|
|
29341
|
+
}
|
|
28967
29342
|
return evaluator.evaluate({
|
|
28968
29343
|
...evalContext,
|
|
28969
|
-
|
|
29344
|
+
evalCase,
|
|
29345
|
+
evaluatorTemplateOverride,
|
|
28970
29346
|
evaluator: c
|
|
28971
29347
|
});
|
|
28972
29348
|
}
|
|
@@ -29179,11 +29555,11 @@ function createBuiltinRegistry() {
|
|
|
29179
29555
|
async function discoverAssertions(registry, baseDir) {
|
|
29180
29556
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
29181
29557
|
const candidateDirs = [];
|
|
29182
|
-
let dir =
|
|
29183
|
-
const root =
|
|
29558
|
+
let dir = path38.resolve(baseDir);
|
|
29559
|
+
const root = path38.parse(dir).root;
|
|
29184
29560
|
while (dir !== root) {
|
|
29185
|
-
candidateDirs.push(
|
|
29186
|
-
dir =
|
|
29561
|
+
candidateDirs.push(path38.join(dir, ".agentv", "assertions"));
|
|
29562
|
+
dir = path38.dirname(dir);
|
|
29187
29563
|
}
|
|
29188
29564
|
let files = [];
|
|
29189
29565
|
for (const assertionsDir of candidateDirs) {
|
|
@@ -29199,7 +29575,7 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
29199
29575
|
}
|
|
29200
29576
|
const discoveredTypes = [];
|
|
29201
29577
|
for (const filePath of files) {
|
|
29202
|
-
const basename =
|
|
29578
|
+
const basename = path38.basename(filePath);
|
|
29203
29579
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
29204
29580
|
if (registry.has(typeName)) {
|
|
29205
29581
|
continue;
|
|
@@ -29218,12 +29594,12 @@ async function discoverAssertions(registry, baseDir) {
|
|
|
29218
29594
|
async function discoverGraders(registry, baseDir) {
|
|
29219
29595
|
const patterns = ["*.ts", "*.js", "*.mts", "*.mjs"];
|
|
29220
29596
|
const candidateDirs = [];
|
|
29221
|
-
let dir =
|
|
29222
|
-
const root =
|
|
29597
|
+
let dir = path39.resolve(baseDir);
|
|
29598
|
+
const root = path39.parse(dir).root;
|
|
29223
29599
|
while (dir !== root) {
|
|
29224
|
-
candidateDirs.push(
|
|
29225
|
-
candidateDirs.push(
|
|
29226
|
-
dir =
|
|
29600
|
+
candidateDirs.push(path39.join(dir, ".agentv", "graders"));
|
|
29601
|
+
candidateDirs.push(path39.join(dir, ".agentv", "judges"));
|
|
29602
|
+
dir = path39.dirname(dir);
|
|
29227
29603
|
}
|
|
29228
29604
|
let files = [];
|
|
29229
29605
|
for (const gradersDir of candidateDirs) {
|
|
@@ -29239,7 +29615,7 @@ async function discoverGraders(registry, baseDir) {
|
|
|
29239
29615
|
}
|
|
29240
29616
|
const discoveredTypes = [];
|
|
29241
29617
|
for (const filePath of files) {
|
|
29242
|
-
const basename =
|
|
29618
|
+
const basename = path39.basename(filePath);
|
|
29243
29619
|
const typeName = basename.replace(/\.(ts|js|mts|mjs)$/, "");
|
|
29244
29620
|
if (registry.has(typeName)) {
|
|
29245
29621
|
continue;
|
|
@@ -29425,10 +29801,10 @@ async function stageNestedRepoChanges(workspacePath) {
|
|
|
29425
29801
|
}
|
|
29426
29802
|
for (const entry of entries) {
|
|
29427
29803
|
if (entry === ".git" || entry === "node_modules") continue;
|
|
29428
|
-
const childPath =
|
|
29804
|
+
const childPath = path40.join(workspacePath, entry);
|
|
29429
29805
|
try {
|
|
29430
29806
|
if (!statSync(childPath).isDirectory()) continue;
|
|
29431
|
-
if (!statSync(
|
|
29807
|
+
if (!statSync(path40.join(childPath, ".git")).isDirectory()) continue;
|
|
29432
29808
|
} catch {
|
|
29433
29809
|
continue;
|
|
29434
29810
|
}
|
|
@@ -29465,14 +29841,14 @@ async function isDirectory(filePath) {
|
|
|
29465
29841
|
}
|
|
29466
29842
|
function getWorkspacePath(evalRunId, caseId, workspaceRoot) {
|
|
29467
29843
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
29468
|
-
return
|
|
29844
|
+
return path41.join(root, evalRunId, caseId);
|
|
29469
29845
|
}
|
|
29470
29846
|
async function copyDirectoryRecursive(src, dest) {
|
|
29471
29847
|
await mkdir12(dest, { recursive: true });
|
|
29472
29848
|
const entries = await readdir4(src, { withFileTypes: true });
|
|
29473
29849
|
for (const entry of entries) {
|
|
29474
|
-
const srcPath =
|
|
29475
|
-
const destPath =
|
|
29850
|
+
const srcPath = path41.join(src, entry.name);
|
|
29851
|
+
const destPath = path41.join(dest, entry.name);
|
|
29476
29852
|
if (entry.name === ".git") {
|
|
29477
29853
|
continue;
|
|
29478
29854
|
}
|
|
@@ -29484,7 +29860,7 @@ async function copyDirectoryRecursive(src, dest) {
|
|
|
29484
29860
|
}
|
|
29485
29861
|
}
|
|
29486
29862
|
async function createTempWorkspace(templatePath, evalRunId, caseId, workspaceRoot) {
|
|
29487
|
-
const resolvedTemplatePath =
|
|
29863
|
+
const resolvedTemplatePath = path41.resolve(templatePath);
|
|
29488
29864
|
if (!await fileExists(resolvedTemplatePath)) {
|
|
29489
29865
|
throw new TemplateNotFoundError(resolvedTemplatePath);
|
|
29490
29866
|
}
|
|
@@ -29533,7 +29909,7 @@ async function cleanupWorkspace(workspacePath) {
|
|
|
29533
29909
|
}
|
|
29534
29910
|
async function cleanupEvalWorkspaces(evalRunId, workspaceRoot) {
|
|
29535
29911
|
const root = workspaceRoot ?? getWorkspacesRoot();
|
|
29536
|
-
const evalDir =
|
|
29912
|
+
const evalDir = path41.join(root, evalRunId);
|
|
29537
29913
|
if (await fileExists(evalDir)) {
|
|
29538
29914
|
await rm4(evalDir, { recursive: true, force: true });
|
|
29539
29915
|
}
|
|
@@ -29563,12 +29939,14 @@ async function git(args, opts) {
|
|
|
29563
29939
|
return stdout.trim();
|
|
29564
29940
|
}
|
|
29565
29941
|
function normalizeRepoForFingerprint(repo) {
|
|
29566
|
-
const
|
|
29567
|
-
|
|
29568
|
-
path
|
|
29569
|
-
|
|
29570
|
-
|
|
29571
|
-
|
|
29942
|
+
const result = {};
|
|
29943
|
+
if (repo.path) {
|
|
29944
|
+
result.path = repo.path;
|
|
29945
|
+
}
|
|
29946
|
+
if (repo.source) {
|
|
29947
|
+
result.source = repo.source.type === "git" ? { type: "git", url: repo.source.url.toLowerCase().replace(/\.git$/, "") } : { type: "local", path: repo.source.path };
|
|
29948
|
+
}
|
|
29949
|
+
result.ref = getRepoCheckoutRef(repo.checkout);
|
|
29572
29950
|
if (repo.clone?.depth !== void 0) {
|
|
29573
29951
|
result.depth = repo.clone.depth;
|
|
29574
29952
|
}
|
|
@@ -29582,7 +29960,7 @@ function normalizeRepoForFingerprint(repo) {
|
|
|
29582
29960
|
}
|
|
29583
29961
|
function computeWorkspaceFingerprint(repos) {
|
|
29584
29962
|
const canonical = {
|
|
29585
|
-
repos: [...repos].sort((a, b) => a.path.localeCompare(b.path)).map(normalizeRepoForFingerprint)
|
|
29963
|
+
repos: [...repos].sort((a, b) => (a.path ?? "").localeCompare(b.path ?? "")).map(normalizeRepoForFingerprint)
|
|
29586
29964
|
};
|
|
29587
29965
|
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
29588
29966
|
}
|
|
@@ -29590,8 +29968,8 @@ async function copyDirectoryRecursive2(src, dest, skipDirs) {
|
|
|
29590
29968
|
await mkdir13(dest, { recursive: true });
|
|
29591
29969
|
const entries = await readdir5(src, { withFileTypes: true });
|
|
29592
29970
|
for (const entry of entries) {
|
|
29593
|
-
const srcPath =
|
|
29594
|
-
const destPath =
|
|
29971
|
+
const srcPath = path422.join(src, entry.name);
|
|
29972
|
+
const destPath = path422.join(dest, entry.name);
|
|
29595
29973
|
if (entry.name === ".git") {
|
|
29596
29974
|
continue;
|
|
29597
29975
|
}
|
|
@@ -29624,7 +30002,7 @@ var WorkspacePoolManager = class {
|
|
|
29624
30002
|
async acquireWorkspace(options) {
|
|
29625
30003
|
const { templatePath, repos, maxSlots, repoManager, poolReset } = options;
|
|
29626
30004
|
const fingerprint = computeWorkspaceFingerprint(repos);
|
|
29627
|
-
const poolDir =
|
|
30005
|
+
const poolDir = path422.join(this.poolRoot, fingerprint);
|
|
29628
30006
|
await mkdir13(poolDir, { recursive: true });
|
|
29629
30007
|
const drifted = await this.checkDrift(poolDir, fingerprint);
|
|
29630
30008
|
if (drifted) {
|
|
@@ -29634,7 +30012,7 @@ var WorkspacePoolManager = class {
|
|
|
29634
30012
|
await this.removeAllSlots(poolDir);
|
|
29635
30013
|
}
|
|
29636
30014
|
for (let i = 0; i < maxSlots; i++) {
|
|
29637
|
-
const slotPath =
|
|
30015
|
+
const slotPath = path422.join(poolDir, `slot-${i}`);
|
|
29638
30016
|
const lockPath = `${slotPath}.lock`;
|
|
29639
30017
|
const locked = await this.tryLock(lockPath);
|
|
29640
30018
|
if (!locked) {
|
|
@@ -29696,7 +30074,7 @@ var WorkspacePoolManager = class {
|
|
|
29696
30074
|
throw err;
|
|
29697
30075
|
}
|
|
29698
30076
|
try {
|
|
29699
|
-
const pidStr = await
|
|
30077
|
+
const pidStr = await readFile14(lockPath, "utf-8");
|
|
29700
30078
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
29701
30079
|
if (!Number.isNaN(pid)) {
|
|
29702
30080
|
try {
|
|
@@ -29721,9 +30099,9 @@ var WorkspacePoolManager = class {
|
|
|
29721
30099
|
* Returns false (no drift) if metadata.json doesn't exist (first use).
|
|
29722
30100
|
*/
|
|
29723
30101
|
async checkDrift(poolDir, fingerprint) {
|
|
29724
|
-
const metadataPath =
|
|
30102
|
+
const metadataPath = path422.join(poolDir, "metadata.json");
|
|
29725
30103
|
try {
|
|
29726
|
-
const raw = await
|
|
30104
|
+
const raw = await readFile14(metadataPath, "utf-8");
|
|
29727
30105
|
const metadata = JSON.parse(raw);
|
|
29728
30106
|
return metadata.fingerprint !== fingerprint;
|
|
29729
30107
|
} catch {
|
|
@@ -29738,17 +30116,17 @@ var WorkspacePoolManager = class {
|
|
|
29738
30116
|
repos,
|
|
29739
30117
|
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
29740
30118
|
};
|
|
29741
|
-
await writeFile7(
|
|
30119
|
+
await writeFile7(path422.join(poolDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
29742
30120
|
}
|
|
29743
30121
|
/** Remove all slot directories and their lock files from a pool directory. */
|
|
29744
30122
|
async removeAllSlots(poolDir) {
|
|
29745
30123
|
const entries = await readdir5(poolDir);
|
|
29746
30124
|
for (const entry of entries) {
|
|
29747
30125
|
if (entry.startsWith("slot-") && !entry.endsWith(".lock")) {
|
|
29748
|
-
const lockPath =
|
|
30126
|
+
const lockPath = path422.join(poolDir, `${entry}.lock`);
|
|
29749
30127
|
if (existsSync3(lockPath)) {
|
|
29750
30128
|
try {
|
|
29751
|
-
const pidStr = await
|
|
30129
|
+
const pidStr = await readFile14(lockPath, "utf-8");
|
|
29752
30130
|
const pid = Number.parseInt(pidStr.trim(), 10);
|
|
29753
30131
|
if (!Number.isNaN(pid)) {
|
|
29754
30132
|
try {
|
|
@@ -29761,12 +30139,12 @@ var WorkspacePoolManager = class {
|
|
|
29761
30139
|
} catch {
|
|
29762
30140
|
}
|
|
29763
30141
|
}
|
|
29764
|
-
await rm5(
|
|
30142
|
+
await rm5(path422.join(poolDir, entry), { recursive: true, force: true });
|
|
29765
30143
|
await rm5(lockPath, { force: true }).catch(() => {
|
|
29766
30144
|
});
|
|
29767
30145
|
}
|
|
29768
30146
|
}
|
|
29769
|
-
await rm5(
|
|
30147
|
+
await rm5(path422.join(poolDir, "metadata.json"), { force: true }).catch(() => {
|
|
29770
30148
|
});
|
|
29771
30149
|
}
|
|
29772
30150
|
/**
|
|
@@ -29776,14 +30154,15 @@ var WorkspacePoolManager = class {
|
|
|
29776
30154
|
*/
|
|
29777
30155
|
async resetSlot(slotPath, templatePath, repos, poolReset = "fast") {
|
|
29778
30156
|
for (const repo of repos) {
|
|
29779
|
-
|
|
30157
|
+
if (!repo.path || !repo.source) continue;
|
|
30158
|
+
const repoDir = path422.join(slotPath, repo.path);
|
|
29780
30159
|
if (!existsSync3(repoDir)) {
|
|
29781
30160
|
continue;
|
|
29782
30161
|
}
|
|
29783
30162
|
if (poolReset === "none") {
|
|
29784
30163
|
continue;
|
|
29785
30164
|
}
|
|
29786
|
-
const ref = repo.checkout
|
|
30165
|
+
const ref = getRepoCheckoutRef(repo.checkout);
|
|
29787
30166
|
const resolve2 = repo.checkout?.resolve ?? "remote";
|
|
29788
30167
|
if (resolve2 === "remote") {
|
|
29789
30168
|
const fetchArgs = ["fetch", "origin", ref];
|
|
@@ -29800,8 +30179,8 @@ var WorkspacePoolManager = class {
|
|
|
29800
30179
|
}
|
|
29801
30180
|
if (templatePath) {
|
|
29802
30181
|
const repoDirNames = new Set(
|
|
29803
|
-
repos.map((r) => {
|
|
29804
|
-
const normalized = r.path.replace(/^\.\//, "");
|
|
30182
|
+
repos.filter((r) => r.path).map((r) => {
|
|
30183
|
+
const normalized = (r.path ?? "").replace(/^\.\//, "");
|
|
29805
30184
|
return normalized.split("/")[0];
|
|
29806
30185
|
})
|
|
29807
30186
|
);
|
|
@@ -29850,17 +30229,17 @@ var RepoManager = class {
|
|
|
29850
30229
|
static validateLocalPaths(repos) {
|
|
29851
30230
|
const errors = [];
|
|
29852
30231
|
for (const repo of repos) {
|
|
29853
|
-
if (repo.source.type !== "local") continue;
|
|
30232
|
+
if (!repo.source || repo.source.type !== "local") continue;
|
|
29854
30233
|
const sourcePath = repo.source.path;
|
|
29855
30234
|
if (!sourcePath || sourcePath.trim() === "") {
|
|
29856
30235
|
errors.push({
|
|
29857
|
-
repoPath: repo.path,
|
|
30236
|
+
repoPath: repo.path ?? "(none)",
|
|
29858
30237
|
resolvedSourcePath: sourcePath ?? "",
|
|
29859
30238
|
reason: "empty_path"
|
|
29860
30239
|
});
|
|
29861
30240
|
} else if (!existsSync4(sourcePath)) {
|
|
29862
30241
|
errors.push({
|
|
29863
|
-
repoPath: repo.path,
|
|
30242
|
+
repoPath: repo.path ?? "(none)",
|
|
29864
30243
|
resolvedSourcePath: sourcePath,
|
|
29865
30244
|
reason: "not_found"
|
|
29866
30245
|
});
|
|
@@ -29907,7 +30286,13 @@ ${lines.join("\n")}`;
|
|
|
29907
30286
|
* Handles checkout, ref resolution, ancestor walking, shallow clone, sparse checkout.
|
|
29908
30287
|
*/
|
|
29909
30288
|
async materialize(repo, workspacePath) {
|
|
29910
|
-
|
|
30289
|
+
if (!repo.source || !repo.path) {
|
|
30290
|
+
if (this.verbose) {
|
|
30291
|
+
console.log(`[repo] materialize skip path=${repo.path ?? "(none)"} (no source or path)`);
|
|
30292
|
+
}
|
|
30293
|
+
return;
|
|
30294
|
+
}
|
|
30295
|
+
const targetDir = path43.join(workspacePath, repo.path);
|
|
29911
30296
|
const sourceUrl = getSourceUrl(repo.source);
|
|
29912
30297
|
const startedAt = Date.now();
|
|
29913
30298
|
if (this.verbose) {
|
|
@@ -29930,7 +30315,7 @@ ${lines.join("\n")}`;
|
|
|
29930
30315
|
await this.runGit(["sparse-checkout", "init", "--cone"], { cwd: targetDir });
|
|
29931
30316
|
await this.runGit(["sparse-checkout", "set", ...repo.clone.sparse], { cwd: targetDir });
|
|
29932
30317
|
}
|
|
29933
|
-
const ref = repo.checkout
|
|
30318
|
+
const ref = getRepoCheckoutRef(repo.checkout);
|
|
29934
30319
|
const resolve2 = repo.checkout?.resolve ?? "remote";
|
|
29935
30320
|
let resolvedSha;
|
|
29936
30321
|
if (resolve2 === "remote" && repo.source.type === "git") {
|
|
@@ -29982,23 +30367,27 @@ ${lines.join("\n")}`;
|
|
|
29982
30367
|
);
|
|
29983
30368
|
}
|
|
29984
30369
|
}
|
|
29985
|
-
/** Materialize all repos into the workspace. */
|
|
30370
|
+
/** Materialize all repos into the workspace. Skips repos without source (Docker-only repos). */
|
|
29986
30371
|
async materializeAll(repos, workspacePath) {
|
|
30372
|
+
const materializableRepos = repos.filter((r) => r.source);
|
|
29987
30373
|
if (this.verbose) {
|
|
29988
|
-
console.log(
|
|
30374
|
+
console.log(
|
|
30375
|
+
`[repo] materializeAll count=${materializableRepos.length} (${repos.length - materializableRepos.length} skipped, no source) workspace=${workspacePath}`
|
|
30376
|
+
);
|
|
29989
30377
|
}
|
|
29990
|
-
for (const repo of
|
|
30378
|
+
for (const repo of materializableRepos) {
|
|
29991
30379
|
await this.materialize(repo, workspacePath);
|
|
29992
30380
|
}
|
|
29993
30381
|
if (this.verbose) {
|
|
29994
30382
|
console.log("[repo] materializeAll complete");
|
|
29995
30383
|
}
|
|
29996
30384
|
}
|
|
29997
|
-
/** Reset repos in workspace to their checkout state. */
|
|
30385
|
+
/** Reset repos in workspace to their checkout state. Skips repos without path or source. */
|
|
29998
30386
|
async reset(repos, workspacePath, reset) {
|
|
29999
30387
|
const cleanFlag = reset === "strict" ? "-fdx" : "-fd";
|
|
30000
30388
|
for (const repo of repos) {
|
|
30001
|
-
|
|
30389
|
+
if (!repo.path || !repo.source) continue;
|
|
30390
|
+
const targetDir = path43.join(workspacePath, repo.path);
|
|
30002
30391
|
await this.runGit(["reset", "--hard", "HEAD"], { cwd: targetDir });
|
|
30003
30392
|
await this.runGit(["clean", cleanFlag], { cwd: targetDir });
|
|
30004
30393
|
}
|
|
@@ -30008,11 +30397,11 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
30008
30397
|
if (!templatePath) {
|
|
30009
30398
|
return void 0;
|
|
30010
30399
|
}
|
|
30011
|
-
const resolved =
|
|
30400
|
+
const resolved = path44.resolve(templatePath);
|
|
30012
30401
|
const stats = await stat7(resolved);
|
|
30013
30402
|
if (stats.isFile()) {
|
|
30014
30403
|
return {
|
|
30015
|
-
dir:
|
|
30404
|
+
dir: path44.dirname(resolved),
|
|
30016
30405
|
workspaceFile: resolved
|
|
30017
30406
|
};
|
|
30018
30407
|
}
|
|
@@ -30024,14 +30413,14 @@ async function resolveWorkspaceTemplate(templatePath) {
|
|
|
30024
30413
|
if (workspaceFiles.length === 1) {
|
|
30025
30414
|
return {
|
|
30026
30415
|
dir: resolved,
|
|
30027
|
-
workspaceFile:
|
|
30416
|
+
workspaceFile: path44.join(resolved, workspaceFiles[0])
|
|
30028
30417
|
};
|
|
30029
30418
|
}
|
|
30030
30419
|
if (workspaceFiles.length > 1) {
|
|
30031
30420
|
const conventionFile = workspaceFiles.find((f) => f === "template.code-workspace");
|
|
30032
30421
|
return {
|
|
30033
30422
|
dir: resolved,
|
|
30034
|
-
workspaceFile: conventionFile ?
|
|
30423
|
+
workspaceFile: conventionFile ? path44.join(resolved, conventionFile) : void 0
|
|
30035
30424
|
};
|
|
30036
30425
|
}
|
|
30037
30426
|
return { dir: resolved };
|
|
@@ -30246,7 +30635,7 @@ async function runEvaluation(options) {
|
|
|
30246
30635
|
];
|
|
30247
30636
|
const evaluatorRegistry = buildEvaluatorRegistry(evaluators, resolveGraderProvider);
|
|
30248
30637
|
const typeRegistry = createBuiltinRegistry();
|
|
30249
|
-
const discoveryBaseDir = evalFilePath ?
|
|
30638
|
+
const discoveryBaseDir = evalFilePath ? path45.dirname(path45.resolve(evalFilePath)) : process.cwd();
|
|
30250
30639
|
const evalDir = discoveryBaseDir;
|
|
30251
30640
|
await discoverAssertions(typeRegistry, discoveryBaseDir);
|
|
30252
30641
|
await discoverGraders(typeRegistry, discoveryBaseDir);
|
|
@@ -30313,7 +30702,8 @@ async function runEvaluation(options) {
|
|
|
30313
30702
|
for (const ec of filteredEvalCases) {
|
|
30314
30703
|
if (ec.workspace?.repos) {
|
|
30315
30704
|
for (const repo of ec.workspace.repos) {
|
|
30316
|
-
|
|
30705
|
+
if (!repo.source) continue;
|
|
30706
|
+
const key = `${repo.path ?? ""}::${repo.source.type === "local" ? repo.source.path : ""}`;
|
|
30317
30707
|
if (!allRepos.has(key)) {
|
|
30318
30708
|
allRepos.set(key, repo);
|
|
30319
30709
|
}
|
|
@@ -30326,7 +30716,7 @@ async function runEvaluation(options) {
|
|
|
30326
30716
|
const message = RepoManager.formatValidationErrors(localPathErrors);
|
|
30327
30717
|
console.warn(`Warning: ${message}`);
|
|
30328
30718
|
const invalidLocalRepoPaths = new Set(localPathErrors.map((e) => e.repoPath));
|
|
30329
|
-
if (suiteWorkspace?.repos?.some((r) => invalidLocalRepoPaths.has(r.path))) {
|
|
30719
|
+
if (suiteWorkspace?.repos?.some((r) => r.path && invalidLocalRepoPaths.has(r.path))) {
|
|
30330
30720
|
throw new Error(message);
|
|
30331
30721
|
}
|
|
30332
30722
|
}
|
|
@@ -30444,7 +30834,7 @@ async function runEvaluation(options) {
|
|
|
30444
30834
|
}
|
|
30445
30835
|
try {
|
|
30446
30836
|
if (suiteWorkspaceFile && sharedWorkspacePath) {
|
|
30447
|
-
const copiedWorkspaceFile =
|
|
30837
|
+
const copiedWorkspaceFile = path45.join(sharedWorkspacePath, path45.basename(suiteWorkspaceFile));
|
|
30448
30838
|
try {
|
|
30449
30839
|
await stat8(copiedWorkspaceFile);
|
|
30450
30840
|
suiteWorkspaceFile = copiedWorkspaceFile;
|
|
@@ -30459,7 +30849,8 @@ async function runEvaluation(options) {
|
|
|
30459
30849
|
try {
|
|
30460
30850
|
if (needsPerRepoCheck) {
|
|
30461
30851
|
for (const repo of suiteWorkspace.repos) {
|
|
30462
|
-
|
|
30852
|
+
if (!repo.path || !repo.source) continue;
|
|
30853
|
+
const targetDir = path45.join(sharedWorkspacePath, repo.path);
|
|
30463
30854
|
if (existsSync5(targetDir)) {
|
|
30464
30855
|
setupLog(`reusing existing repo at: ${targetDir}`);
|
|
30465
30856
|
continue;
|
|
@@ -30483,6 +30874,19 @@ async function runEvaluation(options) {
|
|
|
30483
30874
|
throw new Error(`Failed to materialize repos: ${message}`);
|
|
30484
30875
|
}
|
|
30485
30876
|
}
|
|
30877
|
+
const suiteDockerConfig = suiteWorkspace?.docker;
|
|
30878
|
+
if (suiteDockerConfig) {
|
|
30879
|
+
setupLog(`pulling Docker image: ${suiteDockerConfig.image}`);
|
|
30880
|
+
const { DockerWorkspaceProvider: DockerWorkspaceProvider2 } = await import("./docker-workspace-RPPXBT27-B4AQHVWA.js");
|
|
30881
|
+
const dockerSetup = new DockerWorkspaceProvider2(suiteDockerConfig);
|
|
30882
|
+
if (!await dockerSetup.isDockerAvailable()) {
|
|
30883
|
+
throw new Error(
|
|
30884
|
+
"Docker workspace configured but Docker CLI is not available. Install Docker and ensure it is running."
|
|
30885
|
+
);
|
|
30886
|
+
}
|
|
30887
|
+
await dockerSetup.pullImage();
|
|
30888
|
+
setupLog("Docker image pull complete");
|
|
30889
|
+
}
|
|
30486
30890
|
const suiteHooksEnabled = hooksEnabled(suiteWorkspace);
|
|
30487
30891
|
const suiteBeforeAllHook = suiteWorkspace?.hooks?.before_all;
|
|
30488
30892
|
if (sharedWorkspacePath && suiteHooksEnabled && hasHookCommand(suiteBeforeAllHook)) {
|
|
@@ -30843,11 +31247,9 @@ async function runBatchEvaluation(options) {
|
|
|
30843
31247
|
const promptInputs = promptInputsList[index];
|
|
30844
31248
|
return {
|
|
30845
31249
|
question: promptInputs.question,
|
|
31250
|
+
systemPrompt: promptInputs.systemMessage,
|
|
30846
31251
|
inputFiles: evalCase.file_paths,
|
|
30847
|
-
evalCaseId: evalCase.id
|
|
30848
|
-
metadata: {
|
|
30849
|
-
systemPrompt: promptInputs.systemMessage ?? ""
|
|
30850
|
-
}
|
|
31252
|
+
evalCaseId: evalCase.id
|
|
30851
31253
|
};
|
|
30852
31254
|
});
|
|
30853
31255
|
const batchResponse = await provider.invokeBatch?.(batchRequests);
|
|
@@ -31046,7 +31448,7 @@ async function runEvalCase(options) {
|
|
|
31046
31448
|
);
|
|
31047
31449
|
}
|
|
31048
31450
|
if (caseWorkspaceFile && workspacePath) {
|
|
31049
|
-
const copiedFile =
|
|
31451
|
+
const copiedFile = path45.join(workspacePath, path45.basename(caseWorkspaceFile));
|
|
31050
31452
|
try {
|
|
31051
31453
|
await stat8(copiedFile);
|
|
31052
31454
|
caseWorkspaceFile = copiedFile;
|
|
@@ -31108,10 +31510,10 @@ async function runEvalCase(options) {
|
|
|
31108
31510
|
const files = evalCase.metadata.agent_skills_files;
|
|
31109
31511
|
if (baseDir && files.length > 0) {
|
|
31110
31512
|
for (const relPath of files) {
|
|
31111
|
-
const srcPath =
|
|
31112
|
-
const destPath =
|
|
31513
|
+
const srcPath = path45.resolve(baseDir, relPath);
|
|
31514
|
+
const destPath = path45.resolve(workspacePath, relPath);
|
|
31113
31515
|
try {
|
|
31114
|
-
await mkdir14(
|
|
31516
|
+
await mkdir14(path45.dirname(destPath), { recursive: true });
|
|
31115
31517
|
await copyFile2(srcPath, destPath);
|
|
31116
31518
|
} catch (error) {
|
|
31117
31519
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -31378,6 +31780,7 @@ async function runEvalCase(options) {
|
|
|
31378
31780
|
availableTargets,
|
|
31379
31781
|
fileChanges,
|
|
31380
31782
|
workspacePath,
|
|
31783
|
+
dockerConfig: evalCase.workspace?.docker,
|
|
31381
31784
|
verbose,
|
|
31382
31785
|
threshold: evalCase.threshold ?? caseThreshold
|
|
31383
31786
|
});
|
|
@@ -31571,6 +31974,7 @@ async function evaluateCandidate(options) {
|
|
|
31571
31974
|
availableTargets,
|
|
31572
31975
|
fileChanges,
|
|
31573
31976
|
workspacePath,
|
|
31977
|
+
dockerConfig,
|
|
31574
31978
|
threshold: evalThreshold
|
|
31575
31979
|
} = options;
|
|
31576
31980
|
const gradeTimestamp = nowFn();
|
|
@@ -31597,6 +32001,7 @@ async function evaluateCandidate(options) {
|
|
|
31597
32001
|
availableTargets,
|
|
31598
32002
|
fileChanges,
|
|
31599
32003
|
workspacePath,
|
|
32004
|
+
dockerConfig,
|
|
31600
32005
|
threshold: evalThreshold
|
|
31601
32006
|
});
|
|
31602
32007
|
const completedAt = nowFn();
|
|
@@ -31672,6 +32077,7 @@ async function runEvaluatorsForCase(options) {
|
|
|
31672
32077
|
availableTargets,
|
|
31673
32078
|
fileChanges,
|
|
31674
32079
|
workspacePath,
|
|
32080
|
+
dockerConfig,
|
|
31675
32081
|
threshold
|
|
31676
32082
|
} = options;
|
|
31677
32083
|
if (evalCase.assertions && evalCase.assertions.length > 0) {
|
|
@@ -31699,6 +32105,7 @@ async function runEvaluatorsForCase(options) {
|
|
|
31699
32105
|
availableTargets,
|
|
31700
32106
|
fileChanges,
|
|
31701
32107
|
workspacePath,
|
|
32108
|
+
dockerConfig,
|
|
31702
32109
|
threshold
|
|
31703
32110
|
});
|
|
31704
32111
|
}
|
|
@@ -31707,6 +32114,7 @@ async function runEvaluatorsForCase(options) {
|
|
|
31707
32114
|
if (!activeEvaluator) {
|
|
31708
32115
|
throw new Error(`No evaluator registered for kind '${evaluatorKind}'`);
|
|
31709
32116
|
}
|
|
32117
|
+
const implicitEvaluator = evaluatorKind === "llm-grader" && !evalCase.assertions ? buildImplicitLlmGraderConfig(evalCase) : void 0;
|
|
31710
32118
|
const score = await activeEvaluator.evaluate({
|
|
31711
32119
|
evalCase,
|
|
31712
32120
|
candidate,
|
|
@@ -31726,10 +32134,22 @@ async function runEvaluatorsForCase(options) {
|
|
|
31726
32134
|
targetResolver,
|
|
31727
32135
|
availableTargets,
|
|
31728
32136
|
fileChanges,
|
|
31729
|
-
workspacePath
|
|
32137
|
+
workspacePath,
|
|
32138
|
+
dockerConfig,
|
|
32139
|
+
...implicitEvaluator ? { evaluator: implicitEvaluator } : {}
|
|
31730
32140
|
});
|
|
31731
32141
|
return { score };
|
|
31732
32142
|
}
|
|
32143
|
+
function buildImplicitLlmGraderConfig(evalCase) {
|
|
32144
|
+
if (!evalCase.preprocessors || evalCase.preprocessors.length === 0) {
|
|
32145
|
+
return void 0;
|
|
32146
|
+
}
|
|
32147
|
+
return {
|
|
32148
|
+
name: "llm-grader",
|
|
32149
|
+
type: "llm-grader",
|
|
32150
|
+
preprocessors: evalCase.preprocessors
|
|
32151
|
+
};
|
|
32152
|
+
}
|
|
31733
32153
|
async function runEvaluatorList(options) {
|
|
31734
32154
|
const {
|
|
31735
32155
|
evalCase,
|
|
@@ -31754,7 +32174,8 @@ async function runEvaluatorList(options) {
|
|
|
31754
32174
|
targetResolver,
|
|
31755
32175
|
availableTargets,
|
|
31756
32176
|
fileChanges,
|
|
31757
|
-
workspacePath
|
|
32177
|
+
workspacePath,
|
|
32178
|
+
dockerConfig
|
|
31758
32179
|
} = options;
|
|
31759
32180
|
const scored = [];
|
|
31760
32181
|
const scores = [];
|
|
@@ -31777,9 +32198,10 @@ async function runEvaluatorList(options) {
|
|
|
31777
32198
|
targetResolver,
|
|
31778
32199
|
availableTargets,
|
|
31779
32200
|
fileChanges,
|
|
31780
|
-
workspacePath
|
|
32201
|
+
workspacePath,
|
|
32202
|
+
dockerConfig
|
|
31781
32203
|
};
|
|
31782
|
-
const evalFileDir = evalCase.file_paths[0] ?
|
|
32204
|
+
const evalFileDir = evalCase.file_paths[0] ? path45.dirname(evalCase.file_paths[0]) : process.cwd();
|
|
31783
32205
|
const dispatchContext = {
|
|
31784
32206
|
graderProvider,
|
|
31785
32207
|
targetResolver,
|
|
@@ -31939,13 +32361,11 @@ async function invokeProvider(provider, options) {
|
|
|
31939
32361
|
const braintrustSpanIds = streamCallbacks?.getActiveSpanIds?.() ?? void 0;
|
|
31940
32362
|
return await provider.invoke({
|
|
31941
32363
|
question: promptInputs.question,
|
|
32364
|
+
systemPrompt: promptInputs.systemMessage,
|
|
31942
32365
|
chatPrompt: promptInputs.chatPrompt,
|
|
31943
32366
|
inputFiles: evalCase.file_paths,
|
|
31944
32367
|
evalCaseId: evalCase.id,
|
|
31945
32368
|
attempt,
|
|
31946
|
-
metadata: {
|
|
31947
|
-
systemPrompt: promptInputs.systemMessage ?? ""
|
|
31948
|
-
},
|
|
31949
32369
|
signal: controller.signal,
|
|
31950
32370
|
cwd,
|
|
31951
32371
|
workspaceFile,
|
|
@@ -32176,7 +32596,7 @@ async function evaluate(config) {
|
|
|
32176
32596
|
}
|
|
32177
32597
|
const gitRoot = await findGitRoot(process.cwd());
|
|
32178
32598
|
const repoRoot = gitRoot ?? process.cwd();
|
|
32179
|
-
const testFilePath = config.specFile ?
|
|
32599
|
+
const testFilePath = config.specFile ? path46.resolve(config.specFile) : path46.join(process.cwd(), "__programmatic__.yaml");
|
|
32180
32600
|
await loadEnvHierarchy(repoRoot, testFilePath);
|
|
32181
32601
|
let resolvedTarget;
|
|
32182
32602
|
let taskProvider;
|
|
@@ -32291,10 +32711,10 @@ function computeSummary(results, durationMs, threshold = DEFAULT_THRESHOLD) {
|
|
|
32291
32711
|
var TARGET_FILE_CANDIDATES = [".agentv/targets.yaml", ".agentv/targets.yml"];
|
|
32292
32712
|
async function discoverDefaultTarget(repoRoot) {
|
|
32293
32713
|
const cwd = process.cwd();
|
|
32294
|
-
const chain = buildDirectoryChain(
|
|
32714
|
+
const chain = buildDirectoryChain(path46.join(cwd, "_placeholder"), repoRoot);
|
|
32295
32715
|
for (const dir of chain) {
|
|
32296
32716
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
32297
|
-
const targetsPath =
|
|
32717
|
+
const targetsPath = path46.join(dir, candidate);
|
|
32298
32718
|
if (!existsSync6(targetsPath)) continue;
|
|
32299
32719
|
try {
|
|
32300
32720
|
const definitions = await readTargetDefinitions(targetsPath);
|
|
@@ -32307,16 +32727,16 @@ async function discoverDefaultTarget(repoRoot) {
|
|
|
32307
32727
|
return null;
|
|
32308
32728
|
}
|
|
32309
32729
|
async function loadEnvHierarchy(repoRoot, startPath) {
|
|
32310
|
-
const { readFileSync:
|
|
32730
|
+
const { readFileSync: readFileSync5 } = await import("node:fs");
|
|
32311
32731
|
const chain = buildDirectoryChain(startPath, repoRoot);
|
|
32312
32732
|
const envFiles = [];
|
|
32313
32733
|
for (const dir of chain) {
|
|
32314
|
-
const envPath =
|
|
32734
|
+
const envPath = path46.join(dir, ".env");
|
|
32315
32735
|
if (existsSync6(envPath)) envFiles.push(envPath);
|
|
32316
32736
|
}
|
|
32317
32737
|
for (let i = 0; i < envFiles.length; i++) {
|
|
32318
32738
|
try {
|
|
32319
|
-
const content =
|
|
32739
|
+
const content = readFileSync5(envFiles[i], "utf8");
|
|
32320
32740
|
for (const line of content.split("\n")) {
|
|
32321
32741
|
const trimmed = line.trim();
|
|
32322
32742
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
@@ -32385,12 +32805,12 @@ var CONFIG_FILE_NAMES = [
|
|
|
32385
32805
|
".agentv/config.js"
|
|
32386
32806
|
];
|
|
32387
32807
|
async function loadTsConfig(projectRoot) {
|
|
32388
|
-
const { existsSync:
|
|
32808
|
+
const { existsSync: existsSync9 } = await import("node:fs");
|
|
32389
32809
|
const { pathToFileURL: pathToFileURL2 } = await import("node:url");
|
|
32390
32810
|
const { join: join2 } = await import("node:path");
|
|
32391
32811
|
for (const fileName of CONFIG_FILE_NAMES) {
|
|
32392
32812
|
const filePath = join2(projectRoot, fileName);
|
|
32393
|
-
if (!
|
|
32813
|
+
if (!existsSync9(filePath)) {
|
|
32394
32814
|
continue;
|
|
32395
32815
|
}
|
|
32396
32816
|
try {
|
|
@@ -32498,7 +32918,7 @@ async function scanRepoDeps(evalFilePaths) {
|
|
|
32498
32918
|
try {
|
|
32499
32919
|
const repos = await extractReposFromEvalFile(filePath);
|
|
32500
32920
|
for (const repo of repos) {
|
|
32501
|
-
if (repo.source.type !== "git") continue;
|
|
32921
|
+
if (!repo.source || repo.source.type !== "git") continue;
|
|
32502
32922
|
const ref = repo.checkout?.ref;
|
|
32503
32923
|
const key = `${normalizeGitUrl(repo.source.url)}\0${ref ?? ""}`;
|
|
32504
32924
|
const existing = seen.get(key);
|
|
@@ -32526,11 +32946,11 @@ async function scanRepoDeps(evalFilePaths) {
|
|
|
32526
32946
|
return { repos: [...seen.values()], errors };
|
|
32527
32947
|
}
|
|
32528
32948
|
async function extractReposFromEvalFile(filePath) {
|
|
32529
|
-
const content = await
|
|
32530
|
-
const parsed = interpolateEnv(
|
|
32949
|
+
const content = await readFile15(filePath, "utf8");
|
|
32950
|
+
const parsed = interpolateEnv(parse6(content), process.env);
|
|
32531
32951
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
32532
32952
|
const obj = parsed;
|
|
32533
|
-
const evalFileDir =
|
|
32953
|
+
const evalFileDir = path47.dirname(path47.resolve(filePath));
|
|
32534
32954
|
const repos = [];
|
|
32535
32955
|
const suiteRepos = await extractReposFromWorkspaceRaw(obj.workspace, evalFileDir);
|
|
32536
32956
|
repos.push(...suiteRepos);
|
|
@@ -32546,9 +32966,9 @@ async function extractReposFromEvalFile(filePath) {
|
|
|
32546
32966
|
}
|
|
32547
32967
|
async function extractReposFromWorkspaceRaw(raw, evalFileDir) {
|
|
32548
32968
|
if (typeof raw === "string") {
|
|
32549
|
-
const workspaceFilePath =
|
|
32550
|
-
const content = await
|
|
32551
|
-
const parsed = interpolateEnv(
|
|
32969
|
+
const workspaceFilePath = path47.resolve(evalFileDir, raw);
|
|
32970
|
+
const content = await readFile15(workspaceFilePath, "utf8");
|
|
32971
|
+
const parsed = interpolateEnv(parse6(content), process.env);
|
|
32552
32972
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return [];
|
|
32553
32973
|
return extractReposFromObject(parsed);
|
|
32554
32974
|
}
|
|
@@ -32582,7 +33002,7 @@ var ResponseCache = class {
|
|
|
32582
33002
|
async get(key) {
|
|
32583
33003
|
const filePath = this.keyToPath(key);
|
|
32584
33004
|
try {
|
|
32585
|
-
const data = await
|
|
33005
|
+
const data = await readFile16(filePath, "utf8");
|
|
32586
33006
|
return JSON.parse(data);
|
|
32587
33007
|
} catch {
|
|
32588
33008
|
return void 0;
|
|
@@ -32590,13 +33010,13 @@ var ResponseCache = class {
|
|
|
32590
33010
|
}
|
|
32591
33011
|
async set(key, value) {
|
|
32592
33012
|
const filePath = this.keyToPath(key);
|
|
32593
|
-
const dir =
|
|
33013
|
+
const dir = path48.dirname(filePath);
|
|
32594
33014
|
await mkdir15(dir, { recursive: true });
|
|
32595
33015
|
await writeFile8(filePath, JSON.stringify(value, null, 2), "utf8");
|
|
32596
33016
|
}
|
|
32597
33017
|
keyToPath(key) {
|
|
32598
33018
|
const prefix = key.slice(0, 2);
|
|
32599
|
-
return
|
|
33019
|
+
return path48.join(this.cachePath, prefix, `${key}.json`);
|
|
32600
33020
|
}
|
|
32601
33021
|
};
|
|
32602
33022
|
function shouldEnableCache(params) {
|
|
@@ -32610,16 +33030,289 @@ function shouldSkipCacheForTemperature(targetConfig) {
|
|
|
32610
33030
|
}
|
|
32611
33031
|
return false;
|
|
32612
33032
|
}
|
|
33033
|
+
var execFileAsync3 = promisify7(execFile3);
|
|
33034
|
+
function sanitizeRepoSlug(repo) {
|
|
33035
|
+
return repo.trim().replace(/[^A-Za-z0-9._-]+/g, "-");
|
|
33036
|
+
}
|
|
33037
|
+
function withFriendlyGitHubAuthError(error) {
|
|
33038
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
33039
|
+
const lower = message.toLowerCase();
|
|
33040
|
+
if (lower.includes("authentication failed") || lower.includes("could not read username") || lower.includes("permission denied") || lower.includes("not logged into any github hosts")) {
|
|
33041
|
+
return new Error(`${message}. Run 'gh auth login' to authenticate.`);
|
|
33042
|
+
}
|
|
33043
|
+
return new Error(message);
|
|
33044
|
+
}
|
|
33045
|
+
function normalizeResultsExportConfig(config) {
|
|
33046
|
+
return {
|
|
33047
|
+
repo: config.repo.trim(),
|
|
33048
|
+
path: config.path.trim().replace(/^\/+|\/+$/g, ""),
|
|
33049
|
+
auto_push: config.auto_push === true,
|
|
33050
|
+
branch_prefix: config.branch_prefix?.trim() || "eval-results"
|
|
33051
|
+
};
|
|
33052
|
+
}
|
|
33053
|
+
function resolveResultsRepoUrl(repo) {
|
|
33054
|
+
if (repo.includes("://") || repo.startsWith("git@")) {
|
|
33055
|
+
return repo;
|
|
33056
|
+
}
|
|
33057
|
+
return `https://github.com/${repo}.git`;
|
|
33058
|
+
}
|
|
33059
|
+
function getResultsRepoCachePaths(repo) {
|
|
33060
|
+
const rootDir = path49.join(getAgentvHome(), "cache", "results-repo", sanitizeRepoSlug(repo));
|
|
33061
|
+
return {
|
|
33062
|
+
rootDir,
|
|
33063
|
+
repoDir: path49.join(rootDir, "repo"),
|
|
33064
|
+
statusFile: path49.join(rootDir, "status.json")
|
|
33065
|
+
};
|
|
33066
|
+
}
|
|
33067
|
+
function readPersistedStatus(statusFile) {
|
|
33068
|
+
if (!existsSync7(statusFile)) {
|
|
33069
|
+
return {};
|
|
33070
|
+
}
|
|
33071
|
+
try {
|
|
33072
|
+
return JSON.parse(readFileSync3(statusFile, "utf8"));
|
|
33073
|
+
} catch {
|
|
33074
|
+
return {};
|
|
33075
|
+
}
|
|
33076
|
+
}
|
|
33077
|
+
function writePersistedStatus(statusFile, status) {
|
|
33078
|
+
mkdirSync2(path49.dirname(statusFile), { recursive: true });
|
|
33079
|
+
writeFileSync(statusFile, `${JSON.stringify(status, null, 2)}
|
|
33080
|
+
`, "utf8");
|
|
33081
|
+
}
|
|
33082
|
+
async function runCommand(executable, args, options) {
|
|
33083
|
+
try {
|
|
33084
|
+
const { stdout, stderr } = await execFileAsync3(executable, [...args], {
|
|
33085
|
+
cwd: options?.cwd,
|
|
33086
|
+
env: process.env
|
|
33087
|
+
});
|
|
33088
|
+
return { stdout, stderr };
|
|
33089
|
+
} catch (error) {
|
|
33090
|
+
if (options?.check === false && error && typeof error === "object") {
|
|
33091
|
+
const execError = error;
|
|
33092
|
+
return {
|
|
33093
|
+
stdout: execError.stdout ?? "",
|
|
33094
|
+
stderr: execError.stderr ?? ""
|
|
33095
|
+
};
|
|
33096
|
+
}
|
|
33097
|
+
throw withFriendlyGitHubAuthError(error);
|
|
33098
|
+
}
|
|
33099
|
+
}
|
|
33100
|
+
async function runGit(args, options) {
|
|
33101
|
+
return runCommand("git", args, options);
|
|
33102
|
+
}
|
|
33103
|
+
async function runGh(args, options) {
|
|
33104
|
+
return runCommand("gh", args, options);
|
|
33105
|
+
}
|
|
33106
|
+
async function resolveDefaultBranch(repoDir) {
|
|
33107
|
+
try {
|
|
33108
|
+
const { stdout } = await runGit(["symbolic-ref", "refs/remotes/origin/HEAD"], { cwd: repoDir });
|
|
33109
|
+
const ref = stdout.trim();
|
|
33110
|
+
const prefix = "refs/remotes/origin/";
|
|
33111
|
+
if (ref.startsWith(prefix)) {
|
|
33112
|
+
return ref.slice(prefix.length);
|
|
33113
|
+
}
|
|
33114
|
+
} catch {
|
|
33115
|
+
}
|
|
33116
|
+
for (const candidate of ["main", "master"]) {
|
|
33117
|
+
try {
|
|
33118
|
+
await runGit(["rev-parse", "--verify", `origin/${candidate}`], { cwd: repoDir });
|
|
33119
|
+
return candidate;
|
|
33120
|
+
} catch {
|
|
33121
|
+
}
|
|
33122
|
+
}
|
|
33123
|
+
return "main";
|
|
33124
|
+
}
|
|
33125
|
+
async function updateCacheRepo(repoDir, baseBranch) {
|
|
33126
|
+
await runGit(["fetch", "origin", "--prune"], { cwd: repoDir });
|
|
33127
|
+
await runGit(["checkout", baseBranch], { cwd: repoDir });
|
|
33128
|
+
await runGit(["pull", "--ff-only", "origin", baseBranch], { cwd: repoDir });
|
|
33129
|
+
}
|
|
33130
|
+
function updateStatusFile(config, patch) {
|
|
33131
|
+
const cachePaths = getResultsRepoCachePaths(config.repo);
|
|
33132
|
+
const current = readPersistedStatus(cachePaths.statusFile);
|
|
33133
|
+
writePersistedStatus(cachePaths.statusFile, {
|
|
33134
|
+
...current,
|
|
33135
|
+
...patch
|
|
33136
|
+
});
|
|
33137
|
+
}
|
|
33138
|
+
async function ensureResultsRepoClone(config) {
|
|
33139
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33140
|
+
const cachePaths = getResultsRepoCachePaths(normalized.repo);
|
|
33141
|
+
mkdirSync2(cachePaths.rootDir, { recursive: true });
|
|
33142
|
+
if (!existsSync7(cachePaths.repoDir)) {
|
|
33143
|
+
try {
|
|
33144
|
+
await runGit([
|
|
33145
|
+
"clone",
|
|
33146
|
+
"--filter=blob:none",
|
|
33147
|
+
resolveResultsRepoUrl(normalized.repo),
|
|
33148
|
+
cachePaths.repoDir
|
|
33149
|
+
]);
|
|
33150
|
+
return cachePaths.repoDir;
|
|
33151
|
+
} catch (error) {
|
|
33152
|
+
updateStatusFile(normalized, { last_error: withFriendlyGitHubAuthError(error).message });
|
|
33153
|
+
throw withFriendlyGitHubAuthError(error);
|
|
33154
|
+
}
|
|
33155
|
+
}
|
|
33156
|
+
if (!existsSync7(path49.join(cachePaths.repoDir, ".git"))) {
|
|
33157
|
+
throw new Error(`Results repo cache is not a git repository: ${cachePaths.repoDir}`);
|
|
33158
|
+
}
|
|
33159
|
+
return cachePaths.repoDir;
|
|
33160
|
+
}
|
|
33161
|
+
function getResultsRepoStatus(config) {
|
|
33162
|
+
if (!config) {
|
|
33163
|
+
return {
|
|
33164
|
+
configured: false,
|
|
33165
|
+
available: false,
|
|
33166
|
+
repo: "",
|
|
33167
|
+
cache_dir: ""
|
|
33168
|
+
};
|
|
33169
|
+
}
|
|
33170
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33171
|
+
const cachePaths = getResultsRepoCachePaths(normalized.repo);
|
|
33172
|
+
const persisted = readPersistedStatus(cachePaths.statusFile);
|
|
33173
|
+
return {
|
|
33174
|
+
configured: true,
|
|
33175
|
+
available: existsSync7(cachePaths.repoDir),
|
|
33176
|
+
repo: normalized.repo,
|
|
33177
|
+
path: normalized.path,
|
|
33178
|
+
auto_push: normalized.auto_push,
|
|
33179
|
+
branch_prefix: normalized.branch_prefix,
|
|
33180
|
+
cache_dir: cachePaths.repoDir,
|
|
33181
|
+
last_synced_at: persisted.last_synced_at,
|
|
33182
|
+
last_error: persisted.last_error
|
|
33183
|
+
};
|
|
33184
|
+
}
|
|
33185
|
+
async function syncResultsRepo(config) {
|
|
33186
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33187
|
+
try {
|
|
33188
|
+
const repoDir = await ensureResultsRepoClone(normalized);
|
|
33189
|
+
const baseBranch = await resolveDefaultBranch(repoDir);
|
|
33190
|
+
await updateCacheRepo(repoDir, baseBranch);
|
|
33191
|
+
updateStatusFile(normalized, {
|
|
33192
|
+
last_synced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33193
|
+
last_error: void 0
|
|
33194
|
+
});
|
|
33195
|
+
} catch (error) {
|
|
33196
|
+
updateStatusFile(normalized, {
|
|
33197
|
+
last_error: withFriendlyGitHubAuthError(error).message
|
|
33198
|
+
});
|
|
33199
|
+
throw withFriendlyGitHubAuthError(error);
|
|
33200
|
+
}
|
|
33201
|
+
return getResultsRepoStatus(normalized);
|
|
33202
|
+
}
|
|
33203
|
+
async function checkoutResultsRepoBranch(config, branchName) {
|
|
33204
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33205
|
+
const repoDir = await ensureResultsRepoClone(normalized);
|
|
33206
|
+
const baseBranch = await resolveDefaultBranch(repoDir);
|
|
33207
|
+
await updateCacheRepo(repoDir, baseBranch);
|
|
33208
|
+
await runGit(["checkout", "-B", branchName, `origin/${baseBranch}`], { cwd: repoDir });
|
|
33209
|
+
updateStatusFile(normalized, { last_error: void 0 });
|
|
33210
|
+
return {
|
|
33211
|
+
branchName,
|
|
33212
|
+
baseBranch,
|
|
33213
|
+
repoDir
|
|
33214
|
+
};
|
|
33215
|
+
}
|
|
33216
|
+
async function prepareResultsRepoBranch(config, branchName) {
|
|
33217
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33218
|
+
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
33219
|
+
const baseBranch = await resolveDefaultBranch(cloneDir);
|
|
33220
|
+
await updateCacheRepo(cloneDir, baseBranch);
|
|
33221
|
+
const worktreeRoot = await mkdtemp3(path49.join(os3.tmpdir(), "agentv-results-repo-"));
|
|
33222
|
+
const worktreeDir = path49.join(worktreeRoot, "repo");
|
|
33223
|
+
await runGit(["worktree", "add", "-B", branchName, worktreeDir, `origin/${baseBranch}`], {
|
|
33224
|
+
cwd: cloneDir
|
|
33225
|
+
});
|
|
33226
|
+
return {
|
|
33227
|
+
branchName,
|
|
33228
|
+
baseBranch,
|
|
33229
|
+
repoDir: worktreeDir,
|
|
33230
|
+
cleanup: async () => {
|
|
33231
|
+
try {
|
|
33232
|
+
await runGit(["worktree", "remove", "--force", worktreeDir], { cwd: cloneDir });
|
|
33233
|
+
} finally {
|
|
33234
|
+
await rm6(worktreeRoot, { recursive: true, force: true }).catch(() => void 0);
|
|
33235
|
+
}
|
|
33236
|
+
}
|
|
33237
|
+
};
|
|
33238
|
+
}
|
|
33239
|
+
async function stageResultsArtifacts(params) {
|
|
33240
|
+
rmSync(params.destinationDir, { recursive: true, force: true });
|
|
33241
|
+
mkdirSync2(path49.dirname(params.destinationDir), { recursive: true });
|
|
33242
|
+
await cp3(params.sourceDir, params.destinationDir, { recursive: true });
|
|
33243
|
+
}
|
|
33244
|
+
function resolveResultsRepoRunsDir(config) {
|
|
33245
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33246
|
+
return path49.join(
|
|
33247
|
+
getResultsRepoCachePaths(normalized.repo).repoDir,
|
|
33248
|
+
...normalized.path.split("/")
|
|
33249
|
+
);
|
|
33250
|
+
}
|
|
33251
|
+
async function directorySizeBytes(targetPath) {
|
|
33252
|
+
const entry = await stat9(targetPath);
|
|
33253
|
+
if (entry.isFile()) {
|
|
33254
|
+
return entry.size;
|
|
33255
|
+
}
|
|
33256
|
+
let total = 0;
|
|
33257
|
+
for (const child of await readdir8(targetPath, { withFileTypes: true })) {
|
|
33258
|
+
total += await directorySizeBytes(path49.join(targetPath, child.name));
|
|
33259
|
+
}
|
|
33260
|
+
return total;
|
|
33261
|
+
}
|
|
33262
|
+
async function commitAndPushResultsBranch(params) {
|
|
33263
|
+
await runGit(["add", "--all"], { cwd: params.repoDir });
|
|
33264
|
+
const { stdout: diffStdout } = await runGit(["status", "--porcelain"], {
|
|
33265
|
+
cwd: params.repoDir,
|
|
33266
|
+
check: false
|
|
33267
|
+
});
|
|
33268
|
+
if (diffStdout.trim().length === 0) {
|
|
33269
|
+
return false;
|
|
33270
|
+
}
|
|
33271
|
+
await runGit(["commit", "-m", params.commitMessage], { cwd: params.repoDir });
|
|
33272
|
+
await runGit(["push", "-u", "origin", params.branchName], { cwd: params.repoDir });
|
|
33273
|
+
return true;
|
|
33274
|
+
}
|
|
33275
|
+
async function pushResultsRepoBranch(config, branchName, cwd) {
|
|
33276
|
+
const normalized = normalizeResultsExportConfig(config);
|
|
33277
|
+
await runGit(["push", "-u", "origin", branchName], {
|
|
33278
|
+
cwd: cwd ?? getResultsRepoCachePaths(normalized.repo).repoDir
|
|
33279
|
+
});
|
|
33280
|
+
updateStatusFile(normalized, {
|
|
33281
|
+
last_synced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
33282
|
+
last_error: void 0
|
|
33283
|
+
});
|
|
33284
|
+
}
|
|
33285
|
+
async function createDraftResultsPr(params) {
|
|
33286
|
+
const { stdout } = await runGh(
|
|
33287
|
+
[
|
|
33288
|
+
"pr",
|
|
33289
|
+
"create",
|
|
33290
|
+
"--draft",
|
|
33291
|
+
"--repo",
|
|
33292
|
+
params.repo,
|
|
33293
|
+
"--base",
|
|
33294
|
+
params.baseBranch,
|
|
33295
|
+
"--head",
|
|
33296
|
+
params.branchName,
|
|
33297
|
+
"--title",
|
|
33298
|
+
params.title,
|
|
33299
|
+
"--body",
|
|
33300
|
+
params.body
|
|
33301
|
+
],
|
|
33302
|
+
{ cwd: params.repoDir }
|
|
33303
|
+
);
|
|
33304
|
+
return stdout.trim();
|
|
33305
|
+
}
|
|
32613
33306
|
function getProjectsRegistryPath() {
|
|
32614
|
-
return
|
|
33307
|
+
return path50.join(getAgentvHome(), "projects.yaml");
|
|
32615
33308
|
}
|
|
32616
33309
|
function loadProjectRegistry() {
|
|
32617
33310
|
const registryPath = getProjectsRegistryPath();
|
|
32618
|
-
if (!
|
|
33311
|
+
if (!existsSync8(registryPath)) {
|
|
32619
33312
|
return { projects: [] };
|
|
32620
33313
|
}
|
|
32621
33314
|
try {
|
|
32622
|
-
const raw =
|
|
33315
|
+
const raw = readFileSync4(registryPath, "utf-8");
|
|
32623
33316
|
const parsed = parseYaml3(raw);
|
|
32624
33317
|
if (!parsed || !Array.isArray(parsed.projects)) {
|
|
32625
33318
|
return { projects: [] };
|
|
@@ -32631,14 +33324,14 @@ function loadProjectRegistry() {
|
|
|
32631
33324
|
}
|
|
32632
33325
|
function saveProjectRegistry(registry) {
|
|
32633
33326
|
const registryPath = getProjectsRegistryPath();
|
|
32634
|
-
const dir =
|
|
32635
|
-
if (!
|
|
32636
|
-
|
|
33327
|
+
const dir = path50.dirname(registryPath);
|
|
33328
|
+
if (!existsSync8(dir)) {
|
|
33329
|
+
mkdirSync3(dir, { recursive: true });
|
|
32637
33330
|
}
|
|
32638
|
-
|
|
33331
|
+
writeFileSync2(registryPath, stringifyYaml(registry), "utf-8");
|
|
32639
33332
|
}
|
|
32640
33333
|
function deriveProjectId(dirPath, existingIds) {
|
|
32641
|
-
const base =
|
|
33334
|
+
const base = path50.basename(dirPath).toLowerCase().replace(/[^a-z0-9-]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
32642
33335
|
let candidate = base || "project";
|
|
32643
33336
|
let suffix = 2;
|
|
32644
33337
|
while (existingIds.includes(candidate)) {
|
|
@@ -32648,11 +33341,11 @@ function deriveProjectId(dirPath, existingIds) {
|
|
|
32648
33341
|
return candidate;
|
|
32649
33342
|
}
|
|
32650
33343
|
function addProject(projectPath) {
|
|
32651
|
-
const absPath =
|
|
32652
|
-
if (!
|
|
33344
|
+
const absPath = path50.resolve(projectPath);
|
|
33345
|
+
if (!existsSync8(absPath)) {
|
|
32653
33346
|
throw new Error(`Directory not found: ${absPath}`);
|
|
32654
33347
|
}
|
|
32655
|
-
if (!
|
|
33348
|
+
if (!existsSync8(path50.join(absPath, ".agentv"))) {
|
|
32656
33349
|
throw new Error(`No .agentv/ directory found in ${absPath}. Run an evaluation first.`);
|
|
32657
33350
|
}
|
|
32658
33351
|
const registry = loadProjectRegistry();
|
|
@@ -32666,7 +33359,7 @@ function addProject(projectPath) {
|
|
|
32666
33359
|
absPath,
|
|
32667
33360
|
registry.projects.map((p) => p.id)
|
|
32668
33361
|
),
|
|
32669
|
-
name:
|
|
33362
|
+
name: path50.basename(absPath),
|
|
32670
33363
|
path: absPath,
|
|
32671
33364
|
addedAt: now2,
|
|
32672
33365
|
lastOpenedAt: now2
|
|
@@ -32695,14 +33388,14 @@ function touchProject(projectId) {
|
|
|
32695
33388
|
}
|
|
32696
33389
|
}
|
|
32697
33390
|
function discoverProjects(rootDir, maxDepth = 2) {
|
|
32698
|
-
const absRoot =
|
|
32699
|
-
if (!
|
|
33391
|
+
const absRoot = path50.resolve(rootDir);
|
|
33392
|
+
if (!existsSync8(absRoot) || !statSync2(absRoot).isDirectory()) {
|
|
32700
33393
|
return [];
|
|
32701
33394
|
}
|
|
32702
33395
|
const results = [];
|
|
32703
33396
|
function scan(dir, depth) {
|
|
32704
33397
|
if (depth > maxDepth) return;
|
|
32705
|
-
if (
|
|
33398
|
+
if (existsSync8(path50.join(dir, ".agentv"))) {
|
|
32706
33399
|
results.push(dir);
|
|
32707
33400
|
return;
|
|
32708
33401
|
}
|
|
@@ -32712,7 +33405,7 @@ function discoverProjects(rootDir, maxDepth = 2) {
|
|
|
32712
33405
|
for (const entry of entries) {
|
|
32713
33406
|
if (!entry.isDirectory()) continue;
|
|
32714
33407
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
32715
|
-
scan(
|
|
33408
|
+
scan(path50.join(dir, entry.name), depth + 1);
|
|
32716
33409
|
}
|
|
32717
33410
|
} catch {
|
|
32718
33411
|
}
|
|
@@ -32816,8 +33509,8 @@ var OtelTraceExporter = class {
|
|
|
32816
33509
|
async init() {
|
|
32817
33510
|
try {
|
|
32818
33511
|
const [sdkTraceNode, resourcesMod, semconvMod, api, coreMod] = await Promise.all([
|
|
32819
|
-
import("./src-
|
|
32820
|
-
import("./esm-
|
|
33512
|
+
import("./src-PXDA7QIS.js"),
|
|
33513
|
+
import("./esm-UYZ3HJBU.js"),
|
|
32821
33514
|
import("./esm-RVQPUGWH.js"),
|
|
32822
33515
|
import("./esm-R77SNOF5.js"),
|
|
32823
33516
|
import("./esm-ZADQ4XQH-5LX2IKZV.js").catch(() => null)
|
|
@@ -33611,30 +34304,30 @@ function extractResponseItemContent(content) {
|
|
|
33611
34304
|
}
|
|
33612
34305
|
return parts.length > 0 ? parts.join("") : void 0;
|
|
33613
34306
|
}
|
|
33614
|
-
var DEFAULT_SESSIONS_DIR = () =>
|
|
34307
|
+
var DEFAULT_SESSIONS_DIR = () => path51.join(homedir3(), ".codex", "sessions");
|
|
33615
34308
|
async function discoverCodexSessions(opts) {
|
|
33616
34309
|
const sessionsDir = opts?.sessionsDir ?? DEFAULT_SESSIONS_DIR();
|
|
33617
34310
|
const limit = opts?.latest ? 1 : opts?.limit ?? 10;
|
|
33618
34311
|
const sessions = [];
|
|
33619
34312
|
let yearDirs;
|
|
33620
34313
|
try {
|
|
33621
|
-
yearDirs = await
|
|
34314
|
+
yearDirs = await readdir9(sessionsDir);
|
|
33622
34315
|
} catch {
|
|
33623
34316
|
return [];
|
|
33624
34317
|
}
|
|
33625
34318
|
for (const year of yearDirs) {
|
|
33626
|
-
const yearPath =
|
|
34319
|
+
const yearPath = path51.join(sessionsDir, year);
|
|
33627
34320
|
let monthDirs;
|
|
33628
34321
|
try {
|
|
33629
|
-
monthDirs = await
|
|
34322
|
+
monthDirs = await readdir9(yearPath);
|
|
33630
34323
|
} catch {
|
|
33631
34324
|
continue;
|
|
33632
34325
|
}
|
|
33633
34326
|
for (const month of monthDirs) {
|
|
33634
|
-
const monthPath =
|
|
34327
|
+
const monthPath = path51.join(yearPath, month);
|
|
33635
34328
|
let dayDirs;
|
|
33636
34329
|
try {
|
|
33637
|
-
dayDirs = await
|
|
34330
|
+
dayDirs = await readdir9(monthPath);
|
|
33638
34331
|
} catch {
|
|
33639
34332
|
continue;
|
|
33640
34333
|
}
|
|
@@ -33643,22 +34336,22 @@ async function discoverCodexSessions(opts) {
|
|
|
33643
34336
|
const dirDate = `${year}-${month}-${day}`;
|
|
33644
34337
|
if (dirDate !== opts.date) continue;
|
|
33645
34338
|
}
|
|
33646
|
-
const dayPath =
|
|
34339
|
+
const dayPath = path51.join(monthPath, day);
|
|
33647
34340
|
let files;
|
|
33648
34341
|
try {
|
|
33649
|
-
files = await
|
|
34342
|
+
files = await readdir9(dayPath);
|
|
33650
34343
|
} catch {
|
|
33651
34344
|
continue;
|
|
33652
34345
|
}
|
|
33653
34346
|
for (const file of files) {
|
|
33654
34347
|
if (!file.startsWith("rollout-") || !file.endsWith(".jsonl")) continue;
|
|
33655
|
-
const filePath =
|
|
34348
|
+
const filePath = path51.join(dayPath, file);
|
|
33656
34349
|
const nameWithoutExt = file.replace(/\.jsonl$/, "");
|
|
33657
34350
|
const parts = nameWithoutExt.split("-");
|
|
33658
34351
|
const sessionId = parts.length >= 6 ? parts.slice(-5).join("-") : nameWithoutExt;
|
|
33659
34352
|
let updatedAt;
|
|
33660
34353
|
try {
|
|
33661
|
-
const fileStat = await
|
|
34354
|
+
const fileStat = await stat10(filePath);
|
|
33662
34355
|
updatedAt = fileStat.mtime;
|
|
33663
34356
|
} catch {
|
|
33664
34357
|
updatedAt = /* @__PURE__ */ new Date(0);
|
|
@@ -33671,7 +34364,7 @@ async function discoverCodexSessions(opts) {
|
|
|
33671
34364
|
sessions.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime());
|
|
33672
34365
|
return sessions.slice(0, limit);
|
|
33673
34366
|
}
|
|
33674
|
-
var DEFAULT_PROJECTS_DIR = () =>
|
|
34367
|
+
var DEFAULT_PROJECTS_DIR = () => path52.join(homedir4(), ".claude", "projects");
|
|
33675
34368
|
function encodeProjectPath(projectPath) {
|
|
33676
34369
|
return projectPath.replace(/\//g, "-");
|
|
33677
34370
|
}
|
|
@@ -33680,7 +34373,7 @@ async function discoverClaudeSessions(opts) {
|
|
|
33680
34373
|
const limit = opts?.latest ? 1 : opts?.limit ?? 10;
|
|
33681
34374
|
let projectDirs;
|
|
33682
34375
|
try {
|
|
33683
|
-
projectDirs = await
|
|
34376
|
+
projectDirs = await readdir10(projectsDir);
|
|
33684
34377
|
} catch {
|
|
33685
34378
|
return [];
|
|
33686
34379
|
}
|
|
@@ -33690,10 +34383,10 @@ async function discoverClaudeSessions(opts) {
|
|
|
33690
34383
|
}
|
|
33691
34384
|
const sessions = [];
|
|
33692
34385
|
for (const projectDir of projectDirs) {
|
|
33693
|
-
const dirPath =
|
|
34386
|
+
const dirPath = path52.join(projectsDir, projectDir);
|
|
33694
34387
|
let entries;
|
|
33695
34388
|
try {
|
|
33696
|
-
entries = await
|
|
34389
|
+
entries = await readdir10(dirPath);
|
|
33697
34390
|
} catch {
|
|
33698
34391
|
continue;
|
|
33699
34392
|
}
|
|
@@ -33701,10 +34394,10 @@ async function discoverClaudeSessions(opts) {
|
|
|
33701
34394
|
if (!entry.endsWith(".jsonl")) continue;
|
|
33702
34395
|
const sessionId = entry.replace(/\.jsonl$/, "");
|
|
33703
34396
|
if (opts?.sessionId && sessionId !== opts.sessionId) continue;
|
|
33704
|
-
const filePath =
|
|
34397
|
+
const filePath = path52.join(dirPath, entry);
|
|
33705
34398
|
let updatedAt;
|
|
33706
34399
|
try {
|
|
33707
|
-
const fileStat = await
|
|
34400
|
+
const fileStat = await stat11(filePath);
|
|
33708
34401
|
updatedAt = fileStat.mtime;
|
|
33709
34402
|
} catch {
|
|
33710
34403
|
updatedAt = /* @__PURE__ */ new Date(0);
|
|
@@ -33745,11 +34438,11 @@ function toTranscriptJsonLine(entry) {
|
|
|
33745
34438
|
};
|
|
33746
34439
|
}
|
|
33747
34440
|
async function readTranscriptJsonl(filePath) {
|
|
33748
|
-
const text2 = await
|
|
34441
|
+
const text2 = await readFile17(filePath, "utf8");
|
|
33749
34442
|
return text2.split("\n").filter((line) => line.trim().length > 0).map((line) => JSON.parse(line));
|
|
33750
34443
|
}
|
|
33751
34444
|
async function readTranscriptFile(filePath) {
|
|
33752
|
-
return
|
|
34445
|
+
return readFile17(filePath, "utf8");
|
|
33753
34446
|
}
|
|
33754
34447
|
var TranscriptProvider = class _TranscriptProvider {
|
|
33755
34448
|
id;
|
|
@@ -33956,6 +34649,20 @@ export {
|
|
|
33956
34649
|
ResponseCache,
|
|
33957
34650
|
shouldEnableCache,
|
|
33958
34651
|
shouldSkipCacheForTemperature,
|
|
34652
|
+
normalizeResultsExportConfig,
|
|
34653
|
+
resolveResultsRepoUrl,
|
|
34654
|
+
getResultsRepoCachePaths,
|
|
34655
|
+
ensureResultsRepoClone,
|
|
34656
|
+
getResultsRepoStatus,
|
|
34657
|
+
syncResultsRepo,
|
|
34658
|
+
checkoutResultsRepoBranch,
|
|
34659
|
+
prepareResultsRepoBranch,
|
|
34660
|
+
stageResultsArtifacts,
|
|
34661
|
+
resolveResultsRepoRunsDir,
|
|
34662
|
+
directorySizeBytes,
|
|
34663
|
+
commitAndPushResultsBranch,
|
|
34664
|
+
pushResultsRepoBranch,
|
|
34665
|
+
createDraftResultsPr,
|
|
33959
34666
|
getProjectsRegistryPath,
|
|
33960
34667
|
loadProjectRegistry,
|
|
33961
34668
|
saveProjectRegistry,
|
|
@@ -33981,4 +34688,4 @@ export {
|
|
|
33981
34688
|
TranscriptProvider,
|
|
33982
34689
|
createAgentKernel
|
|
33983
34690
|
};
|
|
33984
|
-
//# sourceMappingURL=chunk-
|
|
34691
|
+
//# sourceMappingURL=chunk-FQGY6QXQ.js.map
|