@hawkeyexl/manni 4.5.0 → 4.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/batch-OZOACN4G.js +41 -0
- package/dist/build-Z5VOVADH.js +32 -0
- package/dist/calibrate-TZYT42DR.js +36 -0
- package/dist/capture-SL4MNDDP.js +34 -0
- package/dist/check-5OMA42Y6.js +26 -0
- package/dist/{check-ESNS7ZBC.js → check-OAJWC3WD.js} +7 -5
- package/dist/{check-RQIIIURI.js → check-PKY7KTQE.js} +13 -14
- package/dist/check-WB2QDHJQ.js +880 -0
- package/dist/check-WB2QDHJQ.js.map +1 -0
- package/dist/{chunk-F2OVHJNC.js → chunk-2JFPBZJK.js} +11 -11
- package/dist/{chunk-CP4ZM43V.js → chunk-2KRB3EG7.js} +2 -2
- package/dist/{chunk-QLUMKWQQ.js → chunk-2M6DVNI5.js} +10 -6
- package/dist/{chunk-QLUMKWQQ.js.map → chunk-2M6DVNI5.js.map} +1 -1
- package/dist/{chunk-HQFVJDEN.js → chunk-3NWX34ZI.js} +19 -19
- package/dist/{chunk-TCC737IS.js → chunk-5G53XATW.js} +4 -4
- package/dist/{chunk-B7XUEQF3.js → chunk-6KKZXB2O.js} +7 -5
- package/dist/{chunk-B7XUEQF3.js.map → chunk-6KKZXB2O.js.map} +1 -1
- package/dist/{chunk-SNEHX4VE.js → chunk-733Y2STG.js} +2 -2
- package/dist/{chunk-ELZ2IIOF.js → chunk-73TC4DYH.js} +2 -2
- package/dist/{chunk-QY3DOZMS.js → chunk-7WRP5FFP.js} +4 -4
- package/dist/{chunk-3R5Y3XOL.js → chunk-APKH2T6E.js} +29 -591
- package/dist/chunk-APKH2T6E.js.map +1 -0
- package/dist/{chunk-FLF6AECQ.js → chunk-B3TS4WRM.js} +17 -15
- package/dist/{chunk-FLF6AECQ.js.map → chunk-B3TS4WRM.js.map} +1 -1
- package/dist/chunk-BWCWTHYS.js +146 -0
- package/dist/chunk-BWCWTHYS.js.map +1 -0
- package/dist/{chunk-C6HZ6C5T.js → chunk-C3B2VJNC.js} +2 -2
- package/dist/{chunk-NN4ASVTY.js → chunk-C73RSUIC.js} +4 -4
- package/dist/chunk-CU57HT2E.js +116 -0
- package/dist/chunk-CU57HT2E.js.map +1 -0
- package/dist/chunk-CWXZ2MRT.js +222 -0
- package/dist/chunk-CWXZ2MRT.js.map +1 -0
- package/dist/{chunk-GAZFKZHM.js → chunk-DC5DDJVE.js} +6 -6
- package/dist/{chunk-52PDIZKV.js → chunk-ECVQNGTH.js} +3 -3
- package/dist/chunk-FLAO4ZS3.js +502 -0
- package/dist/chunk-FLAO4ZS3.js.map +1 -0
- package/dist/chunk-FQGLRJGX.js +174 -0
- package/dist/chunk-FQGLRJGX.js.map +1 -0
- package/dist/{chunk-XWFNO5PO.js → chunk-GAJIJKET.js} +3 -3
- package/dist/{chunk-QCBICNKC.js → chunk-GREQKO6M.js} +202 -54
- package/dist/chunk-GREQKO6M.js.map +1 -0
- package/dist/chunk-HX44GB4Z.js +30 -0
- package/dist/chunk-HX44GB4Z.js.map +1 -0
- package/dist/{chunk-7ZUCEMCY.js → chunk-IY7NYNPZ.js} +9 -11
- package/dist/{chunk-7ZUCEMCY.js.map → chunk-IY7NYNPZ.js.map} +1 -1
- package/dist/{chunk-527HHQPS.js → chunk-KCOFEWV5.js} +6 -4
- package/dist/{chunk-527HHQPS.js.map → chunk-KCOFEWV5.js.map} +1 -1
- package/dist/chunk-M2RLHZ4J.js +849 -0
- package/dist/chunk-M2RLHZ4J.js.map +1 -0
- package/dist/{chunk-K3U5F2QT.js → chunk-M6GSS72T.js} +12 -12
- package/dist/{chunk-UK2I3ACY.js → chunk-MHM4QEBV.js} +4 -4
- package/dist/{chunk-EL2BZTDL.js → chunk-MOWMCEOJ.js} +13 -11
- package/dist/{chunk-EL2BZTDL.js.map → chunk-MOWMCEOJ.js.map} +1 -1
- package/dist/{chunk-OIZ3XMKB.js → chunk-MWBJ3KTF.js} +4 -4
- package/dist/chunk-N7JSEDSC.js +233 -0
- package/dist/chunk-N7JSEDSC.js.map +1 -0
- package/dist/{chunk-MZC62HPZ.js → chunk-NBMPHD4I.js} +2 -2
- package/dist/{chunk-YA63PVWI.js → chunk-NHHOPZRA.js} +12 -10
- package/dist/{chunk-YA63PVWI.js.map → chunk-NHHOPZRA.js.map} +1 -1
- package/dist/{chunk-DX32P2DT.js → chunk-NTWW7J76.js} +2 -2
- package/dist/{chunk-6XTQDFGP.js → chunk-PCFG7E6B.js} +3 -3
- package/dist/{chunk-6XTQDFGP.js.map → chunk-PCFG7E6B.js.map} +1 -1
- package/dist/{chunk-SJO2FTPG.js → chunk-PL4UAF4S.js} +73 -1304
- package/dist/chunk-PL4UAF4S.js.map +1 -0
- package/dist/chunk-QCHVR4ZM.js +286 -0
- package/dist/chunk-QCHVR4ZM.js.map +1 -0
- package/dist/{chunk-JQU7FHYH.js → chunk-QW77LCOU.js} +10 -10
- package/dist/{chunk-IA7BMIPE.js → chunk-SM6NOMJS.js} +2 -2
- package/dist/{chunk-DE7JSWXV.js → chunk-SWSTHZBI.js} +8 -6
- package/dist/{chunk-DE7JSWXV.js.map → chunk-SWSTHZBI.js.map} +1 -1
- package/dist/{chunk-NGK3N6F2.js → chunk-T6SYGLWR.js} +3 -3
- package/dist/chunk-TH4JVQPF.js +133 -0
- package/dist/chunk-TH4JVQPF.js.map +1 -0
- package/dist/{chunk-TVIX653Q.js → chunk-TLXE7LYK.js} +4 -4
- package/dist/{chunk-HVFR66GQ.js → chunk-TUAAR3MM.js} +3280 -283
- package/dist/chunk-TUAAR3MM.js.map +1 -0
- package/dist/{chunk-TA42STP7.js → chunk-UAKNZVCD.js} +4 -4
- package/dist/{chunk-C3KI45RF.js → chunk-UJSGS2Y5.js} +4 -4
- package/dist/{chunk-ZX43SRZR.js → chunk-UWQ2UVHE.js} +8 -6
- package/dist/{chunk-ZX43SRZR.js.map → chunk-UWQ2UVHE.js.map} +1 -1
- package/dist/chunk-WQWKYAAK.js +86 -0
- package/dist/chunk-WQWKYAAK.js.map +1 -0
- package/dist/{chunk-LJ2RDH3F.js → chunk-XG6PVB5S.js} +1 -23
- package/dist/chunk-XG6PVB5S.js.map +1 -0
- package/dist/{chunk-Q35YIDBV.js → chunk-XKBEHKWO.js} +29 -264
- package/dist/chunk-XKBEHKWO.js.map +1 -0
- package/dist/chunk-XPEWUK7J.js +532 -0
- package/dist/chunk-XPEWUK7J.js.map +1 -0
- package/dist/{chunk-BOIF2XGG.js → chunk-XXDXIIQ4.js} +2 -2
- package/dist/chunk-YLK2J5YK.js +610 -0
- package/dist/chunk-YLK2J5YK.js.map +1 -0
- package/dist/{chunk-7OQAU4EA.js → chunk-Z4IZGEM3.js} +16 -12
- package/dist/{chunk-7OQAU4EA.js.map → chunk-Z4IZGEM3.js.map} +1 -1
- package/dist/{cli-E3H6ZP4I.js → cli-6EWVMEN6.js} +5 -3
- package/dist/{cli-E3H6ZP4I.js.map → cli-6EWVMEN6.js.map} +1 -1
- package/dist/{cli-MHTYPVOD.js → cli-6H7HVDMQ.js} +113 -20
- package/dist/{cli-MHTYPVOD.js.map → cli-6H7HVDMQ.js.map} +1 -1
- package/dist/{cli-HUIMNJT6.js → cli-EPVZHAZS.js} +19 -20
- package/dist/{cli-HUIMNJT6.js.map → cli-EPVZHAZS.js.map} +1 -1
- package/dist/{cli-6PGVHP7Y.js → cli-LUCSGTVI.js} +17 -19
- package/dist/{cli-6PGVHP7Y.js.map → cli-LUCSGTVI.js.map} +1 -1
- package/dist/{cli-PRZHQ3QB.js → cli-NIG4N3XM.js} +14 -15
- package/dist/{cli-PRZHQ3QB.js.map → cli-NIG4N3XM.js.map} +1 -1
- package/dist/{cli-TYGOHJOJ.js → cli-PSKNB66T.js} +225 -58
- package/dist/cli-PSKNB66T.js.map +1 -0
- package/dist/{cli-QKIJOXGW.js → cli-UO4NHIB3.js} +17 -17
- package/dist/{cli-AAG3N4NB.js → cli-UQ7A3YA2.js} +17 -18
- package/dist/{cli-AAG3N4NB.js.map → cli-UQ7A3YA2.js.map} +1 -1
- package/dist/{cli-YW6VWBHX.js → cli-YHYDLQAF.js} +10 -10
- package/dist/cli.js +11 -11
- package/dist/config-P4HDNTYD.js +26 -0
- package/dist/conformance-ZFCMKM6M.js +18 -0
- package/dist/docmeta.js +9 -9
- package/dist/github-7ZCH5PSO.js +26 -0
- package/dist/github-POT6724K.js +31 -0
- package/dist/host-6GURH5HE.js +23 -0
- package/dist/index.d.ts +72 -4
- package/dist/index.js +176 -166
- package/dist/index.js.map +1 -1
- package/dist/{json-4DCKDGUV.js → json-KT2K62L5.js} +12 -13
- package/dist/lint-5AQ5C2BV.js +29 -0
- package/dist/list-5JZKX2NK.js +28 -0
- package/dist/list-SWPFXQMU.js +30 -0
- package/dist/{page-QEDB7EVU.js → page-UPSXRP75.js} +11 -12
- package/dist/prepare-6GNEWCNH.js +155 -0
- package/dist/prepare-6GNEWCNH.js.map +1 -0
- package/dist/{pretty-SQX33UM2.js → pretty-Q6FQAOK4.js} +13 -14
- package/dist/{pretty-5DGEBZSV.js → pretty-SQRPFPSQ.js} +12 -13
- package/dist/provider-AMGRDY6V.js +34 -0
- package/dist/release-IECYCLTD.js +36 -0
- package/dist/release-IECYCLTD.js.map +1 -0
- package/dist/{reporters-OHSI3RMV.js → reporters-45MMC2ID.js} +11 -12
- package/dist/{reporters-YELDDYJD.js → reporters-XUV4MM64.js} +7 -5
- package/dist/reporters-XUV4MM64.js.map +1 -0
- package/dist/run-KMWY2IAN.js +32 -0
- package/dist/run-KMWY2IAN.js.map +1 -0
- package/dist/{term-2WMDZ4PY.js → term-PCJUCWNF.js} +16 -17
- package/dist/{term-2WMDZ4PY.js.map → term-PCJUCWNF.js.map} +1 -1
- package/dist/validate-IMNCZNWG.js +17 -0
- package/dist/validate-IMNCZNWG.js.map +1 -0
- package/package.json +2 -2
- package/plugin/manni/.claude-plugin/plugin.json +1 -1
- package/plugin/manni/hooks/hooks.json +47 -0
- package/dist/batch-GOE3UXOD.js +0 -36
- package/dist/build-M7BMEHL2.js +0 -33
- package/dist/calibrate-6RRAILUY.js +0 -31
- package/dist/capture-RJPQ6B4N.js +0 -32
- package/dist/check-BQD64X3N.js +0 -27
- package/dist/chunk-3R5Y3XOL.js.map +0 -1
- package/dist/chunk-3YHAYXBC.js +0 -164
- package/dist/chunk-3YHAYXBC.js.map +0 -1
- package/dist/chunk-HVFR66GQ.js.map +0 -1
- package/dist/chunk-LJ2RDH3F.js.map +0 -1
- package/dist/chunk-NAZRLJAK.js +0 -3090
- package/dist/chunk-NAZRLJAK.js.map +0 -1
- package/dist/chunk-Q35YIDBV.js.map +0 -1
- package/dist/chunk-QCBICNKC.js.map +0 -1
- package/dist/chunk-SJO2FTPG.js.map +0 -1
- package/dist/chunk-TU2WUBVZ.js +0 -2
- package/dist/cli-TYGOHJOJ.js.map +0 -1
- package/dist/github-3WMSTNTK.js +0 -32
- package/dist/github-OXSJQ4PS.js +0 -27
- package/dist/lint-YDHVVZHT.js +0 -30
- package/dist/list-3ZO2NIYO.js +0 -31
- package/dist/list-EJH6VAOW.js +0 -29
- package/dist/run-IP6NZHAG.js +0 -33
- package/dist/validate-G6HH3YXL.js +0 -16
- /package/dist/{batch-GOE3UXOD.js.map → batch-OZOACN4G.js.map} +0 -0
- /package/dist/{build-M7BMEHL2.js.map → build-Z5VOVADH.js.map} +0 -0
- /package/dist/{calibrate-6RRAILUY.js.map → calibrate-TZYT42DR.js.map} +0 -0
- /package/dist/{capture-RJPQ6B4N.js.map → capture-SL4MNDDP.js.map} +0 -0
- /package/dist/{check-BQD64X3N.js.map → check-5OMA42Y6.js.map} +0 -0
- /package/dist/{check-ESNS7ZBC.js.map → check-OAJWC3WD.js.map} +0 -0
- /package/dist/{check-RQIIIURI.js.map → check-PKY7KTQE.js.map} +0 -0
- /package/dist/{chunk-F2OVHJNC.js.map → chunk-2JFPBZJK.js.map} +0 -0
- /package/dist/{chunk-CP4ZM43V.js.map → chunk-2KRB3EG7.js.map} +0 -0
- /package/dist/{chunk-HQFVJDEN.js.map → chunk-3NWX34ZI.js.map} +0 -0
- /package/dist/{chunk-TCC737IS.js.map → chunk-5G53XATW.js.map} +0 -0
- /package/dist/{chunk-SNEHX4VE.js.map → chunk-733Y2STG.js.map} +0 -0
- /package/dist/{chunk-ELZ2IIOF.js.map → chunk-73TC4DYH.js.map} +0 -0
- /package/dist/{chunk-QY3DOZMS.js.map → chunk-7WRP5FFP.js.map} +0 -0
- /package/dist/{chunk-C6HZ6C5T.js.map → chunk-C3B2VJNC.js.map} +0 -0
- /package/dist/{chunk-NN4ASVTY.js.map → chunk-C73RSUIC.js.map} +0 -0
- /package/dist/{chunk-GAZFKZHM.js.map → chunk-DC5DDJVE.js.map} +0 -0
- /package/dist/{chunk-52PDIZKV.js.map → chunk-ECVQNGTH.js.map} +0 -0
- /package/dist/{chunk-XWFNO5PO.js.map → chunk-GAJIJKET.js.map} +0 -0
- /package/dist/{chunk-K3U5F2QT.js.map → chunk-M6GSS72T.js.map} +0 -0
- /package/dist/{chunk-UK2I3ACY.js.map → chunk-MHM4QEBV.js.map} +0 -0
- /package/dist/{chunk-OIZ3XMKB.js.map → chunk-MWBJ3KTF.js.map} +0 -0
- /package/dist/{chunk-MZC62HPZ.js.map → chunk-NBMPHD4I.js.map} +0 -0
- /package/dist/{chunk-DX32P2DT.js.map → chunk-NTWW7J76.js.map} +0 -0
- /package/dist/{chunk-JQU7FHYH.js.map → chunk-QW77LCOU.js.map} +0 -0
- /package/dist/{chunk-IA7BMIPE.js.map → chunk-SM6NOMJS.js.map} +0 -0
- /package/dist/{chunk-NGK3N6F2.js.map → chunk-T6SYGLWR.js.map} +0 -0
- /package/dist/{chunk-TVIX653Q.js.map → chunk-TLXE7LYK.js.map} +0 -0
- /package/dist/{chunk-TA42STP7.js.map → chunk-UAKNZVCD.js.map} +0 -0
- /package/dist/{chunk-C3KI45RF.js.map → chunk-UJSGS2Y5.js.map} +0 -0
- /package/dist/{chunk-BOIF2XGG.js.map → chunk-XXDXIIQ4.js.map} +0 -0
- /package/dist/{cli-QKIJOXGW.js.map → cli-UO4NHIB3.js.map} +0 -0
- /package/dist/{chunk-TU2WUBVZ.js.map → cli-YHYDLQAF.js.map} +0 -0
- /package/dist/{cli-YW6VWBHX.js.map → config-P4HDNTYD.js.map} +0 -0
- /package/dist/{github-3WMSTNTK.js.map → conformance-ZFCMKM6M.js.map} +0 -0
- /package/dist/{github-OXSJQ4PS.js.map → github-7ZCH5PSO.js.map} +0 -0
- /package/dist/{json-4DCKDGUV.js.map → github-POT6724K.js.map} +0 -0
- /package/dist/{lint-YDHVVZHT.js.map → host-6GURH5HE.js.map} +0 -0
- /package/dist/{list-3ZO2NIYO.js.map → json-KT2K62L5.js.map} +0 -0
- /package/dist/{list-EJH6VAOW.js.map → lint-5AQ5C2BV.js.map} +0 -0
- /package/dist/{page-QEDB7EVU.js.map → list-5JZKX2NK.js.map} +0 -0
- /package/dist/{pretty-5DGEBZSV.js.map → list-SWPFXQMU.js.map} +0 -0
- /package/dist/{pretty-SQX33UM2.js.map → page-UPSXRP75.js.map} +0 -0
- /package/dist/{reporters-OHSI3RMV.js.map → pretty-Q6FQAOK4.js.map} +0 -0
- /package/dist/{reporters-YELDDYJD.js.map → pretty-SQRPFPSQ.js.map} +0 -0
- /package/dist/{run-IP6NZHAG.js.map → provider-AMGRDY6V.js.map} +0 -0
- /package/dist/{validate-G6HH3YXL.js.map → reporters-45MMC2ID.js.map} +0 -0
|
@@ -1,28 +1,57 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
parseTraceFile,
|
|
4
|
+
renderTrace
|
|
5
|
+
} from "./chunk-M2RLHZ4J.js";
|
|
6
|
+
import {
|
|
7
|
+
evaluateWhen,
|
|
8
|
+
matchesGlob,
|
|
9
|
+
rejectWhen,
|
|
10
|
+
skippedTrigger,
|
|
11
|
+
validateWhen
|
|
12
|
+
} from "./chunk-FQGLRJGX.js";
|
|
13
|
+
import {
|
|
14
|
+
fail,
|
|
15
|
+
finding,
|
|
16
|
+
firstError,
|
|
17
|
+
optionalBoolean,
|
|
18
|
+
optionalEnum,
|
|
19
|
+
optionalNumber,
|
|
20
|
+
optionalString,
|
|
21
|
+
optionsError,
|
|
22
|
+
orderedBounds,
|
|
23
|
+
pass,
|
|
24
|
+
requireOneOf,
|
|
25
|
+
requiredString,
|
|
26
|
+
skippedWindow,
|
|
27
|
+
windowFor
|
|
28
|
+
} from "./chunk-CWXZ2MRT.js";
|
|
29
|
+
import {
|
|
5
30
|
extractEvals,
|
|
6
31
|
findManifest,
|
|
7
|
-
foldBranchAvailability,
|
|
8
|
-
makeRedactor,
|
|
9
|
-
newAvailability,
|
|
10
|
-
newReplay,
|
|
11
|
-
remapAvailability,
|
|
12
32
|
resolveArtifacts
|
|
13
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-APKH2T6E.js";
|
|
14
34
|
import {
|
|
15
35
|
discoverTraces
|
|
16
36
|
} from "./chunk-4TSWU2SU.js";
|
|
37
|
+
import {
|
|
38
|
+
discoverConfig
|
|
39
|
+
} from "./chunk-XPEWUK7J.js";
|
|
40
|
+
import {
|
|
41
|
+
announceSelection,
|
|
42
|
+
assertProviderSelection,
|
|
43
|
+
constructProvider,
|
|
44
|
+
resolveProviderIdentity,
|
|
45
|
+
selectProvider
|
|
46
|
+
} from "./chunk-BWCWTHYS.js";
|
|
17
47
|
import {
|
|
18
48
|
TracevalsError
|
|
19
49
|
} from "./chunk-UKZNLVJW.js";
|
|
20
50
|
import {
|
|
21
51
|
isTurnBudgetSkip,
|
|
22
|
-
parseDuration,
|
|
23
52
|
realExec,
|
|
24
53
|
turnBudgetSkipReason
|
|
25
|
-
} from "./chunk-
|
|
54
|
+
} from "./chunk-XG6PVB5S.js";
|
|
26
55
|
import {
|
|
27
56
|
NOT_GRANTED_REASON,
|
|
28
57
|
grantsFor,
|
|
@@ -39,7 +68,7 @@ import {
|
|
|
39
68
|
} from "./chunk-GLP5PSXB.js";
|
|
40
69
|
import {
|
|
41
70
|
declaredAt
|
|
42
|
-
} from "./chunk-
|
|
71
|
+
} from "./chunk-TLXE7LYK.js";
|
|
43
72
|
import {
|
|
44
73
|
PATH_JOIN,
|
|
45
74
|
externalMetadataJoin,
|
|
@@ -47,15 +76,15 @@ import {
|
|
|
47
76
|
loadExternalMetadata,
|
|
48
77
|
memberOf,
|
|
49
78
|
mergeWithMarks
|
|
50
|
-
} from "./chunk-
|
|
79
|
+
} from "./chunk-TUAAR3MM.js";
|
|
51
80
|
import {
|
|
52
81
|
xmlEscape
|
|
53
|
-
} from "./chunk-
|
|
82
|
+
} from "./chunk-5G53XATW.js";
|
|
54
83
|
import {
|
|
55
84
|
DocmetaError,
|
|
56
85
|
classifyRef,
|
|
57
86
|
extractFrontmatter
|
|
58
|
-
} from "./chunk-
|
|
87
|
+
} from "./chunk-KCOFEWV5.js";
|
|
59
88
|
import {
|
|
60
89
|
palette
|
|
61
90
|
} from "./chunk-RNVYGWIC.js";
|
|
@@ -64,18 +93,15 @@ import {
|
|
|
64
93
|
escapeWorkflowCommandProperty
|
|
65
94
|
} from "./chunk-MNH7QO4Z.js";
|
|
66
95
|
import {
|
|
67
|
-
assertKnownProvider,
|
|
68
|
-
assertLocalFlag,
|
|
69
|
-
assertModelHasProvider,
|
|
70
96
|
hasPagePlaceholder,
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
providerSpecFor,
|
|
75
|
-
resolveIdentity,
|
|
76
|
-
selectProvider,
|
|
97
|
+
ownsKey
|
|
98
|
+
} from "./chunk-XKBEHKWO.js";
|
|
99
|
+
import {
|
|
77
100
|
warn
|
|
78
|
-
} from "./chunk-
|
|
101
|
+
} from "./chunk-HX44GB4Z.js";
|
|
102
|
+
import {
|
|
103
|
+
parseDuration
|
|
104
|
+
} from "./chunk-QCHVR4ZM.js";
|
|
79
105
|
import {
|
|
80
106
|
errorMessage
|
|
81
107
|
} from "./chunk-G3T47CFV.js";
|
|
@@ -1300,701 +1326,7 @@ function renderCalibration(report, format, opts = {}) {
|
|
|
1300
1326
|
|
|
1301
1327
|
// src/tracevals/commands/run.ts
|
|
1302
1328
|
import { writeFile } from "node:fs/promises";
|
|
1303
|
-
import { resolve as
|
|
1304
|
-
|
|
1305
|
-
// src/tracevals/trace/claude.ts
|
|
1306
|
-
import { readdir, readFile } from "node:fs/promises";
|
|
1307
|
-
import { basename as basename3, dirname, join, resolve } from "node:path";
|
|
1308
|
-
|
|
1309
|
-
// src/tracevals/trace/detect.ts
|
|
1310
|
-
var SUPPORTED = "supported formats: Claude Code session files (~/.claude/projects/**.jsonl) and claude -p stream-json";
|
|
1311
|
-
function detectFormat(line) {
|
|
1312
|
-
let rec;
|
|
1313
|
-
try {
|
|
1314
|
-
rec = JSON.parse(line);
|
|
1315
|
-
} catch {
|
|
1316
|
-
throw new TracevalsError(`trace is not JSONL (${SUPPORTED})`);
|
|
1317
|
-
}
|
|
1318
|
-
if (typeof rec !== "object" || rec === null) {
|
|
1319
|
-
throw new TracevalsError(`unrecognized trace format (${SUPPORTED})`);
|
|
1320
|
-
}
|
|
1321
|
-
if ("parentUuid" in rec || "sessionId" in rec) return "claude-session";
|
|
1322
|
-
if ("session_id" in rec) return "claude-stream";
|
|
1323
|
-
throw new TracevalsError(`unrecognized trace format (${SUPPORTED})`);
|
|
1324
|
-
}
|
|
1325
|
-
function detectContentFormat(content) {
|
|
1326
|
-
let sawParseable = false;
|
|
1327
|
-
for (const line of content.split("\n")) {
|
|
1328
|
-
const trimmed = line.trim();
|
|
1329
|
-
if (!trimmed) continue;
|
|
1330
|
-
let rec;
|
|
1331
|
-
try {
|
|
1332
|
-
rec = JSON.parse(trimmed);
|
|
1333
|
-
} catch {
|
|
1334
|
-
continue;
|
|
1335
|
-
}
|
|
1336
|
-
if (typeof rec !== "object" || rec === null) continue;
|
|
1337
|
-
sawParseable = true;
|
|
1338
|
-
const r = rec;
|
|
1339
|
-
if ("parentUuid" in r || "sessionId" in r) return "claude-session";
|
|
1340
|
-
if ("session_id" in r) return "claude-stream";
|
|
1341
|
-
}
|
|
1342
|
-
throw new TracevalsError(
|
|
1343
|
-
sawParseable ? `unrecognized trace format (${SUPPORTED})` : `empty or unparseable trace file (${SUPPORTED})`
|
|
1344
|
-
);
|
|
1345
|
-
}
|
|
1346
|
-
|
|
1347
|
-
// src/tracevals/trace/claude.ts
|
|
1348
|
-
async function parseTraceFile(filePath) {
|
|
1349
|
-
const absolute = resolve(filePath);
|
|
1350
|
-
const content = await readFile(absolute, "utf-8");
|
|
1351
|
-
const trace = parseTraceContent(content, absolute);
|
|
1352
|
-
await mergeSidecarBranches(trace, absolute);
|
|
1353
|
-
return trace;
|
|
1354
|
-
}
|
|
1355
|
-
function parseTraceContent(content, filePath) {
|
|
1356
|
-
const format = detectContentFormat(content);
|
|
1357
|
-
const { records, unparseable } = parseLines(content);
|
|
1358
|
-
const trace = format === "claude-session" ? parseSession(records, filePath) : parseStream(records, filePath);
|
|
1359
|
-
if (unparseable > 0) {
|
|
1360
|
-
trace.warnings.push(
|
|
1361
|
-
`${unparseable} unparseable JSONL line(s) were skipped`
|
|
1362
|
-
);
|
|
1363
|
-
}
|
|
1364
|
-
return trace;
|
|
1365
|
-
}
|
|
1366
|
-
function parseLines(content) {
|
|
1367
|
-
const records = [];
|
|
1368
|
-
let unparseable = 0;
|
|
1369
|
-
for (const line of content.split("\n")) {
|
|
1370
|
-
const trimmed = line.trim();
|
|
1371
|
-
if (!trimmed) continue;
|
|
1372
|
-
try {
|
|
1373
|
-
const rec = JSON.parse(trimmed);
|
|
1374
|
-
if (typeof rec === "object" && rec !== null) records.push(rec);
|
|
1375
|
-
else unparseable += 1;
|
|
1376
|
-
} catch {
|
|
1377
|
-
unparseable += 1;
|
|
1378
|
-
}
|
|
1379
|
-
}
|
|
1380
|
-
return { records, unparseable };
|
|
1381
|
-
}
|
|
1382
|
-
var SESSION_META_TYPES = /* @__PURE__ */ new Set([
|
|
1383
|
-
"queue-operation",
|
|
1384
|
-
"attachment",
|
|
1385
|
-
"last-prompt",
|
|
1386
|
-
"custom-title",
|
|
1387
|
-
"summary",
|
|
1388
|
-
"mode",
|
|
1389
|
-
"pr-link"
|
|
1390
|
-
]);
|
|
1391
|
-
var COMMAND_NAME_RE = /<command-name>([^<]+)<\/command-name>/;
|
|
1392
|
-
var COMMAND_ARGS_RE = /<command-args>([^<]*)<\/command-args>/;
|
|
1393
|
-
function newTrace(filePath) {
|
|
1394
|
-
return {
|
|
1395
|
-
source: "claude-code",
|
|
1396
|
-
file: filePath,
|
|
1397
|
-
cwd: "",
|
|
1398
|
-
events: [],
|
|
1399
|
-
toolCalls: [],
|
|
1400
|
-
skillInvocations: [],
|
|
1401
|
-
agentSpawns: [],
|
|
1402
|
-
subagentBranches: [],
|
|
1403
|
-
availability: newAvailability(),
|
|
1404
|
-
fileAccesses: [],
|
|
1405
|
-
userMessages: [],
|
|
1406
|
-
assistantTexts: [],
|
|
1407
|
-
turnCount: 0,
|
|
1408
|
-
warnings: []
|
|
1409
|
-
};
|
|
1410
|
-
}
|
|
1411
|
-
function pushEvent(trace, event) {
|
|
1412
|
-
const index = trace.events.length;
|
|
1413
|
-
trace.events.push({ ...event, index });
|
|
1414
|
-
return index;
|
|
1415
|
-
}
|
|
1416
|
-
function plainMessageText(rec) {
|
|
1417
|
-
const content = rec.message?.content;
|
|
1418
|
-
return typeof content === "string" ? content : void 0;
|
|
1419
|
-
}
|
|
1420
|
-
function resolveBranch(branches, parentUuid, rootText) {
|
|
1421
|
-
if (typeof parentUuid !== "string") return void 0;
|
|
1422
|
-
const inherited = branches.resolved.get(parentUuid);
|
|
1423
|
-
if (inherited !== void 0) return inherited;
|
|
1424
|
-
const pending = branches.spawns.get(parentUuid);
|
|
1425
|
-
if (pending === void 0 || pending.length === 0) return void 0;
|
|
1426
|
-
const byPrompt = rootText !== void 0 ? pending.find((b) => !b.claimed && b.prompt === rootText) : void 0;
|
|
1427
|
-
const branch = byPrompt ?? pending.find((b) => !b.claimed) ?? pending[0];
|
|
1428
|
-
if (branch === void 0) return void 0;
|
|
1429
|
-
branch.claimed = true;
|
|
1430
|
-
return branch.toolUseId;
|
|
1431
|
-
}
|
|
1432
|
-
function parseSession(records, filePath) {
|
|
1433
|
-
const trace = newTrace(filePath);
|
|
1434
|
-
const branches = { spawns: /* @__PURE__ */ new Map(), resolved: /* @__PURE__ */ new Map() };
|
|
1435
|
-
const availability = newReplay();
|
|
1436
|
-
const uuids = /* @__PURE__ */ new Set();
|
|
1437
|
-
for (const rec of records) {
|
|
1438
|
-
if (typeof rec.uuid === "string") uuids.add(rec.uuid);
|
|
1439
|
-
}
|
|
1440
|
-
const continuations = [];
|
|
1441
|
-
let sawMessage = false;
|
|
1442
|
-
let compactions = 0;
|
|
1443
|
-
let inputTokens = 0;
|
|
1444
|
-
let outputTokens = 0;
|
|
1445
|
-
let sawUsage = false;
|
|
1446
|
-
for (const rec of records) {
|
|
1447
|
-
const type = rec.type;
|
|
1448
|
-
const timestamp = rec.timestamp;
|
|
1449
|
-
if (timestamp) {
|
|
1450
|
-
trace.startedAt ??= timestamp;
|
|
1451
|
-
trace.endedAt = timestamp;
|
|
1452
|
-
}
|
|
1453
|
-
if (rec.sessionId) trace.sessionId ??= rec.sessionId;
|
|
1454
|
-
if (rec.subtype === "compact_boundary") compactions += 1;
|
|
1455
|
-
if (type !== void 0 && SESSION_META_TYPES.has(type)) {
|
|
1456
|
-
if (type === "summary" && !sawMessage && typeof rec.leafUuid === "string" && !uuids.has(rec.leafUuid)) {
|
|
1457
|
-
continuations.push(rec.leafUuid);
|
|
1458
|
-
}
|
|
1459
|
-
const index = pushEvent(trace, { kind: "meta", timestamp, raw: rec });
|
|
1460
|
-
if (type === "attachment" && applyAvailabilityRecord(availability, rec.attachment, index)) {
|
|
1461
|
-
availability.roster.recorded = true;
|
|
1462
|
-
}
|
|
1463
|
-
continue;
|
|
1464
|
-
}
|
|
1465
|
-
const sidechain = rec.isSidechain === true;
|
|
1466
|
-
if (!sidechain && typeof rec.cwd === "string") trace.cwd = rec.cwd;
|
|
1467
|
-
if (!sidechain && typeof rec.gitBranch === "string") {
|
|
1468
|
-
trace.gitBranch = rec.gitBranch;
|
|
1469
|
-
}
|
|
1470
|
-
const context = { sidechain, timestamp, branches };
|
|
1471
|
-
if (typeof rec.uuid === "string") context.uuid = rec.uuid;
|
|
1472
|
-
if (sidechain) {
|
|
1473
|
-
const branchId = resolveBranch(
|
|
1474
|
-
branches,
|
|
1475
|
-
rec.parentUuid,
|
|
1476
|
-
plainMessageText(rec)
|
|
1477
|
-
);
|
|
1478
|
-
if (branchId !== void 0) {
|
|
1479
|
-
context.branchId = branchId;
|
|
1480
|
-
if (context.uuid) branches.resolved.set(context.uuid, branchId);
|
|
1481
|
-
}
|
|
1482
|
-
}
|
|
1483
|
-
const message = rec.message;
|
|
1484
|
-
if (type === "user" && message) {
|
|
1485
|
-
sawMessage = true;
|
|
1486
|
-
handleUserMessage(trace, message, context, rec);
|
|
1487
|
-
} else if (type === "assistant" && message) {
|
|
1488
|
-
sawMessage = true;
|
|
1489
|
-
const usage = handleAssistantMessage(trace, message, context, rec);
|
|
1490
|
-
if (usage && !sidechain) {
|
|
1491
|
-
inputTokens += usage.input;
|
|
1492
|
-
outputTokens += usage.output;
|
|
1493
|
-
sawUsage = true;
|
|
1494
|
-
}
|
|
1495
|
-
} else {
|
|
1496
|
-
pushEvent(trace, { kind: "system", timestamp, raw: rec });
|
|
1497
|
-
}
|
|
1498
|
-
}
|
|
1499
|
-
if (sawUsage) trace.usage = { inputTokens, outputTokens };
|
|
1500
|
-
for (const leaf of continuations) {
|
|
1501
|
-
trace.warnings.push(
|
|
1502
|
-
`this trace is a fragment of a longer session: it resumes an earlier one recorded elsewhere (summary leafUuid ${leaf} is not in this file), so evals see only the turns recorded here`
|
|
1503
|
-
);
|
|
1504
|
-
}
|
|
1505
|
-
if (compactions > 0) {
|
|
1506
|
-
trace.warnings.push(
|
|
1507
|
-
`the conversation was compacted ${compactions} time(s): the agent stopped seeing the turns before each boundary, so early transcript content is not what it was working from`
|
|
1508
|
-
);
|
|
1509
|
-
}
|
|
1510
|
-
trace.availability = availability.roster;
|
|
1511
|
-
trace.subagentBranches = buildBranches(trace);
|
|
1512
|
-
return trace;
|
|
1513
|
-
}
|
|
1514
|
-
function buildBranches(trace, sidecars = /* @__PURE__ */ new Map()) {
|
|
1515
|
-
const ranges = /* @__PURE__ */ new Map();
|
|
1516
|
-
for (const event of trace.events) {
|
|
1517
|
-
const id = event.branchId;
|
|
1518
|
-
if (id === void 0) continue;
|
|
1519
|
-
const range = ranges.get(id);
|
|
1520
|
-
if (range === void 0) ranges.set(id, { min: event.index, max: event.index });
|
|
1521
|
-
else range.max = Math.max(range.max, event.index);
|
|
1522
|
-
}
|
|
1523
|
-
if (ranges.size === 0) return [];
|
|
1524
|
-
const spawnFor = /* @__PURE__ */ new Map();
|
|
1525
|
-
for (const spawn of trace.agentSpawns) {
|
|
1526
|
-
if (spawn.toolUseId) spawnFor.set(spawn.toolUseId, spawn);
|
|
1527
|
-
}
|
|
1528
|
-
const parentOf = /* @__PURE__ */ new Map();
|
|
1529
|
-
for (const id of ranges.keys()) {
|
|
1530
|
-
const spawn = spawnFor.get(id);
|
|
1531
|
-
parentOf.set(
|
|
1532
|
-
id,
|
|
1533
|
-
spawn ? trace.events[spawn.index]?.branchId : void 0
|
|
1534
|
-
);
|
|
1535
|
-
}
|
|
1536
|
-
const depthOf = (id) => {
|
|
1537
|
-
let depth = 1;
|
|
1538
|
-
let cursor = parentOf.get(id);
|
|
1539
|
-
const seen = /* @__PURE__ */ new Set([id]);
|
|
1540
|
-
while (cursor !== void 0 && !seen.has(cursor)) {
|
|
1541
|
-
seen.add(cursor);
|
|
1542
|
-
depth += 1;
|
|
1543
|
-
cursor = parentOf.get(cursor);
|
|
1544
|
-
}
|
|
1545
|
-
return depth;
|
|
1546
|
-
};
|
|
1547
|
-
const branches = [];
|
|
1548
|
-
for (const [id, range] of ranges) {
|
|
1549
|
-
const spawn = spawnFor.get(id);
|
|
1550
|
-
const sidecar = sidecars.get(id);
|
|
1551
|
-
const branch = {
|
|
1552
|
-
branchId: id,
|
|
1553
|
-
agentType: sidecar?.agentType ?? spawn?.subagentType ?? id,
|
|
1554
|
-
origin: sidecar ? "sidecar" : "inline",
|
|
1555
|
-
spawnDepth: sidecar?.spawnDepth ?? depthOf(id),
|
|
1556
|
-
spawnIndex: spawn?.index ?? range.min,
|
|
1557
|
-
startIndex: range.min,
|
|
1558
|
-
endIndex: range.max + 1
|
|
1559
|
-
};
|
|
1560
|
-
const description = sidecar?.description ?? spawn?.description;
|
|
1561
|
-
if (description !== void 0) branch.description = description;
|
|
1562
|
-
if (sidecar?.agentId) branch.agentId = sidecar.agentId;
|
|
1563
|
-
if (sidecar?.parentAgentId) branch.parentAgentId = sidecar.parentAgentId;
|
|
1564
|
-
if (sidecar?.file) branch.file = sidecar.file;
|
|
1565
|
-
branches.push(branch);
|
|
1566
|
-
}
|
|
1567
|
-
branches.sort((a, b) => b.spawnDepth - a.spawnDepth);
|
|
1568
|
-
const byId = new Map(branches.map((b) => [b.branchId, b]));
|
|
1569
|
-
for (const branch of branches) {
|
|
1570
|
-
const parentId = parentOf.get(branch.branchId);
|
|
1571
|
-
const parent = parentId === void 0 ? void 0 : byId.get(parentId);
|
|
1572
|
-
if (parent === void 0) continue;
|
|
1573
|
-
parent.startIndex = Math.min(parent.startIndex, branch.startIndex);
|
|
1574
|
-
parent.endIndex = Math.max(parent.endIndex, branch.endIndex);
|
|
1575
|
-
}
|
|
1576
|
-
branches.sort((a, b) => a.startIndex - b.startIndex);
|
|
1577
|
-
return branches;
|
|
1578
|
-
}
|
|
1579
|
-
function handleUserMessage(trace, message, context, raw) {
|
|
1580
|
-
const { sidechain, timestamp, branchId } = context;
|
|
1581
|
-
const content = message.content;
|
|
1582
|
-
if (typeof content === "string") {
|
|
1583
|
-
const index = pushEvent(trace, {
|
|
1584
|
-
kind: "user",
|
|
1585
|
-
timestamp,
|
|
1586
|
-
text: content,
|
|
1587
|
-
sidechain,
|
|
1588
|
-
branchId,
|
|
1589
|
-
raw
|
|
1590
|
-
});
|
|
1591
|
-
if (sidechain) return;
|
|
1592
|
-
const command = COMMAND_NAME_RE.exec(content);
|
|
1593
|
-
if (command?.[1]) {
|
|
1594
|
-
const invocation = {
|
|
1595
|
-
name: command[1].replace(/^\//, ""),
|
|
1596
|
-
via: "command-injection",
|
|
1597
|
-
index
|
|
1598
|
-
};
|
|
1599
|
-
const args = COMMAND_ARGS_RE.exec(content)?.[1]?.trim();
|
|
1600
|
-
if (args) invocation.args = args;
|
|
1601
|
-
trace.skillInvocations.push(invocation);
|
|
1602
|
-
}
|
|
1603
|
-
trace.userMessages.push(content);
|
|
1604
|
-
trace.turnCount += 1;
|
|
1605
|
-
return;
|
|
1606
|
-
}
|
|
1607
|
-
if (Array.isArray(content)) {
|
|
1608
|
-
let hasToolResult = false;
|
|
1609
|
-
const texts = [];
|
|
1610
|
-
for (const block of content) {
|
|
1611
|
-
if (block?.type === "tool_result") hasToolResult = true;
|
|
1612
|
-
if (block?.type === "text" && typeof block.text === "string") {
|
|
1613
|
-
texts.push(block.text);
|
|
1614
|
-
}
|
|
1615
|
-
}
|
|
1616
|
-
if (hasToolResult) {
|
|
1617
|
-
pushEvent(trace, {
|
|
1618
|
-
kind: "tool_result",
|
|
1619
|
-
timestamp,
|
|
1620
|
-
sidechain,
|
|
1621
|
-
branchId,
|
|
1622
|
-
raw
|
|
1623
|
-
});
|
|
1624
|
-
return;
|
|
1625
|
-
}
|
|
1626
|
-
const text = texts.join("\n");
|
|
1627
|
-
pushEvent(trace, {
|
|
1628
|
-
kind: "user",
|
|
1629
|
-
timestamp,
|
|
1630
|
-
text,
|
|
1631
|
-
sidechain,
|
|
1632
|
-
branchId,
|
|
1633
|
-
raw
|
|
1634
|
-
});
|
|
1635
|
-
if (!sidechain && text) {
|
|
1636
|
-
trace.userMessages.push(text);
|
|
1637
|
-
trace.turnCount += 1;
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
}
|
|
1641
|
-
function handleAssistantMessage(trace, message, context, raw) {
|
|
1642
|
-
const { sidechain, timestamp, branchId } = context;
|
|
1643
|
-
if (!sidechain && typeof message.model === "string") {
|
|
1644
|
-
trace.model ??= message.model;
|
|
1645
|
-
}
|
|
1646
|
-
const content = message.content;
|
|
1647
|
-
if (Array.isArray(content)) {
|
|
1648
|
-
for (const block of content) {
|
|
1649
|
-
if (block?.type === "text" && typeof block.text === "string") {
|
|
1650
|
-
pushEvent(trace, {
|
|
1651
|
-
kind: "assistant",
|
|
1652
|
-
timestamp,
|
|
1653
|
-
text: block.text,
|
|
1654
|
-
sidechain,
|
|
1655
|
-
branchId,
|
|
1656
|
-
raw
|
|
1657
|
-
});
|
|
1658
|
-
if (!sidechain) trace.assistantTexts.push(block.text);
|
|
1659
|
-
} else if (block?.type === "tool_use") {
|
|
1660
|
-
extractToolUse(trace, block, context);
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
}
|
|
1664
|
-
const usage = message.usage;
|
|
1665
|
-
if (usage) {
|
|
1666
|
-
return {
|
|
1667
|
-
input: typeof usage.input_tokens === "number" ? usage.input_tokens : 0,
|
|
1668
|
-
output: typeof usage.output_tokens === "number" ? usage.output_tokens : 0
|
|
1669
|
-
};
|
|
1670
|
-
}
|
|
1671
|
-
return void 0;
|
|
1672
|
-
}
|
|
1673
|
-
function extractToolUse(trace, block, context) {
|
|
1674
|
-
const { sidechain, timestamp, branchId } = context;
|
|
1675
|
-
const name = block.name;
|
|
1676
|
-
const input = block.input ?? {};
|
|
1677
|
-
const toolUseId = typeof block.id === "string" ? block.id : void 0;
|
|
1678
|
-
const index = pushEvent(trace, {
|
|
1679
|
-
kind: "tool_call",
|
|
1680
|
-
timestamp,
|
|
1681
|
-
toolName: name,
|
|
1682
|
-
sidechain,
|
|
1683
|
-
branchId,
|
|
1684
|
-
raw: block
|
|
1685
|
-
});
|
|
1686
|
-
const call = { name, input, index, sidechain };
|
|
1687
|
-
if (timestamp) call.timestamp = timestamp;
|
|
1688
|
-
if (branchId) call.branchId = branchId;
|
|
1689
|
-
trace.toolCalls.push(call);
|
|
1690
|
-
switch (name) {
|
|
1691
|
-
case "Skill": {
|
|
1692
|
-
if (typeof input.skill === "string") {
|
|
1693
|
-
const invocation = {
|
|
1694
|
-
name: input.skill,
|
|
1695
|
-
via: "skill-tool",
|
|
1696
|
-
index
|
|
1697
|
-
};
|
|
1698
|
-
if (toolUseId) invocation.toolUseId = toolUseId;
|
|
1699
|
-
if (typeof input.args === "string" && input.args) {
|
|
1700
|
-
invocation.args = input.args;
|
|
1701
|
-
}
|
|
1702
|
-
trace.skillInvocations.push(invocation);
|
|
1703
|
-
}
|
|
1704
|
-
break;
|
|
1705
|
-
}
|
|
1706
|
-
case "Agent": {
|
|
1707
|
-
if (typeof input.subagent_type === "string") {
|
|
1708
|
-
const spawn = {
|
|
1709
|
-
subagentType: input.subagent_type,
|
|
1710
|
-
index
|
|
1711
|
-
};
|
|
1712
|
-
if (typeof input.description === "string") {
|
|
1713
|
-
spawn.description = input.description;
|
|
1714
|
-
}
|
|
1715
|
-
if (toolUseId) spawn.toolUseId = toolUseId;
|
|
1716
|
-
trace.agentSpawns.push(spawn);
|
|
1717
|
-
}
|
|
1718
|
-
if (context.branches && context.uuid && toolUseId) {
|
|
1719
|
-
const pending = context.branches.spawns.get(context.uuid) ?? [];
|
|
1720
|
-
pending.push({
|
|
1721
|
-
toolUseId,
|
|
1722
|
-
prompt: typeof input.prompt === "string" ? input.prompt : void 0,
|
|
1723
|
-
claimed: false
|
|
1724
|
-
});
|
|
1725
|
-
context.branches.spawns.set(context.uuid, pending);
|
|
1726
|
-
}
|
|
1727
|
-
break;
|
|
1728
|
-
}
|
|
1729
|
-
case "Read":
|
|
1730
|
-
case "Write":
|
|
1731
|
-
case "Edit": {
|
|
1732
|
-
if (typeof input.file_path === "string") {
|
|
1733
|
-
const op = name === "Read" ? "read" : name === "Write" ? "write" : "edit";
|
|
1734
|
-
trace.fileAccesses.push({ path: input.file_path, op, index });
|
|
1735
|
-
}
|
|
1736
|
-
break;
|
|
1737
|
-
}
|
|
1738
|
-
}
|
|
1739
|
-
}
|
|
1740
|
-
function parseStream(records, filePath) {
|
|
1741
|
-
const trace = newTrace(filePath);
|
|
1742
|
-
const context = { sidechain: false };
|
|
1743
|
-
let inputTokens = 0;
|
|
1744
|
-
let outputTokens = 0;
|
|
1745
|
-
let sawMessageUsage = false;
|
|
1746
|
-
for (const rec of records) {
|
|
1747
|
-
const type = rec.type;
|
|
1748
|
-
if (type === "system" && (rec.subtype === "init" || !rec.subtype)) {
|
|
1749
|
-
if (typeof rec.cwd === "string") trace.cwd = rec.cwd;
|
|
1750
|
-
if (typeof rec.model === "string") trace.model = rec.model;
|
|
1751
|
-
if (typeof rec.session_id === "string") trace.sessionId = rec.session_id;
|
|
1752
|
-
pushEvent(trace, { kind: "system", raw: rec });
|
|
1753
|
-
continue;
|
|
1754
|
-
}
|
|
1755
|
-
if (type === "assistant" || type === "user") {
|
|
1756
|
-
const message = rec.message ?? rec;
|
|
1757
|
-
if (type === "assistant") {
|
|
1758
|
-
const usage = handleAssistantMessage(trace, message, context, rec);
|
|
1759
|
-
if (usage) {
|
|
1760
|
-
inputTokens += usage.input;
|
|
1761
|
-
outputTokens += usage.output;
|
|
1762
|
-
sawMessageUsage = true;
|
|
1763
|
-
}
|
|
1764
|
-
} else {
|
|
1765
|
-
handleUserMessage(trace, message, context, rec);
|
|
1766
|
-
}
|
|
1767
|
-
continue;
|
|
1768
|
-
}
|
|
1769
|
-
if (type === "result") {
|
|
1770
|
-
trace.isError = rec.is_error === true;
|
|
1771
|
-
if (typeof rec.num_turns === "number") trace.turnCount = rec.num_turns;
|
|
1772
|
-
const usage = rec.usage;
|
|
1773
|
-
const totalCost = typeof rec.total_cost_usd === "number" ? rec.total_cost_usd : void 0;
|
|
1774
|
-
if (usage) {
|
|
1775
|
-
trace.usage = {
|
|
1776
|
-
inputTokens: typeof usage.input_tokens === "number" ? usage.input_tokens : 0,
|
|
1777
|
-
outputTokens: typeof usage.output_tokens === "number" ? usage.output_tokens : 0
|
|
1778
|
-
};
|
|
1779
|
-
if (totalCost !== void 0) trace.usage.totalCostUsd = totalCost;
|
|
1780
|
-
} else if (sawMessageUsage) {
|
|
1781
|
-
trace.usage = { inputTokens, outputTokens };
|
|
1782
|
-
if (totalCost !== void 0) trace.usage.totalCostUsd = totalCost;
|
|
1783
|
-
}
|
|
1784
|
-
pushEvent(trace, { kind: "meta", raw: rec });
|
|
1785
|
-
continue;
|
|
1786
|
-
}
|
|
1787
|
-
pushEvent(trace, { kind: "meta", raw: rec });
|
|
1788
|
-
}
|
|
1789
|
-
if (!trace.usage && sawMessageUsage) {
|
|
1790
|
-
trace.usage = { inputTokens, outputTokens };
|
|
1791
|
-
}
|
|
1792
|
-
return trace;
|
|
1793
|
-
}
|
|
1794
|
-
var AGENT_META_RE = /^agent-(.+)\.meta\.json$/;
|
|
1795
|
-
var AGENT_JSONL_RE = /^agent-(.+)\.jsonl$/;
|
|
1796
|
-
function sidecarDirFor(traceFile) {
|
|
1797
|
-
const stem = basename3(traceFile).replace(/\.jsonl$/i, "");
|
|
1798
|
-
return join(dirname(traceFile), stem, "subagents");
|
|
1799
|
-
}
|
|
1800
|
-
function asSource(id, trace) {
|
|
1801
|
-
return {
|
|
1802
|
-
id,
|
|
1803
|
-
events: trace.events,
|
|
1804
|
-
toolCalls: trace.toolCalls,
|
|
1805
|
-
fileAccesses: trace.fileAccesses,
|
|
1806
|
-
skillInvocations: trace.skillInvocations,
|
|
1807
|
-
agentSpawns: trace.agentSpawns
|
|
1808
|
-
};
|
|
1809
|
-
}
|
|
1810
|
-
function stampBranch(sub, branchId) {
|
|
1811
|
-
for (const event of sub.events) {
|
|
1812
|
-
event.sidechain = true;
|
|
1813
|
-
event.branchId = branchId;
|
|
1814
|
-
}
|
|
1815
|
-
for (const call of sub.toolCalls) {
|
|
1816
|
-
call.sidechain = true;
|
|
1817
|
-
call.branchId = branchId;
|
|
1818
|
-
}
|
|
1819
|
-
}
|
|
1820
|
-
async function loadSidecars(dir, trace) {
|
|
1821
|
-
let names2;
|
|
1822
|
-
try {
|
|
1823
|
-
names2 = await readdir(dir);
|
|
1824
|
-
} catch (error) {
|
|
1825
|
-
const code = error.code;
|
|
1826
|
-
if (code !== "ENOENT" && code !== "ENOTDIR") {
|
|
1827
|
-
trace.warnings.push(
|
|
1828
|
-
`could not read the subagent transcript directory ${dir} (${code ?? "unknown error"}), so subagent turns are missing here`
|
|
1829
|
-
);
|
|
1830
|
-
}
|
|
1831
|
-
return [];
|
|
1832
|
-
}
|
|
1833
|
-
const metas = /* @__PURE__ */ new Set();
|
|
1834
|
-
const transcripts = /* @__PURE__ */ new Set();
|
|
1835
|
-
for (const name of names2) {
|
|
1836
|
-
const meta = AGENT_META_RE.exec(name);
|
|
1837
|
-
if (meta?.[1] !== void 0) {
|
|
1838
|
-
metas.add(meta[1]);
|
|
1839
|
-
continue;
|
|
1840
|
-
}
|
|
1841
|
-
const jsonl = AGENT_JSONL_RE.exec(name);
|
|
1842
|
-
if (jsonl?.[1] !== void 0) transcripts.add(jsonl[1]);
|
|
1843
|
-
}
|
|
1844
|
-
for (const orphan of [...transcripts].sort()) {
|
|
1845
|
-
if (!metas.has(orphan)) {
|
|
1846
|
-
trace.warnings.push(
|
|
1847
|
-
`subagent transcript agent-${orphan}.jsonl has no agent-${orphan}.meta.json, so it names no Agent spawn and was not merged`
|
|
1848
|
-
);
|
|
1849
|
-
}
|
|
1850
|
-
}
|
|
1851
|
-
const loaded = [];
|
|
1852
|
-
for (const agentId of [...metas].sort()) {
|
|
1853
|
-
let meta;
|
|
1854
|
-
try {
|
|
1855
|
-
const raw = await readFile(
|
|
1856
|
-
join(dir, `agent-${agentId}.meta.json`),
|
|
1857
|
-
"utf-8"
|
|
1858
|
-
);
|
|
1859
|
-
const parsed = JSON.parse(raw);
|
|
1860
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
1861
|
-
throw new Error("shape");
|
|
1862
|
-
}
|
|
1863
|
-
meta = parsed;
|
|
1864
|
-
} catch {
|
|
1865
|
-
trace.warnings.push(
|
|
1866
|
-
`subagent metadata agent-${agentId}.meta.json is unreadable or not an object, so that branch was not merged`
|
|
1867
|
-
);
|
|
1868
|
-
continue;
|
|
1869
|
-
}
|
|
1870
|
-
const file = join(dir, `agent-${agentId}.jsonl`);
|
|
1871
|
-
let sub;
|
|
1872
|
-
try {
|
|
1873
|
-
sub = parseTraceContent(await readFile(file, "utf-8"), file);
|
|
1874
|
-
} catch {
|
|
1875
|
-
trace.warnings.push(
|
|
1876
|
-
`subagent transcript agent-${agentId}.jsonl is missing or unreadable, so that branch was not merged`
|
|
1877
|
-
);
|
|
1878
|
-
continue;
|
|
1879
|
-
}
|
|
1880
|
-
for (const warning of sub.warnings) {
|
|
1881
|
-
trace.warnings.push(`subagent ${meta.agentType ?? agentId}: ${warning}`);
|
|
1882
|
-
}
|
|
1883
|
-
loaded.push({ agentId, file, meta, sub, source: asSource(agentId, sub) });
|
|
1884
|
-
}
|
|
1885
|
-
return loaded;
|
|
1886
|
-
}
|
|
1887
|
-
async function mergeSidecarBranches(trace, traceFile) {
|
|
1888
|
-
const loaded = await loadSidecars(sidecarDirFor(traceFile), trace);
|
|
1889
|
-
if (loaded.length === 0) return;
|
|
1890
|
-
const base = asSource("", trace);
|
|
1891
|
-
const sources = /* @__PURE__ */ new Map([["", base]]);
|
|
1892
|
-
const spawnOwner = /* @__PURE__ */ new Map();
|
|
1893
|
-
const registerSpawns = (source) => {
|
|
1894
|
-
for (const spawn of source.agentSpawns) {
|
|
1895
|
-
if (spawn.toolUseId !== void 0) {
|
|
1896
|
-
spawnOwner.set(spawn.toolUseId, {
|
|
1897
|
-
sourceId: source.id,
|
|
1898
|
-
local: spawn.index
|
|
1899
|
-
});
|
|
1900
|
-
}
|
|
1901
|
-
}
|
|
1902
|
-
};
|
|
1903
|
-
registerSpawns(base);
|
|
1904
|
-
const children = /* @__PURE__ */ new Map();
|
|
1905
|
-
const attached = [];
|
|
1906
|
-
const pending = [...loaded].sort(
|
|
1907
|
-
(a, b) => (a.meta.spawnDepth ?? 1) - (b.meta.spawnDepth ?? 1) || (a.agentId < b.agentId ? -1 : a.agentId > b.agentId ? 1 : 0)
|
|
1908
|
-
);
|
|
1909
|
-
let progress = true;
|
|
1910
|
-
while (progress) {
|
|
1911
|
-
progress = false;
|
|
1912
|
-
for (let i = 0; i < pending.length; i += 1) {
|
|
1913
|
-
const sidecar = pending[i];
|
|
1914
|
-
const branchId = sidecar.meta.toolUseId;
|
|
1915
|
-
const owner = branchId === void 0 ? void 0 : spawnOwner.get(branchId);
|
|
1916
|
-
if (branchId === void 0 || owner === void 0) continue;
|
|
1917
|
-
pending.splice(i, 1);
|
|
1918
|
-
i -= 1;
|
|
1919
|
-
progress = true;
|
|
1920
|
-
stampBranch(sidecar.sub, branchId);
|
|
1921
|
-
sources.set(sidecar.agentId, sidecar.source);
|
|
1922
|
-
const key = `${owner.sourceId}\0${owner.local}`;
|
|
1923
|
-
children.set(key, [...children.get(key) ?? [], sidecar]);
|
|
1924
|
-
registerSpawns(sidecar.source);
|
|
1925
|
-
attached.push({ sidecar, branchId });
|
|
1926
|
-
}
|
|
1927
|
-
}
|
|
1928
|
-
for (const sidecar of pending) {
|
|
1929
|
-
trace.warnings.push(
|
|
1930
|
-
`subagent transcript agent-${sidecar.agentId}.jsonl names Agent call ${sidecar.meta.toolUseId ?? "(none recorded)"}, which is not in this trace, so that branch was not merged`
|
|
1931
|
-
);
|
|
1932
|
-
}
|
|
1933
|
-
if (attached.length === 0) return;
|
|
1934
|
-
const slots = [];
|
|
1935
|
-
const emit = (sourceId) => {
|
|
1936
|
-
const source = sources.get(sourceId);
|
|
1937
|
-
if (source === void 0) return;
|
|
1938
|
-
for (let local = 0; local < source.events.length; local += 1) {
|
|
1939
|
-
slots.push({ sourceId, local });
|
|
1940
|
-
for (const kid of children.get(`${sourceId}\0${local}`) ?? []) {
|
|
1941
|
-
emit(kid.agentId);
|
|
1942
|
-
}
|
|
1943
|
-
}
|
|
1944
|
-
};
|
|
1945
|
-
emit("");
|
|
1946
|
-
const remap = /* @__PURE__ */ new Map();
|
|
1947
|
-
slots.forEach((slot, final) => {
|
|
1948
|
-
const row = remap.get(slot.sourceId) ?? [];
|
|
1949
|
-
row[slot.local] = final;
|
|
1950
|
-
remap.set(slot.sourceId, row);
|
|
1951
|
-
});
|
|
1952
|
-
const finalIndex = (sourceId, local) => remap.get(sourceId)?.[local] ?? local;
|
|
1953
|
-
trace.events = slots.map((slot, final) => {
|
|
1954
|
-
const event = sources.get(slot.sourceId)?.events[slot.local];
|
|
1955
|
-
return { ...event, index: final };
|
|
1956
|
-
});
|
|
1957
|
-
const gather = (pick) => [...sources.values()].flatMap(
|
|
1958
|
-
(source) => pick(source).map((item) => ({
|
|
1959
|
-
...item,
|
|
1960
|
-
index: finalIndex(source.id, item.index)
|
|
1961
|
-
}))
|
|
1962
|
-
).sort((a, b) => a.index - b.index);
|
|
1963
|
-
trace.toolCalls = gather((s) => s.toolCalls);
|
|
1964
|
-
trace.fileAccesses = gather((s) => s.fileAccesses);
|
|
1965
|
-
trace.skillInvocations = gather((s) => s.skillInvocations);
|
|
1966
|
-
trace.agentSpawns = gather((s) => s.agentSpawns);
|
|
1967
|
-
trace.availability = remapAvailability(
|
|
1968
|
-
trace.availability,
|
|
1969
|
-
(index) => finalIndex("", index)
|
|
1970
|
-
);
|
|
1971
|
-
for (const { sidecar, branchId } of attached) {
|
|
1972
|
-
const owner = spawnOwner.get(branchId);
|
|
1973
|
-
foldBranchAvailability(
|
|
1974
|
-
trace.availability,
|
|
1975
|
-
sidecar.sub.availability,
|
|
1976
|
-
branchId,
|
|
1977
|
-
owner === void 0 ? 0 : finalIndex(owner.sourceId, owner.local)
|
|
1978
|
-
);
|
|
1979
|
-
}
|
|
1980
|
-
const described = /* @__PURE__ */ new Map();
|
|
1981
|
-
for (const { sidecar, branchId } of attached) {
|
|
1982
|
-
const info = {
|
|
1983
|
-
agentId: sidecar.agentId,
|
|
1984
|
-
file: sidecar.file
|
|
1985
|
-
};
|
|
1986
|
-
if (sidecar.meta.agentType) info.agentType = sidecar.meta.agentType;
|
|
1987
|
-
if (sidecar.meta.description) info.description = sidecar.meta.description;
|
|
1988
|
-
if (sidecar.meta.parentAgentId) {
|
|
1989
|
-
info.parentAgentId = sidecar.meta.parentAgentId;
|
|
1990
|
-
}
|
|
1991
|
-
if (typeof sidecar.meta.spawnDepth === "number") {
|
|
1992
|
-
info.spawnDepth = sidecar.meta.spawnDepth;
|
|
1993
|
-
}
|
|
1994
|
-
described.set(branchId, info);
|
|
1995
|
-
}
|
|
1996
|
-
trace.subagentBranches = buildBranches(trace, described);
|
|
1997
|
-
}
|
|
1329
|
+
import { resolve as resolve4 } from "node:path";
|
|
1998
1330
|
|
|
1999
1331
|
// src/tracevals/core/plan.ts
|
|
2000
1332
|
var IMPLICIT_EVAL_NAME = "adheres-to-artifact";
|
|
@@ -2145,346 +1477,6 @@ ${graded}`
|
|
|
2145
1477
|
return parts.join("\n\n");
|
|
2146
1478
|
}
|
|
2147
1479
|
|
|
2148
|
-
// src/tracevals/graders/glob.ts
|
|
2149
|
-
var literal = (ch) => ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
2150
|
-
var compiled = /* @__PURE__ */ new Map();
|
|
2151
|
-
function globToRegExp(glob) {
|
|
2152
|
-
const cached = compiled.get(glob);
|
|
2153
|
-
if (cached !== void 0) return cached;
|
|
2154
|
-
const built = compile(glob);
|
|
2155
|
-
compiled.set(glob, built);
|
|
2156
|
-
return built;
|
|
2157
|
-
}
|
|
2158
|
-
function compile(glob) {
|
|
2159
|
-
const spec = glob.replace(/\\/g, "/").toLowerCase().replace(/(?:\*\*\/)+/g, "**/");
|
|
2160
|
-
let out = "";
|
|
2161
|
-
for (let i = 0; i < spec.length; i += 1) {
|
|
2162
|
-
const ch = spec[i];
|
|
2163
|
-
if (ch === "?") {
|
|
2164
|
-
out += "[^/]";
|
|
2165
|
-
continue;
|
|
2166
|
-
}
|
|
2167
|
-
if (ch !== "*") {
|
|
2168
|
-
out += literal(ch);
|
|
2169
|
-
continue;
|
|
2170
|
-
}
|
|
2171
|
-
if (spec[i + 1] !== "*") {
|
|
2172
|
-
out += "[^/]*";
|
|
2173
|
-
continue;
|
|
2174
|
-
}
|
|
2175
|
-
i += 1;
|
|
2176
|
-
if (spec[i + 1] === "/") {
|
|
2177
|
-
i += 1;
|
|
2178
|
-
if (out !== "") out += "(?:.*/)?";
|
|
2179
|
-
} else if (out.endsWith("/")) {
|
|
2180
|
-
out = `${out.slice(0, -1)}(?:/.*)?`;
|
|
2181
|
-
} else {
|
|
2182
|
-
out += ".*";
|
|
2183
|
-
}
|
|
2184
|
-
}
|
|
2185
|
-
return new RegExp(`^(?:.*/)?${out}$`);
|
|
2186
|
-
}
|
|
2187
|
-
function normalizePath(path) {
|
|
2188
|
-
return path.replace(/\\/g, "/").toLowerCase();
|
|
2189
|
-
}
|
|
2190
|
-
function matchesGlob(path, glob) {
|
|
2191
|
-
return globToRegExp(glob).test(normalizePath(path));
|
|
2192
|
-
}
|
|
2193
|
-
|
|
2194
|
-
// src/tracevals/graders/when.ts
|
|
2195
|
-
var stringValue = (kind) => (value) => typeof value === "string" && value.length > 0 ? void 0 : `options.when.${kind} must be a string`;
|
|
2196
|
-
var prompts = /* @__PURE__ */ new Map();
|
|
2197
|
-
function promptRe(pattern) {
|
|
2198
|
-
const cached = prompts.get(pattern);
|
|
2199
|
-
if (cached !== void 0) return cached;
|
|
2200
|
-
const built = new RegExp(pattern);
|
|
2201
|
-
prompts.set(pattern, built);
|
|
2202
|
-
return built;
|
|
2203
|
-
}
|
|
2204
|
-
var CONDITIONS = {
|
|
2205
|
-
"file-access": {
|
|
2206
|
-
validate: stringValue("file-access"),
|
|
2207
|
-
test: (value, window) => window.fileAccesses.some((a) => matchesGlob(a.path, value)),
|
|
2208
|
-
reason: (value) => `no file matching "${value}" was accessed`
|
|
2209
|
-
},
|
|
2210
|
-
"tool-used": {
|
|
2211
|
-
validate: stringValue("tool-used"),
|
|
2212
|
-
test: (value, window) => window.toolCalls.some((c) => c.name === value),
|
|
2213
|
-
reason: (value) => `the ${value} tool was never used`
|
|
2214
|
-
},
|
|
2215
|
-
"prompt-matches": {
|
|
2216
|
-
validate: (value) => {
|
|
2217
|
-
const shape = stringValue("prompt-matches")(value);
|
|
2218
|
-
if (shape !== void 0) return shape;
|
|
2219
|
-
try {
|
|
2220
|
-
new RegExp(value);
|
|
2221
|
-
} catch (err) {
|
|
2222
|
-
return `options.when.prompt-matches is not a valid regular expression: ${err.message}`;
|
|
2223
|
-
}
|
|
2224
|
-
return void 0;
|
|
2225
|
-
},
|
|
2226
|
-
test: (value, window) => {
|
|
2227
|
-
const re = promptRe(value);
|
|
2228
|
-
return window.userMessages.some((text) => re.test(text));
|
|
2229
|
-
},
|
|
2230
|
-
reason: (value) => `no prompt matched /${value}/`
|
|
2231
|
-
},
|
|
2232
|
-
"turn-count-above": {
|
|
2233
|
-
validate: (value) => {
|
|
2234
|
-
if (typeof value !== "number" || !Number.isInteger(value)) {
|
|
2235
|
-
return "options.when.turn-count-above must be a whole number";
|
|
2236
|
-
}
|
|
2237
|
-
if (value < 0) return "options.when.turn-count-above must be at least 0";
|
|
2238
|
-
return void 0;
|
|
2239
|
-
},
|
|
2240
|
-
test: (value, window) => window.turnCount > value,
|
|
2241
|
-
reason: (value, window) => `${window.turnCount} turn(s) is not more than ${value}`
|
|
2242
|
-
}
|
|
2243
|
-
};
|
|
2244
|
-
var WHEN_CONDITIONS = Object.keys(
|
|
2245
|
-
CONDITIONS
|
|
2246
|
-
);
|
|
2247
|
-
function validateWhen(options) {
|
|
2248
|
-
const when = options.when;
|
|
2249
|
-
if (when === void 0) return void 0;
|
|
2250
|
-
if (typeof when !== "object" || when === null || Array.isArray(when)) {
|
|
2251
|
-
return "options.when must be an object of conditions";
|
|
2252
|
-
}
|
|
2253
|
-
const conditions = when;
|
|
2254
|
-
const keys = Object.keys(conditions);
|
|
2255
|
-
if (keys.length === 0) {
|
|
2256
|
-
return `options.when needs at least one of: ${WHEN_CONDITIONS.join(", ")}`;
|
|
2257
|
-
}
|
|
2258
|
-
for (const key of keys) {
|
|
2259
|
-
const condition = CONDITIONS[key];
|
|
2260
|
-
if (condition === void 0) {
|
|
2261
|
-
return `options.when.${key} is not a known condition; expected one of: ${WHEN_CONDITIONS.join(", ")}`;
|
|
2262
|
-
}
|
|
2263
|
-
const invalid = condition.validate(conditions[key]);
|
|
2264
|
-
if (invalid !== void 0) return invalid;
|
|
2265
|
-
}
|
|
2266
|
-
return void 0;
|
|
2267
|
-
}
|
|
2268
|
-
function evaluateWhen(options, window) {
|
|
2269
|
-
const when = options.when;
|
|
2270
|
-
if (when === void 0) return { armed: true };
|
|
2271
|
-
for (const name of WHEN_CONDITIONS) {
|
|
2272
|
-
const value = when[name];
|
|
2273
|
-
if (value === void 0) continue;
|
|
2274
|
-
const condition = CONDITIONS[name];
|
|
2275
|
-
if (condition.test(value, window)) continue;
|
|
2276
|
-
return {
|
|
2277
|
-
armed: false,
|
|
2278
|
-
reason: `trigger not met (${window.label}): ${condition.reason(value, window)}`
|
|
2279
|
-
};
|
|
2280
|
-
}
|
|
2281
|
-
return { armed: true };
|
|
2282
|
-
}
|
|
2283
|
-
function rejectWhen(kind, options) {
|
|
2284
|
-
if (options.when === void 0) return void 0;
|
|
2285
|
-
return `options.when is not supported by the ${kind} grader, which grades the whole session rather than a window`;
|
|
2286
|
-
}
|
|
2287
|
-
function skippedTrigger(trigger) {
|
|
2288
|
-
return { findings: [], skipped: trigger.reason ?? "trigger not met" };
|
|
2289
|
-
}
|
|
2290
|
-
|
|
2291
|
-
// src/tracevals/graders/util.ts
|
|
2292
|
-
function finding(plan, message) {
|
|
2293
|
-
return {
|
|
2294
|
-
evalName: plan.evalName,
|
|
2295
|
-
artifact: plan.artifact.path,
|
|
2296
|
-
message,
|
|
2297
|
-
severity: plan.severity
|
|
2298
|
-
};
|
|
2299
|
-
}
|
|
2300
|
-
function fail(plan, message) {
|
|
2301
|
-
return { findings: [finding(plan, message)] };
|
|
2302
|
-
}
|
|
2303
|
-
var pass = { findings: [] };
|
|
2304
|
-
function optionsError(kind, message) {
|
|
2305
|
-
return { findings: [], error: `${kind}: ${message}` };
|
|
2306
|
-
}
|
|
2307
|
-
function firstError(...checks) {
|
|
2308
|
-
return checks.find((check) => check !== void 0);
|
|
2309
|
-
}
|
|
2310
|
-
function requiredString(options, key) {
|
|
2311
|
-
const value = options[key];
|
|
2312
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
2313
|
-
return `options.${key} is required`;
|
|
2314
|
-
}
|
|
2315
|
-
return void 0;
|
|
2316
|
-
}
|
|
2317
|
-
function optionalString(options, key) {
|
|
2318
|
-
const value = options[key];
|
|
2319
|
-
if (value === void 0) return void 0;
|
|
2320
|
-
if (typeof value !== "string" || value.length === 0) {
|
|
2321
|
-
return `options.${key} must be a non-empty string`;
|
|
2322
|
-
}
|
|
2323
|
-
return void 0;
|
|
2324
|
-
}
|
|
2325
|
-
function optionalEnum(options, key, allowed) {
|
|
2326
|
-
const value = options[key];
|
|
2327
|
-
if (value === void 0) return void 0;
|
|
2328
|
-
if (typeof value !== "string" || !allowed.includes(value)) {
|
|
2329
|
-
return `options.${key} must be one of: ${allowed.join(", ")}`;
|
|
2330
|
-
}
|
|
2331
|
-
return void 0;
|
|
2332
|
-
}
|
|
2333
|
-
function optionalNumber(options, key, bounds = {}) {
|
|
2334
|
-
const value = options[key];
|
|
2335
|
-
if (value === void 0) return void 0;
|
|
2336
|
-
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
2337
|
-
return `options.${key} must be a finite number`;
|
|
2338
|
-
}
|
|
2339
|
-
if (bounds.integer === true && !Number.isInteger(value)) {
|
|
2340
|
-
return `options.${key} must be a whole number`;
|
|
2341
|
-
}
|
|
2342
|
-
if (bounds.min !== void 0 && value < bounds.min) {
|
|
2343
|
-
return `options.${key} must be at least ${bounds.min}`;
|
|
2344
|
-
}
|
|
2345
|
-
return void 0;
|
|
2346
|
-
}
|
|
2347
|
-
function optionalBoolean(options, key) {
|
|
2348
|
-
if (options[key] !== void 0 && typeof options[key] !== "boolean") {
|
|
2349
|
-
return `options.${key} must be a boolean`;
|
|
2350
|
-
}
|
|
2351
|
-
return void 0;
|
|
2352
|
-
}
|
|
2353
|
-
function orderedBounds(options, minKey, maxKey) {
|
|
2354
|
-
const min = options[minKey];
|
|
2355
|
-
const max = options[maxKey];
|
|
2356
|
-
if (typeof min === "number" && typeof max === "number" && min > max) {
|
|
2357
|
-
return `options.${maxKey} must be greater than or equal to options.${minKey}`;
|
|
2358
|
-
}
|
|
2359
|
-
return void 0;
|
|
2360
|
-
}
|
|
2361
|
-
function requireOneOf(options, keys) {
|
|
2362
|
-
if (keys.some((key) => options[key] !== void 0)) return void 0;
|
|
2363
|
-
return `at least one of ${keys.map((k) => `options.${k}`).join(" or ")} is required`;
|
|
2364
|
-
}
|
|
2365
|
-
function windowFor(trace, plan) {
|
|
2366
|
-
const { name, type } = plan.artifact;
|
|
2367
|
-
if (type === "skill") return skillWindow(trace, name);
|
|
2368
|
-
if (type === "agent") return agentWindow(trace, name);
|
|
2369
|
-
if (type === "slash-command") return slashCommandWindow(trace, name);
|
|
2370
|
-
return {
|
|
2371
|
-
scope: "session",
|
|
2372
|
-
label: "whole session",
|
|
2373
|
-
empty: false,
|
|
2374
|
-
events: trace.events,
|
|
2375
|
-
toolCalls: trace.toolCalls,
|
|
2376
|
-
fileAccesses: trace.fileAccesses,
|
|
2377
|
-
skillInvocations: trace.skillInvocations,
|
|
2378
|
-
userMessages: trace.userMessages,
|
|
2379
|
-
assistantTexts: trace.assistantTexts,
|
|
2380
|
-
turnCount: trace.turnCount
|
|
2381
|
-
};
|
|
2382
|
-
}
|
|
2383
|
-
function skippedWindow(window) {
|
|
2384
|
-
return { findings: [], skipped: window.reason ?? "the artifact governed no turns" };
|
|
2385
|
-
}
|
|
2386
|
-
function skillWindow(trace, name) {
|
|
2387
|
-
return injectionWindow(trace, "skill", `skill "${name}"`, (s) => s.name === name);
|
|
2388
|
-
}
|
|
2389
|
-
function slashCommandWindow(trace, name) {
|
|
2390
|
-
return injectionWindow(
|
|
2391
|
-
trace,
|
|
2392
|
-
"slash-command",
|
|
2393
|
-
`slash command "/${name}"`,
|
|
2394
|
-
(s) => s.name === name && s.via === "command-injection"
|
|
2395
|
-
);
|
|
2396
|
-
}
|
|
2397
|
-
function injectionWindow(trace, scope, label, opensHere) {
|
|
2398
|
-
const opens = trace.skillInvocations.filter(opensHere).map((s) => s.index);
|
|
2399
|
-
if (opens.length === 0) {
|
|
2400
|
-
return emptyWindow(
|
|
2401
|
-
scope,
|
|
2402
|
-
label,
|
|
2403
|
-
`${label} was never invoked in this trace, so it governed no turns`
|
|
2404
|
-
);
|
|
2405
|
-
}
|
|
2406
|
-
const chainOf = (index) => trace.events[index]?.branchId;
|
|
2407
|
-
const own = new Set(opens.map(chainOf));
|
|
2408
|
-
const boundaries = trace.skillInvocations.filter((s) => own.has(chainOf(s.index))).map((s) => s.index).sort((a, b) => a - b);
|
|
2409
|
-
const spans = opens.map((start) => ({
|
|
2410
|
-
start,
|
|
2411
|
-
end: boundaries.find((b) => b > start) ?? Number.POSITIVE_INFINITY
|
|
2412
|
-
}));
|
|
2413
|
-
return materialize(trace, scope, label, spans, void 0, own);
|
|
2414
|
-
}
|
|
2415
|
-
function agentWindow(trace, name) {
|
|
2416
|
-
const label = `agent "${name}"`;
|
|
2417
|
-
const mine = trace.subagentBranches.filter((b) => b.agentType === name);
|
|
2418
|
-
if (mine.length === 0) {
|
|
2419
|
-
const spawned = trace.agentSpawns.some((a) => a.subagentType === name);
|
|
2420
|
-
return emptyWindow(
|
|
2421
|
-
"agent",
|
|
2422
|
-
label,
|
|
2423
|
-
spawned ? `${label} was spawned but recorded no subagent turns (no inline sidechain records and no sidecar transcript), so it governed no turns` : `${label} was never spawned in this trace, so it governed no turns`
|
|
2424
|
-
);
|
|
2425
|
-
}
|
|
2426
|
-
const branchIds = new Set(mine.map((b) => b.branchId));
|
|
2427
|
-
for (const branch of trace.subagentBranches) {
|
|
2428
|
-
if (branchIds.has(branch.branchId)) continue;
|
|
2429
|
-
const nested = mine.some(
|
|
2430
|
-
(m) => m.startIndex <= branch.startIndex && branch.endIndex <= m.endIndex
|
|
2431
|
-
);
|
|
2432
|
-
if (nested) branchIds.add(branch.branchId);
|
|
2433
|
-
}
|
|
2434
|
-
const spans = mine.map((b) => ({ start: b.startIndex, end: b.endIndex }));
|
|
2435
|
-
return materialize(
|
|
2436
|
-
trace,
|
|
2437
|
-
"agent",
|
|
2438
|
-
label,
|
|
2439
|
-
spans,
|
|
2440
|
-
branchIds,
|
|
2441
|
-
new Set(mine.map((b) => b.branchId))
|
|
2442
|
-
);
|
|
2443
|
-
}
|
|
2444
|
-
function emptyWindow(scope, label, reason) {
|
|
2445
|
-
return {
|
|
2446
|
-
scope,
|
|
2447
|
-
label,
|
|
2448
|
-
empty: true,
|
|
2449
|
-
reason,
|
|
2450
|
-
events: [],
|
|
2451
|
-
toolCalls: [],
|
|
2452
|
-
fileAccesses: [],
|
|
2453
|
-
skillInvocations: [],
|
|
2454
|
-
userMessages: [],
|
|
2455
|
-
assistantTexts: [],
|
|
2456
|
-
turnCount: 0
|
|
2457
|
-
};
|
|
2458
|
-
}
|
|
2459
|
-
function materialize(trace, scope, label, spans, branchIds, own) {
|
|
2460
|
-
const inSpan = (index) => spans.some((s) => index >= s.start && index < s.end);
|
|
2461
|
-
const excluded = /* @__PURE__ */ new Set();
|
|
2462
|
-
if (branchIds !== void 0) {
|
|
2463
|
-
for (const event of trace.events) {
|
|
2464
|
-
if (!inSpan(event.index)) continue;
|
|
2465
|
-
if (event.branchId === void 0 || !branchIds.has(event.branchId)) {
|
|
2466
|
-
excluded.add(event.index);
|
|
2467
|
-
}
|
|
2468
|
-
}
|
|
2469
|
-
}
|
|
2470
|
-
const inWindow = (index) => inSpan(index) && !excluded.has(index);
|
|
2471
|
-
const events = trace.events.filter((e) => inWindow(e.index));
|
|
2472
|
-
const textOf = (kind) => events.filter((e) => e.kind === kind && own.has(e.branchId) && e.text).map((e) => e.text);
|
|
2473
|
-
const userMessages = textOf("user");
|
|
2474
|
-
return {
|
|
2475
|
-
scope,
|
|
2476
|
-
label,
|
|
2477
|
-
empty: false,
|
|
2478
|
-
events,
|
|
2479
|
-
toolCalls: trace.toolCalls.filter((c) => inWindow(c.index)),
|
|
2480
|
-
fileAccesses: trace.fileAccesses.filter((a) => inWindow(a.index)),
|
|
2481
|
-
skillInvocations: trace.skillInvocations.filter((s) => inWindow(s.index)),
|
|
2482
|
-
userMessages,
|
|
2483
|
-
assistantTexts: textOf("assistant"),
|
|
2484
|
-
turnCount: userMessages.length
|
|
2485
|
-
};
|
|
2486
|
-
}
|
|
2487
|
-
|
|
2488
1480
|
// src/tracevals/graders/tool-usage.ts
|
|
2489
1481
|
function checkContradiction(options) {
|
|
2490
1482
|
const expect = options.expect ?? "used";
|
|
@@ -2789,7 +1781,7 @@ var costGrader = {
|
|
|
2789
1781
|
|
|
2790
1782
|
// src/tracevals/core/target.ts
|
|
2791
1783
|
import { readFileSync } from "node:fs";
|
|
2792
|
-
import { isAbsolute as isAbsolute2, relative as relative2, resolve
|
|
1784
|
+
import { isAbsolute as isAbsolute2, relative as relative2, resolve, sep } from "node:path";
|
|
2793
1785
|
var DEFAULT_TARGET = "transcript";
|
|
2794
1786
|
function describeTarget(target) {
|
|
2795
1787
|
const t = target ?? DEFAULT_TARGET;
|
|
@@ -2817,7 +1809,7 @@ function readTarget(target, ctx) {
|
|
|
2817
1809
|
reason: `target file "${t.path}" is an absolute path; name it relative to the project root`
|
|
2818
1810
|
};
|
|
2819
1811
|
}
|
|
2820
|
-
const abs =
|
|
1812
|
+
const abs = resolve(ctx.root, t.path);
|
|
2821
1813
|
const rel = relative2(ctx.root, abs);
|
|
2822
1814
|
if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute2(rel)) {
|
|
2823
1815
|
return {
|
|
@@ -2913,13 +1905,13 @@ var regexGrader = {
|
|
|
2913
1905
|
|
|
2914
1906
|
// src/tracevals/graders/json-output.ts
|
|
2915
1907
|
import { Ajv2020 } from "ajv/dist/2020.js";
|
|
2916
|
-
var
|
|
1908
|
+
var compiled = /* @__PURE__ */ new Map();
|
|
2917
1909
|
function compileSchema(schema) {
|
|
2918
1910
|
const key = JSON.stringify(schema);
|
|
2919
|
-
const hit =
|
|
1911
|
+
const hit = compiled.get(key);
|
|
2920
1912
|
if (hit !== void 0) return hit;
|
|
2921
1913
|
const built = new Ajv2020({ allErrors: true, strict: false }).compile(schema);
|
|
2922
|
-
|
|
1914
|
+
compiled.set(key, built);
|
|
2923
1915
|
return built;
|
|
2924
1916
|
}
|
|
2925
1917
|
function validateOptions8(options) {
|
|
@@ -3060,133 +2052,6 @@ function listGraderKinds() {
|
|
|
3060
2052
|
return [...graders.keys()];
|
|
3061
2053
|
}
|
|
3062
2054
|
|
|
3063
|
-
// src/tracevals/judge/render.ts
|
|
3064
|
-
var DEFAULTS = {
|
|
3065
|
-
maxBlockChars: 2e3,
|
|
3066
|
-
maxTotalChars: 15e4,
|
|
3067
|
-
redact: []
|
|
3068
|
-
};
|
|
3069
|
-
var MIN_TIMELINE_CHARS = 200;
|
|
3070
|
-
var HEADER_TRUNCATED = "\n[... header truncated ...]";
|
|
3071
|
-
var TIMELINE_SHELL = "\n\n## Timeline\n";
|
|
3072
|
-
function renderTrace(trace, options = {}, plan) {
|
|
3073
|
-
const { maxBlockChars, maxTotalChars, redact } = { ...DEFAULTS, ...options };
|
|
3074
|
-
const scrub = makeRedactor(redact);
|
|
3075
|
-
const clip = (text, max) => {
|
|
3076
|
-
const safe = scrub(text);
|
|
3077
|
-
if (safe.length <= max) return safe;
|
|
3078
|
-
return `${safe.slice(0, max)} [... truncated ${safe.length - max} chars ...]`;
|
|
3079
|
-
};
|
|
3080
|
-
const window = plan === void 0 ? void 0 : windowFor(trace, plan);
|
|
3081
|
-
const scoped = window !== void 0 && window.scope !== "session" ? window : void 0;
|
|
3082
|
-
const events = scoped?.events ?? trace.events;
|
|
3083
|
-
const fullHeader = scrub(
|
|
3084
|
-
[
|
|
3085
|
-
"# Session",
|
|
3086
|
-
`source: ${trace.source}`,
|
|
3087
|
-
trace.model ? `model: ${trace.model}` : void 0,
|
|
3088
|
-
`cwd: ${trace.cwd}`,
|
|
3089
|
-
trace.gitBranch ? `branch: ${trace.gitBranch}` : void 0,
|
|
3090
|
-
scoped ? `scope: ${scoped.label} \u2014 ${events.length} of ${trace.events.length} session events` : void 0,
|
|
3091
|
-
// Everything below describes what the judge can actually see. On a
|
|
3092
|
-
// scoped render the timeline is the window, so a session-wide count here
|
|
3093
|
-
// reads as evidence that the rest was truncated away — the judge would
|
|
3094
|
-
// discount a digest that is complete for what it was asked about. The
|
|
3095
|
-
// session totals stay, in parentheses, because "3 of 87" is context the
|
|
3096
|
-
// window count alone does not carry.
|
|
3097
|
-
scoped ? `turns: ${scoped.turnCount} (of ${trace.turnCount} in the session)` : `turns: ${trace.turnCount}`,
|
|
3098
|
-
(() => {
|
|
3099
|
-
const used = (scoped ?? trace).skillInvocations.map((s) => s.name);
|
|
3100
|
-
return used.length > 0 ? `skills used: ${used.join(", ")}` : "skills used: none";
|
|
3101
|
-
})(),
|
|
3102
|
-
(() => {
|
|
3103
|
-
const inWindow = new Set(events.map((e) => e.index));
|
|
3104
|
-
const spawns = trace.agentSpawns.filter(
|
|
3105
|
-
(a) => !scoped || inWindow.has(a.index)
|
|
3106
|
-
);
|
|
3107
|
-
return spawns.length > 0 ? `agents spawned: ${spawns.map((a) => a.subagentType).join(", ")}` : void 0;
|
|
3108
|
-
})(),
|
|
3109
|
-
(() => {
|
|
3110
|
-
const inWindow = new Set(events.map((e) => e.index));
|
|
3111
|
-
const branches = trace.subagentBranches.filter(
|
|
3112
|
-
(b) => !scoped || inWindow.has(b.spawnIndex)
|
|
3113
|
-
);
|
|
3114
|
-
return branches.length > 0 ? `subagent transcripts: ${branches.map((b) => `${b.agentType} (depth ${b.spawnDepth})`).join(", ")}` : void 0;
|
|
3115
|
-
})()
|
|
3116
|
-
].filter((line) => line !== void 0).join("\n")
|
|
3117
|
-
);
|
|
3118
|
-
const headerRoom = Math.max(
|
|
3119
|
-
maxTotalChars - TIMELINE_SHELL.length - MIN_TIMELINE_CHARS,
|
|
3120
|
-
0
|
|
3121
|
-
);
|
|
3122
|
-
const header = fullHeader.length <= headerRoom ? fullHeader : fullHeader.slice(
|
|
3123
|
-
0,
|
|
3124
|
-
Math.max(headerRoom - HEADER_TRUNCATED.length, 0)
|
|
3125
|
-
) + HEADER_TRUNCATED;
|
|
3126
|
-
const branchTypes = /* @__PURE__ */ new Map();
|
|
3127
|
-
for (const spawn of trace.agentSpawns) {
|
|
3128
|
-
if (spawn.toolUseId) branchTypes.set(spawn.toolUseId, spawn.subagentType);
|
|
3129
|
-
}
|
|
3130
|
-
for (const branch2 of trace.subagentBranches) {
|
|
3131
|
-
branchTypes.set(branch2.branchId, branch2.agentType);
|
|
3132
|
-
}
|
|
3133
|
-
const label = (branchId) => branchTypes.get(branchId) ?? branchId;
|
|
3134
|
-
const callAt = new Map(trace.toolCalls.map((call) => [call.index, call]));
|
|
3135
|
-
if (scoped?.empty === true) {
|
|
3136
|
-
return `${header}${TIMELINE_SHELL}[no turns: ${scrub(scoped.reason ?? "")}]`;
|
|
3137
|
-
}
|
|
3138
|
-
const lines = [];
|
|
3139
|
-
let branch;
|
|
3140
|
-
for (const event of events) {
|
|
3141
|
-
const tag = event.branchId ? `:${label(event.branchId)}` : event.sidechain ? ":sidechain" : "";
|
|
3142
|
-
let line;
|
|
3143
|
-
switch (event.kind) {
|
|
3144
|
-
case "user":
|
|
3145
|
-
if (event.text) line = `[user${tag}] ${clip(event.text, maxBlockChars)}`;
|
|
3146
|
-
break;
|
|
3147
|
-
case "assistant":
|
|
3148
|
-
if (event.text) {
|
|
3149
|
-
line = `[assistant${tag}] ${clip(event.text, maxBlockChars)}`;
|
|
3150
|
-
}
|
|
3151
|
-
break;
|
|
3152
|
-
case "tool_call": {
|
|
3153
|
-
const call = callAt.get(event.index);
|
|
3154
|
-
const input = call ? JSON.stringify(call.input) : "";
|
|
3155
|
-
line = `[tool${tag}] ${event.toolName ?? call?.name ?? "?"} ${clip(input, Math.min(maxBlockChars, 500))}`;
|
|
3156
|
-
break;
|
|
3157
|
-
}
|
|
3158
|
-
default:
|
|
3159
|
-
break;
|
|
3160
|
-
}
|
|
3161
|
-
if (line === void 0) continue;
|
|
3162
|
-
if (event.branchId !== branch) {
|
|
3163
|
-
if (branch !== void 0) lines.push(`[/subagent ${label(branch)}]`);
|
|
3164
|
-
branch = event.branchId;
|
|
3165
|
-
if (branch !== void 0) {
|
|
3166
|
-
lines.push(`[subagent ${label(branch)} (${branch})]`);
|
|
3167
|
-
}
|
|
3168
|
-
}
|
|
3169
|
-
lines.push(line);
|
|
3170
|
-
}
|
|
3171
|
-
if (branch !== void 0) lines.push(`[/subagent ${label(branch)}]`);
|
|
3172
|
-
let timeline = scrub(lines.join("\n"));
|
|
3173
|
-
const budget = Math.max(
|
|
3174
|
-
maxTotalChars - header.length - TIMELINE_SHELL.length,
|
|
3175
|
-
MIN_TIMELINE_CHARS
|
|
3176
|
-
);
|
|
3177
|
-
if (timeline.length > budget) {
|
|
3178
|
-
const marker = (omitted2) => `
|
|
3179
|
-
[... truncated ${omitted2} chars of transcript ...]
|
|
3180
|
-
`;
|
|
3181
|
-
const room = Math.max(budget - marker(timeline.length).length, 0);
|
|
3182
|
-
const headLen = Math.floor(room * 0.6);
|
|
3183
|
-
const tailLen = room - headLen;
|
|
3184
|
-
const omitted = timeline.length - headLen - tailLen;
|
|
3185
|
-
timeline = `${timeline.slice(0, headLen)}${marker(omitted)}${tailLen > 0 ? timeline.slice(-tailLen) : ""}`;
|
|
3186
|
-
}
|
|
3187
|
-
return `${header}${TIMELINE_SHELL}${timeline}`;
|
|
3188
|
-
}
|
|
3189
|
-
|
|
3190
2055
|
// src/tracevals/core/engine.ts
|
|
3191
2056
|
async function runEvals(options) {
|
|
3192
2057
|
const start = Date.now();
|
|
@@ -3495,7 +2360,7 @@ function summariseManifest(found, coverage) {
|
|
|
3495
2360
|
}
|
|
3496
2361
|
|
|
3497
2362
|
// src/tracevals/evals/external.ts
|
|
3498
|
-
import { relative as relative3, resolve as
|
|
2363
|
+
import { relative as relative3, resolve as resolve2, sep as sep2 } from "node:path";
|
|
3499
2364
|
var METADATA_KEY = "metadata";
|
|
3500
2365
|
var MARKED = /* @__PURE__ */ new Set([METADATA_KEY]);
|
|
3501
2366
|
var toPosix2 = (path) => path.split(sep2).join("/");
|
|
@@ -3561,7 +2426,7 @@ function reader(scoped, opts) {
|
|
|
3561
2426
|
const merged = await mergeWithMarks(
|
|
3562
2427
|
label,
|
|
3563
2428
|
extracted,
|
|
3564
|
-
await indexFor(
|
|
2429
|
+
await indexFor(resolve2(base, label)),
|
|
3565
2430
|
[first.name],
|
|
3566
2431
|
base,
|
|
3567
2432
|
{ encryptionKey: () => opts.key, marks: await marks() }
|
|
@@ -3569,7 +2434,7 @@ function reader(scoped, opts) {
|
|
|
3569
2434
|
const locate2 = (pointer) => {
|
|
3570
2435
|
const found = merged.locate(pointer);
|
|
3571
2436
|
if (found === void 0 || classifyRef(found.file).kind === "url") return found;
|
|
3572
|
-
return { ...found, file:
|
|
2437
|
+
return { ...found, file: resolve2(base, found.file) };
|
|
3573
2438
|
};
|
|
3574
2439
|
const at = merged.locate(`/${METADATA_KEY}`);
|
|
3575
2440
|
const [declaration] = first.externalMetadata;
|
|
@@ -3579,14 +2444,14 @@ function reader(scoped, opts) {
|
|
|
3579
2444
|
const url = classifyRef(at.file).kind === "url";
|
|
3580
2445
|
const owner = {
|
|
3581
2446
|
collection: first.name,
|
|
3582
|
-
path: url ? at.file :
|
|
2447
|
+
path: url ? at.file : resolve2(base, at.file),
|
|
3583
2448
|
file: at.file,
|
|
3584
2449
|
join: externalMetadataJoin(declaration),
|
|
3585
2450
|
url
|
|
3586
2451
|
};
|
|
3587
2452
|
const out = { extracted: merged.extracted, locate: locate2, owner };
|
|
3588
2453
|
if (owner.join === PATH_JOIN) {
|
|
3589
|
-
out.entry = toPosix2(relative3(configDir,
|
|
2454
|
+
out.entry = toPosix2(relative3(configDir, resolve2(base, label)));
|
|
3590
2455
|
} else {
|
|
3591
2456
|
const value = merged.joins.find((j) => j.field === owner.join)?.value ?? extracted.data[owner.join];
|
|
3592
2457
|
if (typeof value === "string" && value !== "") out.entry = value;
|
|
@@ -3601,18 +2466,18 @@ function reader(scoped, opts) {
|
|
|
3601
2466
|
|
|
3602
2467
|
// src/tracevals/graders/plugins.ts
|
|
3603
2468
|
import { createRequire } from "node:module";
|
|
3604
|
-
import { isAbsolute as isAbsolute3, join
|
|
2469
|
+
import { isAbsolute as isAbsolute3, join, resolve as resolve3 } from "node:path";
|
|
3605
2470
|
import { pathToFileURL } from "node:url";
|
|
3606
2471
|
var alreadyLoaded = /* @__PURE__ */ new Set();
|
|
3607
2472
|
var RELATIVE = /^\.\.?[/\\]/;
|
|
3608
2473
|
function locate(specifier, configDir) {
|
|
3609
2474
|
if (specifier.startsWith("file:")) return { href: specifier };
|
|
3610
2475
|
if (RELATIVE.test(specifier) || isAbsolute3(specifier)) {
|
|
3611
|
-
const path =
|
|
2476
|
+
const path = resolve3(configDir, specifier);
|
|
3612
2477
|
return { href: pathToFileURL(path).href, shown: path };
|
|
3613
2478
|
}
|
|
3614
2479
|
try {
|
|
3615
|
-
const path = createRequire(
|
|
2480
|
+
const path = createRequire(join(resolve3(configDir), "package.json")).resolve(
|
|
3616
2481
|
specifier
|
|
3617
2482
|
);
|
|
3618
2483
|
return { href: pathToFileURL(path).href, shown: path };
|
|
@@ -3691,8 +2556,8 @@ async function loadGraderPlugins(options) {
|
|
|
3691
2556
|
}
|
|
3692
2557
|
|
|
3693
2558
|
// src/tracevals/history.ts
|
|
3694
|
-
import { appendFile, mkdir, readFile
|
|
3695
|
-
import { dirname
|
|
2559
|
+
import { appendFile, mkdir, readFile } from "node:fs/promises";
|
|
2560
|
+
import { dirname } from "node:path";
|
|
3696
2561
|
function entryFor(report) {
|
|
3697
2562
|
return {
|
|
3698
2563
|
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -3711,7 +2576,7 @@ function entryFor(report) {
|
|
|
3711
2576
|
}
|
|
3712
2577
|
async function appendHistory(file, report) {
|
|
3713
2578
|
try {
|
|
3714
|
-
await mkdir(
|
|
2579
|
+
await mkdir(dirname(file), { recursive: true });
|
|
3715
2580
|
await appendFile(file, `${JSON.stringify(entryFor(report))}
|
|
3716
2581
|
`, "utf-8");
|
|
3717
2582
|
} catch (err) {
|
|
@@ -3723,7 +2588,7 @@ async function appendHistory(file, report) {
|
|
|
3723
2588
|
async function loadHistory(file) {
|
|
3724
2589
|
let content;
|
|
3725
2590
|
try {
|
|
3726
|
-
content = await
|
|
2591
|
+
content = await readFile(file, "utf-8");
|
|
3727
2592
|
} catch {
|
|
3728
2593
|
return [];
|
|
3729
2594
|
}
|
|
@@ -3771,83 +2636,6 @@ function compareToLast(history, report) {
|
|
|
3771
2636
|
};
|
|
3772
2637
|
}
|
|
3773
2638
|
|
|
3774
|
-
// src/tracevals/judge/provider.ts
|
|
3775
|
-
import {
|
|
3776
|
-
makeProvider as makeInferenceProvider,
|
|
3777
|
-
mockVerdict
|
|
3778
|
-
} from "@hawkeyexl/inference";
|
|
3779
|
-
var CONFIG_KEY = "tracevals.provider";
|
|
3780
|
-
var toTracevalsError = (message) => new TracevalsError(message);
|
|
3781
|
-
function selectProvider2(config, flags = {}, ev = {}) {
|
|
3782
|
-
assertLocalFlag(flags, toTracevalsError);
|
|
3783
|
-
return selectProvider(flags, [
|
|
3784
|
-
{
|
|
3785
|
-
...ev.provider !== void 0 ? { provider: ev.provider } : {},
|
|
3786
|
-
...ev.model !== void 0 ? { model: ev.model } : {},
|
|
3787
|
-
...ev.origin !== void 0 ? { origin: ev.origin } : {}
|
|
3788
|
-
},
|
|
3789
|
-
{
|
|
3790
|
-
...config.provider !== null ? { provider: config.provider } : {},
|
|
3791
|
-
...config.model !== null ? { model: config.model } : {},
|
|
3792
|
-
origin: CONFIG_KEY
|
|
3793
|
-
},
|
|
3794
|
-
{
|
|
3795
|
-
...config.providers.provider !== void 0 ? { provider: config.providers.provider } : {},
|
|
3796
|
-
...config.providers.model !== void 0 ? { model: config.providers.model } : {},
|
|
3797
|
-
origin: "providers.provider"
|
|
3798
|
-
}
|
|
3799
|
-
]);
|
|
3800
|
-
}
|
|
3801
|
-
function announceSelection(selection) {
|
|
3802
|
-
const message = localNotice(selection);
|
|
3803
|
-
if (message !== void 0) noticeOnce(message);
|
|
3804
|
-
}
|
|
3805
|
-
function assertProviderSelection(selection) {
|
|
3806
|
-
assertKnownProvider(selection.provider, toTracevalsError);
|
|
3807
|
-
assertModelHasProvider(selection.provider, selection.model, CONFIG_KEY, toTracevalsError);
|
|
3808
|
-
}
|
|
3809
|
-
function providerSpecFor2(config, selection, options = {}) {
|
|
3810
|
-
const spec = providerSpecFor(config.providers, selection);
|
|
3811
|
-
switch (selection.provider) {
|
|
3812
|
-
case "anthropic":
|
|
3813
|
-
return { ...spec, anthropic: { toolName: "record_verdict" } };
|
|
3814
|
-
case "openai":
|
|
3815
|
-
return { ...spec, openai: { schemaName: "verdict" } };
|
|
3816
|
-
case "mock":
|
|
3817
|
-
return {
|
|
3818
|
-
...spec,
|
|
3819
|
-
mockResponses: options.mockResponses ?? [mockVerdict("pass", 0.95)]
|
|
3820
|
-
};
|
|
3821
|
-
default:
|
|
3822
|
-
return spec;
|
|
3823
|
-
}
|
|
3824
|
-
}
|
|
3825
|
-
async function resolveProviderIdentity(config, flags = {}, ev = {}) {
|
|
3826
|
-
const selection = selectProvider2(config, flags, ev);
|
|
3827
|
-
const { provider, model } = selection;
|
|
3828
|
-
assertKnownProvider(provider, toTracevalsError);
|
|
3829
|
-
assertModelHasProvider(provider, model, CONFIG_KEY, toTracevalsError);
|
|
3830
|
-
announceSelection(selection);
|
|
3831
|
-
return resolveIdentity(
|
|
3832
|
-
providerSpecFor2(config, { provider, model: model ?? null }),
|
|
3833
|
-
toTracevalsError
|
|
3834
|
-
);
|
|
3835
|
-
}
|
|
3836
|
-
function constructProvider(config, identity, options = {}) {
|
|
3837
|
-
try {
|
|
3838
|
-
return makeInferenceProvider(providerSpecFor2(config, identity, options));
|
|
3839
|
-
} catch (e) {
|
|
3840
|
-
throw new TracevalsError(errorMessage(e));
|
|
3841
|
-
}
|
|
3842
|
-
}
|
|
3843
|
-
async function makeJudgeProvider(config, flags = {}, ev = {}, options = {}) {
|
|
3844
|
-
return constructProvider(
|
|
3845
|
-
config,
|
|
3846
|
-
await resolveProviderIdentity(config, flags, ev),
|
|
3847
|
-
options
|
|
3848
|
-
);
|
|
3849
|
-
}
|
|
3850
|
-
|
|
3851
2639
|
// src/tracevals/judge/trace-judge.ts
|
|
3852
2640
|
import {
|
|
3853
2641
|
computeConsensus,
|
|
@@ -4113,7 +2901,7 @@ async function prepareRun(options) {
|
|
|
4113
2901
|
...options.model !== void 0 ? { model: options.model } : {},
|
|
4114
2902
|
...options.local !== void 0 ? { local: options.local } : {}
|
|
4115
2903
|
};
|
|
4116
|
-
const selection =
|
|
2904
|
+
const selection = selectProvider(config, flags);
|
|
4117
2905
|
assertProviderSelection(selection);
|
|
4118
2906
|
let judge = options.judge;
|
|
4119
2907
|
if (judge === void 0 && options.deterministicOnly !== true) {
|
|
@@ -4131,7 +2919,7 @@ async function prepareRun(options) {
|
|
|
4131
2919
|
// then the family's `providers:` — so an eval cannot quietly outrank a
|
|
4132
2920
|
// flag someone just typed.
|
|
4133
2921
|
providerFor: async (ev) => {
|
|
4134
|
-
const chosen =
|
|
2922
|
+
const chosen = selectProvider(config, flags, ev);
|
|
4135
2923
|
announceSelection(chosen);
|
|
4136
2924
|
const key = keyOf(chosen);
|
|
4137
2925
|
if (key === defaultKey) return provider;
|
|
@@ -4151,7 +2939,7 @@ async function prepareRun(options) {
|
|
|
4151
2939
|
runs: options.runs ?? config.judge.ensembleRuns,
|
|
4152
2940
|
temperature: config.judge.temperature,
|
|
4153
2941
|
zones: config.judge.zones,
|
|
4154
|
-
cacheDir:
|
|
2942
|
+
cacheDir: resolve4(configDir, config.judge.cacheDir),
|
|
4155
2943
|
...options.noCache !== void 0 ? { noCache: options.noCache } : {},
|
|
4156
2944
|
maxTurns: options.maxTurns ?? config.judge.maxTurns
|
|
4157
2945
|
});
|
|
@@ -4187,7 +2975,7 @@ async function runOne(options, context) {
|
|
|
4187
2975
|
});
|
|
4188
2976
|
let comparison;
|
|
4189
2977
|
if (options.history) {
|
|
4190
|
-
const historyFile =
|
|
2978
|
+
const historyFile = resolve4(context.configDir, config.history.file);
|
|
4191
2979
|
comparison = compareToLast(await loadHistory(historyFile), report) ?? void 0;
|
|
4192
2980
|
await appendHistory(historyFile, report);
|
|
4193
2981
|
}
|
|
@@ -4314,27 +3102,15 @@ export {
|
|
|
4314
3102
|
render,
|
|
4315
3103
|
renderBatch,
|
|
4316
3104
|
renderCalibration,
|
|
4317
|
-
detectFormat,
|
|
4318
|
-
detectContentFormat,
|
|
4319
|
-
parseTraceFile,
|
|
4320
|
-
parseTraceContent,
|
|
4321
3105
|
IMPLICIT_EVAL_NAME,
|
|
4322
3106
|
planEvals,
|
|
4323
3107
|
PROMPT_VERSION,
|
|
4324
3108
|
JUDGE_SYSTEM_PROMPT,
|
|
4325
3109
|
buildUserContent,
|
|
4326
|
-
globToRegExp,
|
|
4327
|
-
matchesGlob,
|
|
4328
|
-
WHEN_CONDITIONS,
|
|
4329
|
-
validateWhen,
|
|
4330
|
-
evaluateWhen,
|
|
4331
|
-
windowFor,
|
|
4332
|
-
skippedWindow,
|
|
4333
3110
|
BUILTIN_GRADER_KINDS,
|
|
4334
3111
|
registerGrader,
|
|
4335
3112
|
graderFor,
|
|
4336
3113
|
listGraderKinds,
|
|
4337
|
-
renderTrace,
|
|
4338
3114
|
runEvals,
|
|
4339
3115
|
METADATA_KEY,
|
|
4340
3116
|
urlManifestRefusal,
|
|
@@ -4344,13 +3120,6 @@ export {
|
|
|
4344
3120
|
appendHistory,
|
|
4345
3121
|
loadHistory,
|
|
4346
3122
|
compareToLast,
|
|
4347
|
-
selectProvider2 as selectProvider,
|
|
4348
|
-
announceSelection,
|
|
4349
|
-
assertProviderSelection,
|
|
4350
|
-
providerSpecFor2 as providerSpecFor,
|
|
4351
|
-
resolveProviderIdentity,
|
|
4352
|
-
constructProvider,
|
|
4353
|
-
makeJudgeProvider,
|
|
4354
3123
|
sha256,
|
|
4355
3124
|
VerdictCache,
|
|
4356
3125
|
cacheKey,
|
|
@@ -4363,4 +3132,4 @@ export {
|
|
|
4363
3132
|
resolveBatchTraces,
|
|
4364
3133
|
runBatch
|
|
4365
3134
|
};
|
|
4366
|
-
//# sourceMappingURL=chunk-
|
|
3135
|
+
//# sourceMappingURL=chunk-PL4UAF4S.js.map
|