agentv 4.42.4 → 4.44.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{artifact-writer-7PXSB6LM.js → artifact-writer-72MQQELS.js} +4 -4
- package/dist/{chunk-ISPJY2AX.js → chunk-2N3YSEMI.js} +2454 -415
- package/dist/chunk-2N3YSEMI.js.map +1 -0
- package/dist/{chunk-ZPFM52HV.js → chunk-5DYGN547.js} +2379 -635
- package/dist/chunk-5DYGN547.js.map +1 -0
- package/dist/{chunk-N6U3GXMF.js → chunk-CXHYTYHU.js} +620 -342
- package/dist/chunk-CXHYTYHU.js.map +1 -0
- package/dist/{chunk-6SQDBXT7.js → chunk-KAGZRUO6.js} +2 -2
- package/dist/chunk-KAGZRUO6.js.map +1 -0
- package/dist/{chunk-RBOP5X36.js → chunk-NCNDA3F6.js} +2605 -171
- package/dist/chunk-NCNDA3F6.js.map +1 -0
- package/dist/cli.js +5 -5
- package/dist/dashboard/assets/index-1Hvy8sBy.js +119 -0
- package/dist/dashboard/assets/{index-uXJsSG83.js → index-DitffAxS.js} +1 -1
- package/dist/dashboard/assets/index-J5sPcQO3.css +1 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-VQWTETNY.js → dist-L4YCQOQ6.js} +67 -3
- package/dist/index.js +9 -5
- package/dist/{interactive-TNRJEFA2.js → interactive-DRNRKLEQ.js} +5 -5
- package/dist/skills/agentv-bench/SKILL.md +2 -2
- package/dist/skills/agentv-bench/agents/executor.md +1 -1
- package/dist/skills/agentv-bench/agents/grader.md +1 -1
- package/dist/skills/agentv-bench/references/autoresearch.md +4 -4
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +2 -1
- package/dist/skills/agentv-bench/references/schemas.md +1 -1
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +4 -4
- package/dist/skills/agentv-eval-writer/SKILL.md +24 -17
- package/dist/skills/agentv-eval-writer/references/config-schema.json +130 -0
- package/dist/skills/agentv-eval-writer/references/eval-schema.json +192 -0
- package/dist/{ts-eval-loader-PYEU2PMP-YLXIQTEQ.js → ts-eval-loader-S7ESROX2-6CB3HB6H.js} +2 -2
- package/package.json +6 -5
- package/dist/chunk-6SQDBXT7.js.map +0 -1
- package/dist/chunk-ISPJY2AX.js.map +0 -1
- package/dist/chunk-N6U3GXMF.js.map +0 -1
- package/dist/chunk-RBOP5X36.js.map +0 -1
- package/dist/chunk-ZPFM52HV.js.map +0 -1
- package/dist/dashboard/assets/index-BwBJwHp4.js +0 -118
- package/dist/dashboard/assets/index-C5v6_zhR.css +0 -1
- /package/dist/{artifact-writer-7PXSB6LM.js.map → artifact-writer-72MQQELS.js.map} +0 -0
- /package/dist/{dist-VQWTETNY.js.map → dist-L4YCQOQ6.js.map} +0 -0
- /package/dist/{interactive-TNRJEFA2.js.map → interactive-DRNRKLEQ.js.map} +0 -0
- /package/dist/{ts-eval-loader-PYEU2PMP-YLXIQTEQ.js.map → ts-eval-loader-S7ESROX2-6CB3HB6H.js.map} +0 -0
|
@@ -3,11 +3,13 @@ import {
|
|
|
3
3
|
Mutex,
|
|
4
4
|
RESULT_INDEX_FILENAME,
|
|
5
5
|
TARGET_FILE_CANDIDATES,
|
|
6
|
+
TagRevisionConflictError,
|
|
7
|
+
assertExpectedTagRevision,
|
|
6
8
|
buildDefaultRunDir,
|
|
9
|
+
buildDefaultRunDirFromName,
|
|
7
10
|
c,
|
|
8
11
|
clearRemoteRunTags,
|
|
9
12
|
createRunDirName,
|
|
10
|
-
deleteRunTags,
|
|
11
13
|
detectFileType,
|
|
12
14
|
discoverEvalFiles,
|
|
13
15
|
discoverTargetsFile,
|
|
@@ -24,14 +26,19 @@ import {
|
|
|
24
26
|
getTraceSpans,
|
|
25
27
|
getTraceSummary,
|
|
26
28
|
isRemoteRunId,
|
|
29
|
+
isReservedResultsNamespace,
|
|
27
30
|
listMergedResultFiles,
|
|
28
31
|
listResultFiles,
|
|
29
32
|
loadEnvFromHierarchy,
|
|
30
33
|
loadLightweightResults,
|
|
31
34
|
loadManifestResults,
|
|
35
|
+
loadNormalizedResultsConfig,
|
|
32
36
|
loadResultFile,
|
|
33
37
|
loadRunCache,
|
|
38
|
+
materializeRunState,
|
|
34
39
|
maybeAutoExportRunArtifacts,
|
|
40
|
+
normalizeExperimentName,
|
|
41
|
+
normalizeTags,
|
|
35
42
|
package_default,
|
|
36
43
|
padLeft,
|
|
37
44
|
padRight,
|
|
@@ -39,6 +46,7 @@ import {
|
|
|
39
46
|
readRemoteRunTagState,
|
|
40
47
|
readRunTags,
|
|
41
48
|
readTestSuiteTarget,
|
|
49
|
+
relativeRunPathFromCwd,
|
|
42
50
|
resolveEvalPaths,
|
|
43
51
|
resolveResultSourcePath,
|
|
44
52
|
resolveRunCacheFile,
|
|
@@ -56,39 +64,46 @@ import {
|
|
|
56
64
|
validateTargetsFile,
|
|
57
65
|
validateWorkspacePaths,
|
|
58
66
|
writeRunTags
|
|
59
|
-
} from "./chunk-
|
|
67
|
+
} from "./chunk-CXHYTYHU.js";
|
|
60
68
|
import {
|
|
61
69
|
materializeEvalBundle,
|
|
62
70
|
toSnakeCaseDeep as toSnakeCaseDeep2,
|
|
63
71
|
writeArtifactsFromResults as writeArtifactsFromResults2
|
|
64
|
-
} from "./chunk-
|
|
72
|
+
} from "./chunk-KAGZRUO6.js";
|
|
65
73
|
import {
|
|
66
74
|
DEFAULT_CATEGORY,
|
|
67
75
|
deriveCategory,
|
|
68
76
|
discoverClaudeSessions,
|
|
69
77
|
discoverCodexSessions,
|
|
70
78
|
getOutputFilenames,
|
|
79
|
+
normalizeTraceArtifactToTraceSessionResponse,
|
|
71
80
|
parseClaudeSession,
|
|
72
81
|
parseCodexSession,
|
|
73
82
|
prepareEvalWorkspace,
|
|
83
|
+
readGitResultArtifact,
|
|
74
84
|
runBeforeSessionHook,
|
|
75
85
|
scanRepoDeps,
|
|
76
86
|
syncProjects,
|
|
77
87
|
transpileEvalYamlFile,
|
|
78
88
|
trimBaselineResult
|
|
79
|
-
} from "./chunk-
|
|
89
|
+
} from "./chunk-NCNDA3F6.js";
|
|
80
90
|
import {
|
|
91
|
+
AGENTV_RESULTS_ARTIFACTS_REF,
|
|
81
92
|
DEFAULT_THRESHOLD,
|
|
82
93
|
EXECUTION_TRACE_SCHEMA_VERSION,
|
|
83
94
|
addProject,
|
|
84
95
|
aggregateRunDir,
|
|
85
96
|
buildBenchmarkArtifact,
|
|
97
|
+
buildResultIndexArtifact,
|
|
86
98
|
buildTestTargetKey,
|
|
87
99
|
buildTimingArtifact,
|
|
100
|
+
buildTraceEnvelopeFromEvaluationResult,
|
|
88
101
|
buildTraceFromMessages,
|
|
89
102
|
createBuiltinRegistry,
|
|
90
103
|
discoverCopilotSessions,
|
|
91
104
|
executeScript,
|
|
105
|
+
externalTraceMetadataFromRecord,
|
|
106
|
+
external_exports,
|
|
92
107
|
getAgentvConfigDir,
|
|
93
108
|
getProject,
|
|
94
109
|
getWorkspacePoolRoot,
|
|
@@ -100,6 +115,7 @@ import {
|
|
|
100
115
|
loadTestSuite,
|
|
101
116
|
normalizeLineEndings,
|
|
102
117
|
normalizeResultRow,
|
|
118
|
+
omitExternalTraceMetadataKeys,
|
|
103
119
|
parseAgentSkillsEvals,
|
|
104
120
|
parseCopilotEvents,
|
|
105
121
|
parseYamlValue,
|
|
@@ -120,14 +136,16 @@ import {
|
|
|
120
136
|
runRegexAssertion,
|
|
121
137
|
runStartsWithAssertion,
|
|
122
138
|
toCamelCaseDeep,
|
|
139
|
+
toExternalTraceMetadataWire,
|
|
123
140
|
toSnakeCaseDeep,
|
|
141
|
+
toTraceEnvelopeWire,
|
|
124
142
|
toTranscriptJsonLines,
|
|
125
143
|
touchProject,
|
|
126
144
|
traceEnvelopeToTraceSummary,
|
|
127
145
|
traceEnvelopeToTranscriptMessages,
|
|
128
146
|
traceFromTranscriptJsonLines,
|
|
129
147
|
writeArtifactsFromResults
|
|
130
|
-
} from "./chunk-
|
|
148
|
+
} from "./chunk-5DYGN547.js";
|
|
131
149
|
import {
|
|
132
150
|
__commonJS,
|
|
133
151
|
__require,
|
|
@@ -888,7 +906,7 @@ var require_src = __commonJS({
|
|
|
888
906
|
});
|
|
889
907
|
|
|
890
908
|
// src/index.ts
|
|
891
|
-
import
|
|
909
|
+
import path36 from "node:path";
|
|
892
910
|
|
|
893
911
|
// ../../node_modules/.bun/chalk@5.6.2/node_modules/chalk/source/vendor/ansi-styles/index.js
|
|
894
912
|
var ANSI_BACKGROUND_OFFSET = 10;
|
|
@@ -3320,7 +3338,9 @@ ${STYLES}
|
|
|
3320
3338
|
<body>
|
|
3321
3339
|
<header class="header">
|
|
3322
3340
|
<div class="header-left">
|
|
3323
|
-
<h1 class="header-title"
|
|
3341
|
+
<h1 class="header-title" aria-label="AgentV">
|
|
3342
|
+
<span class="brand-letter" aria-hidden="true">A</span><span class="brand-middle" aria-hidden="true">GENT</span><span class="brand-letter" aria-hidden="true">V</span>
|
|
3343
|
+
</h1>
|
|
3324
3344
|
<span class="header-subtitle">Evaluation Report</span>
|
|
3325
3345
|
</div>
|
|
3326
3346
|
<div class="header-right">${liveIndicator}</div>
|
|
@@ -3345,7 +3365,7 @@ var STYLES = `
|
|
|
3345
3365
|
*{margin:0;padding:0;box-sizing:border-box}
|
|
3346
3366
|
:root{
|
|
3347
3367
|
--bg:#f6f8fa;--surface:#fff;--border:#d0d7de;--border-light:#e8ebee;
|
|
3348
|
-
--text:#1f2328;--text-muted:#656d76;
|
|
3368
|
+
--text:#1f2328;--text-muted:#656d76;--brand-middle:#656d76;
|
|
3349
3369
|
--primary:#0969da;--primary-bg:#ddf4ff;
|
|
3350
3370
|
--success:#1a7f37;--success-bg:#dafbe1;
|
|
3351
3371
|
--danger:#cf222e;--danger-bg:#ffebe9;
|
|
@@ -3361,6 +3381,8 @@ body{font-family:var(--font);background:var(--bg);color:var(--text);line-height:
|
|
|
3361
3381
|
.header{background:var(--surface);border-bottom:1px solid var(--border);padding:12px 24px;display:flex;align-items:center;justify-content:space-between}
|
|
3362
3382
|
.header-left{display:flex;align-items:baseline;gap:12px}
|
|
3363
3383
|
.header-title{font-size:18px;font-weight:600}
|
|
3384
|
+
.brand-letter{color:var(--primary)}
|
|
3385
|
+
.brand-middle{color:var(--brand-middle)}
|
|
3364
3386
|
.header-subtitle{font-size:14px;color:var(--text-muted)}
|
|
3365
3387
|
.live-badge{color:var(--success);font-size:12px;font-weight:600;animation:pulse 2s infinite}
|
|
3366
3388
|
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
|
|
@@ -4632,7 +4654,7 @@ var evalRunCommand = command({
|
|
|
4632
4654
|
resultsRemote: option({
|
|
4633
4655
|
type: optional(string),
|
|
4634
4656
|
long: "results-remote",
|
|
4635
|
-
description: "Git remote name for results push/fetch
|
|
4657
|
+
description: "Advanced compatibility override: local Git remote name for results push/fetch"
|
|
4636
4658
|
}),
|
|
4637
4659
|
resultsPush: flag({
|
|
4638
4660
|
long: "results-push",
|
|
@@ -4806,7 +4828,7 @@ var evalRunCommand = command({
|
|
|
4806
4828
|
},
|
|
4807
4829
|
handler: async (args) => {
|
|
4808
4830
|
if (args.evalPaths.length === 0 && process.stdin.isTTY) {
|
|
4809
|
-
const { launchInteractiveWizard } = await import("./interactive-
|
|
4831
|
+
const { launchInteractiveWizard } = await import("./interactive-DRNRKLEQ.js");
|
|
4810
4832
|
await launchInteractiveWizard();
|
|
4811
4833
|
return;
|
|
4812
4834
|
}
|
|
@@ -4882,15 +4904,701 @@ var evalRunCommand = command({
|
|
|
4882
4904
|
}
|
|
4883
4905
|
});
|
|
4884
4906
|
|
|
4907
|
+
// ../../packages/sdk/dist/index.js
|
|
4908
|
+
import { stringify as stringifyYaml2 } from "yaml";
|
|
4909
|
+
import { readFileSync as readFileSync3 } from "node:fs";
|
|
4910
|
+
import { spawn } from "node:child_process";
|
|
4911
|
+
import { copyFile, mkdtemp, readFile as readFile2, rm } from "node:fs/promises";
|
|
4912
|
+
import { tmpdir } from "node:os";
|
|
4913
|
+
import nodePath2 from "node:path";
|
|
4914
|
+
var TokenUsageSchema = external_exports.object({
|
|
4915
|
+
input: external_exports.number(),
|
|
4916
|
+
output: external_exports.number(),
|
|
4917
|
+
cached: external_exports.number().optional(),
|
|
4918
|
+
reasoning: external_exports.number().optional()
|
|
4919
|
+
});
|
|
4920
|
+
var TraceSummarySchema = external_exports.object({
|
|
4921
|
+
eventCount: external_exports.number(),
|
|
4922
|
+
toolCalls: external_exports.record(external_exports.string(), external_exports.number()),
|
|
4923
|
+
errorCount: external_exports.number(),
|
|
4924
|
+
toolDurations: external_exports.record(external_exports.string(), external_exports.array(external_exports.number())).optional(),
|
|
4925
|
+
llmCallCount: external_exports.number().optional()
|
|
4926
|
+
});
|
|
4927
|
+
var TRACE_SOURCE_KINDS = [
|
|
4928
|
+
"agentv_run",
|
|
4929
|
+
"otlp",
|
|
4930
|
+
"phoenix",
|
|
4931
|
+
"langfuse",
|
|
4932
|
+
"pi_session",
|
|
4933
|
+
"imported_transcript",
|
|
4934
|
+
"compact_transcript"
|
|
4935
|
+
];
|
|
4936
|
+
var TRACE_EVENT_TYPES = [
|
|
4937
|
+
"message",
|
|
4938
|
+
"model_turn",
|
|
4939
|
+
"tool_call",
|
|
4940
|
+
"tool_result",
|
|
4941
|
+
"final_response",
|
|
4942
|
+
"error"
|
|
4943
|
+
];
|
|
4944
|
+
var TRACE_TOOL_STATUSES = ["ok", "error", "timeout", "cancelled", "unknown"];
|
|
4945
|
+
var TRACE_REDACTION_LEVELS = ["none", "partial", "full"];
|
|
4946
|
+
var MetadataSchema = external_exports.record(external_exports.string(), external_exports.unknown());
|
|
4947
|
+
var TraceRedactionStateSchema = external_exports.object({
|
|
4948
|
+
level: external_exports.enum(TRACE_REDACTION_LEVELS),
|
|
4949
|
+
fields: external_exports.array(external_exports.string()).optional(),
|
|
4950
|
+
reason: external_exports.string().optional()
|
|
4951
|
+
});
|
|
4952
|
+
var TraceErrorSchema = external_exports.object({
|
|
4953
|
+
message: external_exports.string(),
|
|
4954
|
+
name: external_exports.string().optional(),
|
|
4955
|
+
code: external_exports.string().optional(),
|
|
4956
|
+
stack: external_exports.string().optional(),
|
|
4957
|
+
metadata: MetadataSchema.optional()
|
|
4958
|
+
});
|
|
4959
|
+
var TraceSourceSchema = external_exports.object({
|
|
4960
|
+
kind: external_exports.enum(TRACE_SOURCE_KINDS),
|
|
4961
|
+
path: external_exports.string().optional(),
|
|
4962
|
+
url: external_exports.string().optional(),
|
|
4963
|
+
provider: external_exports.string().optional(),
|
|
4964
|
+
format: external_exports.string().optional(),
|
|
4965
|
+
version: external_exports.string().optional(),
|
|
4966
|
+
metadata: MetadataSchema.optional()
|
|
4967
|
+
});
|
|
4968
|
+
var TraceSessionSchema = external_exports.object({
|
|
4969
|
+
sessionId: external_exports.string().optional(),
|
|
4970
|
+
conversationId: external_exports.string().optional(),
|
|
4971
|
+
cwd: external_exports.string().optional(),
|
|
4972
|
+
startedAt: external_exports.string().optional(),
|
|
4973
|
+
endedAt: external_exports.string().optional(),
|
|
4974
|
+
metadata: MetadataSchema.optional()
|
|
4975
|
+
});
|
|
4976
|
+
var TraceBranchSchema = external_exports.object({
|
|
4977
|
+
selectedLeafId: external_exports.string().optional(),
|
|
4978
|
+
selectedPathIds: external_exports.array(external_exports.string()).optional(),
|
|
4979
|
+
includedEventIds: external_exports.array(external_exports.string()).optional(),
|
|
4980
|
+
omittedEventIds: external_exports.array(external_exports.string()).optional(),
|
|
4981
|
+
selectionReason: external_exports.string().optional()
|
|
4982
|
+
});
|
|
4983
|
+
var TraceSourceRefSchema = external_exports.object({
|
|
4984
|
+
eventId: external_exports.string().optional(),
|
|
4985
|
+
messageId: external_exports.string().optional(),
|
|
4986
|
+
spanId: external_exports.string().optional(),
|
|
4987
|
+
traceId: external_exports.string().optional(),
|
|
4988
|
+
rawKind: external_exports.string().optional(),
|
|
4989
|
+
path: external_exports.string().optional(),
|
|
4990
|
+
line: external_exports.number().int().nonnegative().optional(),
|
|
4991
|
+
metadata: MetadataSchema.optional()
|
|
4992
|
+
});
|
|
4993
|
+
var TraceRawEvidenceSchema = external_exports.object({
|
|
4994
|
+
kind: external_exports.string(),
|
|
4995
|
+
ref: external_exports.string().optional(),
|
|
4996
|
+
mediaType: external_exports.string().optional(),
|
|
4997
|
+
content: external_exports.unknown().optional(),
|
|
4998
|
+
redacted: external_exports.boolean().optional(),
|
|
4999
|
+
metadata: MetadataSchema.optional()
|
|
5000
|
+
});
|
|
5001
|
+
var TraceMessageSchema = external_exports.object({
|
|
5002
|
+
role: external_exports.string(),
|
|
5003
|
+
name: external_exports.string().optional(),
|
|
5004
|
+
content: external_exports.unknown().optional(),
|
|
5005
|
+
redaction: TraceRedactionStateSchema.optional(),
|
|
5006
|
+
tokenUsage: TokenUsageSchema.optional(),
|
|
5007
|
+
metadata: MetadataSchema.optional()
|
|
5008
|
+
});
|
|
5009
|
+
var TraceModelSchema = external_exports.object({
|
|
5010
|
+
provider: external_exports.string().optional(),
|
|
5011
|
+
name: external_exports.string().optional(),
|
|
5012
|
+
invocationId: external_exports.string().optional(),
|
|
5013
|
+
tokenUsage: TokenUsageSchema.optional(),
|
|
5014
|
+
metadata: MetadataSchema.optional()
|
|
5015
|
+
});
|
|
5016
|
+
var TraceToolSchema = external_exports.object({
|
|
5017
|
+
name: external_exports.string(),
|
|
5018
|
+
callId: external_exports.string().optional(),
|
|
5019
|
+
input: external_exports.unknown().optional(),
|
|
5020
|
+
output: external_exports.unknown().optional(),
|
|
5021
|
+
status: external_exports.enum(TRACE_TOOL_STATUSES).optional(),
|
|
5022
|
+
error: TraceErrorSchema.optional(),
|
|
5023
|
+
redaction: TraceRedactionStateSchema.optional(),
|
|
5024
|
+
metadata: MetadataSchema.optional()
|
|
5025
|
+
});
|
|
5026
|
+
var TraceEventSchema = external_exports.object({
|
|
5027
|
+
eventId: external_exports.string(),
|
|
5028
|
+
parentEventId: external_exports.string().optional(),
|
|
5029
|
+
ordinal: external_exports.number().int().nonnegative(),
|
|
5030
|
+
type: external_exports.enum(TRACE_EVENT_TYPES),
|
|
5031
|
+
timestamp: external_exports.string().optional(),
|
|
5032
|
+
durationMs: external_exports.number().nonnegative().optional(),
|
|
5033
|
+
durationInferred: external_exports.boolean().optional(),
|
|
5034
|
+
turnIndex: external_exports.number().int().nonnegative().optional(),
|
|
5035
|
+
message: TraceMessageSchema.optional(),
|
|
5036
|
+
model: TraceModelSchema.optional(),
|
|
5037
|
+
tool: TraceToolSchema.optional(),
|
|
5038
|
+
error: TraceErrorSchema.optional(),
|
|
5039
|
+
sourceRef: TraceSourceRefSchema.optional(),
|
|
5040
|
+
rawEvidence: external_exports.array(TraceRawEvidenceSchema).optional(),
|
|
5041
|
+
redaction: TraceRedactionStateSchema.optional(),
|
|
5042
|
+
metadata: MetadataSchema.optional()
|
|
5043
|
+
});
|
|
5044
|
+
var TraceArtifactSchema = external_exports.object({
|
|
5045
|
+
source: TraceSourceSchema,
|
|
5046
|
+
session: TraceSessionSchema,
|
|
5047
|
+
branch: TraceBranchSchema.optional(),
|
|
5048
|
+
events: external_exports.array(TraceEventSchema),
|
|
5049
|
+
tokenUsage: TokenUsageSchema.optional(),
|
|
5050
|
+
costUsd: external_exports.number().optional(),
|
|
5051
|
+
durationMs: external_exports.number().optional(),
|
|
5052
|
+
startedAt: external_exports.string().optional(),
|
|
5053
|
+
endedAt: external_exports.string().optional(),
|
|
5054
|
+
metadata: MetadataSchema.optional()
|
|
5055
|
+
});
|
|
5056
|
+
var ToolCallSchema = external_exports.object({
|
|
5057
|
+
tool: external_exports.string(),
|
|
5058
|
+
input: external_exports.unknown().optional(),
|
|
5059
|
+
output: external_exports.unknown().optional(),
|
|
5060
|
+
id: external_exports.string().optional(),
|
|
5061
|
+
startTime: external_exports.string().optional(),
|
|
5062
|
+
endTime: external_exports.string().optional(),
|
|
5063
|
+
durationMs: external_exports.number().optional()
|
|
5064
|
+
});
|
|
5065
|
+
var ContentTextSchema = external_exports.object({
|
|
5066
|
+
type: external_exports.literal("text"),
|
|
5067
|
+
text: external_exports.string()
|
|
5068
|
+
});
|
|
5069
|
+
var ContentImageSchema = external_exports.object({
|
|
5070
|
+
type: external_exports.literal("image"),
|
|
5071
|
+
media_type: external_exports.string(),
|
|
5072
|
+
path: external_exports.string()
|
|
5073
|
+
});
|
|
5074
|
+
var ContentFileSchema = external_exports.object({
|
|
5075
|
+
type: external_exports.literal("file"),
|
|
5076
|
+
media_type: external_exports.string(),
|
|
5077
|
+
path: external_exports.string()
|
|
5078
|
+
});
|
|
5079
|
+
var ContentSchema = external_exports.discriminatedUnion("type", [
|
|
5080
|
+
ContentTextSchema,
|
|
5081
|
+
ContentImageSchema,
|
|
5082
|
+
ContentFileSchema
|
|
5083
|
+
]);
|
|
5084
|
+
var MessageContentBlockSchema = external_exports.union([ContentSchema, external_exports.record(external_exports.unknown())]);
|
|
5085
|
+
var MessageSchema = external_exports.object({
|
|
5086
|
+
role: external_exports.enum(["assistant", "user", "system", "tool"]),
|
|
5087
|
+
content: external_exports.union([external_exports.string(), external_exports.array(MessageContentBlockSchema), external_exports.record(external_exports.unknown())]).optional(),
|
|
5088
|
+
toolCalls: external_exports.array(ToolCallSchema).optional(),
|
|
5089
|
+
name: external_exports.string().optional(),
|
|
5090
|
+
startTime: external_exports.string().optional(),
|
|
5091
|
+
endTime: external_exports.string().optional(),
|
|
5092
|
+
durationMs: external_exports.number().optional(),
|
|
5093
|
+
metadata: external_exports.record(external_exports.unknown()).optional()
|
|
5094
|
+
});
|
|
5095
|
+
var TraceSchema = TraceSummarySchema.extend({
|
|
5096
|
+
messages: external_exports.array(MessageSchema),
|
|
5097
|
+
events: external_exports.array(TraceEventSchema),
|
|
5098
|
+
tokenUsage: TokenUsageSchema.optional(),
|
|
5099
|
+
costUsd: external_exports.number().optional(),
|
|
5100
|
+
durationMs: external_exports.number().optional(),
|
|
5101
|
+
startTime: external_exports.string().optional(),
|
|
5102
|
+
endTime: external_exports.string().optional(),
|
|
5103
|
+
metadata: MetadataSchema.optional()
|
|
5104
|
+
});
|
|
5105
|
+
var CodeGraderInputSchema = external_exports.object({
|
|
5106
|
+
criteria: external_exports.string(),
|
|
5107
|
+
expectedOutput: external_exports.array(MessageSchema),
|
|
5108
|
+
output: external_exports.string().nullable().optional(),
|
|
5109
|
+
messages: external_exports.array(MessageSchema).optional().default([]),
|
|
5110
|
+
/** Path to a temp file containing the output JSON (used for large payloads). */
|
|
5111
|
+
outputPath: external_exports.string().optional(),
|
|
5112
|
+
inputFiles: external_exports.array(external_exports.string()),
|
|
5113
|
+
input: external_exports.array(MessageSchema),
|
|
5114
|
+
metadata: external_exports.record(external_exports.unknown()).nullable().optional(),
|
|
5115
|
+
trace: TraceSchema.nullable().optional(),
|
|
5116
|
+
traceSummary: TraceSummarySchema.nullable().optional(),
|
|
5117
|
+
tokenUsage: TokenUsageSchema.nullable().optional(),
|
|
5118
|
+
costUsd: external_exports.number().nullable().optional(),
|
|
5119
|
+
durationMs: external_exports.number().nullable().optional(),
|
|
5120
|
+
startTime: external_exports.string().nullable().optional(),
|
|
5121
|
+
endTime: external_exports.string().nullable().optional(),
|
|
5122
|
+
fileChanges: external_exports.string().nullable().optional(),
|
|
5123
|
+
workspacePath: external_exports.string().nullable().optional(),
|
|
5124
|
+
config: external_exports.record(external_exports.unknown()).nullable().optional()
|
|
5125
|
+
});
|
|
5126
|
+
var CodeGraderResultSchema = external_exports.object({
|
|
5127
|
+
score: external_exports.number().min(0).max(1),
|
|
5128
|
+
assertions: external_exports.array(
|
|
5129
|
+
external_exports.object({
|
|
5130
|
+
text: external_exports.string(),
|
|
5131
|
+
passed: external_exports.boolean(),
|
|
5132
|
+
evidence: external_exports.string().optional()
|
|
5133
|
+
})
|
|
5134
|
+
).optional().default([]),
|
|
5135
|
+
/** Optional structured details for domain-specific metrics (e.g., TP/TN/FP/FN counts, alignments). */
|
|
5136
|
+
details: external_exports.record(external_exports.unknown()).optional()
|
|
5137
|
+
});
|
|
5138
|
+
var EVAL_SUITE_SYMBOL = Symbol.for("@agentv/sdk/eval-suite");
|
|
5139
|
+
var TO_EVAL_YAML_OBJECT_SYMBOL = Symbol.for("@agentv/sdk/to-eval-yaml-object");
|
|
5140
|
+
function withCommon(config, options = {}) {
|
|
5141
|
+
return {
|
|
5142
|
+
...options.name !== void 0 ? { name: options.name } : {},
|
|
5143
|
+
...config,
|
|
5144
|
+
...options.weight !== void 0 ? { weight: options.weight } : {},
|
|
5145
|
+
...options.required !== void 0 ? { required: options.required } : {},
|
|
5146
|
+
...options.minScore !== void 0 ? { minScore: options.minScore } : {},
|
|
5147
|
+
...options.negate !== void 0 ? { negate: options.negate } : {}
|
|
5148
|
+
};
|
|
5149
|
+
}
|
|
5150
|
+
function containsGrader(value, options) {
|
|
5151
|
+
return withCommon({ type: "contains", value }, options);
|
|
5152
|
+
}
|
|
5153
|
+
function equalsGrader(value, options) {
|
|
5154
|
+
return withCommon({ type: "equals", value }, options);
|
|
5155
|
+
}
|
|
5156
|
+
function exactGrader(value, options) {
|
|
5157
|
+
return equalsGrader(value, options);
|
|
5158
|
+
}
|
|
5159
|
+
function regexGrader(pattern, options = {}) {
|
|
5160
|
+
const value = pattern instanceof RegExp ? pattern.source : pattern;
|
|
5161
|
+
const flags = options.flags ?? (pattern instanceof RegExp ? pattern.flags : void 0);
|
|
5162
|
+
return withCommon(
|
|
5163
|
+
{
|
|
5164
|
+
type: "regex",
|
|
5165
|
+
value,
|
|
5166
|
+
...flags ? { flags } : {}
|
|
5167
|
+
},
|
|
5168
|
+
options
|
|
5169
|
+
);
|
|
5170
|
+
}
|
|
5171
|
+
function isJsonGrader(options) {
|
|
5172
|
+
return withCommon({ type: "is-json" }, options);
|
|
5173
|
+
}
|
|
5174
|
+
function jsonGrader(options) {
|
|
5175
|
+
return isJsonGrader(options);
|
|
5176
|
+
}
|
|
5177
|
+
function rubricsGrader(criteria, options) {
|
|
5178
|
+
return withCommon({ type: "rubrics", criteria }, options);
|
|
5179
|
+
}
|
|
5180
|
+
function llmGrader(options = {}) {
|
|
5181
|
+
return withCommon(
|
|
5182
|
+
{
|
|
5183
|
+
type: "llm-grader",
|
|
5184
|
+
...options.prompt !== void 0 ? { prompt: options.prompt } : {},
|
|
5185
|
+
...options.rubrics !== void 0 ? { rubrics: options.rubrics } : {},
|
|
5186
|
+
...options.target !== void 0 ? { target: options.target } : {},
|
|
5187
|
+
...options.config !== void 0 ? { config: options.config } : {},
|
|
5188
|
+
...options.maxSteps !== void 0 ? { maxSteps: options.maxSteps } : {},
|
|
5189
|
+
...options.temperature !== void 0 ? { temperature: options.temperature } : {},
|
|
5190
|
+
...options.preprocessors !== void 0 ? { preprocessors: options.preprocessors } : {}
|
|
5191
|
+
},
|
|
5192
|
+
options
|
|
5193
|
+
);
|
|
5194
|
+
}
|
|
5195
|
+
function codeGrader(command2, options = {}) {
|
|
5196
|
+
return withCommon(
|
|
5197
|
+
{
|
|
5198
|
+
type: "code-grader",
|
|
5199
|
+
command: command2,
|
|
5200
|
+
...options.cwd !== void 0 ? { cwd: options.cwd } : {},
|
|
5201
|
+
...options.target !== void 0 ? { target: options.target } : {},
|
|
5202
|
+
...options.config !== void 0 ? { config: options.config } : {},
|
|
5203
|
+
...options.preprocessors !== void 0 ? { preprocessors: options.preprocessors } : {}
|
|
5204
|
+
},
|
|
5205
|
+
options
|
|
5206
|
+
);
|
|
5207
|
+
}
|
|
5208
|
+
var graders = Object.freeze({
|
|
5209
|
+
contains: containsGrader,
|
|
5210
|
+
equals: equalsGrader,
|
|
5211
|
+
exact: exactGrader,
|
|
5212
|
+
regex: regexGrader,
|
|
5213
|
+
isJson: isJsonGrader,
|
|
5214
|
+
json: jsonGrader,
|
|
5215
|
+
rubrics: rubricsGrader,
|
|
5216
|
+
llmGrader,
|
|
5217
|
+
codeGrader
|
|
5218
|
+
});
|
|
5219
|
+
function toCamelCase(str) {
|
|
5220
|
+
if (/^[A-Z]/.test(str)) {
|
|
5221
|
+
return str;
|
|
5222
|
+
}
|
|
5223
|
+
return str.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());
|
|
5224
|
+
}
|
|
5225
|
+
function toCamelCaseDeep2(obj) {
|
|
5226
|
+
if (obj === null || obj === void 0) {
|
|
5227
|
+
return obj;
|
|
5228
|
+
}
|
|
5229
|
+
if (Array.isArray(obj)) {
|
|
5230
|
+
return obj.map((item) => toCamelCaseDeep2(item));
|
|
5231
|
+
}
|
|
5232
|
+
if (typeof obj === "object") {
|
|
5233
|
+
const result = {};
|
|
5234
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
5235
|
+
const camelKey = toCamelCase(key);
|
|
5236
|
+
result[camelKey] = toCamelCaseDeep2(value);
|
|
5237
|
+
}
|
|
5238
|
+
return result;
|
|
5239
|
+
}
|
|
5240
|
+
return obj;
|
|
5241
|
+
}
|
|
5242
|
+
function enrichInput(input) {
|
|
5243
|
+
return input;
|
|
5244
|
+
}
|
|
5245
|
+
function readStdin() {
|
|
5246
|
+
return readFileSync3(0, "utf8");
|
|
5247
|
+
}
|
|
5248
|
+
function clampScore(value) {
|
|
5249
|
+
if (Number.isNaN(value) || !Number.isFinite(value)) {
|
|
5250
|
+
return 0;
|
|
5251
|
+
}
|
|
5252
|
+
return Math.max(0, Math.min(1, value));
|
|
5253
|
+
}
|
|
5254
|
+
function formatError(error) {
|
|
5255
|
+
if (error instanceof Error) {
|
|
5256
|
+
return error.message;
|
|
5257
|
+
}
|
|
5258
|
+
return String(error);
|
|
5259
|
+
}
|
|
5260
|
+
async function runCodeGrader(handler) {
|
|
5261
|
+
try {
|
|
5262
|
+
const stdin = readStdin();
|
|
5263
|
+
const rawInput = JSON.parse(stdin);
|
|
5264
|
+
const camelInput = toCamelCaseDeep2(rawInput);
|
|
5265
|
+
const input = CodeGraderInputSchema.parse(camelInput);
|
|
5266
|
+
if (input.outputPath && (input.output === null || input.output === void 0)) {
|
|
5267
|
+
let cachedOutput;
|
|
5268
|
+
const filePath = input.outputPath;
|
|
5269
|
+
Object.defineProperty(input, "output", {
|
|
5270
|
+
get() {
|
|
5271
|
+
if (cachedOutput === void 0) {
|
|
5272
|
+
cachedOutput = JSON.parse(readFileSync3(filePath, "utf8"));
|
|
5273
|
+
}
|
|
5274
|
+
return cachedOutput;
|
|
5275
|
+
},
|
|
5276
|
+
configurable: true,
|
|
5277
|
+
enumerable: true
|
|
5278
|
+
});
|
|
5279
|
+
}
|
|
5280
|
+
enrichInput(input);
|
|
5281
|
+
const rawResult = await handler(input);
|
|
5282
|
+
const result = CodeGraderResultSchema.parse({
|
|
5283
|
+
...rawResult,
|
|
5284
|
+
score: clampScore(rawResult.score)
|
|
5285
|
+
});
|
|
5286
|
+
console.log(JSON.stringify(result, null, 2));
|
|
5287
|
+
} catch (error) {
|
|
5288
|
+
const errorMessage = formatError(error);
|
|
5289
|
+
const errorResult = {
|
|
5290
|
+
score: 0,
|
|
5291
|
+
assertions: [{ text: `Evaluation failed: ${errorMessage}`, passed: false }]
|
|
5292
|
+
};
|
|
5293
|
+
console.log(JSON.stringify(errorResult, null, 2));
|
|
5294
|
+
process.exit(1);
|
|
5295
|
+
}
|
|
5296
|
+
}
|
|
5297
|
+
function workspacePathFrom2(input) {
|
|
5298
|
+
const workspacePath = input.workspacePath ?? process.env.AGENTV_WORKSPACE_PATH;
|
|
5299
|
+
return workspacePath?.trim() ? workspacePath : void 0;
|
|
5300
|
+
}
|
|
5301
|
+
function truncate(value, maxLength = 2e3) {
|
|
5302
|
+
if (value.length <= maxLength) {
|
|
5303
|
+
return value;
|
|
5304
|
+
}
|
|
5305
|
+
return `${value.slice(0, maxLength)}
|
|
5306
|
+
...(truncated)`;
|
|
5307
|
+
}
|
|
5308
|
+
function resolveInsideWorkspace(workspacePath, relativePath, label, options = {}) {
|
|
5309
|
+
if (!relativePath.trim()) {
|
|
5310
|
+
throw new Error(`${label} must not be empty.`);
|
|
5311
|
+
}
|
|
5312
|
+
if (nodePath2.isAbsolute(relativePath)) {
|
|
5313
|
+
throw new Error(`${label} must be relative to the workspace: ${relativePath}`);
|
|
5314
|
+
}
|
|
5315
|
+
const root = nodePath2.resolve(workspacePath);
|
|
5316
|
+
const resolvedPath = nodePath2.resolve(root, relativePath);
|
|
5317
|
+
const relativeToRoot = nodePath2.relative(root, resolvedPath);
|
|
5318
|
+
if (relativeToRoot === "" && options.allowRoot === true) {
|
|
5319
|
+
return resolvedPath;
|
|
5320
|
+
}
|
|
5321
|
+
if (relativeToRoot === "" || relativeToRoot.startsWith("..") || nodePath2.isAbsolute(relativeToRoot)) {
|
|
5322
|
+
throw new Error(`${label} must stay inside the workspace: ${relativePath}`);
|
|
5323
|
+
}
|
|
5324
|
+
return resolvedPath;
|
|
5325
|
+
}
|
|
5326
|
+
function normalizeTestFiles(testFile) {
|
|
5327
|
+
if (testFile === void 0) {
|
|
5328
|
+
return [];
|
|
5329
|
+
}
|
|
5330
|
+
return typeof testFile === "string" ? [testFile] : [...testFile];
|
|
5331
|
+
}
|
|
5332
|
+
function buildCommand(options, testFiles = normalizeTestFiles(options.testFile)) {
|
|
5333
|
+
if (options.command && options.command.length > 0) {
|
|
5334
|
+
return [...options.command];
|
|
5335
|
+
}
|
|
5336
|
+
const command2 = [...options.vitestCommand ?? ["bunx", "vitest", "run"]];
|
|
5337
|
+
command2.push(...testFiles);
|
|
5338
|
+
return command2;
|
|
5339
|
+
}
|
|
5340
|
+
function parseJsonObjectFromText(text) {
|
|
5341
|
+
const trimmed = text.trim();
|
|
5342
|
+
if (!trimmed) {
|
|
5343
|
+
return void 0;
|
|
5344
|
+
}
|
|
5345
|
+
try {
|
|
5346
|
+
return JSON.parse(trimmed);
|
|
5347
|
+
} catch {
|
|
5348
|
+
const start = trimmed.indexOf("{");
|
|
5349
|
+
const end = trimmed.lastIndexOf("}");
|
|
5350
|
+
if (start >= 0 && end > start) {
|
|
5351
|
+
return JSON.parse(trimmed.slice(start, end + 1));
|
|
5352
|
+
}
|
|
5353
|
+
throw new Error("Vitest output did not contain a JSON object.");
|
|
5354
|
+
}
|
|
5355
|
+
}
|
|
5356
|
+
function isVitestJsonReport(value) {
|
|
5357
|
+
return typeof value === "object" && value !== null && Array.isArray(value.testResults);
|
|
5358
|
+
}
|
|
5359
|
+
function assertionText(file, assertion2) {
|
|
5360
|
+
const title = assertion2.fullName ?? [...assertion2.ancestorTitles ?? [], assertion2.title].filter(Boolean).join(" ");
|
|
5361
|
+
return title || file.name || "Vitest assertion";
|
|
5362
|
+
}
|
|
5363
|
+
function vitestReportToCodeGraderResult(report, options = {}) {
|
|
5364
|
+
const assertions = (report.testResults ?? []).flatMap(
|
|
5365
|
+
(file) => (file.assertionResults ?? []).map((item) => {
|
|
5366
|
+
const passed = item.status === "passed";
|
|
5367
|
+
const evidence = item.failureMessages && item.failureMessages.length > 0 ? truncate(item.failureMessages.join("\n\n")) : void 0;
|
|
5368
|
+
return {
|
|
5369
|
+
text: assertionText(file, item),
|
|
5370
|
+
passed,
|
|
5371
|
+
...evidence !== void 0 ? { evidence } : {}
|
|
5372
|
+
};
|
|
5373
|
+
})
|
|
5374
|
+
);
|
|
5375
|
+
if (assertions.length === 0) {
|
|
5376
|
+
const passed = options.passWithNoTests === true;
|
|
5377
|
+
return CodeGraderResultSchema.parse({
|
|
5378
|
+
score: passed ? 1 : 0,
|
|
5379
|
+
assertions: [{ text: "Vitest reported no tests", passed }],
|
|
5380
|
+
details: {
|
|
5381
|
+
vitest_success: report.success ?? false,
|
|
5382
|
+
num_total_tests: report.numTotalTests ?? 0,
|
|
5383
|
+
num_passed_tests: report.numPassedTests ?? 0,
|
|
5384
|
+
num_failed_tests: report.numFailedTests ?? 0,
|
|
5385
|
+
num_pending_tests: report.numPendingTests ?? 0,
|
|
5386
|
+
num_todo_tests: report.numTodoTests ?? 0
|
|
5387
|
+
}
|
|
5388
|
+
});
|
|
5389
|
+
}
|
|
5390
|
+
const passedCount = assertions.filter((item) => item.passed).length;
|
|
5391
|
+
return CodeGraderResultSchema.parse({
|
|
5392
|
+
score: passedCount / assertions.length,
|
|
5393
|
+
assertions,
|
|
5394
|
+
details: {
|
|
5395
|
+
vitest_success: report.success ?? passedCount === assertions.length,
|
|
5396
|
+
num_total_tests: report.numTotalTests ?? assertions.length,
|
|
5397
|
+
num_passed_tests: report.numPassedTests ?? passedCount,
|
|
5398
|
+
num_failed_tests: report.numFailedTests ?? assertions.length - passedCount,
|
|
5399
|
+
num_pending_tests: report.numPendingTests ?? 0,
|
|
5400
|
+
num_todo_tests: report.numTodoTests ?? 0
|
|
5401
|
+
}
|
|
5402
|
+
});
|
|
5403
|
+
}
|
|
5404
|
+
function runCommand(command2, options) {
|
|
5405
|
+
if (command2.length === 0) {
|
|
5406
|
+
return Promise.reject(new Error("Vitest command must not be empty."));
|
|
5407
|
+
}
|
|
5408
|
+
return new Promise((resolve3, reject) => {
|
|
5409
|
+
const child = spawn(command2[0], command2.slice(1), {
|
|
5410
|
+
cwd: options.cwd,
|
|
5411
|
+
env: { ...process.env, ...options.env },
|
|
5412
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
5413
|
+
});
|
|
5414
|
+
let stdout = "";
|
|
5415
|
+
let stderr = "";
|
|
5416
|
+
let timedOut = false;
|
|
5417
|
+
const timeout = options.timeoutMs !== void 0 ? setTimeout(() => {
|
|
5418
|
+
timedOut = true;
|
|
5419
|
+
child.kill("SIGTERM");
|
|
5420
|
+
}, options.timeoutMs) : void 0;
|
|
5421
|
+
child.stdout.setEncoding("utf8");
|
|
5422
|
+
child.stderr.setEncoding("utf8");
|
|
5423
|
+
child.stdout.on("data", (chunk) => {
|
|
5424
|
+
stdout += chunk;
|
|
5425
|
+
});
|
|
5426
|
+
child.stderr.on("data", (chunk) => {
|
|
5427
|
+
stderr += chunk;
|
|
5428
|
+
});
|
|
5429
|
+
child.on("error", reject);
|
|
5430
|
+
child.on("close", (exitCode) => {
|
|
5431
|
+
if (timeout) {
|
|
5432
|
+
clearTimeout(timeout);
|
|
5433
|
+
}
|
|
5434
|
+
if (timedOut) {
|
|
5435
|
+
reject(new Error(`Vitest command timed out after ${options.timeoutMs}ms.`));
|
|
5436
|
+
return;
|
|
5437
|
+
}
|
|
5438
|
+
resolve3({ exitCode, stdout, stderr });
|
|
5439
|
+
});
|
|
5440
|
+
});
|
|
5441
|
+
}
|
|
5442
|
+
function resolveSourceTestFile(testFileRoot, testFile) {
|
|
5443
|
+
if (!testFile.trim()) {
|
|
5444
|
+
throw new Error("testFile entries must not be empty.");
|
|
5445
|
+
}
|
|
5446
|
+
return nodePath2.isAbsolute(testFile) ? nodePath2.resolve(testFile) : nodePath2.resolve(testFileRoot, testFile);
|
|
5447
|
+
}
|
|
5448
|
+
async function copyTestFilesIntoWorkspace(testFiles, options, cwd) {
|
|
5449
|
+
if (testFiles.length === 0) {
|
|
5450
|
+
return { testFiles };
|
|
5451
|
+
}
|
|
5452
|
+
const tempDir = await mkdtemp(nodePath2.join(cwd, ".agentv-vitest-"));
|
|
5453
|
+
const testFileRoot = nodePath2.resolve(options.testFileRoot ?? process.cwd());
|
|
5454
|
+
const copiedFiles = await Promise.all(
|
|
5455
|
+
testFiles.map(async (testFile, index) => {
|
|
5456
|
+
const sourcePath = resolveSourceTestFile(testFileRoot, testFile);
|
|
5457
|
+
const destinationPath = nodePath2.join(tempDir, `${index}-${nodePath2.basename(testFile)}`);
|
|
5458
|
+
await copyFile(sourcePath, destinationPath);
|
|
5459
|
+
return nodePath2.relative(cwd, destinationPath);
|
|
5460
|
+
})
|
|
5461
|
+
);
|
|
5462
|
+
return { testFiles: copiedFiles, tempDir };
|
|
5463
|
+
}
|
|
5464
|
+
async function readVitestReport(commandResult, outputFile) {
|
|
5465
|
+
const rawReport = outputFile !== void 0 ? await readFile2(outputFile, "utf8") : commandResult.stdout;
|
|
5466
|
+
const parsed = parseJsonObjectFromText(rawReport);
|
|
5467
|
+
if (!isVitestJsonReport(parsed)) {
|
|
5468
|
+
throw new Error("Vitest JSON report did not include testResults[].");
|
|
5469
|
+
}
|
|
5470
|
+
return parsed;
|
|
5471
|
+
}
|
|
5472
|
+
async function runVitestWorkspaceGrader(options, input) {
|
|
5473
|
+
const workspacePath = workspacePathFrom2(input);
|
|
5474
|
+
if (!workspacePath) {
|
|
5475
|
+
return {
|
|
5476
|
+
score: 0,
|
|
5477
|
+
assertions: [
|
|
5478
|
+
{
|
|
5479
|
+
text: "Vitest workspace verifier requires workspace_path",
|
|
5480
|
+
passed: false,
|
|
5481
|
+
evidence: "Configure workspace in the eval YAML so AgentV can pass workspace_path."
|
|
5482
|
+
}
|
|
5483
|
+
]
|
|
5484
|
+
};
|
|
5485
|
+
}
|
|
5486
|
+
const tempDirs = [];
|
|
5487
|
+
try {
|
|
5488
|
+
const cwd = options.cwd ? resolveInsideWorkspace(workspacePath, options.cwd, "cwd", { allowRoot: true }) : workspacePath;
|
|
5489
|
+
const testFiles = normalizeTestFiles(options.testFile);
|
|
5490
|
+
const preparedTestFiles = options.copyTestFilesToWorkspace === true ? await copyTestFilesIntoWorkspace(testFiles, options, cwd) : { testFiles };
|
|
5491
|
+
if (preparedTestFiles.tempDir) {
|
|
5492
|
+
tempDirs.push(preparedTestFiles.tempDir);
|
|
5493
|
+
}
|
|
5494
|
+
const command2 = buildCommand(options, preparedTestFiles.testFiles);
|
|
5495
|
+
const appendReporterArgs = options.appendReporterArgs ?? options.command === void 0;
|
|
5496
|
+
let outputFile = options.outputFile ? resolveInsideWorkspace(cwd, options.outputFile, "outputFile") : void 0;
|
|
5497
|
+
if (appendReporterArgs) {
|
|
5498
|
+
const tempDir = await mkdtemp(nodePath2.join(tmpdir(), "agentv-vitest-"));
|
|
5499
|
+
tempDirs.push(tempDir);
|
|
5500
|
+
outputFile = nodePath2.join(tempDir, "results.json");
|
|
5501
|
+
command2.push("--reporter=json", `--outputFile=${outputFile}`);
|
|
5502
|
+
}
|
|
5503
|
+
const result = await runCommand(command2, {
|
|
5504
|
+
cwd,
|
|
5505
|
+
timeoutMs: options.timeoutMs,
|
|
5506
|
+
env: {
|
|
5507
|
+
AGENTV_WORKSPACE_PATH: workspacePath,
|
|
5508
|
+
...outputFile !== void 0 ? { AGENTV_VITEST_JSON_PATH: outputFile } : {},
|
|
5509
|
+
...options.env
|
|
5510
|
+
}
|
|
5511
|
+
});
|
|
5512
|
+
const report = await readVitestReport(result, outputFile);
|
|
5513
|
+
return vitestReportToCodeGraderResult(report, options);
|
|
5514
|
+
} catch (error) {
|
|
5515
|
+
return {
|
|
5516
|
+
score: 0,
|
|
5517
|
+
assertions: [
|
|
5518
|
+
{
|
|
5519
|
+
text: "Vitest workspace verifier failed to run",
|
|
5520
|
+
passed: false,
|
|
5521
|
+
evidence: error instanceof Error ? error.message : String(error)
|
|
5522
|
+
}
|
|
5523
|
+
]
|
|
5524
|
+
};
|
|
5525
|
+
} finally {
|
|
5526
|
+
for (const tempDir of tempDirs.reverse()) {
|
|
5527
|
+
await rm(tempDir, { recursive: true, force: true }).catch(() => {
|
|
5528
|
+
});
|
|
5529
|
+
}
|
|
5530
|
+
}
|
|
5531
|
+
}
|
|
5532
|
+
|
|
5533
|
+
// src/commands/eval/commands/vitest.ts
|
|
5534
|
+
function parseCommand(value) {
|
|
5535
|
+
const trimmed = value?.trim();
|
|
5536
|
+
return trimmed ? trimmed.split(/\s+/) : void 0;
|
|
5537
|
+
}
|
|
5538
|
+
var evalVitestCommand = command({
|
|
5539
|
+
name: "vitest",
|
|
5540
|
+
description: "Run Vitest workspace verifier files as an AgentV code-grader protocol adapter",
|
|
5541
|
+
args: {
|
|
5542
|
+
testFiles: restPositionals({
|
|
5543
|
+
type: string,
|
|
5544
|
+
displayName: "test-files",
|
|
5545
|
+
description: "Vitest verifier file(s) to run"
|
|
5546
|
+
}),
|
|
5547
|
+
cwd: option({
|
|
5548
|
+
type: optional(string),
|
|
5549
|
+
long: "cwd",
|
|
5550
|
+
description: "Workspace-relative directory where Vitest should run"
|
|
5551
|
+
}),
|
|
5552
|
+
vitestCommand: option({
|
|
5553
|
+
type: optional(string),
|
|
5554
|
+
long: "vitest-command",
|
|
5555
|
+
description: 'Vitest command to execute, defaults to "bunx vitest run"'
|
|
5556
|
+
}),
|
|
5557
|
+
timeoutMs: option({
|
|
5558
|
+
type: optional(number),
|
|
5559
|
+
long: "timeout-ms",
|
|
5560
|
+
description: "Timeout for the Vitest command in milliseconds"
|
|
5561
|
+
}),
|
|
5562
|
+
inWorkspace: flag({
|
|
5563
|
+
long: "in-workspace",
|
|
5564
|
+
description: "Treat test files as already present in the prepared workspace instead of copying them from the current directory"
|
|
5565
|
+
}),
|
|
5566
|
+
passWithNoTests: flag({
|
|
5567
|
+
long: "pass-with-no-tests",
|
|
5568
|
+
description: "Return score 1 when Vitest reports zero tests"
|
|
5569
|
+
})
|
|
5570
|
+
},
|
|
5571
|
+
handler: async ({ testFiles, cwd, vitestCommand, timeoutMs, inWorkspace, passWithNoTests }) => {
|
|
5572
|
+
await runCodeGrader((input) => {
|
|
5573
|
+
if (testFiles.length === 0) {
|
|
5574
|
+
throw new Error("Provide at least one Vitest verifier file.");
|
|
5575
|
+
}
|
|
5576
|
+
return runVitestWorkspaceGrader(
|
|
5577
|
+
{
|
|
5578
|
+
testFile: testFiles,
|
|
5579
|
+
cwd,
|
|
5580
|
+
vitestCommand: parseCommand(vitestCommand),
|
|
5581
|
+
timeoutMs,
|
|
5582
|
+
passWithNoTests,
|
|
5583
|
+
copyTestFilesToWorkspace: !inWorkspace,
|
|
5584
|
+
testFileRoot: process.cwd()
|
|
5585
|
+
},
|
|
5586
|
+
input
|
|
5587
|
+
);
|
|
5588
|
+
});
|
|
5589
|
+
}
|
|
5590
|
+
});
|
|
5591
|
+
|
|
4885
5592
|
// src/commands/eval/index.ts
|
|
4886
5593
|
var evalCommand = subcommands({
|
|
4887
5594
|
name: "eval",
|
|
4888
|
-
description: "Evaluation commands. Shorthand:
|
|
5595
|
+
description: "Evaluation commands. Shorthand: eval files run with `eval run`; verifier-looking test files run with `eval vitest`.",
|
|
4889
5596
|
cmds: {
|
|
4890
5597
|
run: evalRunCommand,
|
|
4891
5598
|
assert: evalAssertCommand,
|
|
4892
5599
|
aggregate: evalAggregateCommand,
|
|
4893
|
-
bundle: evalBundleCommand
|
|
5600
|
+
bundle: evalBundleCommand,
|
|
5601
|
+
vitest: evalVitestCommand
|
|
4894
5602
|
}
|
|
4895
5603
|
});
|
|
4896
5604
|
|
|
@@ -5625,7 +6333,7 @@ function formatDurationMs2(ms) {
|
|
|
5625
6333
|
}
|
|
5626
6334
|
|
|
5627
6335
|
// src/commands/import/copilot.ts
|
|
5628
|
-
import { mkdir as mkdir5, readFile as
|
|
6336
|
+
import { mkdir as mkdir5, readFile as readFile3, writeFile as writeFile5 } from "node:fs/promises";
|
|
5629
6337
|
import path11 from "node:path";
|
|
5630
6338
|
var importCopilotCommand = command({
|
|
5631
6339
|
name: "copilot",
|
|
@@ -5692,7 +6400,7 @@ var importCopilotCommand = command({
|
|
|
5692
6400
|
process.exit(1);
|
|
5693
6401
|
}
|
|
5694
6402
|
const eventsPath = path11.join(sessionDir, "events.jsonl");
|
|
5695
|
-
const rawJsonl = await
|
|
6403
|
+
const rawJsonl = await readFile3(eventsPath, "utf8");
|
|
5696
6404
|
const parsed = parseCopilotEvents(rawJsonl);
|
|
5697
6405
|
const transcript = {
|
|
5698
6406
|
messages: parsed.messages,
|
|
@@ -5858,11 +6566,11 @@ Imported ${summary.files_created} eval(s) from ${summary.dataset} \u2192 ${summa
|
|
|
5858
6566
|
});
|
|
5859
6567
|
|
|
5860
6568
|
// src/commands/import/promptfoo.ts
|
|
5861
|
-
import { mkdir as mkdir6, readFile as
|
|
6569
|
+
import { mkdir as mkdir6, readFile as readFile4, writeFile as writeFile6 } from "node:fs/promises";
|
|
5862
6570
|
import path13 from "node:path";
|
|
5863
6571
|
import fg2 from "fast-glob";
|
|
5864
6572
|
import JSON5 from "json5";
|
|
5865
|
-
import { parse as parseYaml, stringify as
|
|
6573
|
+
import { parse as parseYaml, stringify as stringifyYaml3 } from "yaml";
|
|
5866
6574
|
var FILE_PREFIX = "file://";
|
|
5867
6575
|
var PROMPTFOO_COMMENT_PREFIX = "# Converted from promptfoo config: ";
|
|
5868
6576
|
async function convertPromptfooToAgentvSuite(options) {
|
|
@@ -5906,7 +6614,7 @@ async function convertPromptfooToAgentvSuite(options) {
|
|
|
5906
6614
|
}
|
|
5907
6615
|
async function convertPromptfooToAgentvYaml(inputPath) {
|
|
5908
6616
|
const suite = await convertPromptfooToAgentvSuite({ inputPath });
|
|
5909
|
-
const yaml =
|
|
6617
|
+
const yaml = stringifyYaml3(suite, {
|
|
5910
6618
|
indent: 2,
|
|
5911
6619
|
lineWidth: 0
|
|
5912
6620
|
});
|
|
@@ -5947,7 +6655,7 @@ var importPromptfooCommand = command({
|
|
|
5947
6655
|
}
|
|
5948
6656
|
});
|
|
5949
6657
|
async function loadPromptfooConfig(filePath) {
|
|
5950
|
-
const content = await
|
|
6658
|
+
const content = await readFile4(filePath, "utf8");
|
|
5951
6659
|
const parsed = parseStructuredText(content, filePath);
|
|
5952
6660
|
if (!isJsonObject(parsed)) {
|
|
5953
6661
|
throw new Error(`promptfoo config must be an object: ${filePath}`);
|
|
@@ -6030,7 +6738,7 @@ async function loadPromptFromReference(reference, baseDir, identity2) {
|
|
|
6030
6738
|
for (let index = 0; index < files.length; index++) {
|
|
6031
6739
|
const filePath = files[index];
|
|
6032
6740
|
const ext = path13.extname(filePath).toLowerCase();
|
|
6033
|
-
const raw = await
|
|
6741
|
+
const raw = await readFile4(filePath, "utf8");
|
|
6034
6742
|
if (ext === ".json" || ext === ".json5" || ext === ".jsonc") {
|
|
6035
6743
|
const parsed = parseStructuredText(raw, filePath);
|
|
6036
6744
|
if (!Array.isArray(parsed) || !parsed.every(isPromptMessage)) {
|
|
@@ -6084,7 +6792,7 @@ async function expandProviderEntry(rawProvider, baseDir) {
|
|
|
6084
6792
|
const files = await resolvePromptfooFileReference(rawProvider, baseDir);
|
|
6085
6793
|
const providers = [];
|
|
6086
6794
|
for (const filePath of files) {
|
|
6087
|
-
const parsed = parseStructuredText(await
|
|
6795
|
+
const parsed = parseStructuredText(await readFile4(filePath, "utf8"), filePath);
|
|
6088
6796
|
if (!isJsonObject(parsed)) {
|
|
6089
6797
|
throw new Error(`Provider file must be an object: ${filePath}`);
|
|
6090
6798
|
}
|
|
@@ -6123,7 +6831,7 @@ async function loadDefaultTest(rawDefaultTest, baseDir) {
|
|
|
6123
6831
|
if (files.length !== 1) {
|
|
6124
6832
|
throw new Error(`defaultTest must resolve to exactly one file: ${rawDefaultTest}`);
|
|
6125
6833
|
}
|
|
6126
|
-
const parsed = parseStructuredText(await
|
|
6834
|
+
const parsed = parseStructuredText(await readFile4(files[0], "utf8"), files[0]);
|
|
6127
6835
|
if (!isJsonObject(parsed)) {
|
|
6128
6836
|
throw new Error(`defaultTest file must contain an object: ${files[0]}`);
|
|
6129
6837
|
}
|
|
@@ -6178,7 +6886,7 @@ async function loadPromptfooTestsFromReference(reference, baseDir) {
|
|
|
6178
6886
|
`Unsupported test dataset '${path13.basename(filePath)}': XLSX promptfoo datasets are not imported yet`
|
|
6179
6887
|
);
|
|
6180
6888
|
}
|
|
6181
|
-
const content = await
|
|
6889
|
+
const content = await readFile4(filePath, "utf8");
|
|
6182
6890
|
if (ext === ".csv") {
|
|
6183
6891
|
tests.push(...parseCsvPromptfooTests(content, filePath));
|
|
6184
6892
|
continue;
|
|
@@ -6870,7 +7578,7 @@ import path15 from "node:path";
|
|
|
6870
7578
|
import * as readline from "node:readline/promises";
|
|
6871
7579
|
|
|
6872
7580
|
// src/templates/index.ts
|
|
6873
|
-
import { readFileSync as
|
|
7581
|
+
import { readFileSync as readFileSync4, readdirSync, statSync } from "node:fs";
|
|
6874
7582
|
import path14 from "node:path";
|
|
6875
7583
|
import { fileURLToPath } from "node:url";
|
|
6876
7584
|
function getAgentvTemplates() {
|
|
@@ -6879,7 +7587,7 @@ function getAgentvTemplates() {
|
|
|
6879
7587
|
function getEnvExampleTemplate() {
|
|
6880
7588
|
const currentDir = path14.dirname(fileURLToPath(import.meta.url));
|
|
6881
7589
|
const templatesBase = currentDir.includes(`${path14.sep}dist`) ? path14.join(currentDir, "templates") : currentDir;
|
|
6882
|
-
const content =
|
|
7590
|
+
const content = readFileSync4(path14.join(templatesBase, ".env.example"), "utf-8");
|
|
6883
7591
|
return { path: ".env.example", content };
|
|
6884
7592
|
}
|
|
6885
7593
|
function getTemplatesFromDir(subdir) {
|
|
@@ -6902,7 +7610,7 @@ function readTemplatesRecursively(dir, relativePath) {
|
|
|
6902
7610
|
if (stat4.isDirectory()) {
|
|
6903
7611
|
templates.push(...readTemplatesRecursively(fullPath, entryRelativePath));
|
|
6904
7612
|
} else {
|
|
6905
|
-
const content =
|
|
7613
|
+
const content = readFileSync4(fullPath, "utf-8");
|
|
6906
7614
|
templates.push({
|
|
6907
7615
|
path: entryRelativePath.split(path14.sep).join("/"),
|
|
6908
7616
|
// Normalize to forward slashes
|
|
@@ -7022,7 +7730,7 @@ var initCmdTsCommand = command({
|
|
|
7022
7730
|
});
|
|
7023
7731
|
|
|
7024
7732
|
// src/commands/inspect/filter.ts
|
|
7025
|
-
import { existsSync as existsSync4, readFileSync as
|
|
7733
|
+
import { existsSync as existsSync4, readFileSync as readFileSync5, readdirSync as readdirSync2, statSync as statSync2 } from "node:fs";
|
|
7026
7734
|
import path16 from "node:path";
|
|
7027
7735
|
function collectIndexFiles(dir) {
|
|
7028
7736
|
const files = [];
|
|
@@ -7040,6 +7748,24 @@ function collectIndexFiles(dir) {
|
|
|
7040
7748
|
}
|
|
7041
7749
|
return files;
|
|
7042
7750
|
}
|
|
7751
|
+
function collectCurrentResultIndexFiles(cwd) {
|
|
7752
|
+
const resultsDir = path16.join(cwd, ".agentv", "results");
|
|
7753
|
+
const files = [];
|
|
7754
|
+
try {
|
|
7755
|
+
const entries2 = readdirSync2(resultsDir, { withFileTypes: true });
|
|
7756
|
+
for (const entry of entries2) {
|
|
7757
|
+
if (isReservedResultsNamespace(entry.name)) continue;
|
|
7758
|
+
const fullPath = path16.join(resultsDir, entry.name);
|
|
7759
|
+
if (entry.isDirectory()) {
|
|
7760
|
+
files.push(...collectIndexFiles(fullPath));
|
|
7761
|
+
} else if (entry.name === "index.jsonl") {
|
|
7762
|
+
files.push(fullPath);
|
|
7763
|
+
}
|
|
7764
|
+
}
|
|
7765
|
+
} catch {
|
|
7766
|
+
}
|
|
7767
|
+
return files;
|
|
7768
|
+
}
|
|
7043
7769
|
function extractToolNames(record) {
|
|
7044
7770
|
const tools = /* @__PURE__ */ new Set();
|
|
7045
7771
|
const trace = record.trace;
|
|
@@ -7062,10 +7788,23 @@ function extractToolNames(record) {
|
|
|
7062
7788
|
}
|
|
7063
7789
|
return [...tools];
|
|
7064
7790
|
}
|
|
7791
|
+
function inferExperimentFromPath(filePath) {
|
|
7792
|
+
const parts = filePath.split(path16.sep);
|
|
7793
|
+
const resultsIdx = parts.lastIndexOf("results");
|
|
7794
|
+
if (resultsIdx === -1) return void 0;
|
|
7795
|
+
const runPath = parts.slice(resultsIdx + 1, -1);
|
|
7796
|
+
if (isReservedResultsNamespace(runPath[0])) return void 0;
|
|
7797
|
+
if (runPath.length < 2) return void 0;
|
|
7798
|
+
const candidate = runPath[0];
|
|
7799
|
+
if (candidate && !/^\d{4}-\d{2}-\d{2}T/.test(candidate)) {
|
|
7800
|
+
return candidate;
|
|
7801
|
+
}
|
|
7802
|
+
return void 0;
|
|
7803
|
+
}
|
|
7065
7804
|
function parseFilterableRecords(filePath) {
|
|
7066
7805
|
let content;
|
|
7067
7806
|
try {
|
|
7068
|
-
content =
|
|
7807
|
+
content = readFileSync5(filePath, "utf8");
|
|
7069
7808
|
} catch {
|
|
7070
7809
|
return [];
|
|
7071
7810
|
}
|
|
@@ -7082,14 +7821,7 @@ function parseFilterableRecords(filePath) {
|
|
|
7082
7821
|
raw = normalizeResultRow(raw, { lineNumber: i + 1, sourceLabel: filePath });
|
|
7083
7822
|
let experiment = typeof raw.experiment === "string" ? raw.experiment : void 0;
|
|
7084
7823
|
if (!experiment) {
|
|
7085
|
-
|
|
7086
|
-
const runsIdx = parts.indexOf("runs");
|
|
7087
|
-
if (runsIdx !== -1 && parts.length - runsIdx >= 3) {
|
|
7088
|
-
const candidate = parts[runsIdx + 1];
|
|
7089
|
-
if (candidate && !/^\d{4}-\d{2}-\d{2}T/.test(candidate)) {
|
|
7090
|
-
experiment = candidate;
|
|
7091
|
-
}
|
|
7092
|
-
}
|
|
7824
|
+
experiment = inferExperimentFromPath(filePath);
|
|
7093
7825
|
}
|
|
7094
7826
|
records.push({
|
|
7095
7827
|
file: filePath,
|
|
@@ -7150,7 +7882,7 @@ function discoverFilterSources(searchPath, cwd) {
|
|
|
7150
7882
|
}
|
|
7151
7883
|
return [resolved];
|
|
7152
7884
|
}
|
|
7153
|
-
return
|
|
7885
|
+
return collectCurrentResultIndexFiles(cwd);
|
|
7154
7886
|
}
|
|
7155
7887
|
function formatFilterTable(records) {
|
|
7156
7888
|
const lines = [];
|
|
@@ -7194,7 +7926,7 @@ var inspectFilterCommand = command({
|
|
|
7194
7926
|
path: positional({
|
|
7195
7927
|
type: optional(string),
|
|
7196
7928
|
displayName: "path",
|
|
7197
|
-
description: "Directory or file to filter (default: .agentv/results/
|
|
7929
|
+
description: "Directory or file to filter (default: .agentv/results/)"
|
|
7198
7930
|
}),
|
|
7199
7931
|
target: option({
|
|
7200
7932
|
type: optional(string),
|
|
@@ -7286,12 +8018,12 @@ var inspectFilterCommand = command({
|
|
|
7286
8018
|
function formatListTable(metas) {
|
|
7287
8019
|
const lines = [];
|
|
7288
8020
|
if (metas.length === 0) {
|
|
7289
|
-
lines.push(`${c.yellow}No run workspaces found in .agentv/results
|
|
8021
|
+
lines.push(`${c.yellow}No run workspaces found in .agentv/results/${c.reset}`);
|
|
7290
8022
|
lines.push(`${c.dim}Run an evaluation first: agentv run <eval-file>${c.reset}`);
|
|
7291
8023
|
return lines.join("\n");
|
|
7292
8024
|
}
|
|
7293
8025
|
lines.push("");
|
|
7294
|
-
lines.push(`${c.bold}Evaluation Runs${c.reset} ${c.dim}(.agentv/results/
|
|
8026
|
+
lines.push(`${c.bold}Evaluation Runs${c.reset} ${c.dim}(.agentv/results/)${c.reset}`);
|
|
7295
8027
|
lines.push("");
|
|
7296
8028
|
const maxFileLen = Math.max(4, ...metas.map((m) => m.filename.length));
|
|
7297
8029
|
const header = ` ${padRight("File", maxFileLen)} ${padLeft("Tests", 5)} ${padLeft("Pass", 5)} ${padLeft("Score", 6)} ${padLeft("Size", 7)} Timestamp`;
|
|
@@ -7314,7 +8046,7 @@ function formatListTable(metas) {
|
|
|
7314
8046
|
}
|
|
7315
8047
|
var traceListCommand = command({
|
|
7316
8048
|
name: "list",
|
|
7317
|
-
description: "List recent evaluation run workspaces from .agentv/results/
|
|
8049
|
+
description: "List recent evaluation run workspaces from .agentv/results/",
|
|
7318
8050
|
args: {
|
|
7319
8051
|
limit: option({
|
|
7320
8052
|
type: optional(number),
|
|
@@ -7629,7 +8361,7 @@ var traceScoreCommand = command({
|
|
|
7629
8361
|
});
|
|
7630
8362
|
|
|
7631
8363
|
// src/commands/inspect/search.ts
|
|
7632
|
-
import { existsSync as existsSync5, readFileSync as
|
|
8364
|
+
import { existsSync as existsSync5, readFileSync as readFileSync6, readdirSync as readdirSync3, statSync as statSync3 } from "node:fs";
|
|
7633
8365
|
import path17 from "node:path";
|
|
7634
8366
|
function collectJsonlFiles(dir) {
|
|
7635
8367
|
const files = [];
|
|
@@ -7647,6 +8379,24 @@ function collectJsonlFiles(dir) {
|
|
|
7647
8379
|
}
|
|
7648
8380
|
return files;
|
|
7649
8381
|
}
|
|
8382
|
+
function collectCurrentResultJsonlFiles(cwd) {
|
|
8383
|
+
const resultsDir = path17.join(cwd, ".agentv", "results");
|
|
8384
|
+
const files = [];
|
|
8385
|
+
try {
|
|
8386
|
+
const entries2 = readdirSync3(resultsDir, { withFileTypes: true });
|
|
8387
|
+
for (const entry of entries2) {
|
|
8388
|
+
if (isReservedResultsNamespace(entry.name)) continue;
|
|
8389
|
+
const fullPath = path17.join(resultsDir, entry.name);
|
|
8390
|
+
if (entry.isDirectory()) {
|
|
8391
|
+
files.push(...collectJsonlFiles(fullPath));
|
|
8392
|
+
} else if (entry.name.endsWith(".jsonl")) {
|
|
8393
|
+
files.push(fullPath);
|
|
8394
|
+
}
|
|
8395
|
+
}
|
|
8396
|
+
} catch {
|
|
8397
|
+
}
|
|
8398
|
+
return files;
|
|
8399
|
+
}
|
|
7650
8400
|
function extractSnippet(text, matchIndex, matchLength, contextChars = 60) {
|
|
7651
8401
|
const start = Math.max(0, matchIndex - contextChars);
|
|
7652
8402
|
const end = Math.min(text.length, matchIndex + matchLength + contextChars);
|
|
@@ -7659,7 +8409,7 @@ function searchJsonlFile(filePath, regex2, targetFilter, experimentFilter) {
|
|
|
7659
8409
|
const matches = [];
|
|
7660
8410
|
let content;
|
|
7661
8411
|
try {
|
|
7662
|
-
content =
|
|
8412
|
+
content = readFileSync6(filePath, "utf8");
|
|
7663
8413
|
} catch {
|
|
7664
8414
|
return matches;
|
|
7665
8415
|
}
|
|
@@ -7709,7 +8459,7 @@ function discoverSources(basePath, cwd) {
|
|
|
7709
8459
|
return [resolved];
|
|
7710
8460
|
}
|
|
7711
8461
|
const sources = [];
|
|
7712
|
-
sources.push(...
|
|
8462
|
+
sources.push(...collectCurrentResultJsonlFiles(cwd));
|
|
7713
8463
|
sources.push(...collectJsonlFiles(path17.join(cwd, ".agentv", "transcripts")));
|
|
7714
8464
|
return sources;
|
|
7715
8465
|
}
|
|
@@ -7761,7 +8511,7 @@ var inspectSearchCommand = command({
|
|
|
7761
8511
|
path: positional({
|
|
7762
8512
|
type: optional(string),
|
|
7763
8513
|
displayName: "path",
|
|
7764
|
-
description: "Directory or file to search (default: .agentv/results/
|
|
8514
|
+
description: "Directory or file to search (default: .agentv/results/ and .agentv/transcripts/)"
|
|
7765
8515
|
}),
|
|
7766
8516
|
target: option({
|
|
7767
8517
|
type: optional(string),
|
|
@@ -8258,7 +9008,7 @@ var inspectCommand = subcommands({
|
|
|
8258
9008
|
|
|
8259
9009
|
// src/commands/pipeline/bench.ts
|
|
8260
9010
|
import { existsSync as existsSync6 } from "node:fs";
|
|
8261
|
-
import { readFile as
|
|
9011
|
+
import { readFile as readFile5, readdir, writeFile as writeFile7 } from "node:fs/promises";
|
|
8262
9012
|
import { join } from "node:path";
|
|
8263
9013
|
var evalBenchCommand = command({
|
|
8264
9014
|
name: "bench",
|
|
@@ -8271,7 +9021,7 @@ var evalBenchCommand = command({
|
|
|
8271
9021
|
})
|
|
8272
9022
|
},
|
|
8273
9023
|
handler: async ({ exportDir }) => {
|
|
8274
|
-
const manifest = JSON.parse(await
|
|
9024
|
+
const manifest = JSON.parse(await readFile5(join(exportDir, "manifest.json"), "utf8"));
|
|
8275
9025
|
const testIds = manifest.test_ids;
|
|
8276
9026
|
const targetName2 = manifest.target?.name ?? "unknown";
|
|
8277
9027
|
const suiteName = manifest.suite ?? "";
|
|
@@ -8289,7 +9039,7 @@ var evalBenchCommand = command({
|
|
|
8289
9039
|
try {
|
|
8290
9040
|
const resultFiles = (await readdir(codeResultsDir)).filter((f) => f.endsWith(".json"));
|
|
8291
9041
|
for (const file of resultFiles) {
|
|
8292
|
-
const result = JSON.parse(await
|
|
9042
|
+
const result = JSON.parse(await readFile5(join(codeResultsDir, file), "utf8"));
|
|
8293
9043
|
evaluators.push({
|
|
8294
9044
|
name: result.name,
|
|
8295
9045
|
type: result.type ?? "code-grader",
|
|
@@ -8307,12 +9057,12 @@ var evalBenchCommand = command({
|
|
|
8307
9057
|
try {
|
|
8308
9058
|
const graderFiles = (await readdir(llmGradersDir)).filter((f) => f.endsWith(".json"));
|
|
8309
9059
|
for (const file of graderFiles) {
|
|
8310
|
-
const graderMeta = JSON.parse(await
|
|
9060
|
+
const graderMeta = JSON.parse(await readFile5(join(llmGradersDir, file), "utf8"));
|
|
8311
9061
|
const graderName = graderMeta.name;
|
|
8312
9062
|
const diskResultPath = join(testDir, "llm_grader_results", `${graderName}.json`);
|
|
8313
9063
|
let llmResult;
|
|
8314
9064
|
try {
|
|
8315
|
-
llmResult = JSON.parse(await
|
|
9065
|
+
llmResult = JSON.parse(await readFile5(diskResultPath, "utf8"));
|
|
8316
9066
|
} catch {
|
|
8317
9067
|
}
|
|
8318
9068
|
if (llmResult) {
|
|
@@ -8370,7 +9120,7 @@ var evalBenchCommand = command({
|
|
|
8370
9120
|
const timingPath = join(testDir, "timing.json");
|
|
8371
9121
|
if (existsSync6(timingPath)) {
|
|
8372
9122
|
try {
|
|
8373
|
-
const timing = JSON.parse(await
|
|
9123
|
+
const timing = JSON.parse(await readFile5(timingPath, "utf8"));
|
|
8374
9124
|
if (typeof timing.execution_status === "string") {
|
|
8375
9125
|
executionStatus = timing.execution_status;
|
|
8376
9126
|
}
|
|
@@ -8465,7 +9215,7 @@ function computeStats(values) {
|
|
|
8465
9215
|
}
|
|
8466
9216
|
|
|
8467
9217
|
// src/commands/pipeline/grade.ts
|
|
8468
|
-
import { mkdir as mkdir7, readFile as
|
|
9218
|
+
import { mkdir as mkdir7, readFile as readFile6, readdir as readdir2, writeFile as writeFile8 } from "node:fs/promises";
|
|
8469
9219
|
import { join as join2 } from "node:path";
|
|
8470
9220
|
var DEFAULT_CONCURRENCY = 10;
|
|
8471
9221
|
async function runCodeGraders(tasks, concurrency) {
|
|
@@ -8481,7 +9231,7 @@ async function runCodeGraders(tasks, concurrency) {
|
|
|
8481
9231
|
const executeGrader = async (task) => {
|
|
8482
9232
|
const { testDir, resultsDir, graderFile, responseText } = task;
|
|
8483
9233
|
const graderConfig = JSON.parse(
|
|
8484
|
-
await
|
|
9234
|
+
await readFile6(join2(testDir, "code_graders", graderFile), "utf8")
|
|
8485
9235
|
);
|
|
8486
9236
|
if (graderConfig.command) {
|
|
8487
9237
|
await executeCodeGrader(graderConfig, task);
|
|
@@ -8640,7 +9390,7 @@ var evalGradeCommand = command({
|
|
|
8640
9390
|
handler: async ({ exportDir, concurrency }) => {
|
|
8641
9391
|
const maxWorkers = concurrency ?? DEFAULT_CONCURRENCY;
|
|
8642
9392
|
const manifestPath = join2(exportDir, "manifest.json");
|
|
8643
|
-
const manifest = JSON.parse(await
|
|
9393
|
+
const manifest = JSON.parse(await readFile6(manifestPath, "utf8"));
|
|
8644
9394
|
const testIds = manifest.test_ids;
|
|
8645
9395
|
const suiteName = manifest.suite ?? "";
|
|
8646
9396
|
const safeSuiteName = suiteName ? suiteName.replace(/[\/\\:*?"<>|]/g, "_") : "";
|
|
@@ -8658,8 +9408,8 @@ var evalGradeCommand = command({
|
|
|
8658
9408
|
}
|
|
8659
9409
|
if (graderFiles.length === 0) continue;
|
|
8660
9410
|
await mkdir7(resultsDir, { recursive: true });
|
|
8661
|
-
const responseText = await
|
|
8662
|
-
const inputData = JSON.parse(await
|
|
9411
|
+
const responseText = await readFile6(join2(testDir, "response.md"), "utf8");
|
|
9412
|
+
const inputData = JSON.parse(await readFile6(join2(testDir, "input.json"), "utf8"));
|
|
8663
9413
|
for (const graderFile of graderFiles) {
|
|
8664
9414
|
tasks.push({ testId, testDir, resultsDir, graderFile, responseText, inputData });
|
|
8665
9415
|
}
|
|
@@ -8670,7 +9420,7 @@ var evalGradeCommand = command({
|
|
|
8670
9420
|
});
|
|
8671
9421
|
|
|
8672
9422
|
// src/commands/pipeline/input.ts
|
|
8673
|
-
import { readFile as
|
|
9423
|
+
import { readFile as readFile7 } from "node:fs/promises";
|
|
8674
9424
|
import { mkdir as mkdir8, writeFile as writeFile9 } from "node:fs/promises";
|
|
8675
9425
|
import { dirname, join as join3, relative, resolve } from "node:path";
|
|
8676
9426
|
var BUILTIN_ASSERTION_TYPES = /* @__PURE__ */ new Set([
|
|
@@ -8698,7 +9448,7 @@ var evalInputCommand = command({
|
|
|
8698
9448
|
out: option({
|
|
8699
9449
|
type: optional(string),
|
|
8700
9450
|
long: "out",
|
|
8701
|
-
description: "Output directory for extracted inputs (default: .agentv/results
|
|
9451
|
+
description: "Output directory for extracted inputs (default: .agentv/results/<experiment>/<timestamp>)"
|
|
8702
9452
|
}),
|
|
8703
9453
|
experiment: option({
|
|
8704
9454
|
type: optional(string),
|
|
@@ -8872,7 +9622,7 @@ async function writeGraderConfigs(testDir, assertions, evalDir) {
|
|
|
8872
9622
|
let promptContent = "";
|
|
8873
9623
|
if (config.resolvedPromptPath) {
|
|
8874
9624
|
try {
|
|
8875
|
-
promptContent = await
|
|
9625
|
+
promptContent = await readFile7(config.resolvedPromptPath, "utf8");
|
|
8876
9626
|
} catch {
|
|
8877
9627
|
promptContent = typeof config.prompt === "string" ? config.prompt : "";
|
|
8878
9628
|
}
|
|
@@ -8921,9 +9671,9 @@ async function writeJson(filePath, data) {
|
|
|
8921
9671
|
|
|
8922
9672
|
// src/commands/pipeline/run.ts
|
|
8923
9673
|
import { exec } from "node:child_process";
|
|
8924
|
-
import { existsSync as existsSync7, readFileSync as
|
|
8925
|
-
import { mkdir as mkdir9, readFile as
|
|
8926
|
-
import { tmpdir } from "node:os";
|
|
9674
|
+
import { existsSync as existsSync7, readFileSync as readFileSync7, unlinkSync } from "node:fs";
|
|
9675
|
+
import { mkdir as mkdir9, readFile as readFile8, readdir as readdir3, writeFile as writeFile10 } from "node:fs/promises";
|
|
9676
|
+
import { tmpdir as tmpdir2 } from "node:os";
|
|
8927
9677
|
import { dirname as dirname2, join as join4, relative as relative2, resolve as resolve2 } from "node:path";
|
|
8928
9678
|
function extractInputText(input) {
|
|
8929
9679
|
if (!input || input.length === 0) return "";
|
|
@@ -8937,7 +9687,7 @@ function loadEnvFile(dir) {
|
|
|
8937
9687
|
const candidate = join4(current, ".env");
|
|
8938
9688
|
if (existsSync7(candidate)) {
|
|
8939
9689
|
const env2 = {};
|
|
8940
|
-
for (const line of
|
|
9690
|
+
for (const line of readFileSync7(candidate, "utf8").split("\n")) {
|
|
8941
9691
|
const trimmed = line.trim();
|
|
8942
9692
|
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
8943
9693
|
const eqIdx = trimmed.indexOf("=");
|
|
@@ -8964,7 +9714,7 @@ var evalRunCommand2 = command({
|
|
|
8964
9714
|
out: option({
|
|
8965
9715
|
type: optional(string),
|
|
8966
9716
|
long: "out",
|
|
8967
|
-
description: "Output directory for results (default: .agentv/results
|
|
9717
|
+
description: "Output directory for results (default: .agentv/results/<experiment>/<timestamp>)"
|
|
8968
9718
|
}),
|
|
8969
9719
|
workers: option({
|
|
8970
9720
|
type: optional(number),
|
|
@@ -9089,7 +9839,7 @@ var evalRunCommand2 = command({
|
|
|
9089
9839
|
if (!process.env.AGENTV_RUN_TIMESTAMP) {
|
|
9090
9840
|
process.env.AGENTV_RUN_TIMESTAMP = (/* @__PURE__ */ new Date()).toISOString().replace(/:/g, "-").replace(/\./g, "-");
|
|
9091
9841
|
}
|
|
9092
|
-
const mergedEnv = { ...process.env, ...envVars };
|
|
9842
|
+
const mergedEnv = { ...process.env, ...envVars, AGENTV_RUN_DIR: outDir };
|
|
9093
9843
|
const maxWorkers = workers ?? targetWorkers ?? 5;
|
|
9094
9844
|
let invCompleted = 0;
|
|
9095
9845
|
const invTotal = testIds.length;
|
|
@@ -9101,14 +9851,14 @@ var evalRunCommand2 = command({
|
|
|
9101
9851
|
const invokeTarget = async (testId) => {
|
|
9102
9852
|
const subpath = safeSuiteName ? [safeSuiteName, testId] : [testId];
|
|
9103
9853
|
const testDir = join4(outDir, ...subpath);
|
|
9104
|
-
const invoke = JSON.parse(await
|
|
9854
|
+
const invoke = JSON.parse(await readFile8(join4(testDir, "invoke.json"), "utf8"));
|
|
9105
9855
|
if (invoke.kind !== "cli") return;
|
|
9106
|
-
const inputData = JSON.parse(await
|
|
9856
|
+
const inputData = JSON.parse(await readFile8(join4(testDir, "input.json"), "utf8"));
|
|
9107
9857
|
const template = invoke.command;
|
|
9108
9858
|
const cwd = invoke.cwd;
|
|
9109
9859
|
const timeoutMs = invoke.timeout_ms ?? 12e4;
|
|
9110
|
-
const promptFile = join4(
|
|
9111
|
-
const outputFile = join4(
|
|
9860
|
+
const promptFile = join4(tmpdir2(), `agentv-prompt-${testId}-${Date.now()}.txt`);
|
|
9861
|
+
const outputFile = join4(tmpdir2(), `agentv-output-${testId}-${Date.now()}.txt`);
|
|
9112
9862
|
const inputText = extractInputText(inputData.input);
|
|
9113
9863
|
await writeFile10(promptFile, inputText, "utf8");
|
|
9114
9864
|
let rendered = template;
|
|
@@ -9135,7 +9885,7 @@ var evalRunCommand2 = command({
|
|
|
9135
9885
|
const durationMs = Math.round(performance.now() - start);
|
|
9136
9886
|
let response;
|
|
9137
9887
|
if (existsSync7(outputFile)) {
|
|
9138
|
-
response =
|
|
9888
|
+
response = readFileSync7(outputFile, "utf8");
|
|
9139
9889
|
} else {
|
|
9140
9890
|
response = "ERROR: No output file generated";
|
|
9141
9891
|
}
|
|
@@ -9242,8 +9992,8 @@ Done. Results in ${outDir}`);
|
|
|
9242
9992
|
}
|
|
9243
9993
|
if (graderFiles.length === 0) continue;
|
|
9244
9994
|
await mkdir9(resultsDir, { recursive: true });
|
|
9245
|
-
const responseText = await
|
|
9246
|
-
const inputData = JSON.parse(await
|
|
9995
|
+
const responseText = await readFile8(join4(testDir, "response.md"), "utf8");
|
|
9996
|
+
const inputData = JSON.parse(await readFile8(join4(testDir, "input.json"), "utf8"));
|
|
9247
9997
|
for (const graderFile of graderFiles) {
|
|
9248
9998
|
graderTasks.push({ testId, testDir, resultsDir, graderFile, responseText, inputData });
|
|
9249
9999
|
}
|
|
@@ -9294,7 +10044,7 @@ async function writeGraderConfigs2(testDir, assertions, evalDir) {
|
|
|
9294
10044
|
let promptContent = "";
|
|
9295
10045
|
if (config.resolvedPromptPath) {
|
|
9296
10046
|
try {
|
|
9297
|
-
promptContent =
|
|
10047
|
+
promptContent = readFileSync7(config.resolvedPromptPath, "utf8");
|
|
9298
10048
|
} catch {
|
|
9299
10049
|
promptContent = typeof config.prompt === "string" ? config.prompt : "";
|
|
9300
10050
|
}
|
|
@@ -9335,7 +10085,7 @@ var pipelineCommand = subcommands({
|
|
|
9335
10085
|
});
|
|
9336
10086
|
|
|
9337
10087
|
// src/commands/prepare/index.ts
|
|
9338
|
-
import { cp, mkdir as mkdir10, rename, rm, writeFile as writeFile11 } from "node:fs/promises";
|
|
10088
|
+
import { cp, mkdir as mkdir10, rename, rm as rm2, writeFile as writeFile11 } from "node:fs/promises";
|
|
9339
10089
|
import path18 from "node:path";
|
|
9340
10090
|
function setupStatusFromHook(status) {
|
|
9341
10091
|
if (status === "success") {
|
|
@@ -9378,7 +10128,7 @@ async function moveDirectory(sourcePath, destinationPath) {
|
|
|
9378
10128
|
throw error;
|
|
9379
10129
|
}
|
|
9380
10130
|
await cp(sourcePath, destinationPath, { recursive: true });
|
|
9381
|
-
await
|
|
10131
|
+
await rm2(sourcePath, { recursive: true, force: true });
|
|
9382
10132
|
}
|
|
9383
10133
|
}
|
|
9384
10134
|
async function placePreparedWorkspace(prepared, destinationPath) {
|
|
@@ -9387,7 +10137,7 @@ async function placePreparedWorkspace(prepared, destinationPath) {
|
|
|
9387
10137
|
return destinationPath;
|
|
9388
10138
|
}
|
|
9389
10139
|
await mkdir10(path18.dirname(destinationPath), { recursive: true });
|
|
9390
|
-
await
|
|
10140
|
+
await rm2(destinationPath, { recursive: true, force: true });
|
|
9391
10141
|
if (prepared.cleanupPolicy.mode !== "static" && prepared.pool === void 0) {
|
|
9392
10142
|
await moveDirectory(sourcePath, destinationPath);
|
|
9393
10143
|
return destinationPath;
|
|
@@ -9591,7 +10341,7 @@ import {
|
|
|
9591
10341
|
cpSync,
|
|
9592
10342
|
existsSync as existsSync8,
|
|
9593
10343
|
mkdirSync as mkdirSync2,
|
|
9594
|
-
readFileSync as
|
|
10344
|
+
readFileSync as readFileSync8,
|
|
9595
10345
|
statSync as statSync4,
|
|
9596
10346
|
writeFileSync as writeFileSync3
|
|
9597
10347
|
} from "node:fs";
|
|
@@ -9607,12 +10357,12 @@ function parseJsonlLine(line) {
|
|
|
9607
10357
|
return JSON.parse(line);
|
|
9608
10358
|
}
|
|
9609
10359
|
function readManifestRecords(manifestPath) {
|
|
9610
|
-
return
|
|
10360
|
+
return readFileSync8(manifestPath, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map(parseJsonlLine);
|
|
9611
10361
|
}
|
|
9612
10362
|
function readBenchmarkMetadata(manifestPath) {
|
|
9613
10363
|
try {
|
|
9614
10364
|
const benchmarkPath = path19.join(path19.dirname(manifestPath), "benchmark.json");
|
|
9615
|
-
const parsed = JSON.parse(
|
|
10365
|
+
const parsed = JSON.parse(readFileSync8(benchmarkPath, "utf8"));
|
|
9616
10366
|
return {
|
|
9617
10367
|
timestamp: parsed.metadata?.timestamp,
|
|
9618
10368
|
displayName: parsed.metadata?.display_name
|
|
@@ -9727,10 +10477,10 @@ function sanitizePathSegment(value) {
|
|
|
9727
10477
|
function defaultDisplayName(sources) {
|
|
9728
10478
|
return `Combined run (${sources.map((source) => source.displayName).join(" + ")})`;
|
|
9729
10479
|
}
|
|
9730
|
-
function
|
|
10480
|
+
function defaultCombinedRunDirForExperiment(cwd, experiment, startedAt) {
|
|
9731
10481
|
const parsed = startedAt ? new Date(startedAt) : void 0;
|
|
9732
10482
|
const timestamp = parsed && !Number.isNaN(parsed.getTime()) ? createRunDirName(parsed) : sanitizePathSegment(startedAt ?? "unknown-time");
|
|
9733
|
-
return
|
|
10483
|
+
return buildDefaultRunDirFromName(cwd, experiment, timestamp);
|
|
9734
10484
|
}
|
|
9735
10485
|
function uniqueRunDir(baseDir) {
|
|
9736
10486
|
if (!existsSync8(baseDir)) return baseDir;
|
|
@@ -9741,29 +10491,69 @@ function uniqueRunDir(baseDir) {
|
|
|
9741
10491
|
return `${baseDir}-${suffix}`;
|
|
9742
10492
|
}
|
|
9743
10493
|
function toRunId(cwd, runDir) {
|
|
9744
|
-
const
|
|
9745
|
-
|
|
9746
|
-
if (relative3 === "" || relative3.startsWith("..") || path19.isAbsolute(relative3)) {
|
|
10494
|
+
const relative3 = relativeRunPathFromCwd(cwd, runDir);
|
|
10495
|
+
if (!relative3) {
|
|
9747
10496
|
return path19.basename(runDir);
|
|
9748
10497
|
}
|
|
9749
|
-
const parts = relative3.split(path19.sep);
|
|
9750
|
-
|
|
10498
|
+
const parts = relative3.split(path19.posix.sep);
|
|
10499
|
+
if (parts.length <= 1) {
|
|
10500
|
+
return relative3;
|
|
10501
|
+
}
|
|
10502
|
+
const experiment = parts[0];
|
|
10503
|
+
const timestamp = parts.slice(1).join(path19.posix.sep);
|
|
10504
|
+
return experiment === "default" ? timestamp : `${experiment}::${timestamp}`;
|
|
10505
|
+
}
|
|
10506
|
+
function experimentFromRelativeRunPath(relativeRunPath) {
|
|
10507
|
+
const experiment = relativeRunPath.split(path19.posix.sep).filter(Boolean)[0];
|
|
10508
|
+
return normalizeExperimentName(experiment);
|
|
10509
|
+
}
|
|
10510
|
+
function resolveCombinedExperiment(sources, requestedExperiment) {
|
|
10511
|
+
const experiments = [...new Set(sources.map((source) => source.experiment))];
|
|
10512
|
+
const normalizedRequested = requestedExperiment?.trim() ? normalizeExperimentName(requestedExperiment) : void 0;
|
|
10513
|
+
if (experiments.length === 1) {
|
|
10514
|
+
const sourceExperiment = experiments[0];
|
|
10515
|
+
if (normalizedRequested === void 0) {
|
|
10516
|
+
return sourceExperiment;
|
|
10517
|
+
}
|
|
10518
|
+
if (normalizedRequested !== sourceExperiment) {
|
|
10519
|
+
throw new Error(`Combined runs from the same experiment must inherit "${sourceExperiment}".`);
|
|
10520
|
+
}
|
|
10521
|
+
return sourceExperiment;
|
|
10522
|
+
}
|
|
10523
|
+
if (normalizedRequested !== void 0) {
|
|
10524
|
+
return normalizedRequested;
|
|
10525
|
+
}
|
|
10526
|
+
throw new Error(
|
|
10527
|
+
`Combining runs from multiple experiments requires an experiment name. Source experiments: ${experiments.sort().join(", ")}`
|
|
10528
|
+
);
|
|
9751
10529
|
}
|
|
9752
10530
|
var MANIFEST_PATH_FIELDS = [
|
|
10531
|
+
"artifact_dir",
|
|
9753
10532
|
"grading_path",
|
|
9754
10533
|
"timing_path",
|
|
9755
10534
|
"input_path",
|
|
9756
10535
|
"output_path",
|
|
9757
10536
|
"response_path",
|
|
10537
|
+
"trace_path",
|
|
10538
|
+
"transcript_path",
|
|
10539
|
+
"metrics_path",
|
|
10540
|
+
"raw_provider_log_path",
|
|
9758
10541
|
"task_dir",
|
|
9759
10542
|
"eval_path",
|
|
9760
10543
|
"targets_path",
|
|
9761
10544
|
"files_path",
|
|
9762
10545
|
"graders_path"
|
|
9763
10546
|
];
|
|
10547
|
+
var POINTER_FAMILIES = {
|
|
10548
|
+
trace: "traces",
|
|
10549
|
+
transcript: "transcripts"
|
|
10550
|
+
};
|
|
10551
|
+
function isSafeRelativeArtifactPath(relativePath) {
|
|
10552
|
+
return !path19.isAbsolute(relativePath) && !relativePath.split(/[\\/]+/).includes("..");
|
|
10553
|
+
}
|
|
9764
10554
|
function copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, relativePath) {
|
|
9765
10555
|
if (!relativePath) return void 0;
|
|
9766
|
-
if (
|
|
10556
|
+
if (!isSafeRelativeArtifactPath(relativePath)) {
|
|
9767
10557
|
throw new Error(`Unsafe artifact path in source manifest: ${relativePath}`);
|
|
9768
10558
|
}
|
|
9769
10559
|
const sourcePath = path19.join(sourceBaseDir, relativePath);
|
|
@@ -9781,9 +10571,49 @@ function copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, relativeP
|
|
|
9781
10571
|
}
|
|
9782
10572
|
return rewritten;
|
|
9783
10573
|
}
|
|
9784
|
-
function
|
|
10574
|
+
function rewriteArtifactPointer(pointerName, pointer, sourceBaseDir, outputDir, sourceIndex) {
|
|
10575
|
+
if (!pointer) {
|
|
10576
|
+
return void 0;
|
|
10577
|
+
}
|
|
10578
|
+
if (!isSafeRelativeArtifactPath(pointer.path)) {
|
|
10579
|
+
throw new Error(`Unsafe artifact path in source manifest: ${pointer.path}`);
|
|
10580
|
+
}
|
|
10581
|
+
const sourcePath = path19.join(sourceBaseDir, pointer.path);
|
|
10582
|
+
if (!existsSync8(sourcePath)) {
|
|
10583
|
+
return { ...pointer };
|
|
10584
|
+
}
|
|
10585
|
+
const rewrittenPath = copyReferencedArtifact(sourceBaseDir, outputDir, sourceIndex, pointer.path);
|
|
10586
|
+
if (!rewrittenPath) {
|
|
10587
|
+
return { ...pointer };
|
|
10588
|
+
}
|
|
10589
|
+
const family = pointer.family ?? POINTER_FAMILIES[pointerName];
|
|
10590
|
+
return {
|
|
10591
|
+
...pointer,
|
|
10592
|
+
path: rewrittenPath,
|
|
10593
|
+
key: path19.posix.join(family, rewrittenPath)
|
|
10594
|
+
};
|
|
10595
|
+
}
|
|
10596
|
+
function rewriteTranscriptArtifactPointer(pointer, sourceBaseDir, outputDir, sourceIndex) {
|
|
10597
|
+
return rewriteArtifactPointer("transcript", pointer, sourceBaseDir, outputDir, sourceIndex);
|
|
10598
|
+
}
|
|
10599
|
+
function rewriteArtifactPointers(pointers, sourceBaseDir, outputDir, sourceIndex) {
|
|
10600
|
+
if (!pointers) {
|
|
10601
|
+
return void 0;
|
|
10602
|
+
}
|
|
10603
|
+
return {
|
|
10604
|
+
trace: rewriteArtifactPointer("trace", pointers.trace, sourceBaseDir, outputDir, sourceIndex),
|
|
10605
|
+
transcript: rewriteTranscriptArtifactPointer(
|
|
10606
|
+
pointers.transcript,
|
|
10607
|
+
sourceBaseDir,
|
|
10608
|
+
outputDir,
|
|
10609
|
+
sourceIndex
|
|
10610
|
+
)
|
|
10611
|
+
};
|
|
10612
|
+
}
|
|
10613
|
+
function rewriteAndCopyRecord(row, outputDir, experiment) {
|
|
9785
10614
|
const sourceBaseDir = path19.dirname(row.source.manifestPath);
|
|
9786
10615
|
const rewritten = { ...row.record };
|
|
10616
|
+
rewritten.experiment = experiment;
|
|
9787
10617
|
for (const field of MANIFEST_PATH_FIELDS) {
|
|
9788
10618
|
rewritten[field] = copyReferencedArtifact(
|
|
9789
10619
|
sourceBaseDir,
|
|
@@ -9792,6 +10622,19 @@ function rewriteAndCopyRecord(row, outputDir) {
|
|
|
9792
10622
|
row.record[field]
|
|
9793
10623
|
);
|
|
9794
10624
|
}
|
|
10625
|
+
const artifactPointers = rewriteArtifactPointers(
|
|
10626
|
+
row.record.artifact_pointers,
|
|
10627
|
+
sourceBaseDir,
|
|
10628
|
+
outputDir,
|
|
10629
|
+
row.source.index
|
|
10630
|
+
);
|
|
10631
|
+
rewritten.artifact_pointers = artifactPointers;
|
|
10632
|
+
if (row.record.trace_path && rewritten.trace_path === row.record.trace_path && artifactPointers?.trace?.path) {
|
|
10633
|
+
rewritten.trace_path = artifactPointers.trace.path;
|
|
10634
|
+
}
|
|
10635
|
+
if (row.record.transcript_path && rewritten.transcript_path === row.record.transcript_path && artifactPointers?.transcript?.path) {
|
|
10636
|
+
rewritten.transcript_path = artifactPointers.transcript.path;
|
|
10637
|
+
}
|
|
9795
10638
|
return rewritten;
|
|
9796
10639
|
}
|
|
9797
10640
|
function writeJson3(filePath, value) {
|
|
@@ -9809,10 +10652,18 @@ function writeJsonl(filePath, records) {
|
|
|
9809
10652
|
function buildCombineRunSources(sourcePaths, cwd, options) {
|
|
9810
10653
|
return sourcePaths.map((sourcePath, index) => {
|
|
9811
10654
|
const manifestPath = resolveResultSourcePath(sourcePath, cwd);
|
|
10655
|
+
const runDir = path19.dirname(resolveRunManifestPath(manifestPath));
|
|
10656
|
+
const relativeRunPath = relativeRunPathFromCwd(cwd, runDir);
|
|
10657
|
+
if (!relativeRunPath) {
|
|
10658
|
+
throw new Error(
|
|
10659
|
+
`Run workspace is outside the canonical results layout: ${runDir}. Expected .agentv/results/<experiment>/<timestamp>`
|
|
10660
|
+
);
|
|
10661
|
+
}
|
|
9812
10662
|
return {
|
|
9813
|
-
id: options?.ids?.[index] ??
|
|
9814
|
-
displayName: options?.displayNames?.[index] ?? path19.basename(
|
|
10663
|
+
id: options?.ids?.[index] ?? toRunId(cwd, runDir),
|
|
10664
|
+
displayName: options?.displayNames?.[index] ?? path19.basename(runDir),
|
|
9815
10665
|
manifestPath,
|
|
10666
|
+
experiment: experimentFromRelativeRunPath(relativeRunPath),
|
|
9816
10667
|
tags: options?.tags?.[index]
|
|
9817
10668
|
};
|
|
9818
10669
|
});
|
|
@@ -9823,10 +10674,23 @@ function combineRunSources(options) {
|
|
|
9823
10674
|
}
|
|
9824
10675
|
const loadedSources = loadSources(options.sources);
|
|
9825
10676
|
const startedAt = earliestTimestamp(loadedSources.map((source) => source.startedAt));
|
|
10677
|
+
const experiment = resolveCombinedExperiment(loadedSources, options.experiment);
|
|
9826
10678
|
const displayName = options.displayName?.trim() || defaultDisplayName(loadedSources);
|
|
9827
10679
|
const runDir = uniqueRunDir(
|
|
9828
|
-
options.outputDir ? path19.resolve(options.cwd, options.outputDir) :
|
|
10680
|
+
options.outputDir ? path19.resolve(options.cwd, options.outputDir) : defaultCombinedRunDirForExperiment(options.cwd, experiment, startedAt)
|
|
9829
10681
|
);
|
|
10682
|
+
const relativeOutputRunPath = relativeRunPathFromCwd(options.cwd, runDir);
|
|
10683
|
+
if (!relativeOutputRunPath) {
|
|
10684
|
+
throw new Error(
|
|
10685
|
+
`Output run workspace must use .agentv/results/<experiment>/<timestamp>: ${runDir}`
|
|
10686
|
+
);
|
|
10687
|
+
}
|
|
10688
|
+
const outputExperiment = experimentFromRelativeRunPath(relativeOutputRunPath);
|
|
10689
|
+
if (outputExperiment !== experiment) {
|
|
10690
|
+
throw new Error(
|
|
10691
|
+
`Output run workspace experiment "${outputExperiment}" must match combined experiment "${experiment}".`
|
|
10692
|
+
);
|
|
10693
|
+
}
|
|
9830
10694
|
const { rows, conflicts } = selectRows(
|
|
9831
10695
|
loadedSources,
|
|
9832
10696
|
options.duplicatePolicy,
|
|
@@ -9838,7 +10702,7 @@ function combineRunSources(options) {
|
|
|
9838
10702
|
return left.localeCompare(right);
|
|
9839
10703
|
});
|
|
9840
10704
|
mkdirSync2(runDir, { recursive: true });
|
|
9841
|
-
const records = rows.map((row) => rewriteAndCopyRecord(row, runDir));
|
|
10705
|
+
const records = rows.map((row) => rewriteAndCopyRecord(row, runDir, experiment));
|
|
9842
10706
|
const manifestPath = path19.join(runDir, "index.jsonl");
|
|
9843
10707
|
writeJsonl(manifestPath, records);
|
|
9844
10708
|
const timing = buildTimingArtifact(results);
|
|
@@ -9851,6 +10715,7 @@ function combineRunSources(options) {
|
|
|
9851
10715
|
...benchmark.metadata,
|
|
9852
10716
|
timestamp: startedAt ?? benchmark.metadata.timestamp,
|
|
9853
10717
|
display_name: displayName,
|
|
10718
|
+
experiment,
|
|
9854
10719
|
combined_from_run_ids: loadedSources.map((source) => source.id),
|
|
9855
10720
|
combined_from_display_names: loadedSources.map((source) => source.displayName),
|
|
9856
10721
|
duplicate_policy: options.duplicatePolicy
|
|
@@ -9866,6 +10731,7 @@ function combineRunSources(options) {
|
|
|
9866
10731
|
benchmarkPath,
|
|
9867
10732
|
timingPath,
|
|
9868
10733
|
displayName,
|
|
10734
|
+
experiment,
|
|
9869
10735
|
combinedFromRunIds: loadedSources.map((source) => source.id),
|
|
9870
10736
|
duplicateConflicts: conflicts,
|
|
9871
10737
|
testCount: rows.length,
|
|
@@ -9916,8 +10782,22 @@ function parseDuplicatePolicy(policy) {
|
|
|
9916
10782
|
}
|
|
9917
10783
|
return void 0;
|
|
9918
10784
|
}
|
|
9919
|
-
|
|
9920
|
-
|
|
10785
|
+
function uniqueSourceExperiments(sources) {
|
|
10786
|
+
return [...new Set(sources.map((source) => source.experiment))].sort();
|
|
10787
|
+
}
|
|
10788
|
+
async function promptExperimentName(experiments) {
|
|
10789
|
+
const rl = readline2.createInterface({ input: process.stdin, output: process.stdout });
|
|
10790
|
+
try {
|
|
10791
|
+
const answer = await rl.question(
|
|
10792
|
+
`Source runs span experiments (${experiments.join(", ")}). New experiment name: `
|
|
10793
|
+
);
|
|
10794
|
+
return answer.trim() || void 0;
|
|
10795
|
+
} finally {
|
|
10796
|
+
rl.close();
|
|
10797
|
+
}
|
|
10798
|
+
}
|
|
10799
|
+
var resultsCombineCommand = command({
|
|
10800
|
+
name: "combine",
|
|
9921
10801
|
description: "Combine two or more partial local run workspaces into a new run workspace",
|
|
9922
10802
|
args: {
|
|
9923
10803
|
sources: restPositionals({
|
|
@@ -9929,7 +10809,12 @@ var resultsCombineCommand = command({
|
|
|
9929
10809
|
type: optional(string),
|
|
9930
10810
|
long: "output",
|
|
9931
10811
|
short: "o",
|
|
9932
|
-
description: "Output run workspace directory (defaults to .agentv/results
|
|
10812
|
+
description: "Output run workspace directory (defaults to .agentv/results/<experiment>/<earliest-source-time>)"
|
|
10813
|
+
}),
|
|
10814
|
+
experiment: option({
|
|
10815
|
+
type: optional(string),
|
|
10816
|
+
long: "experiment",
|
|
10817
|
+
description: "Experiment namespace for the combined run. Required when sources span multiple experiments."
|
|
9933
10818
|
}),
|
|
9934
10819
|
displayName: option({
|
|
9935
10820
|
type: optional(string),
|
|
@@ -9949,6 +10834,17 @@ var resultsCombineCommand = command({
|
|
|
9949
10834
|
}
|
|
9950
10835
|
const cwd = process.cwd();
|
|
9951
10836
|
const sources = buildCombineRunSources(args.sources, cwd);
|
|
10837
|
+
const sourceExperiments = uniqueSourceExperiments(sources);
|
|
10838
|
+
let experiment = args.experiment;
|
|
10839
|
+
if (!experiment && sourceExperiments.length > 1 && process.stdin.isTTY && process.stdout.isTTY) {
|
|
10840
|
+
experiment = await promptExperimentName(sourceExperiments);
|
|
10841
|
+
}
|
|
10842
|
+
if (!experiment && sourceExperiments.length > 1) {
|
|
10843
|
+
console.error(
|
|
10844
|
+
`Error: combining runs from multiple experiments requires --experiment <name>. Source experiments: ${sourceExperiments.join(", ")}`
|
|
10845
|
+
);
|
|
10846
|
+
process.exit(1);
|
|
10847
|
+
}
|
|
9952
10848
|
const duplicatePolicy = defaultDuplicatePolicy(parseDuplicatePolicy(args.duplicatePolicy));
|
|
9953
10849
|
let promptChoices;
|
|
9954
10850
|
if (duplicatePolicy === "prompt") {
|
|
@@ -9968,6 +10864,7 @@ var resultsCombineCommand = command({
|
|
|
9968
10864
|
cwd,
|
|
9969
10865
|
sources,
|
|
9970
10866
|
outputDir: args.output,
|
|
10867
|
+
experiment,
|
|
9971
10868
|
displayName: args.displayName,
|
|
9972
10869
|
duplicatePolicy,
|
|
9973
10870
|
promptChoices
|
|
@@ -9993,18 +10890,13 @@ import * as readline3 from "node:readline/promises";
|
|
|
9993
10890
|
// src/commands/results/delete-run.ts
|
|
9994
10891
|
import { existsSync as existsSync9, rmSync } from "node:fs";
|
|
9995
10892
|
import path20 from "node:path";
|
|
9996
|
-
function localRunsRoot(cwd) {
|
|
9997
|
-
return path20.resolve(cwd, ".agentv", "results", "runs");
|
|
9998
|
-
}
|
|
9999
10893
|
function assertLocalRunManifest(cwd, manifestPath, runId) {
|
|
10000
10894
|
const resolvedManifestPath = path20.resolve(manifestPath);
|
|
10001
10895
|
if (path20.basename(resolvedManifestPath) !== RESULT_INDEX_FILENAME) {
|
|
10002
10896
|
throw new Error("Expected a run workspace directory or index.jsonl manifest");
|
|
10003
10897
|
}
|
|
10004
10898
|
const runDir = path20.dirname(resolvedManifestPath);
|
|
10005
|
-
|
|
10006
|
-
const relativeRunDir = path20.relative(runsRoot, runDir);
|
|
10007
|
-
if (relativeRunDir === "" || relativeRunDir.startsWith("..") || path20.isAbsolute(relativeRunDir)) {
|
|
10899
|
+
if (!relativeRunPathFromCwd(cwd, runDir)) {
|
|
10008
10900
|
throw new Error("Run workspace is outside the local results directory");
|
|
10009
10901
|
}
|
|
10010
10902
|
if (!existsSync9(resolvedManifestPath)) {
|
|
@@ -10090,14 +10982,249 @@ var resultsDeleteCommand = command({
|
|
|
10090
10982
|
});
|
|
10091
10983
|
|
|
10092
10984
|
// src/commands/results/export.ts
|
|
10985
|
+
import { readFileSync as readFileSync9 } from "node:fs";
|
|
10986
|
+
import path22 from "node:path";
|
|
10987
|
+
|
|
10988
|
+
// src/commands/results/projection-bundle.ts
|
|
10989
|
+
import { createHash } from "node:crypto";
|
|
10990
|
+
import { mkdir as mkdir11, writeFile as writeFile12 } from "node:fs/promises";
|
|
10093
10991
|
import path21 from "node:path";
|
|
10992
|
+
var PROJECTION_BUNDLE_FILENAME = "projection_bundle.json";
|
|
10993
|
+
var PROJECTION_BUNDLE_SCHEMA_VERSION = "agentv.projection_bundle.v1";
|
|
10994
|
+
function dropUndefined(value) {
|
|
10995
|
+
return Object.fromEntries(
|
|
10996
|
+
Object.entries(value).filter(([, entryValue]) => entryValue !== void 0)
|
|
10997
|
+
);
|
|
10998
|
+
}
|
|
10999
|
+
function toPortablePath(filePath, cwd) {
|
|
11000
|
+
const absolutePath = path21.resolve(filePath);
|
|
11001
|
+
const absoluteCwd = path21.resolve(cwd ?? process.cwd());
|
|
11002
|
+
const relative3 = path21.relative(absoluteCwd, absolutePath);
|
|
11003
|
+
const portable = relative3 && !relative3.startsWith("..") && !path21.isAbsolute(relative3) ? relative3 : absolutePath;
|
|
11004
|
+
return portable.split(path21.sep).join("/");
|
|
11005
|
+
}
|
|
11006
|
+
function stableDate(value) {
|
|
11007
|
+
const parsed = value ? Date.parse(value) : Number.NaN;
|
|
11008
|
+
return Number.isFinite(parsed) ? new Date(parsed) : /* @__PURE__ */ new Date(0);
|
|
11009
|
+
}
|
|
11010
|
+
function bundleCreatedAt(results) {
|
|
11011
|
+
const timestamps = results.map((result) => stableDate(result.timestamp).toISOString()).sort((a, b) => a.localeCompare(b));
|
|
11012
|
+
return timestamps[0] ?? (/* @__PURE__ */ new Date(0)).toISOString();
|
|
11013
|
+
}
|
|
11014
|
+
function shortHash(parts, length = 20) {
|
|
11015
|
+
return createHash("sha256").update(parts.join("\n")).digest("hex").slice(0, length);
|
|
11016
|
+
}
|
|
11017
|
+
function tracePathFor(indexEntry) {
|
|
11018
|
+
return indexEntry.trace_path ?? (indexEntry.artifact_dir ? path21.posix.join(indexEntry.artifact_dir, "trace.json") : void 0);
|
|
11019
|
+
}
|
|
11020
|
+
function artifactRefs(indexEntry, options) {
|
|
11021
|
+
const metadataRefs = dropUndefined({
|
|
11022
|
+
status: options.status,
|
|
11023
|
+
timing_path: indexEntry.timing_path
|
|
11024
|
+
});
|
|
11025
|
+
if (!options.includeRawContent) {
|
|
11026
|
+
return metadataRefs;
|
|
11027
|
+
}
|
|
11028
|
+
return dropUndefined({
|
|
11029
|
+
...metadataRefs,
|
|
11030
|
+
artifact_dir: indexEntry.artifact_dir,
|
|
11031
|
+
grading_path: indexEntry.grading_path,
|
|
11032
|
+
input_path: indexEntry.input_path,
|
|
11033
|
+
output_path: indexEntry.output_path,
|
|
11034
|
+
answer_path: indexEntry.answer_path,
|
|
11035
|
+
response_path: indexEntry.response_path,
|
|
11036
|
+
transcript_path: indexEntry.transcript_path,
|
|
11037
|
+
metrics_path: indexEntry.metrics_path,
|
|
11038
|
+
trace_path: tracePathFor(indexEntry),
|
|
11039
|
+
task_dir: indexEntry.task_dir,
|
|
11040
|
+
eval_path: indexEntry.eval_path,
|
|
11041
|
+
targets_path: indexEntry.targets_path,
|
|
11042
|
+
files_path: indexEntry.files_path,
|
|
11043
|
+
graders_path: indexEntry.graders_path
|
|
11044
|
+
});
|
|
11045
|
+
}
|
|
11046
|
+
function removeTranscriptMessageMetadata(envelope) {
|
|
11047
|
+
return {
|
|
11048
|
+
...envelope,
|
|
11049
|
+
trace: {
|
|
11050
|
+
...envelope.trace,
|
|
11051
|
+
spans: envelope.trace.spans.map((span) => ({
|
|
11052
|
+
...span,
|
|
11053
|
+
events: span.events?.map((event) => {
|
|
11054
|
+
const transcriptMessage = event.attributes?.["agentv.transcript.message"];
|
|
11055
|
+
if (!transcriptMessage || typeof transcriptMessage !== "object" || Array.isArray(transcriptMessage)) {
|
|
11056
|
+
return event;
|
|
11057
|
+
}
|
|
11058
|
+
const { metadata: _metadata, ...safeMessage } = transcriptMessage;
|
|
11059
|
+
return {
|
|
11060
|
+
...event,
|
|
11061
|
+
attributes: {
|
|
11062
|
+
...event.attributes,
|
|
11063
|
+
"agentv.transcript.message": safeMessage
|
|
11064
|
+
}
|
|
11065
|
+
};
|
|
11066
|
+
})
|
|
11067
|
+
}))
|
|
11068
|
+
}
|
|
11069
|
+
};
|
|
11070
|
+
}
|
|
11071
|
+
function safeEnvelope(envelope, options) {
|
|
11072
|
+
if (options.includeRawContent) {
|
|
11073
|
+
return envelope;
|
|
11074
|
+
}
|
|
11075
|
+
const withoutRawEvidence = removeTranscriptMessageMetadata({
|
|
11076
|
+
...envelope,
|
|
11077
|
+
source: {
|
|
11078
|
+
...envelope.source,
|
|
11079
|
+
metadata: void 0
|
|
11080
|
+
},
|
|
11081
|
+
artifacts: void 0,
|
|
11082
|
+
scores: envelope.scores?.map(({ evidence: _evidence, ...score }) => score)
|
|
11083
|
+
});
|
|
11084
|
+
return JSON.parse(JSON.stringify(withoutRawEvidence));
|
|
11085
|
+
}
|
|
11086
|
+
function safeScores(scores, options) {
|
|
11087
|
+
if (!scores) {
|
|
11088
|
+
return void 0;
|
|
11089
|
+
}
|
|
11090
|
+
return options.includeRawContent ? scores : scores.map(({ evidence: _evidence, ...score }) => score);
|
|
11091
|
+
}
|
|
11092
|
+
function captureOptions(includeRawContent) {
|
|
11093
|
+
return includeRawContent ? { content: "full", redactionLevel: "none", redactedFields: [] } : void 0;
|
|
11094
|
+
}
|
|
11095
|
+
function rawContent(result) {
|
|
11096
|
+
return dropUndefined({
|
|
11097
|
+
input: result.input,
|
|
11098
|
+
output: result.output,
|
|
11099
|
+
trace_messages: result.trace.messages
|
|
11100
|
+
});
|
|
11101
|
+
}
|
|
11102
|
+
function buildEntry(result, options, indexRecord) {
|
|
11103
|
+
const includeRawContent = options.includeRawContent ?? false;
|
|
11104
|
+
const sourcePath = toPortablePath(options.sourceFile, options.cwd);
|
|
11105
|
+
const plannedIndexEntry = buildResultIndexArtifact(result);
|
|
11106
|
+
const envelope = buildTraceEnvelopeFromEvaluationResult(result, {
|
|
11107
|
+
evalPath: sourcePath,
|
|
11108
|
+
runId: options.runId,
|
|
11109
|
+
source: { kind: "agentv_run", path: sourcePath, format: "agentv_result" },
|
|
11110
|
+
artifacts: {
|
|
11111
|
+
trace_path: tracePathFor(indexRecord ?? plannedIndexEntry),
|
|
11112
|
+
answer_path: result.output.length > 0 ? "outputs/answer.md" : void 0,
|
|
11113
|
+
response_path: result.output.length > 0 ? "outputs/response.md" : void 0
|
|
11114
|
+
},
|
|
11115
|
+
duplicatePolicy: options.duplicatePolicy,
|
|
11116
|
+
capture: captureOptions(includeRawContent),
|
|
11117
|
+
now: () => stableDate(result.timestamp)
|
|
11118
|
+
});
|
|
11119
|
+
const projectionIdentity = envelope.projectionIdentity;
|
|
11120
|
+
if (!projectionIdentity) {
|
|
11121
|
+
throw new Error(`Result ${result.testId ?? "unknown"} is missing projection identity`);
|
|
11122
|
+
}
|
|
11123
|
+
const indexEntry = indexRecord ?? buildResultIndexArtifact(result, void 0, {
|
|
11124
|
+
projectionIdentity,
|
|
11125
|
+
duplicatePolicy: options.duplicatePolicy
|
|
11126
|
+
});
|
|
11127
|
+
const refs = artifactRefs(indexEntry, {
|
|
11128
|
+
includeRawContent,
|
|
11129
|
+
status: options.artifactRefStatus ?? "planned_export"
|
|
11130
|
+
});
|
|
11131
|
+
const safeEnvelopeWire = safeEnvelope(toTraceEnvelopeWire(envelope), { includeRawContent });
|
|
11132
|
+
const projectionIdentityWire = indexEntry.projection_identity ?? safeEnvelopeWire.projection_identity;
|
|
11133
|
+
if (!projectionIdentityWire) {
|
|
11134
|
+
throw new Error(`Result ${result.testId ?? "unknown"} is missing projection identity`);
|
|
11135
|
+
}
|
|
11136
|
+
const envelopeWire = {
|
|
11137
|
+
...safeEnvelopeWire,
|
|
11138
|
+
projection_identity: projectionIdentityWire
|
|
11139
|
+
};
|
|
11140
|
+
const scores = safeScores(envelopeWire.scores, { includeRawContent });
|
|
11141
|
+
const feedback = dropUndefined({
|
|
11142
|
+
source: "agentv_grading_artifacts",
|
|
11143
|
+
result_score: result.score,
|
|
11144
|
+
execution_status: result.executionStatus,
|
|
11145
|
+
grading_path: refs.grading_path,
|
|
11146
|
+
timing_path: refs.timing_path,
|
|
11147
|
+
assertion_count: result.assertions?.length ?? 0,
|
|
11148
|
+
scores
|
|
11149
|
+
});
|
|
11150
|
+
return {
|
|
11151
|
+
projection_id: projectionIdentity.id,
|
|
11152
|
+
projection_identity: projectionIdentityWire,
|
|
11153
|
+
eval: envelopeWire.eval,
|
|
11154
|
+
artifact_refs: refs,
|
|
11155
|
+
trace: dropUndefined({
|
|
11156
|
+
format: envelopeWire.trace.format,
|
|
11157
|
+
trace_id: envelopeWire.trace.trace_id,
|
|
11158
|
+
root_span_id: envelopeWire.trace.root_span_id,
|
|
11159
|
+
span_count: envelopeWire.trace.spans.length,
|
|
11160
|
+
envelope_ref: refs.trace_path
|
|
11161
|
+
}),
|
|
11162
|
+
trace_envelope: envelopeWire,
|
|
11163
|
+
feedback,
|
|
11164
|
+
capture: envelopeWire.capture,
|
|
11165
|
+
...envelopeWire.conversion_warnings ? { conversion_warnings: envelopeWire.conversion_warnings } : {},
|
|
11166
|
+
...includeRawContent ? { raw_content: rawContent(result) } : {}
|
|
11167
|
+
};
|
|
11168
|
+
}
|
|
11169
|
+
function buildProjectionBundle(results, options) {
|
|
11170
|
+
if (results.length === 0) {
|
|
11171
|
+
throw new Error(`No results found in ${options.sourceFile}`);
|
|
11172
|
+
}
|
|
11173
|
+
const entries2 = results.map(
|
|
11174
|
+
(result, index) => buildEntry(result, options, options.indexRecords?.[index])
|
|
11175
|
+
);
|
|
11176
|
+
const includeRawContent = options.includeRawContent ?? false;
|
|
11177
|
+
const artifactRefStatus = options.artifactRefStatus ?? "planned_export";
|
|
11178
|
+
const conversionWarnings = entries2.flatMap((entry) => entry.conversion_warnings ?? []);
|
|
11179
|
+
const bundleId = `projection-bundle-${shortHash([
|
|
11180
|
+
PROJECTION_BUNDLE_SCHEMA_VERSION,
|
|
11181
|
+
toPortablePath(options.sourceFile, options.cwd),
|
|
11182
|
+
options.runId,
|
|
11183
|
+
artifactRefStatus,
|
|
11184
|
+
includeRawContent ? "raw" : "metadata",
|
|
11185
|
+
...entries2.map((entry) => entry.projection_id)
|
|
11186
|
+
])}`;
|
|
11187
|
+
return {
|
|
11188
|
+
schema_version: PROJECTION_BUNDLE_SCHEMA_VERSION,
|
|
11189
|
+
bundle_id: bundleId,
|
|
11190
|
+
created_at: bundleCreatedAt(results),
|
|
11191
|
+
source: {
|
|
11192
|
+
kind: "agentv_run",
|
|
11193
|
+
path: toPortablePath(options.sourceFile, options.cwd),
|
|
11194
|
+
run_id: options.runId,
|
|
11195
|
+
result_count: results.length
|
|
11196
|
+
},
|
|
11197
|
+
content_policy: {
|
|
11198
|
+
raw_content: includeRawContent ? "included" : "excluded",
|
|
11199
|
+
raw_content_opt_in: includeRawContent,
|
|
11200
|
+
default_capture: includeRawContent ? "full" : "metadata",
|
|
11201
|
+
backend_anonymizer_boundary: "adapter"
|
|
11202
|
+
},
|
|
11203
|
+
capture_summary: entries2[0]?.capture ?? {
|
|
11204
|
+
content: includeRawContent ? "full" : "metadata",
|
|
11205
|
+
redaction_level: includeRawContent ? "none" : "partial"
|
|
11206
|
+
},
|
|
11207
|
+
entries: entries2,
|
|
11208
|
+
...conversionWarnings.length > 0 ? { conversion_warnings: conversionWarnings } : {}
|
|
11209
|
+
};
|
|
11210
|
+
}
|
|
11211
|
+
function serializeProjectionBundle(bundle) {
|
|
11212
|
+
return `${JSON.stringify(bundle, null, 2)}
|
|
11213
|
+
`;
|
|
11214
|
+
}
|
|
11215
|
+
async function writeProjectionBundle(bundle, outputDir) {
|
|
11216
|
+
const bundlePath = path21.join(outputDir, PROJECTION_BUNDLE_FILENAME);
|
|
11217
|
+
await mkdir11(outputDir, { recursive: true });
|
|
11218
|
+
await writeFile12(bundlePath, serializeProjectionBundle(bundle), "utf8");
|
|
11219
|
+
return bundlePath;
|
|
11220
|
+
}
|
|
10094
11221
|
|
|
10095
11222
|
// src/commands/results/shared.ts
|
|
10096
11223
|
import { existsSync as existsSync10 } from "node:fs";
|
|
10097
11224
|
var sourceArg = positional({
|
|
10098
11225
|
type: optional(string),
|
|
10099
11226
|
displayName: "source",
|
|
10100
|
-
description: "Run workspace directory or index.jsonl manifest (defaults to most recent in .agentv/results/
|
|
11227
|
+
description: "Run workspace directory or index.jsonl manifest (defaults to most recent in .agentv/results/)"
|
|
10101
11228
|
});
|
|
10102
11229
|
async function resolveSourceFile(source, cwd) {
|
|
10103
11230
|
let sourceFile;
|
|
@@ -10116,7 +11243,7 @@ async function resolveSourceFile(source, cwd) {
|
|
|
10116
11243
|
} else {
|
|
10117
11244
|
const metas = listResultFiles(cwd, 1);
|
|
10118
11245
|
if (metas.length === 0) {
|
|
10119
|
-
console.error("Error: No run workspaces found in .agentv/results/
|
|
11246
|
+
console.error("Error: No run workspaces found in .agentv/results/");
|
|
10120
11247
|
console.error("Run an evaluation first: agentv eval <eval-file>");
|
|
10121
11248
|
process.exit(1);
|
|
10122
11249
|
}
|
|
@@ -10137,32 +11264,52 @@ async function loadResults(source, cwd) {
|
|
|
10137
11264
|
|
|
10138
11265
|
// src/commands/results/export.ts
|
|
10139
11266
|
function deriveOutputDir(cwd, sourceFile) {
|
|
10140
|
-
if (
|
|
11267
|
+
if (path22.basename(sourceFile) !== RESULT_INDEX_FILENAME) {
|
|
10141
11268
|
throw new Error(`Expected a run manifest named ${RESULT_INDEX_FILENAME}: ${sourceFile}`);
|
|
10142
11269
|
}
|
|
10143
|
-
const runDir =
|
|
10144
|
-
const segments =
|
|
10145
|
-
const
|
|
10146
|
-
if (
|
|
10147
|
-
|
|
11270
|
+
const runDir = path22.dirname(sourceFile);
|
|
11271
|
+
const segments = path22.normalize(runDir).split(path22.sep).filter(Boolean);
|
|
11272
|
+
const resultsIndex = segments.lastIndexOf("results");
|
|
11273
|
+
if (resultsIndex >= 0 && resultsIndex < segments.length - 2) {
|
|
11274
|
+
const runSegments = segments.slice(resultsIndex + 1);
|
|
11275
|
+
if (!isReservedResultsNamespace(runSegments[0])) {
|
|
11276
|
+
return path22.join(cwd, ".agentv", "results", "export", ...runSegments);
|
|
11277
|
+
}
|
|
10148
11278
|
}
|
|
10149
|
-
const parentDir =
|
|
11279
|
+
const parentDir = path22.basename(runDir);
|
|
10150
11280
|
if (parentDir.startsWith("eval_")) {
|
|
10151
|
-
return
|
|
11281
|
+
return path22.join(cwd, ".agentv", "results", "export", parentDir.slice(5));
|
|
10152
11282
|
}
|
|
10153
|
-
return
|
|
11283
|
+
return path22.join(cwd, ".agentv", "results", "export", parentDir);
|
|
10154
11284
|
}
|
|
10155
11285
|
function deriveExportRunId(sourceFile) {
|
|
10156
|
-
if (
|
|
10157
|
-
return
|
|
11286
|
+
if (path22.basename(sourceFile) === RESULT_INDEX_FILENAME) {
|
|
11287
|
+
return path22.basename(path22.dirname(sourceFile));
|
|
10158
11288
|
}
|
|
10159
|
-
return
|
|
11289
|
+
return path22.basename(sourceFile, path22.extname(sourceFile));
|
|
10160
11290
|
}
|
|
10161
11291
|
async function loadExportSource(source, cwd) {
|
|
10162
11292
|
const { sourceFile } = await resolveSourceFile(source, cwd);
|
|
10163
11293
|
const { results } = await loadResults(source, cwd);
|
|
10164
11294
|
return { sourceFile, results };
|
|
10165
11295
|
}
|
|
11296
|
+
function readIndexArtifactEntries(indexPath) {
|
|
11297
|
+
return readFileSync9(indexPath, "utf8").split(/\r?\n/).map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
11298
|
+
}
|
|
11299
|
+
function buildProjectionBundleFromExportedIndex(options) {
|
|
11300
|
+
const indexPath = path22.join(options.outputDir, RESULT_INDEX_FILENAME);
|
|
11301
|
+
const indexRecords = readIndexArtifactEntries(indexPath);
|
|
11302
|
+
const emittedResults = loadManifestResults(indexPath);
|
|
11303
|
+
return buildProjectionBundle(emittedResults, {
|
|
11304
|
+
sourceFile: options.sourceFile,
|
|
11305
|
+
runId: deriveExportRunId(options.sourceFile),
|
|
11306
|
+
cwd: options.cwd,
|
|
11307
|
+
duplicatePolicy: options.duplicatePolicy,
|
|
11308
|
+
includeRawContent: options.includeRawContent,
|
|
11309
|
+
artifactRefStatus: "emitted",
|
|
11310
|
+
indexRecords
|
|
11311
|
+
});
|
|
11312
|
+
}
|
|
10166
11313
|
var resultsExportCommand = command({
|
|
10167
11314
|
name: "export",
|
|
10168
11315
|
description: "Export a run workspace or index.jsonl manifest into a per-test directory structure",
|
|
@@ -10170,7 +11317,7 @@ var resultsExportCommand = command({
|
|
|
10170
11317
|
source: positional({
|
|
10171
11318
|
type: optional(string),
|
|
10172
11319
|
displayName: "source",
|
|
10173
|
-
description: "Run workspace directory or index.jsonl manifest to export (defaults to most recent in .agentv/results/
|
|
11320
|
+
description: "Run workspace directory or index.jsonl manifest to export (defaults to most recent in .agentv/results/)"
|
|
10174
11321
|
}),
|
|
10175
11322
|
out: option({
|
|
10176
11323
|
type: optional(string),
|
|
@@ -10188,20 +11335,67 @@ var resultsExportCommand = command({
|
|
|
10188
11335
|
type: optional(oneOf(["skip", "update", "error"])),
|
|
10189
11336
|
long: "duplicate-policy",
|
|
10190
11337
|
description: "How to handle duplicate projection identities in the output: update (default), skip, or error"
|
|
11338
|
+
}),
|
|
11339
|
+
projectionBundle: flag({
|
|
11340
|
+
long: "projection-bundle",
|
|
11341
|
+
description: "Write a local vendor-neutral projection_bundle.json for non-Phoenix adapters; no service calls"
|
|
11342
|
+
}),
|
|
11343
|
+
dryRun: flag({
|
|
11344
|
+
long: "dry-run",
|
|
11345
|
+
description: "Print deterministic projection bundle JSON without writing export artifacts"
|
|
11346
|
+
}),
|
|
11347
|
+
includeRawContent: flag({
|
|
11348
|
+
long: "include-raw-content",
|
|
11349
|
+
description: "Include raw prompt, output, and tool payload content in the projection bundle (off by default)"
|
|
10191
11350
|
})
|
|
10192
11351
|
},
|
|
10193
|
-
handler: async ({
|
|
11352
|
+
handler: async ({
|
|
11353
|
+
source,
|
|
11354
|
+
out,
|
|
11355
|
+
dir,
|
|
11356
|
+
duplicatePolicy,
|
|
11357
|
+
projectionBundle,
|
|
11358
|
+
dryRun: dryRun2,
|
|
11359
|
+
includeRawContent
|
|
11360
|
+
}) => {
|
|
10194
11361
|
const cwd = dir ?? process.cwd();
|
|
10195
11362
|
const policy = duplicatePolicy ?? "update";
|
|
11363
|
+
const shouldWriteProjectionBundle = projectionBundle;
|
|
11364
|
+
const shouldDryRun = dryRun2;
|
|
11365
|
+
const shouldIncludeRawContent = includeRawContent;
|
|
10196
11366
|
try {
|
|
10197
11367
|
const { sourceFile, results } = await loadExportSource(source, cwd);
|
|
10198
|
-
const outputDir = out ?
|
|
11368
|
+
const outputDir = out ? path22.isAbsolute(out) ? out : path22.resolve(cwd, out) : deriveOutputDir(cwd, sourceFile);
|
|
11369
|
+
const buildBundle = () => buildProjectionBundle(results, {
|
|
11370
|
+
sourceFile,
|
|
11371
|
+
runId: deriveExportRunId(sourceFile),
|
|
11372
|
+
cwd,
|
|
11373
|
+
duplicatePolicy: policy,
|
|
11374
|
+
includeRawContent: shouldIncludeRawContent
|
|
11375
|
+
});
|
|
11376
|
+
if (shouldDryRun) {
|
|
11377
|
+
process.stdout.write(serializeProjectionBundle(buildBundle()));
|
|
11378
|
+
return;
|
|
11379
|
+
}
|
|
10199
11380
|
await writeArtifactsFromResults2(results, outputDir, {
|
|
10200
11381
|
evalFile: sourceFile,
|
|
10201
11382
|
runId: deriveExportRunId(sourceFile),
|
|
10202
11383
|
duplicatePolicy: policy
|
|
10203
11384
|
});
|
|
11385
|
+
const bundlePath = shouldWriteProjectionBundle ? await writeProjectionBundle(
|
|
11386
|
+
buildProjectionBundleFromExportedIndex({
|
|
11387
|
+
sourceFile,
|
|
11388
|
+
outputDir,
|
|
11389
|
+
cwd,
|
|
11390
|
+
duplicatePolicy: policy,
|
|
11391
|
+
includeRawContent: shouldIncludeRawContent
|
|
11392
|
+
}),
|
|
11393
|
+
outputDir
|
|
11394
|
+
) : void 0;
|
|
10204
11395
|
console.log(`Exported ${results.length} test(s) to ${outputDir}`);
|
|
11396
|
+
if (bundlePath) {
|
|
11397
|
+
console.log(`Projection bundle written to ${bundlePath}`);
|
|
11398
|
+
}
|
|
10205
11399
|
for (const result of results) {
|
|
10206
11400
|
console.log(` ${result.testId ?? "unknown"}`);
|
|
10207
11401
|
}
|
|
@@ -10257,8 +11451,8 @@ var resultsFailuresCommand = command({
|
|
|
10257
11451
|
});
|
|
10258
11452
|
|
|
10259
11453
|
// src/commands/results/report.ts
|
|
10260
|
-
import { existsSync as existsSync11, mkdirSync as mkdirSync3, readFileSync as
|
|
10261
|
-
import
|
|
11454
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync3, readFileSync as readFileSync10, writeFileSync as writeFileSync4 } from "node:fs";
|
|
11455
|
+
import path23 from "node:path";
|
|
10262
11456
|
|
|
10263
11457
|
// src/commands/results/report-template.ts
|
|
10264
11458
|
var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
@@ -10281,6 +11475,7 @@ var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
|
10281
11475
|
--text-muted: #9ca3af;
|
|
10282
11476
|
--text-subtle: #6b7280;
|
|
10283
11477
|
--heading: #ffffff;
|
|
11478
|
+
--brand-middle: #d1d5db;
|
|
10284
11479
|
--accent: #22d3ee;
|
|
10285
11480
|
--accent-weak: rgba(8, 145, 178, 0.2);
|
|
10286
11481
|
--pass: #34d399;
|
|
@@ -10304,6 +11499,7 @@ var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
|
10304
11499
|
--text-muted: #64748b;
|
|
10305
11500
|
--text-subtle: #94a3b8;
|
|
10306
11501
|
--heading: #0f172a;
|
|
11502
|
+
--brand-middle: #64748b;
|
|
10307
11503
|
--accent: #0891b2;
|
|
10308
11504
|
--accent-weak: rgba(8, 145, 178, 0.1);
|
|
10309
11505
|
--pass: #16a34a;
|
|
@@ -10401,6 +11597,18 @@ var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
|
10401
11597
|
margin: 0;
|
|
10402
11598
|
}
|
|
10403
11599
|
|
|
11600
|
+
.brand-wordmark {
|
|
11601
|
+
white-space: nowrap;
|
|
11602
|
+
}
|
|
11603
|
+
|
|
11604
|
+
.brand-letter {
|
|
11605
|
+
color: var(--accent);
|
|
11606
|
+
}
|
|
11607
|
+
|
|
11608
|
+
.brand-middle {
|
|
11609
|
+
color: var(--brand-middle);
|
|
11610
|
+
}
|
|
11611
|
+
|
|
10404
11612
|
.header-subtitle {
|
|
10405
11613
|
color: var(--text-muted);
|
|
10406
11614
|
margin: 0;
|
|
@@ -11034,7 +12242,7 @@ var RESULTS_REPORT_TEMPLATE = `<!DOCTYPE html>
|
|
|
11034
12242
|
<div class="shell">
|
|
11035
12243
|
<header class="header">
|
|
11036
12244
|
<div class="header-title-wrap">
|
|
11037
|
-
<div class="eyebrow"
|
|
12245
|
+
<div class="eyebrow"><span class="brand-wordmark" aria-label="AgentV"><span class="brand-letter" aria-hidden="true">A</span><span class="brand-middle" aria-hidden="true">GENT</span><span class="brand-letter" aria-hidden="true">V</span></span> static export</div>
|
|
11038
12246
|
<h1 class="header-title">Evaluation Report</h1>
|
|
11039
12247
|
<p class="header-subtitle">Dashboard-themed HTML generated from an existing AgentV results workspace.</p>
|
|
11040
12248
|
</div>
|
|
@@ -11882,25 +13090,25 @@ function normalizeEvalFileLabel(value) {
|
|
|
11882
13090
|
if (!trimmed) {
|
|
11883
13091
|
return void 0;
|
|
11884
13092
|
}
|
|
11885
|
-
return
|
|
13093
|
+
return path23.basename(trimmed).replace(/\.results\.jsonl$/i, "").replace(/\.eval\.ya?ml$/i, "").replace(/\.ya?ml$/i, "").replace(/\.jsonl$/i, "");
|
|
11886
13094
|
}
|
|
11887
13095
|
function readBenchmarkEvalFile(sourceFile) {
|
|
11888
|
-
const benchmarkPath =
|
|
13096
|
+
const benchmarkPath = path23.join(path23.dirname(sourceFile), "benchmark.json");
|
|
11889
13097
|
if (!existsSync11(benchmarkPath)) {
|
|
11890
13098
|
return void 0;
|
|
11891
13099
|
}
|
|
11892
13100
|
try {
|
|
11893
|
-
const benchmark = JSON.parse(
|
|
13101
|
+
const benchmark = JSON.parse(readFileSync10(benchmarkPath, "utf8"));
|
|
11894
13102
|
return normalizeEvalFileLabel(benchmark.metadata?.eval_file);
|
|
11895
13103
|
} catch {
|
|
11896
13104
|
return void 0;
|
|
11897
13105
|
}
|
|
11898
13106
|
}
|
|
11899
13107
|
function deriveReportPath(sourceFile) {
|
|
11900
|
-
return
|
|
13108
|
+
return path23.join(path23.dirname(sourceFile), "report.html");
|
|
11901
13109
|
}
|
|
11902
13110
|
function serializeReportResult(result, sourceFile, manifestRecord, benchmarkEvalFile) {
|
|
11903
|
-
const fallbackEvalFile = normalizeEvalFileLabel(manifestRecord?.eval_file) ?? benchmarkEvalFile ?? normalizeEvalFileLabel(result.suite) ??
|
|
13111
|
+
const fallbackEvalFile = normalizeEvalFileLabel(manifestRecord?.eval_file) ?? benchmarkEvalFile ?? normalizeEvalFileLabel(result.suite) ?? path23.basename(path23.dirname(sourceFile));
|
|
11904
13112
|
return {
|
|
11905
13113
|
timestamp: result.timestamp,
|
|
11906
13114
|
test_id: result.testId,
|
|
@@ -11923,7 +13131,7 @@ function serializeReportResult(result, sourceFile, manifestRecord, benchmarkEval
|
|
|
11923
13131
|
async function loadReportSource(source, cwd) {
|
|
11924
13132
|
const { sourceFile } = await resolveSourceFile(source, cwd);
|
|
11925
13133
|
const resolvedSourceFile = resolveResultSourcePath(sourceFile, cwd);
|
|
11926
|
-
const content =
|
|
13134
|
+
const content = readFileSync10(resolvedSourceFile, "utf8");
|
|
11927
13135
|
const records = parseResultManifest(content);
|
|
11928
13136
|
const results = loadManifestResults(resolvedSourceFile);
|
|
11929
13137
|
if (results.length === 0) {
|
|
@@ -11948,11 +13156,11 @@ function renderResultsReport(results, sourceFile, records, benchmarkEvalFile) {
|
|
|
11948
13156
|
}
|
|
11949
13157
|
async function writeResultsReport(source, outputPath, cwd) {
|
|
11950
13158
|
const { sourceFile, results, records, benchmarkEvalFile } = await loadReportSource(source, cwd);
|
|
11951
|
-
const resolvedOutputPath = outputPath ?
|
|
13159
|
+
const resolvedOutputPath = outputPath ? path23.isAbsolute(outputPath) ? outputPath : path23.resolve(cwd, outputPath) : deriveReportPath(sourceFile);
|
|
11952
13160
|
const html = renderResultsReport(results, sourceFile, records, benchmarkEvalFile);
|
|
11953
|
-
mkdirSync3(
|
|
13161
|
+
mkdirSync3(path23.dirname(resolvedOutputPath), { recursive: true });
|
|
11954
13162
|
writeFileSync4(resolvedOutputPath, html, "utf8");
|
|
11955
|
-
const written =
|
|
13163
|
+
const written = readFileSync10(resolvedOutputPath, "utf8");
|
|
11956
13164
|
if (written.includes("__DATA_PLACEHOLDER__")) {
|
|
11957
13165
|
throw new Error("Report placeholder substitution failed");
|
|
11958
13166
|
}
|
|
@@ -12071,7 +13279,7 @@ var resultsShowCommand = command({
|
|
|
12071
13279
|
});
|
|
12072
13280
|
|
|
12073
13281
|
// src/commands/results/summary.ts
|
|
12074
|
-
import { existsSync as existsSync12, readFileSync as
|
|
13282
|
+
import { existsSync as existsSync12, readFileSync as readFileSync11 } from "node:fs";
|
|
12075
13283
|
function formatSummary(results, grading) {
|
|
12076
13284
|
const total = results.length;
|
|
12077
13285
|
let passed;
|
|
@@ -12124,7 +13332,7 @@ var resultsSummaryCommand = command({
|
|
|
12124
13332
|
const gradingPath = sourceFile.replace(/\.jsonl$/, ".grading.json");
|
|
12125
13333
|
if (existsSync12(gradingPath)) {
|
|
12126
13334
|
try {
|
|
12127
|
-
grading = JSON.parse(
|
|
13335
|
+
grading = JSON.parse(readFileSync11(gradingPath, "utf8"));
|
|
12128
13336
|
} catch {
|
|
12129
13337
|
}
|
|
12130
13338
|
}
|
|
@@ -12137,22 +13345,26 @@ var resultsSummaryCommand = command({
|
|
|
12137
13345
|
});
|
|
12138
13346
|
|
|
12139
13347
|
// src/commands/results/validate.ts
|
|
12140
|
-
import { existsSync as existsSync13, readFileSync as
|
|
12141
|
-
import
|
|
13348
|
+
import { existsSync as existsSync13, readFileSync as readFileSync12, statSync as statSync5 } from "node:fs";
|
|
13349
|
+
import path24 from "node:path";
|
|
12142
13350
|
function checkDirectoryNaming(runDir) {
|
|
12143
|
-
const dirName =
|
|
12144
|
-
const pathSegments =
|
|
12145
|
-
const
|
|
13351
|
+
const dirName = path24.basename(runDir);
|
|
13352
|
+
const pathSegments = path24.normalize(runDir).split(path24.sep).filter(Boolean);
|
|
13353
|
+
const resultsIndex = pathSegments.lastIndexOf("results");
|
|
12146
13354
|
const diagnostics = [];
|
|
12147
|
-
if (
|
|
13355
|
+
if (resultsIndex >= 0 && pathSegments[resultsIndex + 1] === "runs") {
|
|
13356
|
+
diagnostics.push({
|
|
13357
|
+
severity: "error",
|
|
13358
|
+
message: "The .agentv/results/runs namespace is no longer supported. Expected: .agentv/results/<experiment>/<timestamp>"
|
|
13359
|
+
});
|
|
13360
|
+
} else if (resultsIndex < 0 || resultsIndex > pathSegments.length - 3) {
|
|
12148
13361
|
diagnostics.push({
|
|
12149
13362
|
severity: "warning",
|
|
12150
|
-
message: "Directory is not under
|
|
13363
|
+
message: "Directory is not under the canonical results tree. Expected: .agentv/results/<experiment>/<timestamp>"
|
|
12151
13364
|
});
|
|
12152
13365
|
}
|
|
12153
13366
|
const isNewFormat = /^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z$/.test(dirName);
|
|
12154
|
-
|
|
12155
|
-
if (!isNewFormat && !isLegacyFormat) {
|
|
13367
|
+
if (!isNewFormat) {
|
|
12156
13368
|
diagnostics.push({
|
|
12157
13369
|
severity: "warning",
|
|
12158
13370
|
message: `Directory name '${dirName}' does not match the expected pattern '<ISO-timestamp>'. Example: 2026-03-27T12-42-24-429Z`
|
|
@@ -12171,14 +13383,14 @@ function validateRunDirectory(runDir) {
|
|
|
12171
13383
|
return { diagnostics, entries: entries2 };
|
|
12172
13384
|
}
|
|
12173
13385
|
function checkIndexJsonl(runDir) {
|
|
12174
|
-
const indexPath =
|
|
13386
|
+
const indexPath = path24.join(runDir, "index.jsonl");
|
|
12175
13387
|
const diagnostics = [];
|
|
12176
13388
|
const entries2 = [];
|
|
12177
13389
|
if (!existsSync13(indexPath)) {
|
|
12178
13390
|
diagnostics.push({ severity: "error", message: "index.jsonl is missing" });
|
|
12179
13391
|
return { diagnostics, entries: entries2 };
|
|
12180
13392
|
}
|
|
12181
|
-
const content =
|
|
13393
|
+
const content = readFileSync12(indexPath, "utf8");
|
|
12182
13394
|
const lines = content.split("\n").filter((l) => l.trim().length > 0);
|
|
12183
13395
|
if (lines.length === 0) {
|
|
12184
13396
|
diagnostics.push({ severity: "error", message: "index.jsonl is empty" });
|
|
@@ -12270,7 +13482,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
12270
13482
|
for (const entry of entries2) {
|
|
12271
13483
|
const testId = entry.test_id ?? "?";
|
|
12272
13484
|
if (entry.grading_path) {
|
|
12273
|
-
const gradingPath =
|
|
13485
|
+
const gradingPath = path24.join(runDir, entry.grading_path);
|
|
12274
13486
|
if (!existsSync13(gradingPath)) {
|
|
12275
13487
|
diagnostics.push({
|
|
12276
13488
|
severity: "error",
|
|
@@ -12278,7 +13490,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
12278
13490
|
});
|
|
12279
13491
|
} else {
|
|
12280
13492
|
try {
|
|
12281
|
-
const grading = JSON.parse(
|
|
13493
|
+
const grading = JSON.parse(readFileSync12(gradingPath, "utf8"));
|
|
12282
13494
|
if (!grading.assertions || !Array.isArray(grading.assertions)) {
|
|
12283
13495
|
diagnostics.push({
|
|
12284
13496
|
severity: "error",
|
|
@@ -12300,7 +13512,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
12300
13512
|
}
|
|
12301
13513
|
}
|
|
12302
13514
|
if (entry.timing_path) {
|
|
12303
|
-
const timingPath =
|
|
13515
|
+
const timingPath = path24.join(runDir, entry.timing_path);
|
|
12304
13516
|
if (!existsSync13(timingPath)) {
|
|
12305
13517
|
diagnostics.push({
|
|
12306
13518
|
severity: "warning",
|
|
@@ -12309,7 +13521,7 @@ function checkArtifactFiles(runDir, entries2) {
|
|
|
12309
13521
|
}
|
|
12310
13522
|
}
|
|
12311
13523
|
}
|
|
12312
|
-
const benchmarkPath =
|
|
13524
|
+
const benchmarkPath = path24.join(runDir, "benchmark.json");
|
|
12313
13525
|
if (!existsSync13(benchmarkPath)) {
|
|
12314
13526
|
diagnostics.push({ severity: "warning", message: "benchmark.json is missing" });
|
|
12315
13527
|
}
|
|
@@ -12326,7 +13538,7 @@ var resultsValidateCommand = command({
|
|
|
12326
13538
|
})
|
|
12327
13539
|
},
|
|
12328
13540
|
handler: async ({ runDir }) => {
|
|
12329
|
-
const resolvedDir =
|
|
13541
|
+
const resolvedDir = path24.resolve(runDir);
|
|
12330
13542
|
if (!existsSync13(resolvedDir) || !statSync5(resolvedDir).isDirectory()) {
|
|
12331
13543
|
console.error(`Error: '${runDir}' is not a directory`);
|
|
12332
13544
|
process.exit(1);
|
|
@@ -12371,16 +13583,23 @@ var resultsCommand = subcommands({
|
|
|
12371
13583
|
});
|
|
12372
13584
|
|
|
12373
13585
|
// src/commands/results/serve.ts
|
|
12374
|
-
import {
|
|
13586
|
+
import {
|
|
13587
|
+
existsSync as existsSync16,
|
|
13588
|
+
readFileSync as readFileSync14,
|
|
13589
|
+
readdirSync as readdirSync4,
|
|
13590
|
+
realpathSync,
|
|
13591
|
+
statSync as statSync6,
|
|
13592
|
+
writeFileSync as writeFileSync6
|
|
13593
|
+
} from "node:fs";
|
|
12375
13594
|
import { homedir } from "node:os";
|
|
12376
|
-
import
|
|
13595
|
+
import path27 from "node:path";
|
|
12377
13596
|
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
12378
13597
|
import { Hono } from "hono";
|
|
12379
13598
|
|
|
12380
13599
|
// src/commands/results/eval-runner.ts
|
|
12381
|
-
import { execFileSync as execFileSync2, spawn } from "node:child_process";
|
|
13600
|
+
import { execFileSync as execFileSync2, spawn as spawn2 } from "node:child_process";
|
|
12382
13601
|
import { createWriteStream, existsSync as existsSync14, mkdirSync as mkdirSync4 } from "node:fs";
|
|
12383
|
-
import
|
|
13602
|
+
import path25 from "node:path";
|
|
12384
13603
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
12385
13604
|
var activeRuns = /* @__PURE__ */ new Map();
|
|
12386
13605
|
function generateRunId() {
|
|
@@ -12400,7 +13619,7 @@ function pruneFinishedRuns() {
|
|
|
12400
13619
|
}
|
|
12401
13620
|
function getActiveRunTarget(indexJsonlPath) {
|
|
12402
13621
|
for (const run2 of activeRuns.values()) {
|
|
12403
|
-
if (run2.outputDir &&
|
|
13622
|
+
if (run2.outputDir && path25.join(run2.outputDir, "index.jsonl") === indexJsonlPath) {
|
|
12404
13623
|
return run2.target;
|
|
12405
13624
|
}
|
|
12406
13625
|
}
|
|
@@ -12408,7 +13627,7 @@ function getActiveRunTarget(indexJsonlPath) {
|
|
|
12408
13627
|
}
|
|
12409
13628
|
function getActiveRunStatus(indexJsonlPath) {
|
|
12410
13629
|
for (const run2 of activeRuns.values()) {
|
|
12411
|
-
if (run2.outputDir &&
|
|
13630
|
+
if (run2.outputDir && path25.join(run2.outputDir, "index.jsonl") === indexJsonlPath) {
|
|
12412
13631
|
return run2.status;
|
|
12413
13632
|
}
|
|
12414
13633
|
}
|
|
@@ -12418,7 +13637,7 @@ async function discoverTargetsInProject(cwd) {
|
|
|
12418
13637
|
const repoRoot = await findRepoRoot(cwd) ?? cwd;
|
|
12419
13638
|
let targetsFilePath;
|
|
12420
13639
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
12421
|
-
const fullPath =
|
|
13640
|
+
const fullPath = path25.join(cwd, candidate);
|
|
12422
13641
|
if (existsSync14(fullPath)) {
|
|
12423
13642
|
targetsFilePath = fullPath;
|
|
12424
13643
|
break;
|
|
@@ -12426,7 +13645,7 @@ async function discoverTargetsInProject(cwd) {
|
|
|
12426
13645
|
}
|
|
12427
13646
|
if (!targetsFilePath) {
|
|
12428
13647
|
for (const candidate of TARGET_FILE_CANDIDATES) {
|
|
12429
|
-
const fullPath =
|
|
13648
|
+
const fullPath = path25.join(repoRoot, candidate);
|
|
12430
13649
|
if (existsSync14(fullPath)) {
|
|
12431
13650
|
targetsFilePath = fullPath;
|
|
12432
13651
|
break;
|
|
@@ -12453,7 +13672,25 @@ function validateResumeOptions(req) {
|
|
|
12453
13672
|
}
|
|
12454
13673
|
return void 0;
|
|
12455
13674
|
}
|
|
12456
|
-
function
|
|
13675
|
+
function parseInitialTags(value) {
|
|
13676
|
+
if (value === void 0) return [];
|
|
13677
|
+
if (!Array.isArray(value)) {
|
|
13678
|
+
throw new Error("tags must be an array of strings");
|
|
13679
|
+
}
|
|
13680
|
+
return normalizeTags(value);
|
|
13681
|
+
}
|
|
13682
|
+
function normalizeRunMetadata(req) {
|
|
13683
|
+
const experiment = normalizeExperimentName(req.experiment);
|
|
13684
|
+
const tags = parseInitialTags(req.tags);
|
|
13685
|
+
if ((req.resume || req.rerun_failed) && req.experiment?.trim()) {
|
|
13686
|
+
throw new Error("experiment cannot be changed when resuming an existing run");
|
|
13687
|
+
}
|
|
13688
|
+
if ((req.resume || req.rerun_failed) && tags.length > 0) {
|
|
13689
|
+
throw new Error("initial tags can only be set when creating a new run");
|
|
13690
|
+
}
|
|
13691
|
+
return { experiment, tags };
|
|
13692
|
+
}
|
|
13693
|
+
function buildCliArgs(req, experiment) {
|
|
12457
13694
|
const args = ["eval"];
|
|
12458
13695
|
if (req.suite_filter?.trim()) {
|
|
12459
13696
|
for (const part of req.suite_filter.split(",")) {
|
|
@@ -12472,6 +13709,9 @@ function buildCliArgs(req) {
|
|
|
12472
13709
|
if (req.target?.trim()) {
|
|
12473
13710
|
args.push("--target", req.target.trim());
|
|
12474
13711
|
}
|
|
13712
|
+
if (experiment && req.experiment?.trim()) {
|
|
13713
|
+
args.push("--experiment", experiment);
|
|
13714
|
+
}
|
|
12475
13715
|
if (req.threshold !== void 0 && req.threshold !== null) {
|
|
12476
13716
|
args.push("--threshold", String(req.threshold));
|
|
12477
13717
|
}
|
|
@@ -12500,17 +13740,17 @@ function buildCliPreview(args) {
|
|
|
12500
13740
|
}
|
|
12501
13741
|
function resolveCliPath(cwd) {
|
|
12502
13742
|
const candidates = [
|
|
12503
|
-
|
|
12504
|
-
|
|
13743
|
+
path25.join(cwd, "apps/cli/src/cli.ts"),
|
|
13744
|
+
path25.join(cwd, "apps/cli/dist/cli.js")
|
|
12505
13745
|
];
|
|
12506
13746
|
for (const c4 of candidates) {
|
|
12507
13747
|
if (existsSync14(c4)) {
|
|
12508
13748
|
return { binPath: "bun", args: [c4] };
|
|
12509
13749
|
}
|
|
12510
13750
|
}
|
|
12511
|
-
const currentDir = typeof __dirname !== "undefined" ? __dirname :
|
|
12512
|
-
const fromSrc =
|
|
12513
|
-
const fromDist =
|
|
13751
|
+
const currentDir = typeof __dirname !== "undefined" ? __dirname : path25.dirname(fileURLToPath2(import.meta.url));
|
|
13752
|
+
const fromSrc = path25.resolve(currentDir, "../../cli.ts");
|
|
13753
|
+
const fromDist = path25.resolve(currentDir, "cli.js");
|
|
12514
13754
|
if (existsSync14(fromSrc)) return { binPath: "bun", args: [fromSrc] };
|
|
12515
13755
|
if (existsSync14(fromDist)) return { binPath: "bun", args: [fromDist] };
|
|
12516
13756
|
if (isCommandAvailable("agentv")) {
|
|
@@ -12529,7 +13769,7 @@ function isCommandAvailable(cmd) {
|
|
|
12529
13769
|
function openConsoleLogStream(outputDir) {
|
|
12530
13770
|
try {
|
|
12531
13771
|
mkdirSync4(outputDir, { recursive: true });
|
|
12532
|
-
const stream = createWriteStream(
|
|
13772
|
+
const stream = createWriteStream(path25.join(outputDir, "console.log"), { flags: "w" });
|
|
12533
13773
|
stream.on("error", () => {
|
|
12534
13774
|
});
|
|
12535
13775
|
return stream;
|
|
@@ -12537,6 +13777,11 @@ function openConsoleLogStream(outputDir) {
|
|
|
12537
13777
|
return void 0;
|
|
12538
13778
|
}
|
|
12539
13779
|
}
|
|
13780
|
+
function writeInitialRunTags(outputDir, tags) {
|
|
13781
|
+
if (tags.length === 0) return;
|
|
13782
|
+
mkdirSync4(outputDir, { recursive: true });
|
|
13783
|
+
writeRunTags(path25.join(outputDir, "index.jsonl"), tags);
|
|
13784
|
+
}
|
|
12540
13785
|
function registerEvalRoutes(app2, getCwd, options) {
|
|
12541
13786
|
const readOnly = options?.readOnly === true;
|
|
12542
13787
|
app2.get("/api/eval/discover", async (c4) => {
|
|
@@ -12581,12 +13826,18 @@ function registerEvalRoutes(app2, getCwd, options) {
|
|
|
12581
13826
|
if (resumeError) {
|
|
12582
13827
|
return c4.json({ error: resumeError }, 400);
|
|
12583
13828
|
}
|
|
13829
|
+
let metadata;
|
|
13830
|
+
try {
|
|
13831
|
+
metadata = normalizeRunMetadata(body);
|
|
13832
|
+
} catch (err2) {
|
|
13833
|
+
return c4.json({ error: err2.message }, 400);
|
|
13834
|
+
}
|
|
12584
13835
|
const cliPaths = resolveCliPath(cwd);
|
|
12585
13836
|
if (!cliPaths) {
|
|
12586
13837
|
return c4.json({ error: "Cannot locate agentv CLI entry point" }, 500);
|
|
12587
13838
|
}
|
|
12588
|
-
const args = buildCliArgs(body);
|
|
12589
|
-
const outputDir = body.output?.trim() ?
|
|
13839
|
+
const args = buildCliArgs(body, metadata.experiment);
|
|
13840
|
+
const outputDir = body.output?.trim() ? path25.resolve(cwd, body.output.trim()) : buildDefaultRunDir(cwd, metadata.experiment);
|
|
12590
13841
|
if (!body.output?.trim()) {
|
|
12591
13842
|
args.push("--output", outputDir);
|
|
12592
13843
|
}
|
|
@@ -12604,7 +13855,8 @@ function registerEvalRoutes(app2, getCwd, options) {
|
|
|
12604
13855
|
};
|
|
12605
13856
|
activeRuns.set(runId, run2);
|
|
12606
13857
|
try {
|
|
12607
|
-
|
|
13858
|
+
writeInitialRunTags(outputDir, metadata.tags);
|
|
13859
|
+
const child = spawn2(cliPaths.binPath, [...cliPaths.args, ...args], {
|
|
12608
13860
|
cwd,
|
|
12609
13861
|
stdio: ["ignore", "pipe", "pipe"],
|
|
12610
13862
|
env: { ...process.env },
|
|
@@ -12714,7 +13966,13 @@ Process error: ${err2.message}
|
|
|
12714
13966
|
} catch {
|
|
12715
13967
|
return c4.json({ error: "Invalid JSON body" }, 400);
|
|
12716
13968
|
}
|
|
12717
|
-
|
|
13969
|
+
let metadata;
|
|
13970
|
+
try {
|
|
13971
|
+
metadata = normalizeRunMetadata(body);
|
|
13972
|
+
} catch (err2) {
|
|
13973
|
+
return c4.json({ error: err2.message }, 400);
|
|
13974
|
+
}
|
|
13975
|
+
const args = buildCliArgs(body, metadata.experiment);
|
|
12718
13976
|
return c4.json({ command: buildCliPreview(args) });
|
|
12719
13977
|
});
|
|
12720
13978
|
app2.get("/api/projects/:projectId/eval/discover", async (c4) => {
|
|
@@ -12759,12 +14017,18 @@ Process error: ${err2.message}
|
|
|
12759
14017
|
if (resumeError) {
|
|
12760
14018
|
return c4.json({ error: resumeError }, 400);
|
|
12761
14019
|
}
|
|
14020
|
+
let metadata;
|
|
14021
|
+
try {
|
|
14022
|
+
metadata = normalizeRunMetadata(body);
|
|
14023
|
+
} catch (err2) {
|
|
14024
|
+
return c4.json({ error: err2.message }, 400);
|
|
14025
|
+
}
|
|
12762
14026
|
const cliPaths = resolveCliPath(cwd);
|
|
12763
14027
|
if (!cliPaths) {
|
|
12764
14028
|
return c4.json({ error: "Cannot locate agentv CLI entry point" }, 500);
|
|
12765
14029
|
}
|
|
12766
|
-
const args = buildCliArgs(body);
|
|
12767
|
-
const outputDir = body.output?.trim() ?
|
|
14030
|
+
const args = buildCliArgs(body, metadata.experiment);
|
|
14031
|
+
const outputDir = body.output?.trim() ? path25.resolve(cwd, body.output.trim()) : buildDefaultRunDir(cwd, metadata.experiment);
|
|
12768
14032
|
if (!body.output?.trim()) {
|
|
12769
14033
|
args.push("--output", outputDir);
|
|
12770
14034
|
}
|
|
@@ -12782,7 +14046,8 @@ Process error: ${err2.message}
|
|
|
12782
14046
|
};
|
|
12783
14047
|
activeRuns.set(runId, run2);
|
|
12784
14048
|
try {
|
|
12785
|
-
|
|
14049
|
+
writeInitialRunTags(outputDir, metadata.tags);
|
|
14050
|
+
const child = spawn2(cliPaths.binPath, [...cliPaths.args, ...args], {
|
|
12786
14051
|
cwd,
|
|
12787
14052
|
stdio: ["ignore", "pipe", "pipe"],
|
|
12788
14053
|
env: { ...process.env },
|
|
@@ -12880,24 +14145,30 @@ Process error: ${err2.message}
|
|
|
12880
14145
|
} catch {
|
|
12881
14146
|
return c4.json({ error: "Invalid JSON body" }, 400);
|
|
12882
14147
|
}
|
|
12883
|
-
|
|
14148
|
+
let metadata;
|
|
14149
|
+
try {
|
|
14150
|
+
metadata = normalizeRunMetadata(body);
|
|
14151
|
+
} catch (err2) {
|
|
14152
|
+
return c4.json({ error: err2.message }, 400);
|
|
14153
|
+
}
|
|
14154
|
+
const args = buildCliArgs(body, metadata.experiment);
|
|
12884
14155
|
return c4.json({ command: buildCliPreview(args) });
|
|
12885
14156
|
});
|
|
12886
14157
|
}
|
|
12887
14158
|
|
|
12888
14159
|
// src/commands/results/studio-config.ts
|
|
12889
|
-
import { existsSync as existsSync15, mkdirSync as mkdirSync5, readFileSync as
|
|
12890
|
-
import
|
|
12891
|
-
import { stringify as
|
|
14160
|
+
import { existsSync as existsSync15, mkdirSync as mkdirSync5, readFileSync as readFileSync13, writeFileSync as writeFileSync5 } from "node:fs";
|
|
14161
|
+
import path26 from "node:path";
|
|
14162
|
+
import { stringify as stringifyYaml4 } from "yaml";
|
|
12892
14163
|
var DEFAULTS = {
|
|
12893
14164
|
threshold: DEFAULT_THRESHOLD,
|
|
12894
14165
|
appName: "agentv"
|
|
12895
14166
|
};
|
|
12896
14167
|
function loadStudioConfig(agentvDir) {
|
|
12897
|
-
const localConfigPath =
|
|
12898
|
-
const globalConfigPath =
|
|
14168
|
+
const localConfigPath = path26.join(agentvDir, "config.yaml");
|
|
14169
|
+
const globalConfigPath = path26.join(getAgentvConfigDir(), "config.yaml");
|
|
12899
14170
|
const localConfig = loadParsedConfig(localConfigPath);
|
|
12900
|
-
const globalConfig =
|
|
14171
|
+
const globalConfig = path26.resolve(globalConfigPath) === path26.resolve(localConfigPath) ? void 0 : loadParsedConfig(globalConfigPath);
|
|
12901
14172
|
const threshold = [
|
|
12902
14173
|
readThreshold(localConfig?.dashboard),
|
|
12903
14174
|
readThreshold(localConfig?.studio),
|
|
@@ -12914,7 +14185,7 @@ function loadStudioConfig(agentvDir) {
|
|
|
12914
14185
|
}
|
|
12915
14186
|
function loadParsedConfig(configPath) {
|
|
12916
14187
|
if (!existsSync15(configPath)) return void 0;
|
|
12917
|
-
const raw =
|
|
14188
|
+
const raw = readFileSync13(configPath, "utf-8");
|
|
12918
14189
|
const parsed = parseYamlValue(raw);
|
|
12919
14190
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
12920
14191
|
return parsed;
|
|
@@ -12937,10 +14208,10 @@ function saveStudioConfig(agentvDir, config) {
|
|
|
12937
14208
|
if (!existsSync15(agentvDir)) {
|
|
12938
14209
|
mkdirSync5(agentvDir, { recursive: true });
|
|
12939
14210
|
}
|
|
12940
|
-
const configPath =
|
|
14211
|
+
const configPath = path26.join(agentvDir, "config.yaml");
|
|
12941
14212
|
let existing = {};
|
|
12942
14213
|
if (existsSync15(configPath)) {
|
|
12943
|
-
const raw =
|
|
14214
|
+
const raw = readFileSync13(configPath, "utf-8");
|
|
12944
14215
|
const parsed = parseYamlValue(raw);
|
|
12945
14216
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
12946
14217
|
existing = parsed;
|
|
@@ -12964,20 +14235,20 @@ function saveStudioConfig(agentvDir, config) {
|
|
|
12964
14235
|
app_name: config.appName
|
|
12965
14236
|
}
|
|
12966
14237
|
};
|
|
12967
|
-
const yamlStr =
|
|
14238
|
+
const yamlStr = stringifyYaml4(existing);
|
|
12968
14239
|
writeFileSync5(configPath, yamlStr, "utf-8");
|
|
12969
14240
|
}
|
|
12970
14241
|
|
|
12971
14242
|
// src/commands/results/serve.ts
|
|
12972
14243
|
var DIRECT_DASHBOARD_SOURCE_GUIDANCE = [
|
|
12973
14244
|
"Dashboard reads configured project run sources only.",
|
|
12974
|
-
"Run it from a project root, or pass --dir so Dashboard uses <project>/.agentv/results
|
|
14245
|
+
"Run it from a project root, or pass --dir so Dashboard uses <project>/.agentv/results/:",
|
|
12975
14246
|
" agentv dashboard --dir <project-dir>",
|
|
12976
|
-
"To browse external results, configure results.
|
|
14247
|
+
"To browse external results, configure results.repo.remote or results.repo.path in config YAML.",
|
|
12977
14248
|
"For a one-off run bundle, use: agentv results report <run-workspace-or-index.jsonl>"
|
|
12978
14249
|
].join("\n");
|
|
12979
14250
|
function unsupportedDashboardSourceError(source, cwd) {
|
|
12980
|
-
const resolved =
|
|
14251
|
+
const resolved = path27.isAbsolute(source) ? source : path27.resolve(cwd, source);
|
|
12981
14252
|
return new Error(
|
|
12982
14253
|
`Unsupported Dashboard source: ${resolved}
|
|
12983
14254
|
${DIRECT_DASHBOARD_SOURCE_GUIDANCE}`
|
|
@@ -12989,13 +14260,13 @@ async function resolveSourceFile2(source, cwd) {
|
|
|
12989
14260
|
}
|
|
12990
14261
|
const cache = await loadRunCache(cwd);
|
|
12991
14262
|
const cachedFile = cache ? resolveRunCacheFile(cache) : "";
|
|
12992
|
-
if (cachedFile && existsSync16(cachedFile)) {
|
|
14263
|
+
if (cachedFile && existsSync16(cachedFile) && relativeRunPathFromCwd(cwd, path27.dirname(cachedFile))) {
|
|
12993
14264
|
return cachedFile;
|
|
12994
14265
|
}
|
|
12995
14266
|
const metas = listResultFiles(cwd, 10);
|
|
12996
14267
|
if (metas.length === 0) {
|
|
12997
14268
|
throw new Error(
|
|
12998
|
-
"No run workspaces found in .agentv/results
|
|
14269
|
+
"No run workspaces found in .agentv/results/\nRun an evaluation first: agentv eval <eval-file>"
|
|
12999
14270
|
);
|
|
13000
14271
|
}
|
|
13001
14272
|
if (metas.length > 1) {
|
|
@@ -13017,13 +14288,13 @@ function resolveDashboardMode(_projectCount, options) {
|
|
|
13017
14288
|
}
|
|
13018
14289
|
function bootstrapCurrentProject(cwd, options) {
|
|
13019
14290
|
if (options.single === true) return {};
|
|
13020
|
-
if (!existsSync16(
|
|
14291
|
+
if (!existsSync16(path27.join(cwd, ".agentv"))) return {};
|
|
13021
14292
|
const entry = addProject(cwd);
|
|
13022
14293
|
touchProject(entry.id);
|
|
13023
14294
|
return { currentProjectId: entry.id };
|
|
13024
14295
|
}
|
|
13025
14296
|
function feedbackPath(resultDir) {
|
|
13026
|
-
return
|
|
14297
|
+
return path27.join(resultDir, "feedback.json");
|
|
13027
14298
|
}
|
|
13028
14299
|
function readFeedback(cwd) {
|
|
13029
14300
|
const fp = feedbackPath(cwd);
|
|
@@ -13031,7 +14302,7 @@ function readFeedback(cwd) {
|
|
|
13031
14302
|
return { reviews: [] };
|
|
13032
14303
|
}
|
|
13033
14304
|
try {
|
|
13034
|
-
return JSON.parse(
|
|
14305
|
+
return JSON.parse(readFileSync14(fp, "utf8"));
|
|
13035
14306
|
} catch (err2) {
|
|
13036
14307
|
console.error(`Warning: could not parse ${fp}, starting fresh: ${err2.message}`);
|
|
13037
14308
|
return { reviews: [] };
|
|
@@ -13050,8 +14321,8 @@ function buildFileTree(dirPath, relativeTo) {
|
|
|
13050
14321
|
if (a.isDirectory() !== b.isDirectory()) return a.isDirectory() ? -1 : 1;
|
|
13051
14322
|
return a.name.localeCompare(b.name);
|
|
13052
14323
|
}).map((entry) => {
|
|
13053
|
-
const fullPath =
|
|
13054
|
-
const relPath =
|
|
14324
|
+
const fullPath = path27.join(dirPath, entry.name);
|
|
14325
|
+
const relPath = path27.relative(relativeTo, fullPath);
|
|
13055
14326
|
if (entry.isDirectory()) {
|
|
13056
14327
|
return {
|
|
13057
14328
|
name: entry.name,
|
|
@@ -13060,11 +14331,71 @@ function buildFileTree(dirPath, relativeTo) {
|
|
|
13060
14331
|
children: buildFileTree(fullPath, relativeTo)
|
|
13061
14332
|
};
|
|
13062
14333
|
}
|
|
13063
|
-
return {
|
|
14334
|
+
return {
|
|
14335
|
+
name: entry.name,
|
|
14336
|
+
path: relPath.split(path27.sep).join("/"),
|
|
14337
|
+
type: "file",
|
|
14338
|
+
storage: "local"
|
|
14339
|
+
};
|
|
13064
14340
|
});
|
|
13065
14341
|
}
|
|
14342
|
+
function flattenFileTree(nodes) {
|
|
14343
|
+
const files = [];
|
|
14344
|
+
for (const node of nodes) {
|
|
14345
|
+
if (node.type === "file") {
|
|
14346
|
+
files.push(node);
|
|
14347
|
+
} else if (node.children) {
|
|
14348
|
+
files.push(...flattenFileTree(node.children));
|
|
14349
|
+
}
|
|
14350
|
+
}
|
|
14351
|
+
return files;
|
|
14352
|
+
}
|
|
14353
|
+
function ensureCatalogFileNode(root, entry) {
|
|
14354
|
+
const segments = entry.displayPath.split("/").filter(Boolean);
|
|
14355
|
+
if (segments.length === 0) return;
|
|
14356
|
+
let siblings = root;
|
|
14357
|
+
let currentPath = "";
|
|
14358
|
+
for (let i = 0; i < segments.length; i += 1) {
|
|
14359
|
+
const segment = segments[i];
|
|
14360
|
+
if (!segment) continue;
|
|
14361
|
+
currentPath = currentPath ? `${currentPath}/${segment}` : segment;
|
|
14362
|
+
const isFile = i === segments.length - 1;
|
|
14363
|
+
let node = siblings.find((candidate) => candidate.name === segment);
|
|
14364
|
+
if (!node) {
|
|
14365
|
+
node = {
|
|
14366
|
+
name: segment,
|
|
14367
|
+
path: currentPath,
|
|
14368
|
+
type: isFile ? "file" : "dir",
|
|
14369
|
+
...isFile ? {
|
|
14370
|
+
kind: entry.kind,
|
|
14371
|
+
storage: entry.storage,
|
|
14372
|
+
...entry.ref && { ref: entry.ref },
|
|
14373
|
+
...entry.key && { key: entry.key },
|
|
14374
|
+
...entry.sha256 && { sha256: entry.sha256 },
|
|
14375
|
+
...entry.mediaType && { media_type: entry.mediaType }
|
|
14376
|
+
} : { children: [] }
|
|
14377
|
+
};
|
|
14378
|
+
siblings.push(node);
|
|
14379
|
+
siblings.sort((a, b) => {
|
|
14380
|
+
if (a.type !== b.type) return a.type === "dir" ? -1 : 1;
|
|
14381
|
+
return a.name.localeCompare(b.name);
|
|
14382
|
+
});
|
|
14383
|
+
}
|
|
14384
|
+
if (isFile) {
|
|
14385
|
+
node.kind = entry.kind;
|
|
14386
|
+
node.storage = entry.storage;
|
|
14387
|
+
if (entry.ref) node.ref = entry.ref;
|
|
14388
|
+
if (entry.key) node.key = entry.key;
|
|
14389
|
+
if (entry.sha256) node.sha256 = entry.sha256;
|
|
14390
|
+
if (entry.mediaType) node.media_type = entry.mediaType;
|
|
14391
|
+
return;
|
|
14392
|
+
}
|
|
14393
|
+
node.children ??= [];
|
|
14394
|
+
siblings = node.children;
|
|
14395
|
+
}
|
|
14396
|
+
}
|
|
13066
14397
|
function inferLanguage(filePath) {
|
|
13067
|
-
const ext =
|
|
14398
|
+
const ext = path27.extname(filePath).toLowerCase();
|
|
13068
14399
|
const langMap = {
|
|
13069
14400
|
".json": "json",
|
|
13070
14401
|
".jsonl": "json",
|
|
@@ -13091,22 +14422,383 @@ function inferLanguage(filePath) {
|
|
|
13091
14422
|
return langMap[ext] ?? "plaintext";
|
|
13092
14423
|
}
|
|
13093
14424
|
function inferRawContentType(filePath) {
|
|
13094
|
-
const ext =
|
|
14425
|
+
const ext = path27.extname(filePath).toLowerCase();
|
|
13095
14426
|
if (ext === ".json") return "application/json; charset=utf-8";
|
|
13096
14427
|
if (ext === ".jsonl") return "text/plain; charset=utf-8";
|
|
13097
14428
|
if (ext === ".md") return "text/markdown; charset=utf-8";
|
|
13098
14429
|
return "text/plain; charset=utf-8";
|
|
13099
14430
|
}
|
|
13100
14431
|
function contentDispositionFilename(filePath) {
|
|
13101
|
-
return
|
|
14432
|
+
return path27.basename(filePath).replace(/["\\\r\n]/g, "_");
|
|
14433
|
+
}
|
|
14434
|
+
function isRecord2(value) {
|
|
14435
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
14436
|
+
}
|
|
14437
|
+
function nonEmptyString(value) {
|
|
14438
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
14439
|
+
}
|
|
14440
|
+
function artifactPointerPath(pointer) {
|
|
14441
|
+
if (typeof pointer === "string") return nonEmptyString(pointer);
|
|
14442
|
+
if (!isRecord2(pointer)) return void 0;
|
|
14443
|
+
return nonEmptyString(pointer.path) ?? nonEmptyString(pointer.artifact_path) ?? nonEmptyString(pointer.relative_path);
|
|
14444
|
+
}
|
|
14445
|
+
function artifactPointerDescription(pointer) {
|
|
14446
|
+
if (typeof pointer === "string") return pointer;
|
|
14447
|
+
if (!isRecord2(pointer)) return void 0;
|
|
14448
|
+
const ref = nonEmptyString(pointer.ref);
|
|
14449
|
+
const storage = nonEmptyString(pointer.storage);
|
|
14450
|
+
const uri = nonEmptyString(pointer.uri) ?? nonEmptyString(pointer.href);
|
|
14451
|
+
const pointerPath = artifactPointerPath(pointer);
|
|
14452
|
+
const parts = [
|
|
14453
|
+
ref ? `ref ${ref}` : void 0,
|
|
14454
|
+
storage ? `storage ${storage}` : void 0,
|
|
14455
|
+
uri ? `uri ${uri}` : void 0,
|
|
14456
|
+
pointerPath ? `path ${pointerPath}` : void 0
|
|
14457
|
+
].filter((part) => part !== void 0);
|
|
14458
|
+
return parts.length > 0 ? parts.join(", ") : void 0;
|
|
14459
|
+
}
|
|
14460
|
+
function artifactPointerRef(pointer) {
|
|
14461
|
+
return isRecord2(pointer) ? nonEmptyString(pointer.ref) : void 0;
|
|
14462
|
+
}
|
|
14463
|
+
function artifactPointerKey(pointer) {
|
|
14464
|
+
return isRecord2(pointer) ? nonEmptyString(pointer.key) : void 0;
|
|
14465
|
+
}
|
|
14466
|
+
function artifactPointerSha256(pointer) {
|
|
14467
|
+
return isRecord2(pointer) ? nonEmptyString(pointer.sha256) : void 0;
|
|
14468
|
+
}
|
|
14469
|
+
function artifactPointerMediaType(pointer) {
|
|
14470
|
+
return isRecord2(pointer) ? nonEmptyString(pointer.media_type) : void 0;
|
|
14471
|
+
}
|
|
14472
|
+
function artifactPointerObjectVersion(pointer) {
|
|
14473
|
+
return isRecord2(pointer) ? nonEmptyString(pointer.object_version) : void 0;
|
|
14474
|
+
}
|
|
14475
|
+
function resolveRecordArtifactPointer(record, kind) {
|
|
14476
|
+
const pointer = kind === "transcript" ? record.transcript ?? record.artifacts?.transcript ?? record.artifact_pointers?.transcript : kind === "trace" ? record.artifact_pointers?.trace ?? record.artifacts?.trace : record.artifacts?.answer;
|
|
14477
|
+
const pointerPath = artifactPointerPath(pointer);
|
|
14478
|
+
const description = artifactPointerDescription(pointer);
|
|
14479
|
+
const ref = artifactPointerRef(pointer);
|
|
14480
|
+
const key = artifactPointerKey(pointer);
|
|
14481
|
+
const sha256 = artifactPointerSha256(pointer);
|
|
14482
|
+
const mediaType = artifactPointerMediaType(pointer);
|
|
14483
|
+
const objectVersion = artifactPointerObjectVersion(pointer);
|
|
14484
|
+
if ((kind === "transcript" || kind === "trace") && pointerPath) {
|
|
14485
|
+
return {
|
|
14486
|
+
path: pointerPath,
|
|
14487
|
+
description,
|
|
14488
|
+
ref,
|
|
14489
|
+
...key && { key },
|
|
14490
|
+
...sha256 && { sha256 },
|
|
14491
|
+
...mediaType && { mediaType },
|
|
14492
|
+
...objectVersion && { objectVersion }
|
|
14493
|
+
};
|
|
14494
|
+
}
|
|
14495
|
+
const recordWithTrace = record;
|
|
14496
|
+
const directPath = kind === "transcript" ? record.transcript_path ?? record.artifacts?.transcript_path : kind === "trace" ? recordWithTrace.trace_path ?? nonEmptyString(record.artifacts?.trace_path) : record.answer_path ?? record.artifacts?.answer_path ?? record.output_path;
|
|
14497
|
+
if (directPath) {
|
|
14498
|
+
return { path: directPath, description: directPath };
|
|
14499
|
+
}
|
|
14500
|
+
if (pointerPath) {
|
|
14501
|
+
return {
|
|
14502
|
+
path: pointerPath,
|
|
14503
|
+
description,
|
|
14504
|
+
ref,
|
|
14505
|
+
...key && { key },
|
|
14506
|
+
...sha256 && { sha256 },
|
|
14507
|
+
...mediaType && { mediaType },
|
|
14508
|
+
...objectVersion && { objectVersion }
|
|
14509
|
+
};
|
|
14510
|
+
}
|
|
14511
|
+
if (pointer) {
|
|
14512
|
+
return {
|
|
14513
|
+
description,
|
|
14514
|
+
ref,
|
|
14515
|
+
...key && { key },
|
|
14516
|
+
...sha256 && { sha256 },
|
|
14517
|
+
...mediaType && { mediaType },
|
|
14518
|
+
...objectVersion && { objectVersion },
|
|
14519
|
+
unsupportedReason: description ? `${kind} artifact pointer does not include a local path (${description}).` : `${kind} artifact pointer does not include a local path.`
|
|
14520
|
+
};
|
|
14521
|
+
}
|
|
14522
|
+
return {};
|
|
14523
|
+
}
|
|
14524
|
+
function resolveRunArtifactPath(baseDir, relativePath) {
|
|
14525
|
+
const absolutePath = path27.resolve(baseDir, relativePath);
|
|
14526
|
+
const resolvedBase = path27.resolve(baseDir);
|
|
14527
|
+
if (!isPathInsideDirectory(resolvedBase, absolutePath)) {
|
|
14528
|
+
return { error: "Artifact path is outside the run workspace." };
|
|
14529
|
+
}
|
|
14530
|
+
return { absolutePath };
|
|
14531
|
+
}
|
|
14532
|
+
function isPathInsideDirectory(baseDir, candidatePath) {
|
|
14533
|
+
const relative3 = path27.relative(baseDir, candidatePath);
|
|
14534
|
+
return relative3 === "" || !!relative3 && !relative3.startsWith("..") && !path27.isAbsolute(relative3);
|
|
14535
|
+
}
|
|
14536
|
+
function resolveReadableRunArtifactFile(baseDir, relativePath) {
|
|
14537
|
+
const resolved = resolveRunArtifactPath(baseDir, relativePath);
|
|
14538
|
+
if (!resolved.absolutePath) return { error: resolved.error };
|
|
14539
|
+
let realBase;
|
|
14540
|
+
let realArtifact;
|
|
14541
|
+
try {
|
|
14542
|
+
realBase = realpathSync(baseDir);
|
|
14543
|
+
realArtifact = realpathSync(resolved.absolutePath);
|
|
14544
|
+
} catch {
|
|
14545
|
+
return {};
|
|
14546
|
+
}
|
|
14547
|
+
if (!isPathInsideDirectory(realBase, realArtifact)) {
|
|
14548
|
+
return { error: "Artifact path is outside the run workspace." };
|
|
14549
|
+
}
|
|
14550
|
+
try {
|
|
14551
|
+
if (!statSync6(realArtifact).isFile()) {
|
|
14552
|
+
return {};
|
|
14553
|
+
}
|
|
14554
|
+
} catch {
|
|
14555
|
+
return {};
|
|
14556
|
+
}
|
|
14557
|
+
return { absolutePath: realArtifact };
|
|
14558
|
+
}
|
|
14559
|
+
function normalizeArtifactRelativePath(relativePath) {
|
|
14560
|
+
const normalized = relativePath.split(path27.sep).join("/");
|
|
14561
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
14562
|
+
if (segments.length === 0 || normalized.startsWith("/") || segments.some((segment) => segment === "..")) {
|
|
14563
|
+
return void 0;
|
|
14564
|
+
}
|
|
14565
|
+
return segments.join("/");
|
|
14566
|
+
}
|
|
14567
|
+
function relativeRunPathFromNormalizedManifestPath(manifestPath) {
|
|
14568
|
+
const parts = manifestPath.split("/").filter(Boolean);
|
|
14569
|
+
const runsIndex = parts.lastIndexOf("runs");
|
|
14570
|
+
if (runsIndex === -1 || parts.at(-1) !== "index.jsonl") {
|
|
14571
|
+
return void 0;
|
|
14572
|
+
}
|
|
14573
|
+
const runParts = parts.slice(runsIndex + 1, -1);
|
|
14574
|
+
return runParts.length > 0 ? runParts.join("/") : void 0;
|
|
14575
|
+
}
|
|
14576
|
+
function relativeRunPathFromManifestPath(manifestPath) {
|
|
14577
|
+
return relativeRunPathFromNormalizedManifestPath(manifestPath.split(path27.sep).join("/"));
|
|
14578
|
+
}
|
|
14579
|
+
function relativeRunPathFromManifest(repoDir, manifestPath) {
|
|
14580
|
+
const relativeManifestPath = path27.relative(repoDir, manifestPath).split(path27.sep).join("/");
|
|
14581
|
+
if (relativeManifestPath.length === 0 || relativeManifestPath === manifestPath || relativeManifestPath.startsWith("../")) {
|
|
14582
|
+
return void 0;
|
|
14583
|
+
}
|
|
14584
|
+
return relativeRunPathFromNormalizedManifestPath(relativeManifestPath);
|
|
14585
|
+
}
|
|
14586
|
+
function sidecarArtifactKeyForPointer(repoDir, manifestPath, artifact) {
|
|
14587
|
+
const publishedKey = artifact.key ? normalizeArtifactRelativePath(artifact.key) : void 0;
|
|
14588
|
+
if (publishedKey?.startsWith("runs/")) {
|
|
14589
|
+
return publishedKey;
|
|
14590
|
+
}
|
|
14591
|
+
if (!artifact.path) {
|
|
14592
|
+
return publishedKey;
|
|
14593
|
+
}
|
|
14594
|
+
const relativeArtifactPath = normalizeArtifactRelativePath(artifact.path);
|
|
14595
|
+
const relativeRunPath = relativeRunPathFromManifest(repoDir, manifestPath);
|
|
14596
|
+
if (!relativeArtifactPath || !relativeRunPath) {
|
|
14597
|
+
return void 0;
|
|
14598
|
+
}
|
|
14599
|
+
return ["runs", relativeRunPath, relativeArtifactPath].join("/");
|
|
14600
|
+
}
|
|
14601
|
+
async function readSidecarArtifactText(searchDir, projectId, meta, artifact) {
|
|
14602
|
+
if (artifact.ref !== AGENTV_RESULTS_ARTIFACTS_REF) {
|
|
14603
|
+
return void 0;
|
|
14604
|
+
}
|
|
14605
|
+
const config = await loadNormalizedResultsConfig(searchDir, projectId);
|
|
14606
|
+
if (!config) {
|
|
14607
|
+
return void 0;
|
|
14608
|
+
}
|
|
14609
|
+
const key = sidecarArtifactKeyForPointer(config.path, meta.path, artifact);
|
|
14610
|
+
if (!key) {
|
|
14611
|
+
return void 0;
|
|
14612
|
+
}
|
|
14613
|
+
const bytes = await readGitResultArtifact({
|
|
14614
|
+
repoDir: config.path,
|
|
14615
|
+
key,
|
|
14616
|
+
ref: AGENTV_RESULTS_ARTIFACTS_REF,
|
|
14617
|
+
remote: config.remote,
|
|
14618
|
+
...artifact.sha256 && { sha256: artifact.sha256 },
|
|
14619
|
+
...artifact.objectVersion && { objectVersion: artifact.objectVersion }
|
|
14620
|
+
});
|
|
14621
|
+
return bytes?.toString("utf8");
|
|
14622
|
+
}
|
|
14623
|
+
function addDirectArtifactCatalogEntry(entries2, seen, displayPath, kind) {
|
|
14624
|
+
const normalized = displayPath ? normalizeArtifactRelativePath(displayPath) : void 0;
|
|
14625
|
+
if (!normalized || seen.has(normalized)) return;
|
|
14626
|
+
seen.add(normalized);
|
|
14627
|
+
entries2.push({
|
|
14628
|
+
displayPath: normalized,
|
|
14629
|
+
kind,
|
|
14630
|
+
storage: "local",
|
|
14631
|
+
path: normalized
|
|
14632
|
+
});
|
|
14633
|
+
}
|
|
14634
|
+
function addPointerArtifactCatalogEntry(entries2, seen, artifact, kind, runPath) {
|
|
14635
|
+
const artifactPath = artifact.path ? normalizeArtifactRelativePath(artifact.path) : void 0;
|
|
14636
|
+
const displayPath = artifactPath ?? displayPathFromArtifactKey(artifact.key, runPath);
|
|
14637
|
+
if (!displayPath || seen.has(displayPath)) return;
|
|
14638
|
+
seen.add(displayPath);
|
|
14639
|
+
entries2.push({
|
|
14640
|
+
displayPath,
|
|
14641
|
+
kind,
|
|
14642
|
+
storage: artifact.ref ? "git" : "local",
|
|
14643
|
+
...artifactPath && { path: artifactPath },
|
|
14644
|
+
...artifact.ref && { ref: artifact.ref },
|
|
14645
|
+
...artifact.key && { key: artifact.key },
|
|
14646
|
+
...artifact.sha256 && { sha256: artifact.sha256 },
|
|
14647
|
+
...artifact.mediaType && { mediaType: artifact.mediaType },
|
|
14648
|
+
...artifact.objectVersion && { objectVersion: artifact.objectVersion }
|
|
14649
|
+
});
|
|
14650
|
+
}
|
|
14651
|
+
function displayPathFromArtifactKey(key, runPath) {
|
|
14652
|
+
const normalizedKey = key ? normalizeArtifactRelativePath(key) : void 0;
|
|
14653
|
+
if (!normalizedKey) return void 0;
|
|
14654
|
+
if (!runPath) return normalizedKey;
|
|
14655
|
+
const runPrefix = `runs/${runPath}/`;
|
|
14656
|
+
if (!normalizedKey.startsWith(runPrefix)) return normalizedKey;
|
|
14657
|
+
return normalizeArtifactRelativePath(normalizedKey.slice(runPrefix.length)) ?? normalizedKey;
|
|
14658
|
+
}
|
|
14659
|
+
function buildResultArtifactCatalog(record, options) {
|
|
14660
|
+
const entries2 = [];
|
|
14661
|
+
const seen = /* @__PURE__ */ new Set();
|
|
14662
|
+
const transcript = resolveRecordArtifactPointer(record, "transcript");
|
|
14663
|
+
const trace = resolveRecordArtifactPointer(record, "trace");
|
|
14664
|
+
const answer = resolveRecordArtifactPointer(record, "answer");
|
|
14665
|
+
const recordWithTrace = record;
|
|
14666
|
+
addPointerArtifactCatalogEntry(entries2, seen, transcript, "transcript", options?.runPath);
|
|
14667
|
+
addPointerArtifactCatalogEntry(entries2, seen, trace, "trace", options?.runPath);
|
|
14668
|
+
addPointerArtifactCatalogEntry(entries2, seen, answer, "answer", options?.runPath);
|
|
14669
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.grading_path, "artifact");
|
|
14670
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.timing_path, "artifact");
|
|
14671
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.input_path, "artifact");
|
|
14672
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.output_path, "artifact");
|
|
14673
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.response_path, "artifact");
|
|
14674
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.answer_path, "answer");
|
|
14675
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.transcript_path, "transcript");
|
|
14676
|
+
addDirectArtifactCatalogEntry(entries2, seen, recordWithTrace.trace_path, "trace");
|
|
14677
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.eval_path, "artifact");
|
|
14678
|
+
addDirectArtifactCatalogEntry(entries2, seen, record.targets_path, "artifact");
|
|
14679
|
+
return entries2;
|
|
14680
|
+
}
|
|
14681
|
+
function resultArtifactTreeRootPaths(record, catalog) {
|
|
14682
|
+
return [
|
|
14683
|
+
...catalog.filter((entry) => entry.storage === "local").map((entry) => entry.displayPath),
|
|
14684
|
+
record.task_dir,
|
|
14685
|
+
record.files_path,
|
|
14686
|
+
record.graders_path
|
|
14687
|
+
].filter((p, index, all) => !!p && all.indexOf(p) === index);
|
|
14688
|
+
}
|
|
14689
|
+
function buildLocalResultArtifactTree(baseDir, record, catalog) {
|
|
14690
|
+
const knownPaths = resultArtifactTreeRootPaths(record, catalog);
|
|
14691
|
+
if (knownPaths.length === 0) return [];
|
|
14692
|
+
const artifactDirs = knownPaths.map((p) => path27.dirname(p));
|
|
14693
|
+
let commonDir = artifactDirs[0];
|
|
14694
|
+
for (const dir of artifactDirs) {
|
|
14695
|
+
while (!dir.startsWith(commonDir)) {
|
|
14696
|
+
commonDir = path27.dirname(commonDir);
|
|
14697
|
+
}
|
|
14698
|
+
}
|
|
14699
|
+
return buildFileTree(path27.join(baseDir, commonDir), baseDir);
|
|
14700
|
+
}
|
|
14701
|
+
function catalogEntryForDiscoveredLocalFile(nodes, displayPath) {
|
|
14702
|
+
const normalized = normalizeArtifactRelativePath(displayPath);
|
|
14703
|
+
if (!normalized) return void 0;
|
|
14704
|
+
const file = flattenFileTree(nodes).find((node) => node.path === normalized);
|
|
14705
|
+
return file ? { displayPath: normalized, kind: "artifact", storage: "local", path: normalized } : void 0;
|
|
14706
|
+
}
|
|
14707
|
+
function findArtifactCatalogEntry(catalog, displayPath) {
|
|
14708
|
+
const normalized = normalizeArtifactRelativePath(displayPath);
|
|
14709
|
+
return normalized ? catalog.find((entry) => normalizeArtifactRelativePath(entry.displayPath) === normalized) : void 0;
|
|
14710
|
+
}
|
|
14711
|
+
function findPointerArtifactCatalogEntry(catalog, artifact, kind, runPath) {
|
|
14712
|
+
const displayPath = (artifact.path ? normalizeArtifactRelativePath(artifact.path) : void 0) ?? displayPathFromArtifactKey(artifact.key, runPath);
|
|
14713
|
+
if (displayPath) {
|
|
14714
|
+
const entry = findArtifactCatalogEntry(catalog, displayPath);
|
|
14715
|
+
if (entry) return entry;
|
|
14716
|
+
}
|
|
14717
|
+
return catalog.find(
|
|
14718
|
+
(entry) => entry.kind === kind && artifact.key !== void 0 && entry.key !== void 0 && entry.key === artifact.key
|
|
14719
|
+
);
|
|
14720
|
+
}
|
|
14721
|
+
async function readArtifactCatalogEntryText(searchDir, projectId, meta, entry) {
|
|
14722
|
+
const baseDir = path27.dirname(meta.path);
|
|
14723
|
+
if (entry.storage === "local") {
|
|
14724
|
+
const resolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath);
|
|
14725
|
+
if (resolved.error) return { error: resolved.error };
|
|
14726
|
+
if (!resolved.absolutePath) return {};
|
|
14727
|
+
return { content: readFileSync14(resolved.absolutePath, "utf8") };
|
|
14728
|
+
}
|
|
14729
|
+
const localResolved = resolveReadableRunArtifactFile(baseDir, entry.displayPath);
|
|
14730
|
+
if (localResolved.error) return { error: localResolved.error };
|
|
14731
|
+
if (localResolved.absolutePath) {
|
|
14732
|
+
return { content: readFileSync14(localResolved.absolutePath, "utf8") };
|
|
14733
|
+
}
|
|
14734
|
+
if (entry.ref !== AGENTV_RESULTS_ARTIFACTS_REF) {
|
|
14735
|
+
return {};
|
|
14736
|
+
}
|
|
14737
|
+
const content = await readSidecarArtifactText(searchDir, projectId, meta, {
|
|
14738
|
+
path: entry.path ?? entry.displayPath,
|
|
14739
|
+
ref: entry.ref,
|
|
14740
|
+
...entry.key && { key: entry.key },
|
|
14741
|
+
...entry.sha256 && { sha256: entry.sha256 },
|
|
14742
|
+
...entry.objectVersion && { objectVersion: entry.objectVersion }
|
|
14743
|
+
});
|
|
14744
|
+
return { content };
|
|
14745
|
+
}
|
|
14746
|
+
function artifactFileContentResponse(c4, filePath, fileContent) {
|
|
14747
|
+
if (c4.req.query("raw") === "1" || c4.req.query("download") === "1") {
|
|
14748
|
+
c4.header("Content-Type", inferRawContentType(filePath));
|
|
14749
|
+
if (c4.req.query("download") === "1") {
|
|
14750
|
+
c4.header(
|
|
14751
|
+
"Content-Disposition",
|
|
14752
|
+
`attachment; filename="${contentDispositionFilename(filePath)}"`
|
|
14753
|
+
);
|
|
14754
|
+
}
|
|
14755
|
+
return c4.body(fileContent);
|
|
14756
|
+
}
|
|
14757
|
+
const language = inferLanguage(filePath);
|
|
14758
|
+
return c4.json({ content: fileContent, language });
|
|
14759
|
+
}
|
|
14760
|
+
function missingTranscriptMessage() {
|
|
14761
|
+
return [
|
|
14762
|
+
"This result does not include canonical transcript.jsonl metadata.",
|
|
14763
|
+
"Dashboard does not parse response.md or markdown transcripts for this view."
|
|
14764
|
+
].join(" ");
|
|
14765
|
+
}
|
|
14766
|
+
var TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION = "agentv.dashboard.trace_artifact.v1";
|
|
14767
|
+
function traceSessionArtifactResponse(response) {
|
|
14768
|
+
return {
|
|
14769
|
+
schema_version: TRACE_SESSION_ARTIFACT_RESPONSE_SCHEMA_VERSION,
|
|
14770
|
+
...response
|
|
14771
|
+
};
|
|
14772
|
+
}
|
|
14773
|
+
function missingTraceMessage() {
|
|
14774
|
+
return [
|
|
14775
|
+
"This result does not include canonical trace.json metadata.",
|
|
14776
|
+
"Dashboard trace sessions require an agentv.trace.v1 sidecar artifact."
|
|
14777
|
+
].join(" ");
|
|
14778
|
+
}
|
|
14779
|
+
function parseTraceArtifactJson(content) {
|
|
14780
|
+
try {
|
|
14781
|
+
return { value: JSON.parse(content) };
|
|
14782
|
+
} catch (error) {
|
|
14783
|
+
return {
|
|
14784
|
+
message: `Trace artifact is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
14785
|
+
};
|
|
14786
|
+
}
|
|
13102
14787
|
}
|
|
13103
14788
|
function stripHeavyFields(results) {
|
|
13104
14789
|
return results.map((r) => {
|
|
13105
|
-
const {
|
|
14790
|
+
const {
|
|
14791
|
+
requests,
|
|
14792
|
+
trace,
|
|
14793
|
+
metadata: rawMetadata,
|
|
14794
|
+
...rest
|
|
14795
|
+
} = r;
|
|
14796
|
+
const metadata = omitExternalTraceMetadataKeys(rawMetadata);
|
|
13106
14797
|
const toolCalls = trace?.toolCalls && Object.keys(trace.toolCalls).length > 0 ? trace.toolCalls : void 0;
|
|
13107
14798
|
const graderDurationMs = (r.scores ?? []).reduce((sum, s) => sum + (s.durationMs ?? 0), 0);
|
|
13108
14799
|
return {
|
|
13109
14800
|
...rest,
|
|
14801
|
+
...metadata && { metadata },
|
|
13110
14802
|
...toolCalls && { _toolCalls: toolCalls },
|
|
13111
14803
|
...graderDurationMs > 0 && { _graderDurationMs: graderDurationMs }
|
|
13112
14804
|
};
|
|
@@ -13124,23 +14816,53 @@ function inferExperimentFromRunId(runId) {
|
|
|
13124
14816
|
return experiment;
|
|
13125
14817
|
}
|
|
13126
14818
|
async function readRunTagFields(searchDir, meta, projectId) {
|
|
14819
|
+
if (meta.on_remote) {
|
|
14820
|
+
const state2 = await readRemoteRunTagState(searchDir, meta, projectId);
|
|
14821
|
+
if (state2) {
|
|
14822
|
+
return {
|
|
14823
|
+
tags: state2.tags,
|
|
14824
|
+
remote_tags: state2.remoteTags,
|
|
14825
|
+
metadata_dirty: state2.dirty,
|
|
14826
|
+
...state2.dirty && { pending_tags: state2.pendingTags ?? state2.tags },
|
|
14827
|
+
...materializeRunState({
|
|
14828
|
+
tags: state2.tags,
|
|
14829
|
+
tagRevision: state2.tagRevision,
|
|
14830
|
+
updatedAt: state2.updatedAt
|
|
14831
|
+
})
|
|
14832
|
+
};
|
|
14833
|
+
}
|
|
14834
|
+
}
|
|
13127
14835
|
if (meta.source === "local") {
|
|
13128
14836
|
const tagsEntry = readRunTags(meta.path);
|
|
13129
|
-
|
|
14837
|
+
const runState = materializeRunState({
|
|
14838
|
+
tags: tagsEntry?.tags ?? [],
|
|
14839
|
+
tagRevision: tagsEntry?.tag_revision,
|
|
14840
|
+
updatedAt: tagsEntry?.updated_at || void 0
|
|
14841
|
+
});
|
|
14842
|
+
return {
|
|
14843
|
+
...tagsEntry ? { tags: tagsEntry.tags } : {},
|
|
14844
|
+
...runState
|
|
14845
|
+
};
|
|
13130
14846
|
}
|
|
13131
14847
|
const state = await readRemoteRunTagState(searchDir, meta, projectId);
|
|
13132
14848
|
if (!state) {
|
|
13133
14849
|
return {
|
|
13134
14850
|
tags: [],
|
|
13135
14851
|
remote_tags: [],
|
|
13136
|
-
metadata_dirty: false
|
|
14852
|
+
metadata_dirty: false,
|
|
14853
|
+
...materializeRunState({ tags: [] })
|
|
13137
14854
|
};
|
|
13138
14855
|
}
|
|
13139
14856
|
return {
|
|
13140
14857
|
tags: state.tags,
|
|
13141
14858
|
remote_tags: state.remoteTags,
|
|
13142
14859
|
metadata_dirty: state.dirty,
|
|
13143
|
-
...state.dirty && { pending_tags: state.pendingTags ?? state.tags }
|
|
14860
|
+
...state.dirty && { pending_tags: state.pendingTags ?? state.tags },
|
|
14861
|
+
...materializeRunState({
|
|
14862
|
+
tags: state.tags,
|
|
14863
|
+
tagRevision: state.tagRevision,
|
|
14864
|
+
updatedAt: state.updatedAt
|
|
14865
|
+
})
|
|
13144
14866
|
};
|
|
13145
14867
|
}
|
|
13146
14868
|
function remoteTagMutationResponse(state) {
|
|
@@ -13149,22 +14871,64 @@ function remoteTagMutationResponse(state) {
|
|
|
13149
14871
|
remote_tags: state.remoteTags,
|
|
13150
14872
|
metadata_dirty: state.dirty,
|
|
13151
14873
|
...state.dirty && { pending_tags: state.pendingTags ?? state.tags },
|
|
14874
|
+
...materializeRunState({
|
|
14875
|
+
tags: state.tags,
|
|
14876
|
+
tagRevision: state.tagRevision,
|
|
14877
|
+
updatedAt: state.updatedAt
|
|
14878
|
+
}),
|
|
13152
14879
|
updated_at: state.updatedAt ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
13153
14880
|
};
|
|
13154
14881
|
}
|
|
14882
|
+
function localTagMutationResponse(input) {
|
|
14883
|
+
return materializeRunState({
|
|
14884
|
+
tags: input.tags,
|
|
14885
|
+
tagRevision: input.tagRevision,
|
|
14886
|
+
updatedAt: input.updatedAt
|
|
14887
|
+
});
|
|
14888
|
+
}
|
|
13155
14889
|
function remoteMetadataErrorStatus(error) {
|
|
14890
|
+
if (error instanceof TagRevisionConflictError) {
|
|
14891
|
+
return 409;
|
|
14892
|
+
}
|
|
13156
14893
|
const message = error instanceof Error ? error.message : String(error);
|
|
13157
14894
|
if (message.includes("not configured") || message.includes("not a writable git checkout") || message.includes("outside the results repo runs directory")) {
|
|
13158
14895
|
return 409;
|
|
13159
14896
|
}
|
|
13160
14897
|
return 400;
|
|
13161
14898
|
}
|
|
14899
|
+
function tagMutationErrorBody(error) {
|
|
14900
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
14901
|
+
if (error instanceof TagRevisionConflictError) {
|
|
14902
|
+
return {
|
|
14903
|
+
error: message,
|
|
14904
|
+
expected_tag_revision: error.expectedRevision,
|
|
14905
|
+
current_tag_revision: error.currentRevision
|
|
14906
|
+
};
|
|
14907
|
+
}
|
|
14908
|
+
return { error: message };
|
|
14909
|
+
}
|
|
14910
|
+
function expectedTagRevisionFromRecord(record) {
|
|
14911
|
+
const raw = record.expected_tag_revision ?? record.tag_revision ?? record.etag;
|
|
14912
|
+
if (raw === void 0) return void 0;
|
|
14913
|
+
if (typeof raw !== "string") {
|
|
14914
|
+
throw new Error("expected_tag_revision must be a string");
|
|
14915
|
+
}
|
|
14916
|
+
return raw;
|
|
14917
|
+
}
|
|
14918
|
+
function currentLocalTagRevision(manifestPath) {
|
|
14919
|
+
const tagsEntry = readRunTags(manifestPath);
|
|
14920
|
+
return materializeRunState({
|
|
14921
|
+
tags: tagsEntry?.tags ?? [],
|
|
14922
|
+
tagRevision: tagsEntry?.tag_revision,
|
|
14923
|
+
updatedAt: tagsEntry?.updated_at || void 0
|
|
14924
|
+
}).tag_revision;
|
|
14925
|
+
}
|
|
13162
14926
|
async function ensureRunReadable(searchDir, meta, projectId) {
|
|
13163
14927
|
await ensureRemoteRunAvailable(searchDir, meta, projectId);
|
|
13164
14928
|
}
|
|
13165
14929
|
async function loadManifestResultsForMeta(searchDir, meta, projectId) {
|
|
13166
14930
|
await ensureRunReadable(searchDir, meta, projectId);
|
|
13167
|
-
return loadManifestResults(meta.path);
|
|
14931
|
+
return loadManifestResults(meta.path, { hydrateTranscriptTrace: false });
|
|
13168
14932
|
}
|
|
13169
14933
|
async function loadLightweightResultsForMeta(searchDir, meta, projectId) {
|
|
13170
14934
|
await ensureRunReadable(searchDir, meta, projectId);
|
|
@@ -13172,7 +14936,7 @@ async function loadLightweightResultsForMeta(searchDir, meta, projectId) {
|
|
|
13172
14936
|
}
|
|
13173
14937
|
async function parseManifestForMeta(searchDir, meta, projectId) {
|
|
13174
14938
|
await ensureRunReadable(searchDir, meta, projectId);
|
|
13175
|
-
return parseResultManifest(
|
|
14939
|
+
return parseResultManifest(readFileSync14(meta.path, "utf8"));
|
|
13176
14940
|
}
|
|
13177
14941
|
var DEFAULT_RUN_PAGE_LIMIT = 50;
|
|
13178
14942
|
function parseRunPageLimit(limitParam) {
|
|
@@ -13319,12 +15083,12 @@ async function handleRunLog(c4, { searchDir, projectId }) {
|
|
|
13319
15083
|
if (meta.source === "remote") {
|
|
13320
15084
|
return c4.json({ error: "Run log is not available for remote runs" }, 404);
|
|
13321
15085
|
}
|
|
13322
|
-
const logPath =
|
|
15086
|
+
const logPath = path27.join(path27.dirname(meta.path), "console.log");
|
|
13323
15087
|
if (!existsSync16(logPath)) {
|
|
13324
15088
|
return c4.json({ error: "Run log not found for this run" }, 404);
|
|
13325
15089
|
}
|
|
13326
15090
|
try {
|
|
13327
|
-
const content =
|
|
15091
|
+
const content = readFileSync14(logPath, "utf8");
|
|
13328
15092
|
return c4.text(content);
|
|
13329
15093
|
} catch {
|
|
13330
15094
|
return c4.json({ error: "Failed to read run log" }, 500);
|
|
@@ -13336,11 +15100,12 @@ async function handleRunDetail(c4, { searchDir, projectId }) {
|
|
|
13336
15100
|
if (!meta) return c4.json({ error: "Run not found" }, 404);
|
|
13337
15101
|
try {
|
|
13338
15102
|
const loaded = await loadManifestResultsForMeta(searchDir, meta, projectId);
|
|
15103
|
+
const records = await parseManifestForMeta(searchDir, meta, projectId);
|
|
13339
15104
|
const resumeMeta = meta.source === "local" ? deriveResumeMeta(searchDir, meta.path) : {};
|
|
13340
15105
|
const liveStatus = meta.source === "local" ? getActiveRunStatus(meta.path) : void 0;
|
|
13341
15106
|
const tagFields = await readRunTagFields(searchDir, meta, projectId);
|
|
13342
15107
|
return c4.json({
|
|
13343
|
-
results: stripHeavyFields(loaded),
|
|
15108
|
+
results: attachExternalTraceFields(stripHeavyFields(loaded), records),
|
|
13344
15109
|
source: meta.source,
|
|
13345
15110
|
source_label: meta.displayName,
|
|
13346
15111
|
...tagFields,
|
|
@@ -13351,15 +15116,39 @@ async function handleRunDetail(c4, { searchDir, projectId }) {
|
|
|
13351
15116
|
return c4.json({ error: "Failed to load run" }, 500);
|
|
13352
15117
|
}
|
|
13353
15118
|
}
|
|
15119
|
+
function isPhoenixExternalTrace(metadata) {
|
|
15120
|
+
const provider = metadata.provider?.toLowerCase();
|
|
15121
|
+
return provider === void 0 || provider === "phoenix";
|
|
15122
|
+
}
|
|
15123
|
+
function externalTraceWireFromManifestRecord(record) {
|
|
15124
|
+
if (!record) {
|
|
15125
|
+
return void 0;
|
|
15126
|
+
}
|
|
15127
|
+
const metadata = externalTraceMetadataFromRecord(record) ?? externalTraceMetadataFromRecord(record.metadata);
|
|
15128
|
+
if (!metadata || !isPhoenixExternalTrace(metadata)) {
|
|
15129
|
+
return void 0;
|
|
15130
|
+
}
|
|
15131
|
+
try {
|
|
15132
|
+
return toExternalTraceMetadataWire(metadata);
|
|
15133
|
+
} catch {
|
|
15134
|
+
return void 0;
|
|
15135
|
+
}
|
|
15136
|
+
}
|
|
15137
|
+
function attachExternalTraceFields(results, records) {
|
|
15138
|
+
return results.map((result, index) => {
|
|
15139
|
+
const externalTrace = externalTraceWireFromManifestRecord(records[index]);
|
|
15140
|
+
return externalTrace ? { ...result, external_trace: externalTrace } : result;
|
|
15141
|
+
});
|
|
15142
|
+
}
|
|
13354
15143
|
function deriveResumeMeta(cwd, manifestPath) {
|
|
13355
15144
|
const out = {};
|
|
13356
|
-
const runDir =
|
|
13357
|
-
const relative3 =
|
|
15145
|
+
const runDir = path27.dirname(manifestPath);
|
|
15146
|
+
const relative3 = path27.relative(cwd, runDir);
|
|
13358
15147
|
out.run_dir = relative3 !== "" && !relative3.startsWith("..") ? relative3 : runDir;
|
|
13359
15148
|
try {
|
|
13360
|
-
const benchmarkPath =
|
|
15149
|
+
const benchmarkPath = path27.join(runDir, "benchmark.json");
|
|
13361
15150
|
if (existsSync16(benchmarkPath)) {
|
|
13362
|
-
const parsed = JSON.parse(
|
|
15151
|
+
const parsed = JSON.parse(readFileSync14(benchmarkPath, "utf8"));
|
|
13363
15152
|
const evalFile = parsed.metadata?.eval_file;
|
|
13364
15153
|
if (typeof evalFile === "string" && evalFile.trim()) {
|
|
13365
15154
|
out.suite_filter = evalFile.trim();
|
|
@@ -13494,31 +15283,17 @@ async function handleEvalFiles(c4, { searchDir, projectId }) {
|
|
|
13494
15283
|
const records = await parseManifestForMeta(searchDir, meta, projectId);
|
|
13495
15284
|
const record = records.find((r) => r.test_id === evalId);
|
|
13496
15285
|
if (!record) return c4.json({ error: "Eval not found" }, 404);
|
|
13497
|
-
const baseDir =
|
|
13498
|
-
const
|
|
13499
|
-
|
|
13500
|
-
|
|
13501
|
-
|
|
13502
|
-
|
|
13503
|
-
|
|
13504
|
-
|
|
13505
|
-
|
|
13506
|
-
|
|
13507
|
-
|
|
13508
|
-
record.targets_path,
|
|
13509
|
-
record.files_path,
|
|
13510
|
-
record.graders_path
|
|
13511
|
-
].filter((p) => !!p);
|
|
13512
|
-
if (knownPaths.length === 0) return c4.json({ files: [] });
|
|
13513
|
-
const artifactDirs = knownPaths.map((p) => path26.dirname(p));
|
|
13514
|
-
let commonDir = artifactDirs[0];
|
|
13515
|
-
for (const dir of artifactDirs) {
|
|
13516
|
-
while (!dir.startsWith(commonDir)) {
|
|
13517
|
-
commonDir = path26.dirname(commonDir);
|
|
13518
|
-
}
|
|
13519
|
-
}
|
|
13520
|
-
const artifactAbsDir = path26.join(baseDir, commonDir);
|
|
13521
|
-
const files = buildFileTree(artifactAbsDir, baseDir);
|
|
15286
|
+
const baseDir = path27.dirname(meta.path);
|
|
15287
|
+
const catalog = buildResultArtifactCatalog(record, {
|
|
15288
|
+
runPath: relativeRunPathFromManifestPath(meta.path)
|
|
15289
|
+
});
|
|
15290
|
+
const files = buildLocalResultArtifactTree(baseDir, record, catalog);
|
|
15291
|
+
const localFilePaths = new Set(flattenFileTree(files).map((file) => file.path));
|
|
15292
|
+
for (const entry of catalog) {
|
|
15293
|
+
if (entry.storage === "git" || !localFilePaths.has(entry.displayPath)) {
|
|
15294
|
+
ensureCatalogFileNode(files, entry);
|
|
15295
|
+
}
|
|
15296
|
+
}
|
|
13522
15297
|
return c4.json({ files });
|
|
13523
15298
|
} catch {
|
|
13524
15299
|
return c4.json({ error: "Failed to load file tree" }, 500);
|
|
@@ -13526,6 +15301,7 @@ async function handleEvalFiles(c4, { searchDir, projectId }) {
|
|
|
13526
15301
|
}
|
|
13527
15302
|
async function handleEvalFileContent(c4, { searchDir, projectId }) {
|
|
13528
15303
|
const filename = c4.req.param("filename") ?? "";
|
|
15304
|
+
const evalId = c4.req.param("evalId");
|
|
13529
15305
|
const meta = await findRunById(searchDir, filename, projectId);
|
|
13530
15306
|
if (!meta) return c4.json({ error: "Run not found" }, 404);
|
|
13531
15307
|
const marker = "/files/";
|
|
@@ -13539,30 +15315,213 @@ async function handleEvalFileContent(c4, { searchDir, projectId }) {
|
|
|
13539
15315
|
}
|
|
13540
15316
|
if (!filePath) return c4.json({ error: "No file path specified" }, 400);
|
|
13541
15317
|
await ensureRunReadable(searchDir, meta, projectId);
|
|
13542
|
-
const
|
|
13543
|
-
const
|
|
13544
|
-
if (!
|
|
15318
|
+
const records = parseResultManifest(readFileSync14(meta.path, "utf8"));
|
|
15319
|
+
const record = records.find((r) => r.test_id === evalId);
|
|
15320
|
+
if (!record) return c4.json({ error: "Eval not found" }, 404);
|
|
15321
|
+
const catalog = buildResultArtifactCatalog(record, {
|
|
15322
|
+
runPath: relativeRunPathFromManifestPath(meta.path)
|
|
15323
|
+
});
|
|
15324
|
+
const entry = findArtifactCatalogEntry(catalog, filePath) ?? catalogEntryForDiscoveredLocalFile(
|
|
15325
|
+
buildLocalResultArtifactTree(path27.dirname(meta.path), record, catalog),
|
|
15326
|
+
filePath
|
|
15327
|
+
);
|
|
15328
|
+
if (!entry) {
|
|
15329
|
+
return c4.json({ error: "File not found" }, 404);
|
|
15330
|
+
}
|
|
15331
|
+
const resolved = await readArtifactCatalogEntryText(searchDir, projectId, meta, entry);
|
|
15332
|
+
if (resolved.error) {
|
|
13545
15333
|
return c4.json({ error: "Path traversal not allowed" }, 403);
|
|
13546
15334
|
}
|
|
13547
|
-
if (
|
|
13548
|
-
|
|
15335
|
+
if (resolved.content === void 0) return c4.json({ error: "File not found" }, 404);
|
|
15336
|
+
return artifactFileContentResponse(c4, entry.displayPath, resolved.content);
|
|
15337
|
+
}
|
|
15338
|
+
async function handleEvalTraceSession(c4, { searchDir, projectId }) {
|
|
15339
|
+
const filename = c4.req.param("filename") ?? "";
|
|
15340
|
+
const evalId = c4.req.param("evalId");
|
|
15341
|
+
const meta = await findRunById(searchDir, filename, projectId);
|
|
15342
|
+
if (!meta) return c4.json({ error: "Run not found" }, 404);
|
|
15343
|
+
try {
|
|
15344
|
+
const records = await parseManifestForMeta(searchDir, meta, projectId);
|
|
15345
|
+
const record = records.find((r) => r.test_id === evalId);
|
|
15346
|
+
if (!record) return c4.json({ error: "Eval not found" }, 404);
|
|
15347
|
+
const trace = resolveRecordArtifactPointer(record, "trace");
|
|
15348
|
+
const runPath = relativeRunPathFromManifestPath(meta.path);
|
|
15349
|
+
const traceEntry = findPointerArtifactCatalogEntry(
|
|
15350
|
+
buildResultArtifactCatalog(record, { runPath }),
|
|
15351
|
+
trace,
|
|
15352
|
+
"trace",
|
|
15353
|
+
runPath
|
|
15354
|
+
);
|
|
15355
|
+
if (!traceEntry && !trace.path) {
|
|
15356
|
+
return c4.json(
|
|
15357
|
+
traceSessionArtifactResponse({
|
|
15358
|
+
status: trace.unsupportedReason ? "unsupported" : "missing",
|
|
15359
|
+
message: trace.unsupportedReason ?? missingTraceMessage(),
|
|
15360
|
+
...trace.description && { pointer: trace.description }
|
|
15361
|
+
})
|
|
15362
|
+
);
|
|
15363
|
+
}
|
|
15364
|
+
if (trace.path && !normalizeArtifactRelativePath(trace.path)) {
|
|
15365
|
+
return c4.json(
|
|
15366
|
+
traceSessionArtifactResponse({
|
|
15367
|
+
status: "rejected",
|
|
15368
|
+
trace_path: trace.path,
|
|
15369
|
+
message: "Artifact path is outside the run workspace.",
|
|
15370
|
+
...trace.description && { pointer: trace.description }
|
|
15371
|
+
}),
|
|
15372
|
+
403
|
|
15373
|
+
);
|
|
15374
|
+
}
|
|
15375
|
+
if (!traceEntry) {
|
|
15376
|
+
return c4.json(
|
|
15377
|
+
traceSessionArtifactResponse({
|
|
15378
|
+
status: "dangling",
|
|
15379
|
+
...trace.path && { trace_path: trace.path },
|
|
15380
|
+
message: trace.unsupportedReason ? trace.unsupportedReason : `Trace artifact pointer is present, but ${trace.path} is not available in this run workspace.`,
|
|
15381
|
+
...trace.description && { pointer: trace.description }
|
|
15382
|
+
})
|
|
15383
|
+
);
|
|
15384
|
+
}
|
|
15385
|
+
const resolvedTrace = await readArtifactCatalogEntryText(
|
|
15386
|
+
searchDir,
|
|
15387
|
+
projectId,
|
|
15388
|
+
meta,
|
|
15389
|
+
traceEntry
|
|
15390
|
+
);
|
|
15391
|
+
if (resolvedTrace.error) {
|
|
15392
|
+
return c4.json(
|
|
15393
|
+
traceSessionArtifactResponse({
|
|
15394
|
+
status: "rejected",
|
|
15395
|
+
trace_path: traceEntry.displayPath,
|
|
15396
|
+
message: resolvedTrace.error,
|
|
15397
|
+
...trace.description && { pointer: trace.description }
|
|
15398
|
+
}),
|
|
15399
|
+
403
|
|
15400
|
+
);
|
|
15401
|
+
}
|
|
15402
|
+
const content = resolvedTrace.content;
|
|
15403
|
+
if (content === void 0) {
|
|
15404
|
+
const refMessage = trace.ref ? ` on ${trace.ref}` : "";
|
|
15405
|
+
return c4.json(
|
|
15406
|
+
traceSessionArtifactResponse({
|
|
15407
|
+
status: "dangling",
|
|
15408
|
+
trace_path: traceEntry.displayPath,
|
|
15409
|
+
message: `Trace artifact pointer${refMessage} is present, but ${traceEntry.displayPath} is not available in this run workspace.`,
|
|
15410
|
+
...trace.description && { pointer: trace.description }
|
|
15411
|
+
})
|
|
15412
|
+
);
|
|
15413
|
+
}
|
|
15414
|
+
const parsed = parseTraceArtifactJson(content);
|
|
15415
|
+
if (parsed.message) {
|
|
15416
|
+
return c4.json(
|
|
15417
|
+
traceSessionArtifactResponse({
|
|
15418
|
+
status: "invalid",
|
|
15419
|
+
trace_path: traceEntry.displayPath,
|
|
15420
|
+
message: parsed.message,
|
|
15421
|
+
...trace.description && { pointer: trace.description }
|
|
15422
|
+
})
|
|
15423
|
+
);
|
|
15424
|
+
}
|
|
15425
|
+
const normalized = normalizeTraceArtifactToTraceSessionResponse(parsed.value, {
|
|
15426
|
+
runId: filename,
|
|
15427
|
+
testId: record.test_id,
|
|
15428
|
+
suite: record.suite,
|
|
15429
|
+
target: record.target,
|
|
15430
|
+
artifactPath: traceEntry.displayPath
|
|
15431
|
+
});
|
|
15432
|
+
if (normalized.status === "unsupported") {
|
|
15433
|
+
return c4.json(
|
|
15434
|
+
traceSessionArtifactResponse({
|
|
15435
|
+
status: "unsupported",
|
|
15436
|
+
trace_path: traceEntry.displayPath,
|
|
15437
|
+
message: normalized.message,
|
|
15438
|
+
...trace.description && { pointer: trace.description }
|
|
15439
|
+
})
|
|
15440
|
+
);
|
|
15441
|
+
}
|
|
15442
|
+
return c4.json(
|
|
15443
|
+
traceSessionArtifactResponse({
|
|
15444
|
+
status: "ok",
|
|
15445
|
+
trace_path: traceEntry.displayPath,
|
|
15446
|
+
trace_session: normalized.traceSession,
|
|
15447
|
+
...trace.description && { pointer: trace.description }
|
|
15448
|
+
})
|
|
15449
|
+
);
|
|
15450
|
+
} catch {
|
|
15451
|
+
return c4.json({ error: "Failed to load trace artifact" }, 500);
|
|
13549
15452
|
}
|
|
15453
|
+
}
|
|
15454
|
+
async function handleEvalTranscript(c4, { searchDir, projectId }) {
|
|
15455
|
+
const filename = c4.req.param("filename") ?? "";
|
|
15456
|
+
const evalId = c4.req.param("evalId");
|
|
15457
|
+
const meta = await findRunById(searchDir, filename, projectId);
|
|
15458
|
+
if (!meta) return c4.json({ error: "Run not found" }, 404);
|
|
13550
15459
|
try {
|
|
13551
|
-
const
|
|
13552
|
-
|
|
13553
|
-
|
|
13554
|
-
|
|
13555
|
-
|
|
13556
|
-
|
|
13557
|
-
|
|
13558
|
-
|
|
13559
|
-
|
|
13560
|
-
|
|
15460
|
+
const records = await parseManifestForMeta(searchDir, meta, projectId);
|
|
15461
|
+
const record = records.find((r) => r.test_id === evalId);
|
|
15462
|
+
if (!record) return c4.json({ error: "Eval not found" }, 404);
|
|
15463
|
+
const transcript = resolveRecordArtifactPointer(record, "transcript");
|
|
15464
|
+
const answer = resolveRecordArtifactPointer(record, "answer");
|
|
15465
|
+
const runPath = relativeRunPathFromManifestPath(meta.path);
|
|
15466
|
+
const catalog = buildResultArtifactCatalog(record, { runPath });
|
|
15467
|
+
const transcriptEntry = findPointerArtifactCatalogEntry(
|
|
15468
|
+
catalog,
|
|
15469
|
+
transcript,
|
|
15470
|
+
"transcript",
|
|
15471
|
+
runPath
|
|
15472
|
+
);
|
|
15473
|
+
const answerEntry = findPointerArtifactCatalogEntry(catalog, answer, "answer", runPath);
|
|
15474
|
+
if (!transcriptEntry && !transcript.path) {
|
|
15475
|
+
return c4.json({
|
|
15476
|
+
status: transcript.unsupportedReason ? "unsupported" : "missing",
|
|
15477
|
+
message: transcript.unsupportedReason ?? missingTranscriptMessage(),
|
|
15478
|
+
...transcript.description && { pointer: transcript.description }
|
|
15479
|
+
});
|
|
15480
|
+
}
|
|
15481
|
+
if (!transcriptEntry) {
|
|
15482
|
+
return c4.json({
|
|
15483
|
+
status: "dangling",
|
|
15484
|
+
...transcript.path && { transcript_path: transcript.path },
|
|
15485
|
+
message: transcript.unsupportedReason ? transcript.unsupportedReason : `Transcript artifact pointer is present, but ${transcript.path} is not available in this run workspace.`,
|
|
15486
|
+
...transcript.description && { pointer: transcript.description }
|
|
15487
|
+
});
|
|
13561
15488
|
}
|
|
13562
|
-
const
|
|
13563
|
-
|
|
15489
|
+
const resolvedTranscript = await readArtifactCatalogEntryText(
|
|
15490
|
+
searchDir,
|
|
15491
|
+
projectId,
|
|
15492
|
+
meta,
|
|
15493
|
+
transcriptEntry
|
|
15494
|
+
);
|
|
15495
|
+
if (resolvedTranscript.error) {
|
|
15496
|
+
return c4.json({
|
|
15497
|
+
status: "dangling",
|
|
15498
|
+
transcript_path: transcriptEntry.displayPath,
|
|
15499
|
+
message: resolvedTranscript.error,
|
|
15500
|
+
...transcript.description && { pointer: transcript.description }
|
|
15501
|
+
});
|
|
15502
|
+
}
|
|
15503
|
+
const content = resolvedTranscript.content;
|
|
15504
|
+
if (content === void 0) {
|
|
15505
|
+
const refMessage = transcript.ref ? ` on ${transcript.ref}` : "";
|
|
15506
|
+
return c4.json({
|
|
15507
|
+
status: "dangling",
|
|
15508
|
+
transcript_path: transcriptEntry.displayPath,
|
|
15509
|
+
message: `Transcript artifact pointer${refMessage} is present, but ${transcriptEntry.displayPath} is not available in this run workspace.`,
|
|
15510
|
+
...transcript.description && { pointer: transcript.description }
|
|
15511
|
+
});
|
|
15512
|
+
}
|
|
15513
|
+
const answerContent = answerEntry ? (await readArtifactCatalogEntryText(searchDir, projectId, meta, answerEntry)).content : void 0;
|
|
15514
|
+
return c4.json({
|
|
15515
|
+
status: "ok",
|
|
15516
|
+
transcript_path: transcriptEntry.displayPath,
|
|
15517
|
+
content,
|
|
15518
|
+
language: inferLanguage(transcriptEntry.displayPath),
|
|
15519
|
+
...answerEntry && { answer_path: answerEntry.displayPath },
|
|
15520
|
+
...answerContent !== void 0 && { answer_content: answerContent },
|
|
15521
|
+
...transcript.description && { pointer: transcript.description }
|
|
15522
|
+
});
|
|
13564
15523
|
} catch {
|
|
13565
|
-
return c4.json({ error: "Failed to
|
|
15524
|
+
return c4.json({ error: "Failed to load transcript artifact" }, 500);
|
|
13566
15525
|
}
|
|
13567
15526
|
}
|
|
13568
15527
|
async function handleExperiments(c4, { searchDir, agentvDir, projectId }) {
|
|
@@ -13812,7 +15771,7 @@ function handleConfig(c4, { agentvDir, searchDir }, options) {
|
|
|
13812
15771
|
threshold: config.threshold,
|
|
13813
15772
|
app_name: config.appName,
|
|
13814
15773
|
read_only: options?.readOnly === true,
|
|
13815
|
-
project_name:
|
|
15774
|
+
project_name: path27.basename(searchDir),
|
|
13816
15775
|
project_dashboard: options?.projectDashboard === true,
|
|
13817
15776
|
...options?.currentProjectId && { current_project_id: options.currentProjectId }
|
|
13818
15777
|
});
|
|
@@ -13821,7 +15780,7 @@ function handleFeedbackRead(c4, { searchDir }) {
|
|
|
13821
15780
|
return c4.json(readFeedback(feedbackStoreDir(searchDir)));
|
|
13822
15781
|
}
|
|
13823
15782
|
function feedbackStoreDir(searchDir) {
|
|
13824
|
-
const resultsDir =
|
|
15783
|
+
const resultsDir = path27.join(searchDir, ".agentv", "results");
|
|
13825
15784
|
return existsSync16(resultsDir) ? resultsDir : searchDir;
|
|
13826
15785
|
}
|
|
13827
15786
|
async function handleFeedbackWrite(c4, resultDir) {
|
|
@@ -13865,25 +15824,25 @@ async function handleFeedbackWrite(c4, resultDir) {
|
|
|
13865
15824
|
function expandHomePath(inputPath) {
|
|
13866
15825
|
if (inputPath === "~") return homedir();
|
|
13867
15826
|
if (inputPath.startsWith("~/") || inputPath.startsWith("~\\")) {
|
|
13868
|
-
return
|
|
15827
|
+
return path27.join(homedir(), inputPath.slice(2));
|
|
13869
15828
|
}
|
|
13870
15829
|
return inputPath;
|
|
13871
15830
|
}
|
|
13872
15831
|
function resolveBrowsePath(inputPath, cwd) {
|
|
13873
15832
|
const trimmed = inputPath?.trim() ?? "";
|
|
13874
15833
|
const expanded = trimmed.length > 0 ? expandHomePath(trimmed) : cwd;
|
|
13875
|
-
return
|
|
15834
|
+
return path27.resolve(cwd, expanded);
|
|
13876
15835
|
}
|
|
13877
15836
|
function hasAgentvDir(dirPath) {
|
|
13878
15837
|
try {
|
|
13879
|
-
return statSync6(
|
|
15838
|
+
return statSync6(path27.join(dirPath, ".agentv")).isDirectory();
|
|
13880
15839
|
} catch {
|
|
13881
15840
|
return false;
|
|
13882
15841
|
}
|
|
13883
15842
|
}
|
|
13884
15843
|
function directoryBrowseEntry(dirPath) {
|
|
13885
15844
|
return {
|
|
13886
|
-
name:
|
|
15845
|
+
name: path27.basename(dirPath) || dirPath,
|
|
13887
15846
|
path: dirPath,
|
|
13888
15847
|
hasAgentv: hasAgentvDir(dirPath)
|
|
13889
15848
|
};
|
|
@@ -13905,7 +15864,7 @@ function browseFilesystemDirectories(inputPath, cwd) {
|
|
|
13905
15864
|
let entries2;
|
|
13906
15865
|
try {
|
|
13907
15866
|
entries2 = readdirSync4(browsePath, { withFileTypes: true }).map((entry) => {
|
|
13908
|
-
const entryPath =
|
|
15867
|
+
const entryPath = path27.join(browsePath, entry.name);
|
|
13909
15868
|
if (entry.isDirectory()) return directoryBrowseEntry(entryPath);
|
|
13910
15869
|
if (entry.isSymbolicLink()) {
|
|
13911
15870
|
try {
|
|
@@ -13922,7 +15881,7 @@ function browseFilesystemDirectories(inputPath, cwd) {
|
|
|
13922
15881
|
} catch (err2) {
|
|
13923
15882
|
throw new Error(`Unable to read directory: ${err2.message}`);
|
|
13924
15883
|
}
|
|
13925
|
-
const parentPath =
|
|
15884
|
+
const parentPath = path27.dirname(browsePath);
|
|
13926
15885
|
return {
|
|
13927
15886
|
path: browsePath,
|
|
13928
15887
|
parentPath: parentPath !== browsePath ? parentPath : void 0,
|
|
@@ -13962,36 +15921,82 @@ async function handleRunTagsPut(c4, { searchDir, projectId }) {
|
|
|
13962
15921
|
if (!Array.isArray(tags)) {
|
|
13963
15922
|
return c4.json({ error: "Missing tags array" }, 400);
|
|
13964
15923
|
}
|
|
15924
|
+
let expectedTagRevision;
|
|
15925
|
+
try {
|
|
15926
|
+
expectedTagRevision = expectedTagRevisionFromRecord(body);
|
|
15927
|
+
} catch (err2) {
|
|
15928
|
+
return c4.json({ error: err2.message }, 400);
|
|
15929
|
+
}
|
|
13965
15930
|
try {
|
|
13966
|
-
if (meta.
|
|
13967
|
-
const state = await setRemoteRunTags(
|
|
15931
|
+
if (meta.on_remote) {
|
|
15932
|
+
const state = await setRemoteRunTags(
|
|
15933
|
+
searchDir,
|
|
15934
|
+
meta,
|
|
15935
|
+
tags,
|
|
15936
|
+
projectId,
|
|
15937
|
+
expectedTagRevision
|
|
15938
|
+
);
|
|
13968
15939
|
return c4.json(remoteTagMutationResponse(state));
|
|
13969
15940
|
}
|
|
15941
|
+
assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path));
|
|
13970
15942
|
const entry = writeRunTags(meta.path, tags);
|
|
15943
|
+
const responseState = localTagMutationResponse({
|
|
15944
|
+
tags: entry?.tags ?? [],
|
|
15945
|
+
updatedAt: entry?.updated_at,
|
|
15946
|
+
tagRevision: entry?.tag_revision
|
|
15947
|
+
});
|
|
13971
15948
|
return c4.json({
|
|
13972
15949
|
tags: entry?.tags ?? [],
|
|
15950
|
+
...responseState,
|
|
13973
15951
|
updated_at: entry?.updated_at ?? (/* @__PURE__ */ new Date()).toISOString()
|
|
13974
15952
|
});
|
|
13975
15953
|
} catch (err2) {
|
|
13976
|
-
return c4.json(
|
|
15954
|
+
return c4.json(tagMutationErrorBody(err2), remoteMetadataErrorStatus(err2));
|
|
13977
15955
|
}
|
|
13978
15956
|
}
|
|
13979
15957
|
async function handleRunTagsDelete(c4, { searchDir, projectId }) {
|
|
13980
15958
|
const filename = c4.req.param("filename") ?? "";
|
|
13981
15959
|
const meta = await findRunById(searchDir, filename, projectId);
|
|
13982
15960
|
if (!meta) return c4.json({ error: "Run not found" }, 404);
|
|
15961
|
+
let expectedTagRevision;
|
|
13983
15962
|
try {
|
|
13984
|
-
|
|
13985
|
-
|
|
15963
|
+
const text = await c4.req.text();
|
|
15964
|
+
if (text.trim().length > 0) {
|
|
15965
|
+
const body = JSON.parse(text);
|
|
15966
|
+
if (!body || typeof body !== "object") {
|
|
15967
|
+
return c4.json({ error: "Invalid payload" }, 400);
|
|
15968
|
+
}
|
|
15969
|
+
expectedTagRevision = expectedTagRevisionFromRecord(body);
|
|
15970
|
+
}
|
|
15971
|
+
} catch (err2) {
|
|
15972
|
+
return c4.json(
|
|
15973
|
+
{ error: err2 instanceof SyntaxError ? "Invalid JSON" : err2.message },
|
|
15974
|
+
400
|
|
15975
|
+
);
|
|
15976
|
+
}
|
|
15977
|
+
try {
|
|
15978
|
+
if (meta.on_remote) {
|
|
15979
|
+
const state = await clearRemoteRunTags(searchDir, meta, projectId, expectedTagRevision);
|
|
13986
15980
|
return c4.json({
|
|
13987
15981
|
ok: true,
|
|
13988
15982
|
...remoteTagMutationResponse(state)
|
|
13989
15983
|
});
|
|
13990
15984
|
}
|
|
13991
|
-
|
|
13992
|
-
|
|
15985
|
+
assertExpectedTagRevision(expectedTagRevision, currentLocalTagRevision(meta.path));
|
|
15986
|
+
const entry = writeRunTags(meta.path, []);
|
|
15987
|
+
const responseState = localTagMutationResponse({
|
|
15988
|
+
tags: entry.tags,
|
|
15989
|
+
updatedAt: entry.updated_at,
|
|
15990
|
+
tagRevision: entry.tag_revision
|
|
15991
|
+
});
|
|
15992
|
+
return c4.json({
|
|
15993
|
+
ok: true,
|
|
15994
|
+
tags: entry.tags,
|
|
15995
|
+
...responseState,
|
|
15996
|
+
updated_at: entry.updated_at
|
|
15997
|
+
});
|
|
13993
15998
|
} catch (err2) {
|
|
13994
|
-
return c4.json(
|
|
15999
|
+
return c4.json(tagMutationErrorBody(err2), remoteMetadataErrorStatus(err2));
|
|
13995
16000
|
}
|
|
13996
16001
|
}
|
|
13997
16002
|
async function handleRunDelete(c4, { searchDir, projectId }) {
|
|
@@ -14011,9 +16016,6 @@ async function handleRunDelete(c4, { searchDir, projectId }) {
|
|
|
14011
16016
|
return c4.json({ error: err2.message }, 400);
|
|
14012
16017
|
}
|
|
14013
16018
|
}
|
|
14014
|
-
function getLocalRunsRoot(searchDir) {
|
|
14015
|
-
return path26.join(searchDir, ".agentv", "results", "runs");
|
|
14016
|
-
}
|
|
14017
16019
|
function validateLocalCompletedRun(searchDir, meta, actionName = "Run combine") {
|
|
14018
16020
|
if (meta.source === "remote") {
|
|
14019
16021
|
return { error: `${actionName} is only available for local runs`, status: 400 };
|
|
@@ -14021,13 +16023,12 @@ function validateLocalCompletedRun(searchDir, meta, actionName = "Run combine")
|
|
|
14021
16023
|
if (getActiveRunStatus(meta.path) === "starting" || getActiveRunStatus(meta.path) === "running") {
|
|
14022
16024
|
return { error: "Run is still active", status: 409 };
|
|
14023
16025
|
}
|
|
14024
|
-
const manifestPath =
|
|
14025
|
-
if (
|
|
16026
|
+
const manifestPath = path27.resolve(meta.path);
|
|
16027
|
+
if (path27.basename(manifestPath) !== "index.jsonl") {
|
|
14026
16028
|
return { error: "Run workspace is invalid", status: 400 };
|
|
14027
16029
|
}
|
|
14028
|
-
const runDir =
|
|
14029
|
-
|
|
14030
|
-
if (runDir !== runsRoot && runDir.startsWith(`${runsRoot}${path26.sep}`) && existsSync16(runDir)) {
|
|
16030
|
+
const runDir = path27.dirname(manifestPath);
|
|
16031
|
+
if (relativeRunPathFromCwd(searchDir, runDir) && existsSync16(runDir)) {
|
|
14031
16032
|
return { ok: true };
|
|
14032
16033
|
}
|
|
14033
16034
|
return { error: "Run workspace is outside the local results directory", status: 400 };
|
|
@@ -14058,12 +16059,17 @@ async function handleRunsCombine(c4, { searchDir, projectId }) {
|
|
|
14058
16059
|
if (displayNameValue !== void 0 && typeof displayNameValue !== "string") {
|
|
14059
16060
|
return c4.json({ error: "display_name must be a string" }, 400);
|
|
14060
16061
|
}
|
|
16062
|
+
const experimentValue = payload.experiment;
|
|
16063
|
+
if (experimentValue !== void 0 && typeof experimentValue !== "string") {
|
|
16064
|
+
return c4.json({ error: "experiment must be a string" }, 400);
|
|
16065
|
+
}
|
|
14061
16066
|
const duplicatePolicyValue = payload.duplicate_policy;
|
|
14062
16067
|
const duplicatePolicy = duplicatePolicyValue === void 0 ? "error" : String(duplicatePolicyValue);
|
|
14063
16068
|
if (duplicatePolicy !== "error" && duplicatePolicy !== "latest") {
|
|
14064
16069
|
return c4.json({ error: "duplicate_policy must be error or latest" }, 400);
|
|
14065
16070
|
}
|
|
14066
16071
|
const displayName = displayNameValue?.trim();
|
|
16072
|
+
const experiment = experimentValue?.trim() || void 0;
|
|
14067
16073
|
const metas = [];
|
|
14068
16074
|
for (const runId of runIds) {
|
|
14069
16075
|
const meta = await findRunById(searchDir, runId, projectId);
|
|
@@ -14085,6 +16091,7 @@ async function handleRunsCombine(c4, { searchDir, projectId }) {
|
|
|
14085
16091
|
const combined = combineRunSources({
|
|
14086
16092
|
cwd: searchDir,
|
|
14087
16093
|
sources,
|
|
16094
|
+
experiment,
|
|
14088
16095
|
displayName,
|
|
14089
16096
|
duplicatePolicy
|
|
14090
16097
|
});
|
|
@@ -14094,6 +16101,7 @@ async function handleRunsCombine(c4, { searchDir, projectId }) {
|
|
|
14094
16101
|
ok: true,
|
|
14095
16102
|
run_id: combined.runId,
|
|
14096
16103
|
display_name: combined.displayName,
|
|
16104
|
+
experiment: combined.experiment,
|
|
14097
16105
|
combined_from_run_ids: combined.combinedFromRunIds,
|
|
14098
16106
|
duplicate_conflicts: combined.duplicateConflicts,
|
|
14099
16107
|
...tagEntry && { tags: tagEntry.tags }
|
|
@@ -14104,12 +16112,16 @@ async function handleRunsCombine(c4, { searchDir, projectId }) {
|
|
|
14104
16112
|
if (err2 instanceof CombineDuplicateError) {
|
|
14105
16113
|
return c4.json({ error: err2.message, duplicates: err2.conflicts }, 409);
|
|
14106
16114
|
}
|
|
14107
|
-
|
|
16115
|
+
const message = err2 instanceof Error ? err2.message : String(err2);
|
|
16116
|
+
if (message.includes("requires an experiment name") || message.includes("must inherit") || message.includes("must match combined experiment") || message.includes("Invalid experiment name")) {
|
|
16117
|
+
return c4.json({ error: message }, 400);
|
|
16118
|
+
}
|
|
16119
|
+
return c4.json({ error: message }, 500);
|
|
14108
16120
|
}
|
|
14109
16121
|
}
|
|
14110
16122
|
function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
14111
16123
|
const searchDir = cwd ?? resultDir;
|
|
14112
|
-
const agentvDir =
|
|
16124
|
+
const agentvDir = path27.join(searchDir, ".agentv");
|
|
14113
16125
|
const defaultCtx = { searchDir, agentvDir, projectId: options?.currentProjectId };
|
|
14114
16126
|
const readOnly = options?.readOnly === true;
|
|
14115
16127
|
const app2 = new Hono();
|
|
@@ -14120,7 +16132,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14120
16132
|
}
|
|
14121
16133
|
return handler(c4, {
|
|
14122
16134
|
searchDir: project.path,
|
|
14123
|
-
agentvDir:
|
|
16135
|
+
agentvDir: path27.join(project.path, ".agentv"),
|
|
14124
16136
|
projectId: project.id
|
|
14125
16137
|
});
|
|
14126
16138
|
}
|
|
@@ -14152,7 +16164,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14152
16164
|
}
|
|
14153
16165
|
async function summarizeProjectRunMetas(project) {
|
|
14154
16166
|
const { runs: metas } = await listMergedResultFiles(project.path, void 0, project.id);
|
|
14155
|
-
const threshold = loadStudioConfig(
|
|
16167
|
+
const threshold = loadStudioConfig(path27.join(project.path, ".agentv")).threshold;
|
|
14156
16168
|
let passRateSum = 0;
|
|
14157
16169
|
let executionErrorCount = 0;
|
|
14158
16170
|
for (const meta of metas) {
|
|
@@ -14257,7 +16269,7 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14257
16269
|
if (records.length > 0) {
|
|
14258
16270
|
const qualitySummary = summarizeQualityResults(
|
|
14259
16271
|
records,
|
|
14260
|
-
loadStudioConfig(
|
|
16272
|
+
loadStudioConfig(path27.join(p.path, ".agentv")).threshold
|
|
14261
16273
|
);
|
|
14262
16274
|
target = records[0].target;
|
|
14263
16275
|
experiment = records[0].experiment ?? experiment;
|
|
@@ -14350,6 +16362,14 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14350
16362
|
(c4) => handleCategorySuites(c4, defaultCtx)
|
|
14351
16363
|
);
|
|
14352
16364
|
app2.get("/api/runs/:filename/evals/:evalId", (c4) => handleEvalDetail(c4, defaultCtx));
|
|
16365
|
+
app2.get(
|
|
16366
|
+
"/api/runs/:filename/evals/:evalId/trace-session",
|
|
16367
|
+
(c4) => handleEvalTraceSession(c4, defaultCtx)
|
|
16368
|
+
);
|
|
16369
|
+
app2.get(
|
|
16370
|
+
"/api/runs/:filename/evals/:evalId/transcript",
|
|
16371
|
+
(c4) => handleEvalTranscript(c4, defaultCtx)
|
|
16372
|
+
);
|
|
14353
16373
|
app2.get("/api/runs/:filename/evals/:evalId/files", (c4) => handleEvalFiles(c4, defaultCtx));
|
|
14354
16374
|
app2.get("/api/runs/:filename/evals/:evalId/files/*", (c4) => handleEvalFileContent(c4, defaultCtx));
|
|
14355
16375
|
app2.get("/api/experiments", (c4) => handleExperiments(c4, defaultCtx));
|
|
@@ -14375,11 +16395,11 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14375
16395
|
let testCount = m.testCount;
|
|
14376
16396
|
let executionErrorCount = 0;
|
|
14377
16397
|
try {
|
|
14378
|
-
const
|
|
14379
|
-
totalCostUsd =
|
|
14380
|
-
if (
|
|
16398
|
+
const records = await loadLightweightResultsForMeta(searchDir, m, defaultCtx.projectId);
|
|
16399
|
+
totalCostUsd = records.reduce((sum, r) => sum + (r.costUsd ?? 0), 0);
|
|
16400
|
+
if (records.length > 0) {
|
|
14381
16401
|
const qualitySummary = summarizeQualityResults(
|
|
14382
|
-
|
|
16402
|
+
records,
|
|
14383
16403
|
loadStudioConfig(agentvDir).threshold
|
|
14384
16404
|
);
|
|
14385
16405
|
testCount = qualitySummary.totalCount;
|
|
@@ -14467,6 +16487,14 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14467
16487
|
"/api/projects/:projectId/runs/:filename/evals/:evalId",
|
|
14468
16488
|
(c4) => withProject(c4, handleEvalDetail)
|
|
14469
16489
|
);
|
|
16490
|
+
app2.get(
|
|
16491
|
+
"/api/projects/:projectId/runs/:filename/evals/:evalId/trace-session",
|
|
16492
|
+
(c4) => withProject(c4, handleEvalTraceSession)
|
|
16493
|
+
);
|
|
16494
|
+
app2.get(
|
|
16495
|
+
"/api/projects/:projectId/runs/:filename/evals/:evalId/transcript",
|
|
16496
|
+
(c4) => withProject(c4, handleEvalTranscript)
|
|
16497
|
+
);
|
|
14470
16498
|
app2.get(
|
|
14471
16499
|
"/api/projects/:projectId/runs/:filename/evals/:evalId/files",
|
|
14472
16500
|
(c4) => withProject(c4, handleEvalFiles)
|
|
@@ -14501,22 +16529,22 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14501
16529
|
{ readOnly }
|
|
14502
16530
|
);
|
|
14503
16531
|
const studioDistPath = options?.studioDir ?? resolveStudioDistDir();
|
|
14504
|
-
if (!studioDistPath || !existsSync16(
|
|
16532
|
+
if (!studioDistPath || !existsSync16(path27.join(studioDistPath, "index.html"))) {
|
|
14505
16533
|
throw new Error(
|
|
14506
16534
|
'Dashboard dist not found. Run "bun run build" in apps/dashboard/ to build the SPA.'
|
|
14507
16535
|
);
|
|
14508
16536
|
}
|
|
14509
16537
|
app2.get("/", (c4) => {
|
|
14510
|
-
const indexPath =
|
|
14511
|
-
if (existsSync16(indexPath)) return c4.html(
|
|
16538
|
+
const indexPath = path27.join(studioDistPath, "index.html");
|
|
16539
|
+
if (existsSync16(indexPath)) return c4.html(readFileSync14(indexPath, "utf8"));
|
|
14512
16540
|
return c4.notFound();
|
|
14513
16541
|
});
|
|
14514
16542
|
app2.get("/assets/*", (c4) => {
|
|
14515
16543
|
const assetPath = c4.req.path;
|
|
14516
|
-
const filePath =
|
|
16544
|
+
const filePath = path27.join(studioDistPath, assetPath);
|
|
14517
16545
|
if (!existsSync16(filePath)) return c4.notFound();
|
|
14518
|
-
const content =
|
|
14519
|
-
const ext =
|
|
16546
|
+
const content = readFileSync14(filePath);
|
|
16547
|
+
const ext = path27.extname(filePath);
|
|
14520
16548
|
const mimeTypes = {
|
|
14521
16549
|
".js": "application/javascript",
|
|
14522
16550
|
".css": "text/css",
|
|
@@ -14537,26 +16565,26 @@ function createApp(results, resultDir, cwd, sourceFile, options) {
|
|
|
14537
16565
|
});
|
|
14538
16566
|
app2.get("*", (c4) => {
|
|
14539
16567
|
if (c4.req.path.startsWith("/api/")) return c4.json({ error: "Not found" }, 404);
|
|
14540
|
-
const indexPath =
|
|
14541
|
-
if (existsSync16(indexPath)) return c4.html(
|
|
16568
|
+
const indexPath = path27.join(studioDistPath, "index.html");
|
|
16569
|
+
if (existsSync16(indexPath)) return c4.html(readFileSync14(indexPath, "utf8"));
|
|
14542
16570
|
return c4.notFound();
|
|
14543
16571
|
});
|
|
14544
16572
|
return app2;
|
|
14545
16573
|
}
|
|
14546
16574
|
function resolveStudioDistDir() {
|
|
14547
|
-
const currentDir = typeof __dirname !== "undefined" ? __dirname :
|
|
16575
|
+
const currentDir = typeof __dirname !== "undefined" ? __dirname : path27.dirname(fileURLToPath3(import.meta.url));
|
|
14548
16576
|
const candidates = [
|
|
14549
16577
|
// From src/commands/results/ → sibling apps/dashboard/dist
|
|
14550
|
-
|
|
16578
|
+
path27.resolve(currentDir, "../../../../dashboard/dist"),
|
|
14551
16579
|
// From dist/ → sibling apps/dashboard/dist (monorepo dev)
|
|
14552
|
-
|
|
16580
|
+
path27.resolve(currentDir, "../../dashboard/dist"),
|
|
14553
16581
|
// Bundled inside CLI dist (published package: dist/dashboard/)
|
|
14554
|
-
|
|
16582
|
+
path27.resolve(currentDir, "dashboard"),
|
|
14555
16583
|
// From dist/ in monorepo root context
|
|
14556
|
-
|
|
16584
|
+
path27.resolve(currentDir, "../../../apps/dashboard/dist")
|
|
14557
16585
|
];
|
|
14558
16586
|
for (const candidate of candidates) {
|
|
14559
|
-
if (existsSync16(candidate) && existsSync16(
|
|
16587
|
+
if (existsSync16(candidate) && existsSync16(path27.join(candidate, "index.html"))) {
|
|
14560
16588
|
return candidate;
|
|
14561
16589
|
}
|
|
14562
16590
|
}
|
|
@@ -14564,7 +16592,7 @@ function resolveStudioDistDir() {
|
|
|
14564
16592
|
}
|
|
14565
16593
|
var resultsServeCommand = command({
|
|
14566
16594
|
name: "dashboard",
|
|
14567
|
-
description: "Start AgentV Dashboard
|
|
16595
|
+
description: "Start AgentV Dashboard - the zero-infra local viewer for AgentV-owned result artifacts",
|
|
14568
16596
|
args: {
|
|
14569
16597
|
source: positional({
|
|
14570
16598
|
type: optional(string),
|
|
@@ -14626,7 +16654,7 @@ var resultsServeCommand = command({
|
|
|
14626
16654
|
return;
|
|
14627
16655
|
}
|
|
14628
16656
|
const repoRoot = await findRepoRoot(cwd);
|
|
14629
|
-
const yamlConfig = await loadConfig(
|
|
16657
|
+
const yamlConfig = await loadConfig(path27.join(cwd, "_"), repoRoot);
|
|
14630
16658
|
if (yamlConfig?.required_version) {
|
|
14631
16659
|
await enforceRequiredVersion(yamlConfig.required_version);
|
|
14632
16660
|
}
|
|
@@ -14641,22 +16669,22 @@ var resultsServeCommand = command({
|
|
|
14641
16669
|
let sourceFile;
|
|
14642
16670
|
if (source) {
|
|
14643
16671
|
sourceFile = await resolveSourceFile2(source, cwd);
|
|
14644
|
-
results = loadManifestResults(sourceFile);
|
|
16672
|
+
results = loadManifestResults(sourceFile, { hydrateTranscriptTrace: false });
|
|
14645
16673
|
} else {
|
|
14646
16674
|
const cache = await loadRunCache(cwd);
|
|
14647
16675
|
const cachedFile = cache ? resolveRunCacheFile(cache) : "";
|
|
14648
|
-
if (cachedFile && existsSync16(cachedFile)) {
|
|
16676
|
+
if (cachedFile && existsSync16(cachedFile) && relativeRunPathFromCwd(cwd, path27.dirname(cachedFile))) {
|
|
14649
16677
|
sourceFile = cachedFile;
|
|
14650
|
-
results = loadManifestResults(cachedFile);
|
|
16678
|
+
results = loadManifestResults(cachedFile, { hydrateTranscriptTrace: false });
|
|
14651
16679
|
} else {
|
|
14652
16680
|
const metas = listResultFiles(cwd, 1);
|
|
14653
16681
|
if (metas.length > 0) {
|
|
14654
16682
|
sourceFile = metas[0].path;
|
|
14655
|
-
results = loadManifestResults(metas[0].path);
|
|
16683
|
+
results = loadManifestResults(metas[0].path, { hydrateTranscriptTrace: false });
|
|
14656
16684
|
}
|
|
14657
16685
|
}
|
|
14658
16686
|
}
|
|
14659
|
-
const resultDir = sourceFile ?
|
|
16687
|
+
const resultDir = sourceFile ? path27.dirname(path27.resolve(sourceFile)) : cwd;
|
|
14660
16688
|
const app2 = createApp(results, resultDir, cwd, sourceFile, {
|
|
14661
16689
|
readOnly,
|
|
14662
16690
|
projectDashboard,
|
|
@@ -14692,26 +16720,26 @@ var resultsServeCommand = command({
|
|
|
14692
16720
|
|
|
14693
16721
|
// src/commands/runs/rerun.ts
|
|
14694
16722
|
import { constants } from "node:fs";
|
|
14695
|
-
import { access, readFile as
|
|
14696
|
-
import
|
|
16723
|
+
import { access, readFile as readFile9 } from "node:fs/promises";
|
|
16724
|
+
import path28 from "node:path";
|
|
14697
16725
|
import { config as loadDotenv } from "dotenv";
|
|
14698
16726
|
var TASK_EVAL_FILENAME = "EVAL.yaml";
|
|
14699
16727
|
var TASK_TARGETS_FILENAME = "targets.yaml";
|
|
14700
16728
|
var ENV_REF_PATTERN = /\$\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}/g;
|
|
14701
|
-
function
|
|
16729
|
+
function isRecord3(value) {
|
|
14702
16730
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
14703
16731
|
}
|
|
14704
16732
|
function displayRecord(record) {
|
|
14705
16733
|
return `${record.test_id ?? "unknown"}@${record.target ?? "unknown"}`;
|
|
14706
16734
|
}
|
|
14707
16735
|
function resolveSourcePath(cwd, source) {
|
|
14708
|
-
return
|
|
16736
|
+
return path28.isAbsolute(source) ? source : path28.resolve(cwd, source);
|
|
14709
16737
|
}
|
|
14710
16738
|
function resolveRelativeRunPath(runDir, relativePath) {
|
|
14711
16739
|
if (!relativePath || relativePath.trim().length === 0) {
|
|
14712
16740
|
return void 0;
|
|
14713
16741
|
}
|
|
14714
|
-
return
|
|
16742
|
+
return path28.resolve(runDir, relativePath);
|
|
14715
16743
|
}
|
|
14716
16744
|
async function ensureFile(filePath, label) {
|
|
14717
16745
|
try {
|
|
@@ -14740,25 +16768,25 @@ function stringArray(value) {
|
|
|
14740
16768
|
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string" && entry.length > 0) : [];
|
|
14741
16769
|
}
|
|
14742
16770
|
function readExecutionTarget(parsedEval) {
|
|
14743
|
-
if (!
|
|
16771
|
+
if (!isRecord3(parsedEval)) {
|
|
14744
16772
|
return void 0;
|
|
14745
16773
|
}
|
|
14746
16774
|
const execution = parsedEval.execution;
|
|
14747
|
-
if (
|
|
16775
|
+
if (isRecord3(execution) && typeof execution.target === "string" && execution.target.length > 0) {
|
|
14748
16776
|
return execution.target;
|
|
14749
16777
|
}
|
|
14750
16778
|
return typeof parsedEval.target === "string" && parsedEval.target.length > 0 ? parsedEval.target : void 0;
|
|
14751
16779
|
}
|
|
14752
16780
|
async function readTaskTarget(evalPath, fallback) {
|
|
14753
|
-
const raw = await
|
|
16781
|
+
const raw = await readFile9(evalPath, "utf8");
|
|
14754
16782
|
return readExecutionTarget(parseYamlValue(raw)) ?? fallback;
|
|
14755
16783
|
}
|
|
14756
16784
|
async function readTargetDefinitions2(targetsPath) {
|
|
14757
|
-
const parsed = parseYamlValue(await
|
|
14758
|
-
if (!
|
|
16785
|
+
const parsed = parseYamlValue(await readFile9(targetsPath, "utf8"));
|
|
16786
|
+
if (!isRecord3(parsed) || !Array.isArray(parsed.targets)) {
|
|
14759
16787
|
throw new Error(`Targets file is missing a top-level targets array: ${targetsPath}`);
|
|
14760
16788
|
}
|
|
14761
|
-
return parsed.targets.filter(
|
|
16789
|
+
return parsed.targets.filter(isRecord3);
|
|
14762
16790
|
}
|
|
14763
16791
|
function targetName(definition) {
|
|
14764
16792
|
return typeof definition.name === "string" && definition.name.trim().length > 0 ? definition.name.trim() : void 0;
|
|
@@ -14808,7 +16836,7 @@ function collectEnvRefs(value, names = /* @__PURE__ */ new Set()) {
|
|
|
14808
16836
|
}
|
|
14809
16837
|
return names;
|
|
14810
16838
|
}
|
|
14811
|
-
if (
|
|
16839
|
+
if (isRecord3(value)) {
|
|
14812
16840
|
for (const [key, entry] of Object.entries(value)) {
|
|
14813
16841
|
if (key === "required_env") {
|
|
14814
16842
|
for (const required of stringArray(entry)) {
|
|
@@ -14869,15 +16897,15 @@ async function validateTargetFile(targetsPath, targetNames, label) {
|
|
|
14869
16897
|
}
|
|
14870
16898
|
}
|
|
14871
16899
|
function isInsideOrSame(root, candidate) {
|
|
14872
|
-
const relative3 =
|
|
14873
|
-
return relative3 === "" || !relative3.startsWith("..") && !
|
|
16900
|
+
const relative3 = path28.relative(path28.resolve(root), path28.resolve(candidate));
|
|
16901
|
+
return relative3 === "" || !relative3.startsWith("..") && !path28.isAbsolute(relative3);
|
|
14874
16902
|
}
|
|
14875
16903
|
function forbiddenOutputRoots(sourceRunDir, selected) {
|
|
14876
16904
|
return [
|
|
14877
|
-
|
|
16905
|
+
path28.resolve(sourceRunDir),
|
|
14878
16906
|
...selected.flatMap((bundle) => [
|
|
14879
|
-
|
|
14880
|
-
|
|
16907
|
+
path28.resolve(bundle.artifactDir),
|
|
16908
|
+
path28.resolve(bundle.taskDir)
|
|
14881
16909
|
])
|
|
14882
16910
|
];
|
|
14883
16911
|
}
|
|
@@ -14896,13 +16924,13 @@ function defaultOutputDir(cwd, sourceRunDir, selected, experiment) {
|
|
|
14896
16924
|
if (!roots.some((root) => isInsideOrSame(root, candidate))) {
|
|
14897
16925
|
return candidate;
|
|
14898
16926
|
}
|
|
14899
|
-
return
|
|
16927
|
+
return path28.join(path28.dirname(path28.resolve(sourceRunDir)), `rerun-${createRunDirName()}`);
|
|
14900
16928
|
}
|
|
14901
16929
|
async function loadEnvFile2(envFile, cwd, verbose) {
|
|
14902
16930
|
if (!envFile) {
|
|
14903
16931
|
return;
|
|
14904
16932
|
}
|
|
14905
|
-
const resolved =
|
|
16933
|
+
const resolved = path28.isAbsolute(envFile) ? envFile : path28.resolve(cwd, envFile);
|
|
14906
16934
|
await ensureFile(resolved, "Environment file");
|
|
14907
16935
|
const loaded = loadDotenv({ path: resolved, override: false });
|
|
14908
16936
|
if (loaded.error) {
|
|
@@ -14913,7 +16941,7 @@ async function loadEnvFile2(envFile, cwd, verbose) {
|
|
|
14913
16941
|
}
|
|
14914
16942
|
}
|
|
14915
16943
|
async function loadSelectedTaskBundles(options) {
|
|
14916
|
-
const content = await
|
|
16944
|
+
const content = await readFile9(options.indexPath, "utf8");
|
|
14917
16945
|
const records = parseResultManifest(content);
|
|
14918
16946
|
if (records.length === 0) {
|
|
14919
16947
|
throw new Error(`Run manifest contains no result rows: ${options.indexPath}`);
|
|
@@ -14934,8 +16962,8 @@ async function loadSelectedTaskBundles(options) {
|
|
|
14934
16962
|
options.sourceRunDir,
|
|
14935
16963
|
record.task_dir && `${record.task_dir}/${TASK_TARGETS_FILENAME}`
|
|
14936
16964
|
);
|
|
14937
|
-
const taskDir = resolveRelativeRunPath(options.sourceRunDir, record.task_dir) ?? (evalPath ?
|
|
14938
|
-
const artifactDir = resolveRelativeRunPath(options.sourceRunDir, record.artifact_dir) ?? (taskDir ?
|
|
16965
|
+
const taskDir = resolveRelativeRunPath(options.sourceRunDir, record.task_dir) ?? (evalPath ? path28.dirname(evalPath) : void 0);
|
|
16966
|
+
const artifactDir = resolveRelativeRunPath(options.sourceRunDir, record.artifact_dir) ?? (taskDir ? path28.dirname(taskDir) : void 0);
|
|
14939
16967
|
if (!evalPath || !targetsPath || !taskDir || !artifactDir) {
|
|
14940
16968
|
throw new Error(
|
|
14941
16969
|
`Selected result ${recordLabel} is missing task bundle paths. Re-run requires task/EVAL.yaml and task/targets.yaml.`
|
|
@@ -14965,14 +16993,14 @@ async function loadSelectedTaskBundles(options) {
|
|
|
14965
16993
|
function buildSourceMetadataByEvalFile(sourceRunDir, indexPath, selected) {
|
|
14966
16994
|
return new Map(
|
|
14967
16995
|
selected.map((bundle) => [
|
|
14968
|
-
|
|
16996
|
+
path28.resolve(bundle.evalPath),
|
|
14969
16997
|
{
|
|
14970
16998
|
rerunSource: {
|
|
14971
16999
|
mode: "rerun",
|
|
14972
|
-
sourceRunDir:
|
|
14973
|
-
sourceIndexPath:
|
|
14974
|
-
sourceArtifactDir:
|
|
14975
|
-
sourceTaskDir:
|
|
17000
|
+
sourceRunDir: path28.resolve(sourceRunDir),
|
|
17001
|
+
sourceIndexPath: path28.resolve(indexPath),
|
|
17002
|
+
sourceArtifactDir: path28.resolve(bundle.artifactDir),
|
|
17003
|
+
sourceTaskDir: path28.resolve(bundle.taskDir),
|
|
14976
17004
|
sourceTestId: bundle.testId,
|
|
14977
17005
|
sourceTarget: bundle.sourceTarget,
|
|
14978
17006
|
sourceTimestamp: bundle.record.timestamp
|
|
@@ -15043,7 +17071,7 @@ var runsRerunCommand = command({
|
|
|
15043
17071
|
handler: async (args) => {
|
|
15044
17072
|
const cwd = process.cwd();
|
|
15045
17073
|
const indexPath = resolveRunManifestPath(resolveSourcePath(cwd, args.runDir));
|
|
15046
|
-
const sourceRunDir =
|
|
17074
|
+
const sourceRunDir = path28.dirname(indexPath);
|
|
15047
17075
|
await loadEnvFile2(args.envFile, cwd, args.verbose);
|
|
15048
17076
|
const selected = await loadSelectedTaskBundles({
|
|
15049
17077
|
indexPath,
|
|
@@ -15052,10 +17080,10 @@ var runsRerunCommand = command({
|
|
|
15052
17080
|
sourceTargets: args.sourceTarget
|
|
15053
17081
|
});
|
|
15054
17082
|
const targetOverrides = args.target;
|
|
15055
|
-
const outputDir = args.output ?
|
|
17083
|
+
const outputDir = args.output ? path28.resolve(cwd, args.output) : defaultOutputDir(cwd, sourceRunDir, selected, args.experiment);
|
|
15056
17084
|
assertOutputIsSeparate(outputDir, forbiddenOutputRoots(sourceRunDir, selected));
|
|
15057
17085
|
if (args.targets) {
|
|
15058
|
-
const overrideTargetsPath =
|
|
17086
|
+
const overrideTargetsPath = path28.resolve(cwd, args.targets);
|
|
15059
17087
|
await ensureFile(overrideTargetsPath, "Target override");
|
|
15060
17088
|
const targetNames = targetOverrides.length > 0 ? targetOverrides : selected.map((bundle) => bundle.taskTarget);
|
|
15061
17089
|
await validateTargetFile(overrideTargetsPath, targetNames, "Target override");
|
|
@@ -15079,7 +17107,7 @@ var runsRerunCommand = command({
|
|
|
15079
17107
|
testFiles: selected.map((bundle) => bundle.evalPath),
|
|
15080
17108
|
rawOptions: {
|
|
15081
17109
|
target: targetOverrides,
|
|
15082
|
-
targets: args.targets ?
|
|
17110
|
+
targets: args.targets ? path28.resolve(cwd, args.targets) : void 0,
|
|
15083
17111
|
output: outputDir,
|
|
15084
17112
|
experiment: args.experiment ?? "rerun",
|
|
15085
17113
|
workers: args.workers,
|
|
@@ -15107,7 +17135,7 @@ var runsCommand = subcommands({
|
|
|
15107
17135
|
});
|
|
15108
17136
|
|
|
15109
17137
|
// src/self-update.ts
|
|
15110
|
-
import { spawn as
|
|
17138
|
+
import { spawn as spawn3 } from "node:child_process";
|
|
15111
17139
|
import { existsSync as existsSync17 } from "node:fs";
|
|
15112
17140
|
import { get } from "node:https";
|
|
15113
17141
|
import { basename, dirname as dirname3, join as join5, win32 } from "node:path";
|
|
@@ -15146,9 +17174,9 @@ function detectInstallScopeFromPath(scriptPath, cwd = process.cwd()) {
|
|
|
15146
17174
|
function detectInstallScope() {
|
|
15147
17175
|
return detectInstallScopeFromPath(process.argv[1] ?? "");
|
|
15148
17176
|
}
|
|
15149
|
-
function
|
|
17177
|
+
function runCommand2(cmd, args) {
|
|
15150
17178
|
return new Promise((resolve3, reject) => {
|
|
15151
|
-
const child =
|
|
17179
|
+
const child = spawn3(cmd, args, { stdio: ["inherit", "pipe", "inherit"] });
|
|
15152
17180
|
let stdout = "";
|
|
15153
17181
|
child.stdout?.on("data", (data) => {
|
|
15154
17182
|
process.stdout.write(data);
|
|
@@ -15226,13 +17254,13 @@ async function performSelfUpdate(options) {
|
|
|
15226
17254
|
const args = getInstallArgs(pm, versionSpec, scope);
|
|
15227
17255
|
const command2 = resolvePackageManagerCommand(pm, args);
|
|
15228
17256
|
try {
|
|
15229
|
-
const result = await
|
|
17257
|
+
const result = await runCommand2(command2.cmd, command2.args);
|
|
15230
17258
|
if (result.exitCode !== 0) {
|
|
15231
17259
|
return { success: false, currentVersion, scope };
|
|
15232
17260
|
}
|
|
15233
17261
|
let newVersion;
|
|
15234
17262
|
try {
|
|
15235
|
-
const versionResult = await
|
|
17263
|
+
const versionResult = await runCommand2("agentv", ["--version"]);
|
|
15236
17264
|
newVersion = versionResult.stdout.trim();
|
|
15237
17265
|
} catch {
|
|
15238
17266
|
}
|
|
@@ -15309,14 +17337,14 @@ var selfCommand = subcommands({
|
|
|
15309
17337
|
});
|
|
15310
17338
|
|
|
15311
17339
|
// src/commands/skills/index.ts
|
|
15312
|
-
import { existsSync as existsSync18, readFileSync as
|
|
15313
|
-
import
|
|
17340
|
+
import { existsSync as existsSync18, readFileSync as readFileSync15, readdirSync as readdirSync5 } from "node:fs";
|
|
17341
|
+
import path29 from "node:path";
|
|
15314
17342
|
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
15315
17343
|
function isValidSkillsDir(dir) {
|
|
15316
17344
|
if (!existsSync18(dir)) return false;
|
|
15317
17345
|
try {
|
|
15318
17346
|
return readdirSync5(dir, { withFileTypes: true }).some(
|
|
15319
|
-
(e) => e.isDirectory() && existsSync18(
|
|
17347
|
+
(e) => e.isDirectory() && existsSync18(path29.join(dir, e.name, "SKILL.md"))
|
|
15320
17348
|
);
|
|
15321
17349
|
} catch {
|
|
15322
17350
|
return false;
|
|
@@ -15324,15 +17352,15 @@ function isValidSkillsDir(dir) {
|
|
|
15324
17352
|
}
|
|
15325
17353
|
function findSkillsDir() {
|
|
15326
17354
|
const selfFile = fileURLToPath4(import.meta.url);
|
|
15327
|
-
let dir =
|
|
17355
|
+
let dir = path29.dirname(selfFile);
|
|
15328
17356
|
for (let i = 0; i < 6; i++) {
|
|
15329
|
-
const distCandidate =
|
|
17357
|
+
const distCandidate = path29.join(dir, "dist", "skills");
|
|
15330
17358
|
if (isValidSkillsDir(distCandidate)) return distCandidate;
|
|
15331
|
-
const repoRootCandidate =
|
|
17359
|
+
const repoRootCandidate = path29.join(dir, "skills-data");
|
|
15332
17360
|
if (isValidSkillsDir(repoRootCandidate)) return repoRootCandidate;
|
|
15333
|
-
const legacyCandidate =
|
|
17361
|
+
const legacyCandidate = path29.join(dir, "skills");
|
|
15334
17362
|
if (isValidSkillsDir(legacyCandidate)) return legacyCandidate;
|
|
15335
|
-
dir =
|
|
17363
|
+
dir = path29.dirname(dir);
|
|
15336
17364
|
}
|
|
15337
17365
|
return null;
|
|
15338
17366
|
}
|
|
@@ -15351,9 +17379,9 @@ function listSkillNames(skillsDir) {
|
|
|
15351
17379
|
return readdirSync5(skillsDir, { withFileTypes: true }).filter((e) => e.isDirectory()).map((e) => e.name).sort();
|
|
15352
17380
|
}
|
|
15353
17381
|
function readSkillFile(skillDir, relPath) {
|
|
15354
|
-
const full =
|
|
17382
|
+
const full = path29.join(skillDir, relPath);
|
|
15355
17383
|
if (!existsSync18(full)) return null;
|
|
15356
|
-
return
|
|
17384
|
+
return readFileSync15(full, "utf-8");
|
|
15357
17385
|
}
|
|
15358
17386
|
function collectDir(dir, prefix = "") {
|
|
15359
17387
|
const result = {};
|
|
@@ -15361,9 +17389,9 @@ function collectDir(dir, prefix = "") {
|
|
|
15361
17389
|
for (const entry of readdirSync5(dir, { withFileTypes: true })) {
|
|
15362
17390
|
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
15363
17391
|
if (entry.isDirectory()) {
|
|
15364
|
-
Object.assign(result, collectDir(
|
|
17392
|
+
Object.assign(result, collectDir(path29.join(dir, entry.name), relPath));
|
|
15365
17393
|
} else {
|
|
15366
|
-
result[relPath] =
|
|
17394
|
+
result[relPath] = readFileSync15(path29.join(dir, entry.name), "utf-8");
|
|
15367
17395
|
}
|
|
15368
17396
|
}
|
|
15369
17397
|
return result;
|
|
@@ -15373,14 +17401,14 @@ function listSkillSubdirs(skillDir) {
|
|
|
15373
17401
|
return readdirSync5(skillDir, { withFileTypes: true }).filter((e) => e.isDirectory() && !e.name.startsWith(".")).map((e) => e.name).sort();
|
|
15374
17402
|
}
|
|
15375
17403
|
function readSkill(skillsDir, name, full) {
|
|
15376
|
-
const skillDir =
|
|
17404
|
+
const skillDir = path29.join(skillsDir, name);
|
|
15377
17405
|
if (!existsSync18(skillDir)) return null;
|
|
15378
17406
|
const content = readSkillFile(skillDir, "SKILL.md");
|
|
15379
17407
|
if (content === null) return null;
|
|
15380
17408
|
if (!full) return { name, content };
|
|
15381
17409
|
const files = {};
|
|
15382
17410
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
15383
|
-
const subDir =
|
|
17411
|
+
const subDir = path29.join(skillDir, sub);
|
|
15384
17412
|
const collected = collectDir(subDir, sub);
|
|
15385
17413
|
Object.assign(files, collected);
|
|
15386
17414
|
}
|
|
@@ -15390,16 +17418,16 @@ function findRefFile(skillDir, refName) {
|
|
|
15390
17418
|
const candidates = refName.endsWith(".md") ? [refName] : [refName, `${refName}.md`];
|
|
15391
17419
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
15392
17420
|
for (const candidate of candidates) {
|
|
15393
|
-
const filePath =
|
|
17421
|
+
const filePath = path29.join(skillDir, sub, candidate);
|
|
15394
17422
|
if (existsSync18(filePath)) {
|
|
15395
|
-
return { relPath: `${sub}/${candidate}`, content:
|
|
17423
|
+
return { relPath: `${sub}/${candidate}`, content: readFileSync15(filePath, "utf-8") };
|
|
15396
17424
|
}
|
|
15397
17425
|
}
|
|
15398
17426
|
}
|
|
15399
17427
|
for (const candidate of candidates) {
|
|
15400
|
-
const filePath =
|
|
17428
|
+
const filePath = path29.join(skillDir, candidate);
|
|
15401
17429
|
if (existsSync18(filePath)) {
|
|
15402
|
-
return { relPath: candidate, content:
|
|
17430
|
+
return { relPath: candidate, content: readFileSync15(filePath, "utf-8") };
|
|
15403
17431
|
}
|
|
15404
17432
|
}
|
|
15405
17433
|
return null;
|
|
@@ -15407,7 +17435,7 @@ function findRefFile(skillDir, refName) {
|
|
|
15407
17435
|
function listRefFiles(skillDir) {
|
|
15408
17436
|
const out = [];
|
|
15409
17437
|
for (const sub of listSkillSubdirs(skillDir)) {
|
|
15410
|
-
const subDir =
|
|
17438
|
+
const subDir = path29.join(skillDir, sub);
|
|
15411
17439
|
for (const entry of readdirSync5(subDir, { withFileTypes: true })) {
|
|
15412
17440
|
if (entry.isFile()) out.push(`${sub}/${entry.name}`);
|
|
15413
17441
|
}
|
|
@@ -15491,7 +17519,7 @@ var skillsGetCommand = command({
|
|
|
15491
17519
|
}
|
|
15492
17520
|
process.exit(1);
|
|
15493
17521
|
}
|
|
15494
|
-
const skillDir =
|
|
17522
|
+
const skillDir = path29.join(skillsDir, name);
|
|
15495
17523
|
if (!existsSync18(skillDir)) {
|
|
15496
17524
|
const msg = `skill '${name}' not found`;
|
|
15497
17525
|
if (json) {
|
|
@@ -15580,7 +17608,7 @@ var skillsPathCommand = command({
|
|
|
15580
17608
|
handler: ({ name }) => {
|
|
15581
17609
|
const skillsDir = requireSkillsDir();
|
|
15582
17610
|
if (name) {
|
|
15583
|
-
const skillDir =
|
|
17611
|
+
const skillDir = path29.join(skillsDir, name);
|
|
15584
17612
|
if (!existsSync18(skillDir)) {
|
|
15585
17613
|
console.error(`Error: skill '${name}' not found`);
|
|
15586
17614
|
process.exit(1);
|
|
@@ -15603,7 +17631,7 @@ var skillsCommand = subcommands({
|
|
|
15603
17631
|
|
|
15604
17632
|
// src/commands/transpile/index.ts
|
|
15605
17633
|
import { writeFileSync as writeFileSync7 } from "node:fs";
|
|
15606
|
-
import
|
|
17634
|
+
import path30 from "node:path";
|
|
15607
17635
|
var transpileCommand = command({
|
|
15608
17636
|
name: "transpile",
|
|
15609
17637
|
description: "Convert an EVAL.yaml file to Agent Skills evals.json format",
|
|
@@ -15627,7 +17655,7 @@ var transpileCommand = command({
|
|
|
15627
17655
|
handler: async ({ input, outDir, stdout }) => {
|
|
15628
17656
|
let result;
|
|
15629
17657
|
try {
|
|
15630
|
-
result = transpileEvalYamlFile(
|
|
17658
|
+
result = transpileEvalYamlFile(path30.resolve(input));
|
|
15631
17659
|
} catch (error) {
|
|
15632
17660
|
console.error(`Error: ${error.message}`);
|
|
15633
17661
|
process.exit(1);
|
|
@@ -15651,11 +17679,11 @@ var transpileCommand = command({
|
|
|
15651
17679
|
process.stdout.write("\n");
|
|
15652
17680
|
return;
|
|
15653
17681
|
}
|
|
15654
|
-
const outputDir = outDir ?
|
|
17682
|
+
const outputDir = outDir ? path30.resolve(outDir) : path30.dirname(path30.resolve(input));
|
|
15655
17683
|
const fileNames = getOutputFilenames(result);
|
|
15656
17684
|
for (const [skill, evalsJson] of result.files) {
|
|
15657
17685
|
const fileName = fileNames.get(skill) ?? "evals.json";
|
|
15658
|
-
const outputPath =
|
|
17686
|
+
const outputPath = path30.join(outputDir, fileName);
|
|
15659
17687
|
writeFileSync7(outputPath, `${JSON.stringify(evalsJson, null, 2)}
|
|
15660
17688
|
`);
|
|
15661
17689
|
console.log(`Transpiled to ${outputPath}`);
|
|
@@ -15664,7 +17692,7 @@ var transpileCommand = command({
|
|
|
15664
17692
|
});
|
|
15665
17693
|
|
|
15666
17694
|
// src/commands/trend/index.ts
|
|
15667
|
-
import
|
|
17695
|
+
import path31 from "node:path";
|
|
15668
17696
|
var colors2 = {
|
|
15669
17697
|
reset: "\x1B[0m",
|
|
15670
17698
|
bold: "\x1B[1m",
|
|
@@ -15714,7 +17742,7 @@ function colorizeSlope(value) {
|
|
|
15714
17742
|
}
|
|
15715
17743
|
function ensureTrendIndexPath(source, cwd) {
|
|
15716
17744
|
const resolved = resolveResultSourcePath(source, cwd);
|
|
15717
|
-
if (
|
|
17745
|
+
if (path31.basename(resolved) !== RESULT_INDEX_FILENAME) {
|
|
15718
17746
|
throw new Error(
|
|
15719
17747
|
`Unsupported result source for trend: ${source}. Use a run workspace directory or ${RESULT_INDEX_FILENAME} manifest.`
|
|
15720
17748
|
);
|
|
@@ -15734,10 +17762,10 @@ function resolveTrendSources(cwd, sources, last) {
|
|
|
15734
17762
|
if (last < 2) {
|
|
15735
17763
|
throw new Error("--last must be at least 2");
|
|
15736
17764
|
}
|
|
15737
|
-
const metas = listResultFiles(cwd).filter((meta) =>
|
|
17765
|
+
const metas = listResultFiles(cwd).filter((meta) => path31.basename(meta.path) === RESULT_INDEX_FILENAME).slice(0, last);
|
|
15738
17766
|
if (metas.length < 2) {
|
|
15739
17767
|
throw new Error(
|
|
15740
|
-
"Trend analysis requires at least 2 canonical run workspaces in .agentv/results/
|
|
17768
|
+
"Trend analysis requires at least 2 canonical run workspaces in .agentv/results/"
|
|
15741
17769
|
);
|
|
15742
17770
|
}
|
|
15743
17771
|
return metas.map((meta) => meta.path).reverse();
|
|
@@ -15749,10 +17777,10 @@ function getRunLabel(sourcePath, timestamp) {
|
|
|
15749
17777
|
if (timestamp) {
|
|
15750
17778
|
return timestamp;
|
|
15751
17779
|
}
|
|
15752
|
-
return
|
|
17780
|
+
return path31.basename(path31.dirname(sourcePath));
|
|
15753
17781
|
}
|
|
15754
17782
|
function getRunSortKey(sourcePath, timestamp) {
|
|
15755
|
-
return timestamp ??
|
|
17783
|
+
return timestamp ?? path31.basename(path31.dirname(sourcePath));
|
|
15756
17784
|
}
|
|
15757
17785
|
function mean2(values) {
|
|
15758
17786
|
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
@@ -15931,7 +17959,7 @@ var trendCommand = command({
|
|
|
15931
17959
|
last: option({
|
|
15932
17960
|
type: optional(number),
|
|
15933
17961
|
long: "last",
|
|
15934
|
-
description: "Use the most recent N runs from .agentv/results/
|
|
17962
|
+
description: "Use the most recent N runs from .agentv/results/"
|
|
15935
17963
|
}),
|
|
15936
17964
|
suite: option({
|
|
15937
17965
|
type: optional(string),
|
|
@@ -16007,7 +18035,7 @@ var trendCommand = command({
|
|
|
16007
18035
|
});
|
|
16008
18036
|
|
|
16009
18037
|
// src/commands/trim/index.ts
|
|
16010
|
-
import { readFileSync as
|
|
18038
|
+
import { readFileSync as readFileSync16, writeFileSync as writeFileSync8 } from "node:fs";
|
|
16011
18039
|
var trimCommand = command({
|
|
16012
18040
|
name: "trim",
|
|
16013
18041
|
description: "Trim evaluation results for baseline storage (strips debug/audit fields)",
|
|
@@ -16026,7 +18054,7 @@ var trimCommand = command({
|
|
|
16026
18054
|
},
|
|
16027
18055
|
handler: async ({ input, out }) => {
|
|
16028
18056
|
try {
|
|
16029
|
-
const content =
|
|
18057
|
+
const content = readFileSync16(input, "utf8");
|
|
16030
18058
|
const lines = content.trim().split("\n").filter((line) => line.trim());
|
|
16031
18059
|
const trimmedLines = lines.map((line) => {
|
|
16032
18060
|
const record = JSON.parse(line);
|
|
@@ -16085,12 +18113,12 @@ function formatFileResult(result, useColors) {
|
|
|
16085
18113
|
lines.push(`${statusText} ${fileName}`);
|
|
16086
18114
|
if (result.errors.length > 0) {
|
|
16087
18115
|
for (const error of result.errors) {
|
|
16088
|
-
lines.push(
|
|
18116
|
+
lines.push(formatError2(error, useColors));
|
|
16089
18117
|
}
|
|
16090
18118
|
}
|
|
16091
18119
|
return lines.join("\n");
|
|
16092
18120
|
}
|
|
16093
|
-
function
|
|
18121
|
+
function formatError2(error, useColors) {
|
|
16094
18122
|
const prefix = error.severity === "error" ? " \u2717" : " \u26A0";
|
|
16095
18123
|
const color = error.severity === "error" ? ANSI_RED : ANSI_YELLOW;
|
|
16096
18124
|
const coloredPrefix = useColors ? `${color}${prefix}${ANSI_RESET}` : prefix;
|
|
@@ -16133,7 +18161,7 @@ function isTTY() {
|
|
|
16133
18161
|
// src/commands/validate/validate-files.ts
|
|
16134
18162
|
import { constants as constants2 } from "node:fs";
|
|
16135
18163
|
import { access as access2, readdir as readdir4, stat as stat2 } from "node:fs/promises";
|
|
16136
|
-
import
|
|
18164
|
+
import path32 from "node:path";
|
|
16137
18165
|
import fg3 from "fast-glob";
|
|
16138
18166
|
async function validateFiles(paths) {
|
|
16139
18167
|
const filePaths = await expandPaths(paths);
|
|
@@ -16148,7 +18176,7 @@ async function validateFiles(paths) {
|
|
|
16148
18176
|
};
|
|
16149
18177
|
}
|
|
16150
18178
|
async function validateSingleFile(filePath) {
|
|
16151
|
-
const absolutePath =
|
|
18179
|
+
const absolutePath = path32.resolve(filePath);
|
|
16152
18180
|
const fileType = await detectFileType(absolutePath);
|
|
16153
18181
|
let result;
|
|
16154
18182
|
if (fileType === "eval") {
|
|
@@ -16192,7 +18220,7 @@ async function validateSingleFile(filePath) {
|
|
|
16192
18220
|
async function expandPaths(paths) {
|
|
16193
18221
|
const expanded = /* @__PURE__ */ new Set();
|
|
16194
18222
|
for (const inputPath of paths) {
|
|
16195
|
-
const absolutePath =
|
|
18223
|
+
const absolutePath = path32.resolve(inputPath);
|
|
16196
18224
|
try {
|
|
16197
18225
|
await access2(absolutePath, constants2.F_OK);
|
|
16198
18226
|
const stats = await stat2(absolutePath);
|
|
@@ -16220,7 +18248,7 @@ async function expandPaths(paths) {
|
|
|
16220
18248
|
if (yamlMatches.length === 0) {
|
|
16221
18249
|
console.warn(`Warning: No YAML files matched pattern: ${inputPath}`);
|
|
16222
18250
|
}
|
|
16223
|
-
for (const f of yamlMatches) expanded.add(
|
|
18251
|
+
for (const f of yamlMatches) expanded.add(path32.normalize(f));
|
|
16224
18252
|
}
|
|
16225
18253
|
const sorted = Array.from(expanded);
|
|
16226
18254
|
sorted.sort();
|
|
@@ -16231,7 +18259,7 @@ async function findYamlFiles(dirPath) {
|
|
|
16231
18259
|
try {
|
|
16232
18260
|
const entries2 = await readdir4(dirPath, { withFileTypes: true });
|
|
16233
18261
|
for (const entry of entries2) {
|
|
16234
|
-
const fullPath =
|
|
18262
|
+
const fullPath = path32.join(dirPath, entry.name);
|
|
16235
18263
|
if (entry.isDirectory()) {
|
|
16236
18264
|
if (entry.name === "node_modules" || entry.name.startsWith(".")) {
|
|
16237
18265
|
continue;
|
|
@@ -16248,11 +18276,11 @@ async function findYamlFiles(dirPath) {
|
|
|
16248
18276
|
return results;
|
|
16249
18277
|
}
|
|
16250
18278
|
function isYamlFile(filePath) {
|
|
16251
|
-
const ext =
|
|
18279
|
+
const ext = path32.extname(filePath).toLowerCase();
|
|
16252
18280
|
return ext === ".yaml" || ext === ".yml";
|
|
16253
18281
|
}
|
|
16254
18282
|
function isEvalYamlFile(filePath) {
|
|
16255
|
-
const lower =
|
|
18283
|
+
const lower = path32.basename(filePath).toLowerCase();
|
|
16256
18284
|
return lower.endsWith(".eval.yaml") || lower.endsWith(".eval.yml");
|
|
16257
18285
|
}
|
|
16258
18286
|
|
|
@@ -16308,8 +18336,8 @@ var validateCommand = command({
|
|
|
16308
18336
|
|
|
16309
18337
|
// src/commands/workspace/clean.ts
|
|
16310
18338
|
import { existsSync as existsSync19 } from "node:fs";
|
|
16311
|
-
import { readFile as
|
|
16312
|
-
import
|
|
18339
|
+
import { readFile as readFile10, readdir as readdir5, rm as rm3 } from "node:fs/promises";
|
|
18340
|
+
import path33 from "node:path";
|
|
16313
18341
|
async function confirm2(message) {
|
|
16314
18342
|
const readline4 = await import("node:readline");
|
|
16315
18343
|
const rl = readline4.createInterface({ input: process.stdin, output: process.stdout });
|
|
@@ -16345,10 +18373,10 @@ var cleanCommand = command({
|
|
|
16345
18373
|
const poolDirs = entries2.filter((e) => e.isDirectory());
|
|
16346
18374
|
const matchingDirs = [];
|
|
16347
18375
|
for (const dir of poolDirs) {
|
|
16348
|
-
const poolDir =
|
|
16349
|
-
const metadataPath =
|
|
18376
|
+
const poolDir = path33.join(poolRoot, dir.name);
|
|
18377
|
+
const metadataPath = path33.join(poolDir, "metadata.json");
|
|
16350
18378
|
try {
|
|
16351
|
-
const raw = await
|
|
18379
|
+
const raw = await readFile10(metadataPath, "utf-8");
|
|
16352
18380
|
const metadata = JSON.parse(raw);
|
|
16353
18381
|
const hasRepo = metadata.repos?.some((r) => {
|
|
16354
18382
|
const value = r.repo ?? (r.source?.type === "git" ? r.source.url : r.source?.path);
|
|
@@ -16374,8 +18402,8 @@ var cleanCommand = command({
|
|
|
16374
18402
|
}
|
|
16375
18403
|
}
|
|
16376
18404
|
for (const dir of matchingDirs) {
|
|
16377
|
-
await
|
|
16378
|
-
console.log(`Removed: ${
|
|
18405
|
+
await rm3(dir, { recursive: true, force: true });
|
|
18406
|
+
console.log(`Removed: ${path33.basename(dir).slice(0, 12)}...`);
|
|
16379
18407
|
}
|
|
16380
18408
|
console.log("Done.");
|
|
16381
18409
|
} else {
|
|
@@ -16386,14 +18414,14 @@ var cleanCommand = command({
|
|
|
16386
18414
|
return;
|
|
16387
18415
|
}
|
|
16388
18416
|
}
|
|
16389
|
-
await
|
|
18417
|
+
await rm3(poolRoot, { recursive: true, force: true });
|
|
16390
18418
|
console.log("Workspace pool cleaned.");
|
|
16391
18419
|
}
|
|
16392
18420
|
}
|
|
16393
18421
|
});
|
|
16394
18422
|
|
|
16395
18423
|
// src/commands/workspace/deps.ts
|
|
16396
|
-
import
|
|
18424
|
+
import path34 from "node:path";
|
|
16397
18425
|
var depsCommand = command({
|
|
16398
18426
|
name: "deps",
|
|
16399
18427
|
description: "Scan eval files and list git repo dependencies needed by workspaces",
|
|
@@ -16417,7 +18445,7 @@ var depsCommand = command({
|
|
|
16417
18445
|
const resolvedPaths = await resolveEvalPaths(evalPaths, cwd);
|
|
16418
18446
|
const result = await scanRepoDeps(resolvedPaths);
|
|
16419
18447
|
for (const err2 of result.errors) {
|
|
16420
|
-
console.error(`error: ${
|
|
18448
|
+
console.error(`error: ${path34.relative(cwd, err2.file)}: ${err2.message}`);
|
|
16421
18449
|
}
|
|
16422
18450
|
if (result.errors.length > 0) {
|
|
16423
18451
|
process.exit(1);
|
|
@@ -16428,7 +18456,7 @@ var depsCommand = command({
|
|
|
16428
18456
|
...r.ref !== void 0 && { ref: r.ref },
|
|
16429
18457
|
...r.sparse !== void 0 && { sparse: r.sparse },
|
|
16430
18458
|
...r.ancestor !== void 0 && { ancestor: r.ancestor },
|
|
16431
|
-
...usedBy && { used_by: r.usedBy.map((p) =>
|
|
18459
|
+
...usedBy && { used_by: r.usedBy.map((p) => path34.relative(cwd, p)) }
|
|
16432
18460
|
}))
|
|
16433
18461
|
};
|
|
16434
18462
|
console.log(JSON.stringify(output, null, 2));
|
|
@@ -16437,14 +18465,14 @@ var depsCommand = command({
|
|
|
16437
18465
|
|
|
16438
18466
|
// src/commands/workspace/list.ts
|
|
16439
18467
|
import { existsSync as existsSync20 } from "node:fs";
|
|
16440
|
-
import { readFile as
|
|
16441
|
-
import
|
|
18468
|
+
import { readFile as readFile11, readdir as readdir6, stat as stat3 } from "node:fs/promises";
|
|
18469
|
+
import path35 from "node:path";
|
|
16442
18470
|
async function getDirectorySize(dirPath) {
|
|
16443
18471
|
let totalSize = 0;
|
|
16444
18472
|
try {
|
|
16445
18473
|
const entries2 = await readdir6(dirPath, { withFileTypes: true });
|
|
16446
18474
|
for (const entry of entries2) {
|
|
16447
|
-
const fullPath =
|
|
18475
|
+
const fullPath = path35.join(dirPath, entry.name);
|
|
16448
18476
|
if (entry.isDirectory()) {
|
|
16449
18477
|
totalSize += await getDirectorySize(fullPath);
|
|
16450
18478
|
} else {
|
|
@@ -16479,14 +18507,14 @@ var listCommand = command({
|
|
|
16479
18507
|
return;
|
|
16480
18508
|
}
|
|
16481
18509
|
for (const dir of poolDirs) {
|
|
16482
|
-
const poolDir =
|
|
18510
|
+
const poolDir = path35.join(poolRoot, dir.name);
|
|
16483
18511
|
const fingerprint = dir.name;
|
|
16484
18512
|
const poolEntries = await readdir6(poolDir, { withFileTypes: true });
|
|
16485
18513
|
const slots = poolEntries.filter((e) => e.isDirectory() && e.name.startsWith("slot-"));
|
|
16486
|
-
const metadataPath =
|
|
18514
|
+
const metadataPath = path35.join(poolDir, "metadata.json");
|
|
16487
18515
|
let metadata = null;
|
|
16488
18516
|
try {
|
|
16489
|
-
const raw = await
|
|
18517
|
+
const raw = await readFile11(metadataPath, "utf-8");
|
|
16490
18518
|
metadata = JSON.parse(raw);
|
|
16491
18519
|
} catch {
|
|
16492
18520
|
}
|
|
@@ -16523,17 +18551,17 @@ var workspaceCommand = subcommands({
|
|
|
16523
18551
|
});
|
|
16524
18552
|
|
|
16525
18553
|
// src/update-check.ts
|
|
16526
|
-
import { spawn as
|
|
16527
|
-
import { readFile as
|
|
18554
|
+
import { spawn as spawn4 } from "node:child_process";
|
|
18555
|
+
import { readFile as readFile12 } from "node:fs/promises";
|
|
16528
18556
|
import { join as join6 } from "node:path";
|
|
16529
18557
|
var CHECK_INTERVAL_MS = 24 * 60 * 60 * 1e3;
|
|
16530
18558
|
var CONFIG_DIR = getAgentvConfigDir();
|
|
16531
18559
|
var CACHE_FILE = "version-check.json";
|
|
16532
18560
|
var NPM_REGISTRY_BASE2 = "https://registry.npmjs.org/agentv/";
|
|
16533
|
-
async function getCachedUpdateInfo(
|
|
16534
|
-
const filePath =
|
|
18561
|
+
async function getCachedUpdateInfo(path37) {
|
|
18562
|
+
const filePath = path37 ?? join6(CONFIG_DIR, CACHE_FILE);
|
|
16535
18563
|
try {
|
|
16536
|
-
const raw = await
|
|
18564
|
+
const raw = await readFile12(filePath, "utf-8");
|
|
16537
18565
|
const data = JSON.parse(raw);
|
|
16538
18566
|
if (typeof data.latestVersion === "string" && typeof data.lastCheckedAt === "string") {
|
|
16539
18567
|
return data;
|
|
@@ -16589,7 +18617,7 @@ function backgroundUpdateCheck(distTag = "latest") {
|
|
|
16589
18617
|
}).on('error', () => process.exit()).on('timeout', function() { this.destroy(); process.exit(); });
|
|
16590
18618
|
`;
|
|
16591
18619
|
try {
|
|
16592
|
-
const child =
|
|
18620
|
+
const child = spawn4(process.execPath, ["-e", script], {
|
|
16593
18621
|
detached: true,
|
|
16594
18622
|
stdio: "ignore",
|
|
16595
18623
|
windowsHide: true
|
|
@@ -16640,7 +18668,8 @@ var app = subcommands({
|
|
|
16640
18668
|
workspace: workspaceCommand
|
|
16641
18669
|
}
|
|
16642
18670
|
});
|
|
16643
|
-
var EVAL_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "assert", "aggregate", "bundle"]);
|
|
18671
|
+
var EVAL_SUBCOMMANDS = /* @__PURE__ */ new Set(["run", "assert", "aggregate", "bundle", "vitest"]);
|
|
18672
|
+
var VITEST_VERIFIER_RE = /(?:^|[/\\])(?:EVAL|[^/\\]+[.-](?:test|spec))\.[cm]?[jt]sx?$/i;
|
|
16644
18673
|
var TOP_LEVEL_COMMANDS = /* @__PURE__ */ new Set([
|
|
16645
18674
|
"import",
|
|
16646
18675
|
"inspect",
|
|
@@ -16668,6 +18697,12 @@ var TOP_LEVEL_COMMANDS = /* @__PURE__ */ new Set([
|
|
|
16668
18697
|
function usesDeprecatedStudioAlias(argv) {
|
|
16669
18698
|
return argv[2] === "studio";
|
|
16670
18699
|
}
|
|
18700
|
+
function shouldRunBeforeSessionHook(argv) {
|
|
18701
|
+
return !(argv[2] === "eval" && argv[3] === "vitest");
|
|
18702
|
+
}
|
|
18703
|
+
function inferEvalSubcommand(arg) {
|
|
18704
|
+
return arg && VITEST_VERIFIER_RE.test(arg) ? "vitest" : "run";
|
|
18705
|
+
}
|
|
16671
18706
|
function preprocessArgv(argv) {
|
|
16672
18707
|
const result = [...argv];
|
|
16673
18708
|
if (result[2] === "studio") {
|
|
@@ -16688,7 +18723,7 @@ function preprocessArgv(argv) {
|
|
|
16688
18723
|
const isHelp = nextArg === "--help" || nextArg === "-h";
|
|
16689
18724
|
const isKnownSubcommand = nextArg !== void 0 && EVAL_SUBCOMMANDS.has(nextArg);
|
|
16690
18725
|
if (!isHelp && !isKnownSubcommand) {
|
|
16691
|
-
result.splice(evalIdx + 1, 0,
|
|
18726
|
+
result.splice(evalIdx + 1, 0, inferEvalSubcommand(nextArg));
|
|
16692
18727
|
}
|
|
16693
18728
|
}
|
|
16694
18729
|
}
|
|
@@ -16710,12 +18745,14 @@ ${updateNotice}
|
|
|
16710
18745
|
"Warning: `agentv studio` is deprecated and will be removed in a future release. Use `agentv dashboard` instead.\n"
|
|
16711
18746
|
);
|
|
16712
18747
|
}
|
|
16713
|
-
|
|
16714
|
-
|
|
16715
|
-
|
|
16716
|
-
|
|
16717
|
-
|
|
16718
|
-
|
|
18748
|
+
if (shouldRunBeforeSessionHook(processedArgv)) {
|
|
18749
|
+
const cwd = process.cwd();
|
|
18750
|
+
const repoRoot = await findRepoRoot(cwd);
|
|
18751
|
+
const sessionConfig = await loadConfig(path36.join(cwd, "_"), repoRoot);
|
|
18752
|
+
const beforeSessionCommand = sessionConfig?.hooks?.before_session;
|
|
18753
|
+
if (beforeSessionCommand) {
|
|
18754
|
+
runBeforeSessionHook(beforeSessionCommand);
|
|
18755
|
+
}
|
|
16719
18756
|
}
|
|
16720
18757
|
await run(binary(app), processedArgv);
|
|
16721
18758
|
}
|
|
@@ -16723,7 +18760,9 @@ ${updateNotice}
|
|
|
16723
18760
|
export {
|
|
16724
18761
|
app,
|
|
16725
18762
|
usesDeprecatedStudioAlias,
|
|
18763
|
+
shouldRunBeforeSessionHook,
|
|
18764
|
+
inferEvalSubcommand,
|
|
16726
18765
|
preprocessArgv,
|
|
16727
18766
|
runCli
|
|
16728
18767
|
};
|
|
16729
|
-
//# sourceMappingURL=chunk-
|
|
18768
|
+
//# sourceMappingURL=chunk-2N3YSEMI.js.map
|