agentv 5.3.0-next.1 → 5.3.2-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +70 -60
- package/dist/{artifact-writer-JFNIPMKW.js → artifact-writer-KJEOROKQ.js} +5 -5
- package/dist/{chunk-6ZZCDZPD.js → chunk-6262OKXM.js} +21774 -21034
- package/dist/chunk-6262OKXM.js.map +1 -0
- package/dist/chunk-AQ5BIAXF.js +604 -0
- package/dist/chunk-AQ5BIAXF.js.map +1 -0
- package/dist/chunk-BV5VQLI2.js +2 -0
- package/dist/{chunk-V52ATPTT.js → chunk-JGSRUJZQ.js} +186 -32
- package/dist/chunk-JGSRUJZQ.js.map +1 -0
- package/dist/chunk-MMDLXYBX.js +721 -0
- package/dist/chunk-MMDLXYBX.js.map +1 -0
- package/dist/{chunk-FVR4RQFK.js → chunk-QSK3PC44.js} +1131 -580
- package/dist/chunk-QSK3PC44.js.map +1 -0
- package/dist/chunk-TEEXVJWM.js +2 -0
- package/dist/{chunk-BHKQHG26.js → chunk-WIAJ7JAV.js} +186 -64
- package/dist/chunk-WIAJ7JAV.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +17517 -10
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +2 -0
- package/dist/{ts-eval-loader-DQDYRULE-V3377FOL.js → config.js} +7 -7
- package/dist/config.js.map +1 -0
- package/dist/contracts-DsZmZLl8.d.ts +742 -0
- package/dist/contracts.d.ts +2 -0
- package/dist/contracts.js +28 -0
- package/dist/contracts.js.map +1 -0
- package/dist/dashboard/assets/{index-DNgf3qJ2.js → index-BfqOlLOF.js} +1 -1
- package/dist/dashboard/assets/index-Bh8lpGce.css +1 -0
- package/dist/dashboard/assets/index-oPR3ywQb.js +121 -0
- package/dist/dashboard/index.html +2 -2
- package/dist/{dist-6Z7U473R.js → dist-A3SGR7TW.js} +30 -18
- package/dist/dist-A3SGR7TW.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +137 -18
- package/dist/{interactive-RUY3OCBI.js → interactive-DL7N2C7K.js} +24 -24
- package/dist/interactive-DL7N2C7K.js.map +1 -0
- package/dist/provider.d.ts +2 -0
- package/dist/provider.js +24 -0
- package/dist/provider.js.map +1 -0
- package/dist/sdk.d.ts +802 -0
- package/dist/sdk.js +145 -0
- package/dist/sdk.js.map +1 -0
- package/dist/skills/agentv-bench/SKILL.md +14 -13
- package/dist/skills/agentv-bench/agents/analyzer.md +1 -1
- package/dist/skills/agentv-bench/agents/executor.md +1 -1
- package/dist/skills/agentv-bench/references/autoresearch.md +9 -9
- package/dist/skills/agentv-bench/references/environment-adaptation.md +4 -4
- package/dist/skills/agentv-bench/references/eval-yaml-spec.md +30 -47
- package/dist/skills/agentv-bench/references/schemas.md +44 -60
- package/dist/skills/agentv-bench/references/subagent-pipeline.md +20 -18
- package/dist/skills/agentv-eval-migrations/SKILL.md +13 -0
- package/dist/skills/agentv-eval-migrations/references/breaking-changes.md +56 -39
- package/dist/skills/agentv-eval-writer/SKILL.md +101 -60
- package/dist/skills/agentv-eval-writer/references/custom-evaluators.md +15 -10
- package/dist/skills/agentv-eval-writer/references/eval.schema.json +4543 -5189
- package/dist/skills/agentv-eval-writer/references/python-helpers.md +2 -2
- package/dist/skills/agentv-eval-writer/references/rubric-evaluator.md +20 -3
- package/dist/templates/.agentv/providers.yaml +42 -0
- package/dist/templates/.env.example +2 -2
- package/dist/ts-eval-loader-3G5GEAEC-6F52JLPP.js +18 -0
- package/dist/ts-eval-loader-3G5GEAEC-6F52JLPP.js.map +1 -0
- package/package.json +29 -4
- package/dist/chunk-6ZZCDZPD.js.map +0 -1
- package/dist/chunk-BHKQHG26.js.map +0 -1
- package/dist/chunk-FVR4RQFK.js.map +0 -1
- package/dist/chunk-T32NL3E6.js +0 -17973
- package/dist/chunk-T32NL3E6.js.map +0 -1
- package/dist/chunk-V52ATPTT.js.map +0 -1
- package/dist/dashboard/assets/index-D_bokML8.css +0 -1
- package/dist/dashboard/assets/index-r_jSJmlw.js +0 -121
- package/dist/interactive-RUY3OCBI.js.map +0 -1
- package/dist/templates/.agentv/targets.yaml +0 -97
- /package/dist/{artifact-writer-JFNIPMKW.js.map → artifact-writer-KJEOROKQ.js.map} +0 -0
- /package/dist/{dist-6Z7U473R.js.map → chunk-BV5VQLI2.js.map} +0 -0
- /package/dist/{ts-eval-loader-DQDYRULE-V3377FOL.js.map → chunk-TEEXVJWM.js.map} +0 -0
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>AgentV</title>
|
|
7
|
-
<script type="module" crossorigin src="/assets/index-
|
|
8
|
-
<link rel="stylesheet" crossorigin href="/assets/index-
|
|
7
|
+
<script type="module" crossorigin src="/assets/index-oPR3ywQb.js"></script>
|
|
8
|
+
<link rel="stylesheet" crossorigin href="/assets/index-Bh8lpGce.css">
|
|
9
9
|
</head>
|
|
10
10
|
<body class="bg-gray-950 text-gray-100">
|
|
11
11
|
<div id="root"></div>
|
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
getResultsRepoSyncStatus,
|
|
27
27
|
listGitRuns,
|
|
28
28
|
listGitRunsCached,
|
|
29
|
+
loadEnvPathFiles,
|
|
29
30
|
loadTsConfig,
|
|
30
31
|
materializeGitRun,
|
|
31
32
|
normalizeResultsConfig,
|
|
@@ -34,6 +35,8 @@ import {
|
|
|
34
35
|
parseCodexSession,
|
|
35
36
|
parseCopilotEvents,
|
|
36
37
|
parseEnvOutput,
|
|
38
|
+
parseJsonEnv,
|
|
39
|
+
parseShellExportsEnv,
|
|
37
40
|
prepareEvalWorkspace,
|
|
38
41
|
prepareResultsRepoBranch,
|
|
39
42
|
pushResultsRepoBranch,
|
|
@@ -44,6 +47,7 @@ import {
|
|
|
44
47
|
resolveResultsRepoRunsDir,
|
|
45
48
|
resolveResultsRepoUrl,
|
|
46
49
|
runBeforeSessionHook,
|
|
50
|
+
runEnvFromEntries,
|
|
47
51
|
setupWipWorktree,
|
|
48
52
|
stageResultsArtifacts,
|
|
49
53
|
syncProject,
|
|
@@ -54,7 +58,7 @@ import {
|
|
|
54
58
|
transpileEvalYaml,
|
|
55
59
|
transpileEvalYamlFile,
|
|
56
60
|
trimBaselineResult
|
|
57
|
-
} from "./chunk-
|
|
61
|
+
} from "./chunk-JGSRUJZQ.js";
|
|
58
62
|
import {
|
|
59
63
|
AGENTV_CONFIG_FILE_NAME,
|
|
60
64
|
AGENTV_CONFIG_YML_FILE_NAME,
|
|
@@ -68,7 +72,7 @@ import {
|
|
|
68
72
|
CANONICAL_METRICS_ARTIFACT_PATH,
|
|
69
73
|
CANONICAL_TRANSCRIPT_ARTIFACT_PATH,
|
|
70
74
|
CANONICAL_TRANSCRIPT_TOOL_NAMES,
|
|
71
|
-
|
|
75
|
+
COMMON_PROVIDER_SETTINGS,
|
|
72
76
|
CostGrader,
|
|
73
77
|
DEFAULT_CATEGORY,
|
|
74
78
|
DEFAULT_EVAL_PATTERNS,
|
|
@@ -106,7 +110,6 @@ import {
|
|
|
106
110
|
ResponseCache,
|
|
107
111
|
ResultRowSchemaError,
|
|
108
112
|
ScriptGrader,
|
|
109
|
-
SkillTriggerGrader,
|
|
110
113
|
SkillUsedGrader,
|
|
111
114
|
TEST_MESSAGE_ROLES,
|
|
112
115
|
TRACE_EVENT_TYPES,
|
|
@@ -219,7 +222,7 @@ import {
|
|
|
219
222
|
extractFailOnError,
|
|
220
223
|
extractImageBlocks,
|
|
221
224
|
extractJsonBlob,
|
|
222
|
-
|
|
225
|
+
extractLastAssistantContent2,
|
|
223
226
|
extractTargetFromSuite,
|
|
224
227
|
extractTargetRefsFromSuite,
|
|
225
228
|
extractTargetsFromSuite,
|
|
@@ -266,8 +269,9 @@ import {
|
|
|
266
269
|
isPlainConfigObject,
|
|
267
270
|
isTestMessage,
|
|
268
271
|
isTestMessageRole,
|
|
272
|
+
isTypeScriptEvalConfigFileName,
|
|
269
273
|
killAllTrackedChildren,
|
|
270
|
-
|
|
274
|
+
listProviderLabels,
|
|
271
275
|
loadConfig,
|
|
272
276
|
loadEvalCaseById,
|
|
273
277
|
loadEvalCases,
|
|
@@ -297,9 +301,9 @@ import {
|
|
|
297
301
|
parseTraceSummaryBoundary,
|
|
298
302
|
parseYamlValue,
|
|
299
303
|
readJsonFile,
|
|
304
|
+
readProviderDefinitions,
|
|
300
305
|
readReplayFixtureRecords,
|
|
301
306
|
readRunConfigArtifact,
|
|
302
|
-
readTargetDefinitions,
|
|
303
307
|
readTestSuiteMetadata,
|
|
304
308
|
readTextFile,
|
|
305
309
|
readTraceEnvelopeReplayRecords,
|
|
@@ -310,11 +314,12 @@ import {
|
|
|
310
314
|
replayFixtureRecordToProviderResponse,
|
|
311
315
|
replayLookupIdentityMatches,
|
|
312
316
|
resolveAndCreateProvider,
|
|
313
|
-
|
|
317
|
+
resolveDelegatedProviderDefinition,
|
|
314
318
|
resolveFileReference,
|
|
319
|
+
resolveProviderDefinition,
|
|
320
|
+
resolveProviderDefinitionEnvironments,
|
|
315
321
|
resolveRepoCloneUrl,
|
|
316
322
|
resolveResultsConfigForProject,
|
|
317
|
-
resolveTargetDefinition,
|
|
318
323
|
resolveWorkspaceTemplate,
|
|
319
324
|
rubricEvaluationSchema,
|
|
320
325
|
runContainsAllAssertion,
|
|
@@ -373,16 +378,17 @@ import {
|
|
|
373
378
|
trackChild,
|
|
374
379
|
trackedChildCount,
|
|
375
380
|
transcriptReplayRecordToProviderResponse,
|
|
381
|
+
typeScriptEvalConfigGlob,
|
|
376
382
|
writeArtifacts,
|
|
377
383
|
writeArtifactsFromResults,
|
|
378
384
|
writeInitialRunSummaryArtifact,
|
|
379
385
|
writePerTestArtifacts
|
|
380
|
-
} from "./chunk-
|
|
386
|
+
} from "./chunk-6262OKXM.js";
|
|
387
|
+
import "./chunk-M7BUKBAF.js";
|
|
381
388
|
import "./chunk-7BGERE6L.js";
|
|
382
389
|
import {
|
|
383
390
|
DockerWorkspaceProvider
|
|
384
391
|
} from "./chunk-PEUTJS7B.js";
|
|
385
|
-
import "./chunk-M7BUKBAF.js";
|
|
386
392
|
import "./chunk-QMRVH5ZP.js";
|
|
387
393
|
export {
|
|
388
394
|
AGENTV_CONFIG_FILE_NAME,
|
|
@@ -397,7 +403,7 @@ export {
|
|
|
397
403
|
CANONICAL_METRICS_ARTIFACT_PATH,
|
|
398
404
|
CANONICAL_TRANSCRIPT_ARTIFACT_PATH,
|
|
399
405
|
CANONICAL_TRANSCRIPT_TOOL_NAMES,
|
|
400
|
-
|
|
406
|
+
COMMON_PROVIDER_SETTINGS,
|
|
401
407
|
CostGrader,
|
|
402
408
|
DEFAULT_CATEGORY,
|
|
403
409
|
DEFAULT_EVAL_PATTERNS,
|
|
@@ -437,7 +443,6 @@ export {
|
|
|
437
443
|
ResultRowSchemaError,
|
|
438
444
|
RunBudgetTracker,
|
|
439
445
|
ScriptGrader,
|
|
440
|
-
SkillTriggerGrader,
|
|
441
446
|
SkillUsedGrader,
|
|
442
447
|
TEST_MESSAGE_ROLES,
|
|
443
448
|
TRACE_EVENT_TYPES,
|
|
@@ -568,7 +573,7 @@ export {
|
|
|
568
573
|
extractFailOnError,
|
|
569
574
|
extractImageBlocks,
|
|
570
575
|
extractJsonBlob,
|
|
571
|
-
extractLastAssistantContent,
|
|
576
|
+
extractLastAssistantContent2 as extractLastAssistantContent,
|
|
572
577
|
extractTargetFromSuite,
|
|
573
578
|
extractTargetRefsFromSuite,
|
|
574
579
|
extractTargetsFromSuite,
|
|
@@ -620,11 +625,13 @@ export {
|
|
|
620
625
|
isPlainConfigObject,
|
|
621
626
|
isTestMessage,
|
|
622
627
|
isTestMessageRole,
|
|
628
|
+
isTypeScriptEvalConfigFileName,
|
|
623
629
|
killAllTrackedChildren,
|
|
624
630
|
listGitRuns,
|
|
625
631
|
listGitRunsCached,
|
|
626
|
-
|
|
632
|
+
listProviderLabels,
|
|
627
633
|
loadConfig,
|
|
634
|
+
loadEnvPathFiles,
|
|
628
635
|
loadEvalCaseById,
|
|
629
636
|
loadEvalCases,
|
|
630
637
|
loadEvalSuite,
|
|
@@ -654,9 +661,11 @@ export {
|
|
|
654
661
|
parseCopilotEvents,
|
|
655
662
|
parseEnvOutput,
|
|
656
663
|
parseEvaluationResultBoundary,
|
|
664
|
+
parseJsonEnv,
|
|
657
665
|
parseJsonFromText,
|
|
658
666
|
parseJsonSafe,
|
|
659
667
|
parseJsonlResults,
|
|
668
|
+
parseShellExportsEnv,
|
|
660
669
|
parseTraceBoundary,
|
|
661
670
|
parseTraceSummaryBoundary,
|
|
662
671
|
parseYamlValue,
|
|
@@ -666,9 +675,9 @@ export {
|
|
|
666
675
|
pushWipCheckpoint,
|
|
667
676
|
readGitResultArtifact,
|
|
668
677
|
readJsonFile,
|
|
678
|
+
readProviderDefinitions,
|
|
669
679
|
readReplayFixtureRecords,
|
|
670
680
|
readRunConfigArtifact,
|
|
671
|
-
readTargetDefinitions,
|
|
672
681
|
readTestSuiteMetadata,
|
|
673
682
|
readTextFile,
|
|
674
683
|
readTraceEnvelopeReplayRecords,
|
|
@@ -679,15 +688,16 @@ export {
|
|
|
679
688
|
replayFixtureRecordToProviderResponse,
|
|
680
689
|
replayLookupIdentityMatches,
|
|
681
690
|
resolveAndCreateProvider,
|
|
682
|
-
|
|
691
|
+
resolveDelegatedProviderDefinition,
|
|
683
692
|
resolveFileReference,
|
|
684
693
|
resolveGitResultsIndexCacheFile,
|
|
685
694
|
resolveGitRunsRefCommit,
|
|
695
|
+
resolveProviderDefinition,
|
|
696
|
+
resolveProviderDefinitionEnvironments,
|
|
686
697
|
resolveRepoCloneUrl,
|
|
687
698
|
resolveResultsConfigForProject,
|
|
688
699
|
resolveResultsRepoRunsDir,
|
|
689
700
|
resolveResultsRepoUrl,
|
|
690
|
-
resolveTargetDefinition,
|
|
691
701
|
resolveWorkspaceTemplate,
|
|
692
702
|
rubricEvaluationSchema,
|
|
693
703
|
runBeforeSessionHook,
|
|
@@ -695,6 +705,7 @@ export {
|
|
|
695
705
|
runContainsAnyAssertion,
|
|
696
706
|
runContainsAssertion,
|
|
697
707
|
runEndsWithAssertion,
|
|
708
|
+
runEnvFromEntries,
|
|
698
709
|
runEqualsAssertion,
|
|
699
710
|
runEvalCase,
|
|
700
711
|
runEvaluation,
|
|
@@ -757,9 +768,10 @@ export {
|
|
|
757
768
|
transpileEvalYaml,
|
|
758
769
|
transpileEvalYamlFile,
|
|
759
770
|
trimBaselineResult,
|
|
771
|
+
typeScriptEvalConfigGlob,
|
|
760
772
|
writeArtifacts,
|
|
761
773
|
writeArtifactsFromResults,
|
|
762
774
|
writeInitialRunSummaryArtifact,
|
|
763
775
|
writePerTestArtifacts
|
|
764
776
|
};
|
|
765
|
-
//# sourceMappingURL=dist-
|
|
777
|
+
//# sourceMappingURL=dist-A3SGR7TW.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { AssertionCheck, AssertionContext, AssertionHandler, AssertionScore, AssertionType, CodeGraderConfig, CodeGraderOptions, CodeGraderTargetOptions, ContainsGraderConfig, DefinedEvalSuite, EqualsGraderConfig, EvalAssertionConfig, EvalConfig, EvalDefaultTest, EvalDockerEnvironment, EvalDockerEnvironmentMount, EvalDockerEnvironmentResources, EvalEnvironment, EvalEnvironmentSetup, EvalExecution, EvalHostEnvironment, EvalLifecycleHook, EvalLifecycleHooks, EvalMessage, EvalMessageContent, EvalProviderConfig, EvalProviderEntry, EvalProviderMap, EvalProviderRef, EvalRepeat, EvalRequires, EvalTest, EvalTestOptions, EvalTurn, GraderCatalog, GraderCommand, GraderCommonConfig, GraderHelperConfig, GraderHelperOptions, GraderPromptScriptConfig, GraderRubric, GraderRubricCriterion, GraderRubricOperator, GraderScoreRange, IsJsonGraderConfig, LlmRubricGraderConfig, LowerEvalYamlValue, PromptTemplateHandler, ProviderClient, ProviderInfo, ProviderInvocationError, ProviderInvokeRequest, ProviderInvokeResponse, ProviderNotAvailableError, RegexGraderConfig, RegexGraderOptions, ScriptGraderConfig, ScriptGraderOptions, ScriptGraderProviderOptions, ScriptGraderTargetOptions, Workspace, WorkspaceAssertion, WorkspaceCheck, WorkspaceFile, WorkspaceFileAssertionOptions, WorkspaceGraderContext, WorkspaceGraderHandler, WorkspaceGraderReturn, codeGrader, containsGrader, createProviderClient, createWorkspace, defineAssertion, defineCodeGrader, defineEval, definePromptTemplate, defineScriptGrader, defineWorkspaceGrader, equalsGrader, exactGrader, graders, isJsonGrader, jsonGrader, llmRubricGrader, normalizeWorkspaceGraderResult, regexGrader, runWorkspaceGrader, scriptGrader, serializeEvalYaml, toEvalYamlObject } from './sdk.js';
|
|
2
|
+
export { A as AgentVConfig, d as defineConfig, z } from './contracts-DsZmZLl8.js';
|
|
3
|
+
export { CodeGraderHandler, CodeGraderInput, CodeGraderInputSchema, CodeGraderResult, CodeGraderResultSchema, Content, ContentFile, ContentFileSchema, ContentImage, ContentImageSchema, ContentSchema, ContentText, ContentTextSchema, Message, MessageSchema, PromptTemplateInput, PromptTemplateInputSchema, ScriptGraderCheck, ScriptGraderCheckSchema, ScriptGraderHandler, ScriptGraderInput, ScriptGraderInputSchema, ScriptGraderResult, ScriptGraderResultSchema, TRACE_EVENT_TYPES, TRACE_REDACTION_LEVELS, TRACE_SOURCE_KINDS, TRACE_TOOL_STATUSES, TokenUsage, TokenUsageSchema, ToolCall, ToolCallSchema, Trace, TraceArtifact, TraceArtifactSchema, TraceBranch, TraceBranchSchema, TraceError, TraceErrorSchema, TraceEvent, TraceEventSchema, TraceMessage, TraceMessageSchema, TraceModel, TraceModelSchema, TraceRawEvidence, TraceRawEvidenceSchema, TraceRedactionState, TraceRedactionStateSchema, TraceSchema, TraceSession, TraceSessionSchema, TraceSource, TraceSourceRef, TraceSourceRefSchema, TraceSourceSchema, TraceSummary, TraceSummarySchema, TraceTool, TraceToolSchema, VitestWorkspaceGraderOptions, defineVitestWorkspaceGrader, runCodeGrader, runScriptGrader, runVitestWorkspaceGrader, vitestReportToCodeGraderResult, vitestReportToScriptGraderResult } from '@agentv/core/script-grader';
|
|
4
|
+
export { a3 as AssertEntry, a4 as ConversationTurnInput, a8 as EvalAssertionInput, a6 as EvalRunArtifacts, a9 as EvalRunResult, aa as EvalSummary, a7 as EvalTestInput, a2 as evaluate } from './ts-eval-loader-kf6J7wSc.js';
|
package/dist/index.js
CHANGED
|
@@ -1,26 +1,145 @@
|
|
|
1
1
|
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
3
|
+
ProviderInvocationError,
|
|
4
|
+
ProviderNotAvailableError,
|
|
5
|
+
codeGrader,
|
|
6
|
+
containsGrader,
|
|
7
|
+
createProviderClient,
|
|
8
|
+
createWorkspace,
|
|
9
|
+
defineAssertion,
|
|
10
|
+
defineCodeGrader,
|
|
11
|
+
defineEval,
|
|
12
|
+
definePromptTemplate,
|
|
13
|
+
defineScriptGrader,
|
|
14
|
+
defineWorkspaceGrader,
|
|
15
|
+
equalsGrader,
|
|
16
|
+
exactGrader,
|
|
17
|
+
graders,
|
|
18
|
+
isJsonGrader,
|
|
19
|
+
jsonGrader,
|
|
20
|
+
llmRubricGrader,
|
|
21
|
+
normalizeWorkspaceGraderResult,
|
|
22
|
+
regexGrader,
|
|
23
|
+
runWorkspaceGrader,
|
|
24
|
+
scriptGrader,
|
|
25
|
+
serializeEvalYaml,
|
|
26
|
+
toEvalYamlObject
|
|
27
|
+
} from "./chunk-MMDLXYBX.js";
|
|
28
|
+
import {
|
|
29
|
+
CodeGraderInputSchema,
|
|
30
|
+
CodeGraderResultSchema,
|
|
31
|
+
ContentFileSchema,
|
|
32
|
+
ContentImageSchema,
|
|
33
|
+
ContentSchema,
|
|
34
|
+
ContentTextSchema,
|
|
35
|
+
MessageSchema,
|
|
36
|
+
PromptTemplateInputSchema,
|
|
37
|
+
ScriptGraderCheckSchema,
|
|
38
|
+
ScriptGraderInputSchema,
|
|
39
|
+
ScriptGraderResultSchema,
|
|
40
|
+
TRACE_EVENT_TYPES,
|
|
41
|
+
TRACE_REDACTION_LEVELS,
|
|
42
|
+
TRACE_SOURCE_KINDS,
|
|
43
|
+
TRACE_TOOL_STATUSES,
|
|
44
|
+
TokenUsageSchema,
|
|
45
|
+
ToolCallSchema,
|
|
46
|
+
TraceArtifactSchema,
|
|
47
|
+
TraceBranchSchema,
|
|
48
|
+
TraceErrorSchema,
|
|
49
|
+
TraceEventSchema,
|
|
50
|
+
TraceMessageSchema,
|
|
51
|
+
TraceModelSchema,
|
|
52
|
+
TraceRawEvidenceSchema,
|
|
53
|
+
TraceRedactionStateSchema,
|
|
54
|
+
TraceSchema,
|
|
55
|
+
TraceSessionSchema,
|
|
56
|
+
TraceSourceRefSchema,
|
|
57
|
+
TraceSourceSchema,
|
|
58
|
+
TraceSummarySchema,
|
|
59
|
+
TraceToolSchema,
|
|
60
|
+
defineVitestWorkspaceGrader,
|
|
61
|
+
runCodeGrader,
|
|
62
|
+
runScriptGrader,
|
|
63
|
+
runVitestWorkspaceGrader,
|
|
64
|
+
vitestReportToCodeGraderResult,
|
|
65
|
+
vitestReportToScriptGraderResult
|
|
66
|
+
} from "./chunk-AQ5BIAXF.js";
|
|
67
|
+
import "./chunk-TEEXVJWM.js";
|
|
68
|
+
import {
|
|
69
|
+
defineConfig
|
|
70
|
+
} from "./chunk-JGSRUJZQ.js";
|
|
71
|
+
import {
|
|
72
|
+
evaluate,
|
|
73
|
+
external_exports
|
|
74
|
+
} from "./chunk-6262OKXM.js";
|
|
75
|
+
import "./chunk-M7BUKBAF.js";
|
|
14
76
|
import "./chunk-7BGERE6L.js";
|
|
15
77
|
import "./chunk-PEUTJS7B.js";
|
|
16
|
-
import "./chunk-M7BUKBAF.js";
|
|
17
78
|
import "./chunk-QMRVH5ZP.js";
|
|
18
79
|
export {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
80
|
+
CodeGraderInputSchema,
|
|
81
|
+
CodeGraderResultSchema,
|
|
82
|
+
ContentFileSchema,
|
|
83
|
+
ContentImageSchema,
|
|
84
|
+
ContentSchema,
|
|
85
|
+
ContentTextSchema,
|
|
86
|
+
MessageSchema,
|
|
87
|
+
PromptTemplateInputSchema,
|
|
88
|
+
ProviderInvocationError,
|
|
89
|
+
ProviderNotAvailableError,
|
|
90
|
+
ScriptGraderCheckSchema,
|
|
91
|
+
ScriptGraderInputSchema,
|
|
92
|
+
ScriptGraderResultSchema,
|
|
93
|
+
TRACE_EVENT_TYPES,
|
|
94
|
+
TRACE_REDACTION_LEVELS,
|
|
95
|
+
TRACE_SOURCE_KINDS,
|
|
96
|
+
TRACE_TOOL_STATUSES,
|
|
97
|
+
TokenUsageSchema,
|
|
98
|
+
ToolCallSchema,
|
|
99
|
+
TraceArtifactSchema,
|
|
100
|
+
TraceBranchSchema,
|
|
101
|
+
TraceErrorSchema,
|
|
102
|
+
TraceEventSchema,
|
|
103
|
+
TraceMessageSchema,
|
|
104
|
+
TraceModelSchema,
|
|
105
|
+
TraceRawEvidenceSchema,
|
|
106
|
+
TraceRedactionStateSchema,
|
|
107
|
+
TraceSchema,
|
|
108
|
+
TraceSessionSchema,
|
|
109
|
+
TraceSourceRefSchema,
|
|
110
|
+
TraceSourceSchema,
|
|
111
|
+
TraceSummarySchema,
|
|
112
|
+
TraceToolSchema,
|
|
113
|
+
codeGrader,
|
|
114
|
+
containsGrader,
|
|
115
|
+
createProviderClient,
|
|
116
|
+
createWorkspace,
|
|
117
|
+
defineAssertion,
|
|
118
|
+
defineCodeGrader,
|
|
119
|
+
defineConfig,
|
|
120
|
+
defineEval,
|
|
121
|
+
definePromptTemplate,
|
|
122
|
+
defineScriptGrader,
|
|
123
|
+
defineVitestWorkspaceGrader,
|
|
124
|
+
defineWorkspaceGrader,
|
|
125
|
+
equalsGrader,
|
|
126
|
+
evaluate,
|
|
127
|
+
exactGrader,
|
|
128
|
+
graders,
|
|
129
|
+
isJsonGrader,
|
|
130
|
+
jsonGrader,
|
|
131
|
+
llmRubricGrader,
|
|
132
|
+
normalizeWorkspaceGraderResult,
|
|
133
|
+
regexGrader,
|
|
134
|
+
runCodeGrader,
|
|
135
|
+
runScriptGrader,
|
|
136
|
+
runVitestWorkspaceGrader,
|
|
137
|
+
runWorkspaceGrader,
|
|
138
|
+
scriptGrader,
|
|
139
|
+
serializeEvalYaml,
|
|
140
|
+
toEvalYamlObject,
|
|
141
|
+
vitestReportToCodeGraderResult,
|
|
142
|
+
vitestReportToScriptGraderResult,
|
|
143
|
+
external_exports as z
|
|
25
144
|
};
|
|
26
145
|
//# sourceMappingURL=index.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
2
2
|
import {
|
|
3
|
-
|
|
3
|
+
PROVIDER_FILE_CANDIDATES,
|
|
4
4
|
discoverEvalFiles,
|
|
5
5
|
fileExists,
|
|
6
6
|
filterByCategory,
|
|
@@ -8,17 +8,17 @@ import {
|
|
|
8
8
|
getCategories,
|
|
9
9
|
resolveExistingRunPrimaryPath,
|
|
10
10
|
runEvalCommand
|
|
11
|
-
} from "./chunk-
|
|
12
|
-
import "./chunk-
|
|
13
|
-
import "./chunk-
|
|
11
|
+
} from "./chunk-QSK3PC44.js";
|
|
12
|
+
import "./chunk-WIAJ7JAV.js";
|
|
13
|
+
import "./chunk-JGSRUJZQ.js";
|
|
14
14
|
import {
|
|
15
15
|
getAgentvConfigDir,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
} from "./chunk-
|
|
16
|
+
listProviderLabels,
|
|
17
|
+
readProviderDefinitions
|
|
18
|
+
} from "./chunk-6262OKXM.js";
|
|
19
|
+
import "./chunk-M7BUKBAF.js";
|
|
19
20
|
import "./chunk-7BGERE6L.js";
|
|
20
21
|
import "./chunk-PEUTJS7B.js";
|
|
21
|
-
import "./chunk-M7BUKBAF.js";
|
|
22
22
|
import "./chunk-QMRVH5ZP.js";
|
|
23
23
|
|
|
24
24
|
// src/commands/eval/interactive.ts
|
|
@@ -107,7 +107,7 @@ async function promptMainMenu(lastConfig) {
|
|
|
107
107
|
choices.push({
|
|
108
108
|
name: "\u23EF Resume last run",
|
|
109
109
|
value: "resume",
|
|
110
|
-
description: `${dirLabel} (
|
|
110
|
+
description: `${dirLabel} (provider: ${lastConfig.target})`
|
|
111
111
|
});
|
|
112
112
|
}
|
|
113
113
|
}
|
|
@@ -116,7 +116,7 @@ async function promptMainMenu(lastConfig) {
|
|
|
116
116
|
choices.push({
|
|
117
117
|
name: "\u{1F504} Rerun last config",
|
|
118
118
|
value: "rerun",
|
|
119
|
-
description: `${evalCount} eval file(s),
|
|
119
|
+
description: `${evalCount} eval file(s), provider: ${lastConfig.target}`
|
|
120
120
|
});
|
|
121
121
|
}
|
|
122
122
|
choices.push({ name: "\u{1F680} Run new evaluation", value: "new" }, { name: "\u2715 Exit", value: "exit" });
|
|
@@ -184,22 +184,22 @@ async function promptEvalSelection(allFiles) {
|
|
|
184
184
|
}
|
|
185
185
|
async function promptTargetSelection(cwd, firstEvalPath) {
|
|
186
186
|
const repoRoot = await findRepoRoot(cwd);
|
|
187
|
-
const
|
|
188
|
-
if (!
|
|
189
|
-
console.log(`${ANSI_DIM}No
|
|
187
|
+
const providersPath = await findProvidersFile(cwd, repoRoot, firstEvalPath);
|
|
188
|
+
if (!providersPath) {
|
|
189
|
+
console.log(`${ANSI_DIM}No providers.yaml found. Using default provider.${ANSI_RESET}`);
|
|
190
190
|
return "default";
|
|
191
191
|
}
|
|
192
|
-
const definitions = await
|
|
193
|
-
const targetNames =
|
|
192
|
+
const definitions = await readProviderDefinitions(providersPath);
|
|
193
|
+
const targetNames = listProviderLabels(definitions);
|
|
194
194
|
if (targetNames.length === 0) {
|
|
195
195
|
return "default";
|
|
196
196
|
}
|
|
197
197
|
if (targetNames.length === 1) {
|
|
198
|
-
console.log(`${ANSI_DIM}Using
|
|
198
|
+
console.log(`${ANSI_DIM}Using provider: ${targetNames[0]}${ANSI_RESET}`);
|
|
199
199
|
return targetNames[0];
|
|
200
200
|
}
|
|
201
201
|
return search({
|
|
202
|
-
message: "Select a
|
|
202
|
+
message: "Select a provider (type to search)",
|
|
203
203
|
source: async (term) => {
|
|
204
204
|
const filtered = term ? targetNames.filter((t) => t.toLowerCase().includes(term.toLowerCase())) : targetNames;
|
|
205
205
|
return filtered.map((t) => {
|
|
@@ -207,13 +207,13 @@ async function promptTargetSelection(cwd, firstEvalPath) {
|
|
|
207
207
|
return {
|
|
208
208
|
name: t,
|
|
209
209
|
value: t,
|
|
210
|
-
description: def ? `
|
|
210
|
+
description: def ? `backend: ${def.provider}` : void 0
|
|
211
211
|
};
|
|
212
212
|
});
|
|
213
213
|
}
|
|
214
214
|
});
|
|
215
215
|
}
|
|
216
|
-
async function
|
|
216
|
+
async function findProvidersFile(cwd, repoRoot, evalFilePath) {
|
|
217
217
|
const dirsToSearch = [];
|
|
218
218
|
if (evalFilePath) {
|
|
219
219
|
const evalDir = path2.dirname(evalFilePath);
|
|
@@ -228,7 +228,7 @@ async function findTargetsFile(cwd, repoRoot, evalFilePath) {
|
|
|
228
228
|
dirsToSearch.push(repoRoot);
|
|
229
229
|
}
|
|
230
230
|
for (const dir of dirsToSearch) {
|
|
231
|
-
for (const candidate of
|
|
231
|
+
for (const candidate of PROVIDER_FILE_CANDIDATES) {
|
|
232
232
|
const fullPath = `${dir}/${candidate}`;
|
|
233
233
|
if (await fileExists(fullPath)) {
|
|
234
234
|
return fullPath;
|
|
@@ -267,7 +267,7 @@ ${ANSI_BOLD}Review Configuration${ANSI_RESET}`);
|
|
|
267
267
|
console.log(`${ANSI_DIM}${"\u2500".repeat(40)}${ANSI_RESET}`);
|
|
268
268
|
console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}
|
|
269
269
|
${evalDisplay}`);
|
|
270
|
-
console.log(`${ANSI_GREEN}
|
|
270
|
+
console.log(`${ANSI_GREEN}Provider:${ANSI_RESET} ${config.target}`);
|
|
271
271
|
console.log(`${ANSI_GREEN}Workers:${ANSI_RESET} ${config.workers}`);
|
|
272
272
|
console.log(`${ANSI_GREEN}Cache:${ANSI_RESET} ${config.cache ? "yes" : "no"}`);
|
|
273
273
|
console.log(`${ANSI_DIM}${"\u2500".repeat(40)}${ANSI_RESET}`);
|
|
@@ -279,7 +279,7 @@ ${evalDisplay}`);
|
|
|
279
279
|
async function executeConfig(config, opts) {
|
|
280
280
|
const cwd = process.cwd();
|
|
281
281
|
const rawOptions = {
|
|
282
|
-
|
|
282
|
+
provider: config.target,
|
|
283
283
|
workers: config.workers,
|
|
284
284
|
cache: config.cache,
|
|
285
285
|
...opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {},
|
|
@@ -321,7 +321,7 @@ async function promptRetryErrors(config, outputPath) {
|
|
|
321
321
|
${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
|
|
322
322
|
`);
|
|
323
323
|
const rawOptions = {
|
|
324
|
-
|
|
324
|
+
provider: config.target,
|
|
325
325
|
workers: config.workers,
|
|
326
326
|
cache: config.cache,
|
|
327
327
|
retryErrors: outputPath,
|
|
@@ -344,4 +344,4 @@ ${ANSI_DIM}Retrying execution errors...${ANSI_RESET}
|
|
|
344
344
|
export {
|
|
345
345
|
launchInteractiveWizard
|
|
346
346
|
};
|
|
347
|
-
//# sourceMappingURL=interactive-
|
|
347
|
+
//# sourceMappingURL=interactive-DL7N2C7K.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/commands/eval/interactive.ts","../src/commands/eval/last-config.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport path from 'node:path';\nimport { listProviderLabels, readProviderDefinitions } from '@agentv/core';\nimport { checkbox, confirm, number, search, select } from '@inquirer/prompts';\n\nimport { PROVIDER_FILE_CANDIDATES, fileExists } from '../../utils/providers.js';\nimport {\n type DiscoveredEvalFile,\n discoverEvalFiles,\n filterByCategory,\n getCategories,\n} from './discover.js';\nimport { type LastConfig, loadLastConfig, saveLastConfig } from './last-config.js';\nimport { resolveExistingRunPrimaryPath } from './result-layout.js';\nimport { runEvalCommand } from './run-eval.js';\nimport { findRepoRoot } from './shared.js';\n\nconst ANSI_BOLD = '\\x1b[1m';\nconst ANSI_DIM = '\\x1b[2m';\nconst ANSI_CYAN = '\\x1b[36m';\nconst ANSI_GREEN = '\\x1b[32m';\nconst ANSI_RESET = '\\x1b[0m';\n\nexport interface InteractiveConfig {\n readonly evalPaths: readonly string[];\n readonly target: string;\n readonly workers: number;\n readonly cache: boolean;\n}\n\n/**\n * Launch the interactive wizard when `agentv eval` is called with no arguments.\n */\nexport async function launchInteractiveWizard(): Promise<void> {\n const cwd = process.cwd();\n\n console.log(`\\n${ANSI_BOLD}${ANSI_CYAN}AgentV Interactive Mode${ANSI_RESET}\\n`);\n\n const lastConfig = await loadLastConfig();\n const action = await promptMainMenu(lastConfig);\n\n if (action === 'exit') {\n return;\n }\n\n if (action === 'resume' && lastConfig?.outputDir) {\n const relativeDir = path.relative(cwd, lastConfig.outputDir) || lastConfig.outputDir;\n console.log(`\\n${ANSI_DIM}Resuming run at ${relativeDir}...${ANSI_RESET}\\n`);\n await executeConfig(\n {\n evalPaths: lastConfig.evalPaths,\n target: lastConfig.target,\n workers: lastConfig.workers,\n cache: lastConfig.cache,\n },\n { resumeOutputDir: lastConfig.outputDir },\n );\n return;\n }\n\n if (action === 'rerun' && lastConfig) {\n console.log(`\\n${ANSI_DIM}Rerunning last configuration...${ANSI_RESET}\\n`);\n await executeConfig({\n evalPaths: lastConfig.evalPaths,\n target: lastConfig.target,\n workers: lastConfig.workers,\n cache: lastConfig.cache,\n });\n return;\n }\n\n // Run new evaluation flow\n const config = await promptNewEvaluation(cwd);\n if (!config) {\n return;\n }\n\n // Review & confirm\n const confirmed = await promptReviewAndConfirm(config, cwd);\n if (!confirmed) {\n return;\n }\n\n await executeConfig(config);\n}\n\nasync function promptMainMenu(\n lastConfig: LastConfig | undefined,\n): Promise<'new' | 'rerun' | 'resume' | 'exit'> {\n type MenuChoice = 'new' | 'rerun' | 'resume' | 'exit';\n const choices: Array<{ name: string; value: MenuChoice; description?: string }> = [];\n\n // Resume entry: only when the prior run has a known artifact dir with a manifest.\n if (lastConfig?.outputDir) {\n const indexPath = resolveExistingRunPrimaryPath(lastConfig.outputDir);\n if (indexPath && existsSync(indexPath)) {\n const dirLabel = path.basename(lastConfig.outputDir);\n choices.push({\n name: '⏯ Resume last run',\n value: 'resume',\n description: `${dirLabel} (provider: ${lastConfig.target})`,\n });\n }\n }\n\n if (lastConfig) {\n const evalCount = lastConfig.evalPaths.length;\n choices.push({\n name: '🔄 Rerun last config',\n value: 'rerun',\n description: `${evalCount} eval file(s), provider: ${lastConfig.target}`,\n });\n }\n\n choices.push({ name: '🚀 Run new evaluation', value: 'new' }, { name: '✕ Exit', value: 'exit' });\n\n return select<MenuChoice>({\n message: 'What would you like to do?',\n choices,\n });\n}\n\nasync function promptNewEvaluation(cwd: string): Promise<InteractiveConfig | undefined> {\n // Step 1: Discover eval files\n console.log(`\\n${ANSI_DIM}Scanning for eval files...${ANSI_RESET}`);\n const allFiles = await discoverEvalFiles(cwd);\n\n if (allFiles.length === 0) {\n console.log(\n '\\n⚠ No eval files found in the current directory.\\n' +\n ' Place .yaml or .jsonl eval files in your project, or use:\\n' +\n ' agentv eval <path-to-eval.yaml>\\n',\n );\n return undefined;\n }\n\n console.log(`${ANSI_DIM}Found ${allFiles.length} eval file(s)${ANSI_RESET}\\n`);\n\n // Step 2: Select eval files (optionally filter by category first)\n const selectedFiles = await promptEvalSelection(allFiles);\n if (selectedFiles.length === 0) {\n console.log('\\nNo eval files selected.');\n return undefined;\n }\n\n // Step 3: Select provider\n const target = await promptTargetSelection(cwd, selectedFiles[0].path);\n\n // Step 4: Advanced options\n const advanced = await promptAdvancedOptions();\n\n return {\n evalPaths: selectedFiles.map((f) => f.path),\n target,\n ...advanced,\n };\n}\n\nasync function promptEvalSelection(\n allFiles: readonly DiscoveredEvalFile[],\n): Promise<DiscoveredEvalFile[]> {\n const categories = getCategories(allFiles);\n\n // If only one category or few files, skip category selection\n let filesToSelect: readonly DiscoveredEvalFile[];\n\n if (categories.length > 1) {\n const selectedCategory = await search<string>({\n message: 'Select a category (type to search)',\n source: async (term) => {\n const filtered = term\n ? categories.filter((c) => c.toLowerCase().includes(term.toLowerCase()))\n : categories;\n return [\n { name: '(all categories)', value: '__all__' },\n ...filtered.map((c) => {\n const count = filterByCategory(allFiles, c).length;\n return { name: `${c} (${count} file${count > 1 ? 's' : ''})`, value: c };\n }),\n ];\n },\n });\n\n filesToSelect =\n selectedCategory === '__all__' ? allFiles : filterByCategory(allFiles, selectedCategory);\n } else {\n filesToSelect = allFiles;\n }\n\n return checkbox<DiscoveredEvalFile>({\n message: 'Select eval files to run (space to toggle, enter to confirm)',\n choices: filesToSelect.map((f) => ({\n name: f.relativePath,\n value: f,\n checked: filesToSelect.length <= 5, // auto-select if few files\n })),\n required: true,\n });\n}\n\nasync function promptTargetSelection(cwd: string, firstEvalPath: string): Promise<string> {\n const repoRoot = await findRepoRoot(cwd);\n\n // Try to find providers.yaml near the eval file first, then cwd/repoRoot.\n const providersPath = await findProvidersFile(cwd, repoRoot, firstEvalPath);\n\n if (!providersPath) {\n console.log(`${ANSI_DIM}No providers.yaml found. Using default provider.${ANSI_RESET}`);\n return 'default';\n }\n\n const definitions = await readProviderDefinitions(providersPath);\n const targetNames = listProviderLabels(definitions);\n\n if (targetNames.length === 0) {\n return 'default';\n }\n\n if (targetNames.length === 1) {\n console.log(`${ANSI_DIM}Using provider: ${targetNames[0]}${ANSI_RESET}`);\n return targetNames[0];\n }\n\n return search<string>({\n message: 'Select a provider (type to search)',\n source: async (term) => {\n const filtered = term\n ? targetNames.filter((t) => t.toLowerCase().includes(term.toLowerCase()))\n : targetNames;\n return filtered.map((t) => {\n const def = definitions.find((d) => d.name === t);\n return {\n name: t,\n value: t,\n description: def ? `backend: ${def.provider}` : undefined,\n };\n });\n },\n });\n}\n\nasync function findProvidersFile(\n cwd: string,\n repoRoot: string,\n evalFilePath?: string,\n): Promise<string | undefined> {\n // Build directory chain: eval file dir -> cwd -> repoRoot.\n const dirsToSearch: string[] = [];\n\n if (evalFilePath) {\n const evalDir = path.dirname(evalFilePath);\n if (!dirsToSearch.includes(evalDir)) {\n dirsToSearch.push(evalDir);\n }\n }\n\n if (!dirsToSearch.includes(cwd)) {\n dirsToSearch.push(cwd);\n }\n\n if (repoRoot !== cwd && !dirsToSearch.includes(repoRoot)) {\n dirsToSearch.push(repoRoot);\n }\n\n for (const dir of dirsToSearch) {\n for (const candidate of PROVIDER_FILE_CANDIDATES) {\n const fullPath = `${dir}/${candidate}`;\n if (await fileExists(fullPath)) {\n return fullPath;\n }\n }\n }\n\n return undefined;\n}\n\nasync function promptAdvancedOptions(): Promise<{\n workers: number;\n cache: boolean;\n}> {\n const customize = await confirm({\n message: 'Configure advanced options?',\n default: false,\n });\n\n if (!customize) {\n return { workers: 3, cache: false };\n }\n\n const workers =\n (await number({\n message: 'Number of parallel workers (1-50)',\n default: 3,\n min: 1,\n max: 50,\n })) ?? 3;\n\n const cache = await confirm({\n message: 'Enable response cache?',\n default: false,\n });\n\n return { workers, cache };\n}\n\nasync function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): Promise<boolean> {\n const evalDisplay = config.evalPaths\n .map((p) => {\n const rel = p.startsWith(cwd) ? p.slice(cwd.length + 1) : p;\n return ` ${rel}`;\n })\n .join('\\n');\n\n console.log(`\\n${ANSI_BOLD}Review Configuration${ANSI_RESET}`);\n console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);\n console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}\\n${evalDisplay}`);\n console.log(`${ANSI_GREEN}Provider:${ANSI_RESET} ${config.target}`);\n console.log(`${ANSI_GREEN}Workers:${ANSI_RESET} ${config.workers}`);\n console.log(`${ANSI_GREEN}Cache:${ANSI_RESET} ${config.cache ? 'yes' : 'no'}`);\n console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);\n\n return confirm({\n message: 'Run evaluation with this configuration?',\n default: true,\n });\n}\n\nasync function executeConfig(\n config: InteractiveConfig,\n opts?: { resumeOutputDir?: string },\n): Promise<void> {\n const cwd = process.cwd();\n const rawOptions: Record<string, unknown> = {\n provider: config.target,\n workers: config.workers,\n cache: config.cache,\n ...(opts?.resumeOutputDir ? { output: opts.resumeOutputDir, resume: true } : {}),\n agentTimeout: 120,\n maxRetries: 2,\n verbose: false,\n keepWorkspaces: false,\n cleanupWorkspaces: false,\n trace: false,\n };\n\n const result = await runEvalCommand({\n testFiles: [...config.evalPaths],\n rawOptions,\n });\n\n // Persist config with the resolved artifact dir so the wizard can offer\n // \"Resume last run\" on the next invocation. Done after a successful run so\n // the saved outputDir always points at a real run manifest.\n if (result) {\n await saveLastConfig({\n timestamp: new Date().toISOString(),\n cwd,\n evalPaths: config.evalPaths,\n target: config.target,\n workers: config.workers,\n cache: config.cache,\n outputDir: path.dirname(result.outputPath),\n });\n }\n\n // Prompt to retry errors when execution errors were detected in a TTY\n if (result && result.executionErrorCount > 0 && process.stdin.isTTY) {\n await promptRetryErrors(config, result.outputPath);\n }\n}\n\nasync function promptRetryErrors(config: InteractiveConfig, outputPath: string): Promise<void> {\n const shouldRetry = await confirm({\n message: `Retry ${ANSI_BOLD}execution errors${ANSI_RESET} from this run?`,\n default: true,\n });\n\n if (!shouldRetry) {\n return;\n }\n\n console.log(`\\n${ANSI_DIM}Retrying execution errors...${ANSI_RESET}\\n`);\n\n const rawOptions: Record<string, unknown> = {\n provider: config.target,\n workers: config.workers,\n cache: config.cache,\n retryErrors: outputPath,\n out: outputPath,\n agentTimeout: 120,\n maxRetries: 2,\n verbose: false,\n keepWorkspaces: false,\n cleanupWorkspaces: false,\n trace: false,\n };\n\n const retryResult = await runEvalCommand({\n testFiles: [...config.evalPaths],\n rawOptions,\n });\n\n // Allow chained retries if there are still errors\n if (retryResult && retryResult.executionErrorCount > 0 && process.stdin.isTTY) {\n await promptRetryErrors(config, retryResult.outputPath);\n }\n}\n","import { mkdir, readFile, writeFile } from 'node:fs/promises';\nimport path from 'node:path';\nimport { getAgentvConfigDir } from '@agentv/core';\n\nconst CONFIG_DIR = getAgentvConfigDir();\nconst LAST_CONFIG_PATH = path.join(CONFIG_DIR, 'last-config.json');\n\nexport interface LastConfig {\n readonly timestamp: string;\n readonly cwd: string;\n readonly evalPaths: readonly string[];\n readonly target: string;\n readonly workers: number;\n readonly cache: boolean;\n /**\n * Resolved artifact directory of the last completed wizard run. Used to\n * power the wizard's \"Resume last run\" entry. Optional for backward\n * compatibility with configs saved before this field existed.\n */\n readonly outputDir?: string;\n}\n\nexport async function loadLastConfig(): Promise<LastConfig | undefined> {\n try {\n const content = await readFile(LAST_CONFIG_PATH, 'utf-8');\n return JSON.parse(content) as LastConfig;\n } catch {\n return undefined;\n }\n}\n\nexport async function saveLastConfig(config: LastConfig): Promise<void> {\n await mkdir(CONFIG_DIR, { recursive: true });\n await writeFile(LAST_CONFIG_PATH, JSON.stringify(config, null, 2), 'utf-8');\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,kBAAkB;AAC3B,OAAOA,WAAU;AAEjB,SAAS,UAAU,SAAS,QAAQ,QAAQ,cAAc;;;ACH1D,SAAS,OAAO,UAAU,iBAAiB;AAC3C,OAAO,UAAU;AAGjB,IAAM,aAAa,mBAAmB;AACtC,IAAM,mBAAmB,KAAK,KAAK,YAAY,kBAAkB;AAiBjE,eAAsB,iBAAkD;AACtE,MAAI;AACF,UAAM,UAAU,MAAM,SAAS,kBAAkB,OAAO;AACxD,WAAO,KAAK,MAAM,OAAO;AAAA,EAC3B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,eAAe,QAAmC;AACtE,QAAM,MAAM,YAAY,EAAE,WAAW,KAAK,CAAC;AAC3C,QAAM,UAAU,kBAAkB,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AAC5E;;;ADjBA,IAAM,YAAY;AAClB,IAAM,WAAW;AACjB,IAAM,YAAY;AAClB,IAAM,aAAa;AACnB,IAAM,aAAa;AAYnB,eAAsB,0BAAyC;AAC7D,QAAM,MAAM,QAAQ,IAAI;AAExB,UAAQ,IAAI;AAAA,EAAK,SAAS,GAAG,SAAS,0BAA0B,UAAU;AAAA,CAAI;AAE9E,QAAM,aAAa,MAAM,eAAe;AACxC,QAAM,SAAS,MAAM,eAAe,UAAU;AAE9C,MAAI,WAAW,QAAQ;AACrB;AAAA,EACF;AAEA,MAAI,WAAW,YAAY,YAAY,WAAW;AAChD,UAAM,cAAcC,MAAK,SAAS,KAAK,WAAW,SAAS,KAAK,WAAW;AAC3E,YAAQ,IAAI;AAAA,EAAK,QAAQ,mBAAmB,WAAW,MAAM,UAAU;AAAA,CAAI;AAC3E,UAAM;AAAA,MACJ;AAAA,QACE,WAAW,WAAW;AAAA,QACtB,QAAQ,WAAW;AAAA,QACnB,SAAS,WAAW;AAAA,QACpB,OAAO,WAAW;AAAA,MACpB;AAAA,MACA,EAAE,iBAAiB,WAAW,UAAU;AAAA,IAC1C;AACA;AAAA,EACF;AAEA,MAAI,WAAW,WAAW,YAAY;AACpC,YAAQ,IAAI;AAAA,EAAK,QAAQ,kCAAkC,UAAU;AAAA,CAAI;AACzE,UAAM,cAAc;AAAA,MAClB,WAAW,WAAW;AAAA,MACtB,QAAQ,WAAW;AAAA,MACnB,SAAS,WAAW;AAAA,MACpB,OAAO,WAAW;AAAA,IACpB,CAAC;AACD;AAAA,EACF;AAGA,QAAM,SAAS,MAAM,oBAAoB,GAAG;AAC5C,MAAI,CAAC,QAAQ;AACX;AAAA,EACF;AAGA,QAAM,YAAY,MAAM,uBAAuB,QAAQ,GAAG;AAC1D,MAAI,CAAC,WAAW;AACd;AAAA,EACF;AAEA,QAAM,cAAc,MAAM;AAC5B;AAEA,eAAe,eACb,YAC8C;AAE9C,QAAM,UAA4E,CAAC;AAGnF,MAAI,YAAY,WAAW;AACzB,UAAM,YAAY,8BAA8B,WAAW,SAAS;AACpE,QAAI,aAAa,WAAW,SAAS,GAAG;AACtC,YAAM,WAAWA,MAAK,SAAS,WAAW,SAAS;AACnD,cAAQ,KAAK;AAAA,QACX,MAAM;AAAA,QACN,OAAO;AAAA,QACP,aAAa,GAAG,QAAQ,eAAe,WAAW,MAAM;AAAA,MAC1D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI,YAAY;AACd,UAAM,YAAY,WAAW,UAAU;AACvC,YAAQ,KAAK;AAAA,MACX,MAAM;AAAA,MACN,OAAO;AAAA,MACP,aAAa,GAAG,SAAS,4BAA4B,WAAW,MAAM;AAAA,IACxE,CAAC;AAAA,EACH;AAEA,UAAQ,KAAK,EAAE,MAAM,gCAAyB,OAAO,MAAM,GAAG,EAAE,MAAM,eAAU,OAAO,OAAO,CAAC;AAE/F,SAAO,OAAmB;AAAA,IACxB,SAAS;AAAA,IACT;AAAA,EACF,CAAC;AACH;AAEA,eAAe,oBAAoB,KAAqD;AAEtF,UAAQ,IAAI;AAAA,EAAK,QAAQ,6BAA6B,UAAU,EAAE;AAClE,QAAM,WAAW,MAAM,kBAAkB,GAAG;AAE5C,MAAI,SAAS,WAAW,GAAG;AACzB,YAAQ;AAAA,MACN;AAAA,IAGF;AACA,WAAO;AAAA,EACT;AAEA,UAAQ,IAAI,GAAG,QAAQ,SAAS,SAAS,MAAM,gBAAgB,UAAU;AAAA,CAAI;AAG7E,QAAM,gBAAgB,MAAM,oBAAoB,QAAQ;AACxD,MAAI,cAAc,WAAW,GAAG;AAC9B,YAAQ,IAAI,2BAA2B;AACvC,WAAO;AAAA,EACT;AAGA,QAAM,SAAS,MAAM,sBAAsB,KAAK,cAAc,CAAC,EAAE,IAAI;AAGrE,QAAM,WAAW,MAAM,sBAAsB;AAE7C,SAAO;AAAA,IACL,WAAW,cAAc,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,IAC1C;AAAA,IACA,GAAG;AAAA,EACL;AACF;AAEA,eAAe,oBACb,UAC+B;AAC/B,QAAM,aAAa,cAAc,QAAQ;AAGzC,MAAI;AAEJ,MAAI,WAAW,SAAS,GAAG;AACzB,UAAM,mBAAmB,MAAM,OAAe;AAAA,MAC5C,SAAS;AAAA,MACT,QAAQ,OAAO,SAAS;AACtB,cAAM,WAAW,OACb,WAAW,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC,IACrE;AACJ,eAAO;AAAA,UACL,EAAE,MAAM,oBAAoB,OAAO,UAAU;AAAA,UAC7C,GAAG,SAAS,IAAI,CAAC,MAAM;AACrB,kBAAM,QAAQ,iBAAiB,UAAU,CAAC,EAAE;AAC5C,mBAAO,EAAE,MAAM,GAAG,CAAC,KAAK,KAAK,QAAQ,QAAQ,IAAI,MAAM,EAAE,KAAK,OAAO,EAAE;AAAA,UACzE,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF,CAAC;AAED,oBACE,qBAAqB,YAAY,WAAW,iBAAiB,UAAU,gBAAgB;AAAA,EAC3F,OAAO;AACL,oBAAgB;AAAA,EAClB;AAEA,SAAO,SAA6B;AAAA,IAClC,SAAS;AAAA,IACT,SAAS,cAAc,IAAI,CAAC,OAAO;AAAA,MACjC,MAAM,EAAE;AAAA,MACR,OAAO;AAAA,MACP,SAAS,cAAc,UAAU;AAAA;AAAA,IACnC,EAAE;AAAA,IACF,UAAU;AAAA,EACZ,CAAC;AACH;AAEA,eAAe,sBAAsB,KAAa,eAAwC;AACxF,QAAM,WAAW,MAAM,aAAa,GAAG;AAGvC,QAAM,gBAAgB,MAAM,kBAAkB,KAAK,UAAU,aAAa;AAE1E,MAAI,CAAC,eAAe;AAClB,YAAQ,IAAI,GAAG,QAAQ,mDAAmD,UAAU,EAAE;AACtF,WAAO;AAAA,EACT;AAEA,QAAM,cAAc,MAAM,wBAAwB,aAAa;AAC/D,QAAM,cAAc,mBAAmB,WAAW;AAElD,MAAI,YAAY,WAAW,GAAG;AAC5B,WAAO;AAAA,EACT;AAEA,MAAI,YAAY,WAAW,GAAG;AAC5B,YAAQ,IAAI,GAAG,QAAQ,mBAAmB,YAAY,CAAC,CAAC,GAAG,UAAU,EAAE;AACvE,WAAO,YAAY,CAAC;AAAA,EACtB;AAEA,SAAO,OAAe;AAAA,IACpB,SAAS;AAAA,IACT,QAAQ,OAAO,SAAS;AACtB,YAAM,WAAW,OACb,YAAY,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC,IACtE;AACJ,aAAO,SAAS,IAAI,CAAC,MAAM;AACzB,cAAM,MAAM,YAAY,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC;AAChD,eAAO;AAAA,UACL,MAAM;AAAA,UACN,OAAO;AAAA,UACP,aAAa,MAAM,YAAY,IAAI,QAAQ,KAAK;AAAA,QAClD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;AAEA,eAAe,kBACb,KACA,UACA,cAC6B;AAE7B,QAAM,eAAyB,CAAC;AAEhC,MAAI,cAAc;AAChB,UAAM,UAAUA,MAAK,QAAQ,YAAY;AACzC,QAAI,CAAC,aAAa,SAAS,OAAO,GAAG;AACnC,mBAAa,KAAK,OAAO;AAAA,IAC3B;AAAA,EACF;AAEA,MAAI,CAAC,aAAa,SAAS,GAAG,GAAG;AAC/B,iBAAa,KAAK,GAAG;AAAA,EACvB;AAEA,MAAI,aAAa,OAAO,CAAC,aAAa,SAAS,QAAQ,GAAG;AACxD,iBAAa,KAAK,QAAQ;AAAA,EAC5B;AAEA,aAAW,OAAO,cAAc;AAC9B,eAAW,aAAa,0BAA0B;AAChD,YAAM,WAAW,GAAG,GAAG,IAAI,SAAS;AACpC,UAAI,MAAM,WAAW,QAAQ,GAAG;AAC9B,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAe,wBAGZ;AACD,QAAM,YAAY,MAAM,QAAQ;AAAA,IAC9B,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AAED,MAAI,CAAC,WAAW;AACd,WAAO,EAAE,SAAS,GAAG,OAAO,MAAM;AAAA,EACpC;AAEA,QAAM,UACH,MAAM,OAAO;AAAA,IACZ,SAAS;AAAA,IACT,SAAS;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AAAA,EACP,CAAC,KAAM;AAET,QAAM,QAAQ,MAAM,QAAQ;AAAA,IAC1B,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AAED,SAAO,EAAE,SAAS,MAAM;AAC1B;AAEA,eAAe,uBAAuB,QAA2B,KAA+B;AAC9F,QAAM,cAAc,OAAO,UACxB,IAAI,CAAC,MAAM;AACV,UAAM,MAAM,EAAE,WAAW,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC,IAAI;AAC1D,WAAO,KAAK,GAAG;AAAA,EACjB,CAAC,EACA,KAAK,IAAI;AAEZ,UAAQ,IAAI;AAAA,EAAK,SAAS,uBAAuB,UAAU,EAAE;AAC7D,UAAQ,IAAI,GAAG,QAAQ,GAAG,SAAI,OAAO,EAAE,CAAC,GAAG,UAAU,EAAE;AACvD,UAAQ,IAAI,GAAG,UAAU,cAAc,UAAU;AAAA,EAAK,WAAW,EAAE;AACnE,UAAQ,IAAI,GAAG,UAAU,YAAY,UAAU,KAAK,OAAO,MAAM,EAAE;AACnE,UAAQ,IAAI,GAAG,UAAU,WAAW,UAAU,MAAM,OAAO,OAAO,EAAE;AACpE,UAAQ,IAAI,GAAG,UAAU,SAAS,UAAU,QAAQ,OAAO,QAAQ,QAAQ,IAAI,EAAE;AACjF,UAAQ,IAAI,GAAG,QAAQ,GAAG,SAAI,OAAO,EAAE,CAAC,GAAG,UAAU,EAAE;AAEvD,SAAO,QAAQ;AAAA,IACb,SAAS;AAAA,IACT,SAAS;AAAA,EACX,CAAC;AACH;AAEA,eAAe,cACb,QACA,MACe;AACf,QAAM,MAAM,QAAQ,IAAI;AACxB,QAAM,aAAsC;AAAA,IAC1C,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,IACd,GAAI,MAAM,kBAAkB,EAAE,QAAQ,KAAK,iBAAiB,QAAQ,KAAK,IAAI,CAAC;AAAA,IAC9E,cAAc;AAAA,IACd,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,mBAAmB;AAAA,IACnB,OAAO;AAAA,EACT;AAEA,QAAM,SAAS,MAAM,eAAe;AAAA,IAClC,WAAW,CAAC,GAAG,OAAO,SAAS;AAAA,IAC/B;AAAA,EACF,CAAC;AAKD,MAAI,QAAQ;AACV,UAAM,eAAe;AAAA,MACnB,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,MACA,WAAW,OAAO;AAAA,MAClB,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,MAChB,OAAO,OAAO;AAAA,MACd,WAAWA,MAAK,QAAQ,OAAO,UAAU;AAAA,IAC3C,CAAC;AAAA,EACH;AAGA,MAAI,UAAU,OAAO,sBAAsB,KAAK,QAAQ,MAAM,OAAO;AACnE,UAAM,kBAAkB,QAAQ,OAAO,UAAU;AAAA,EACnD;AACF;AAEA,eAAe,kBAAkB,QAA2B,YAAmC;AAC7F,QAAM,cAAc,MAAM,QAAQ;AAAA,IAChC,SAAS,SAAS,SAAS,mBAAmB,UAAU;AAAA,IACxD,SAAS;AAAA,EACX,CAAC;AAED,MAAI,CAAC,aAAa;AAChB;AAAA,EACF;AAEA,UAAQ,IAAI;AAAA,EAAK,QAAQ,+BAA+B,UAAU;AAAA,CAAI;AAEtE,QAAM,aAAsC;AAAA,IAC1C,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,IACd,aAAa;AAAA,IACb,KAAK;AAAA,IACL,cAAc;AAAA,IACd,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,gBAAgB;AAAA,IAChB,mBAAmB;AAAA,IACnB,OAAO;AAAA,EACT;AAEA,QAAM,cAAc,MAAM,eAAe;AAAA,IACvC,WAAW,CAAC,GAAG,OAAO,SAAS;AAAA,IAC/B;AAAA,EACF,CAAC;AAGD,MAAI,eAAe,YAAY,sBAAsB,KAAK,QAAQ,MAAM,OAAO;AAC7E,UAAM,kBAAkB,QAAQ,YAAY,UAAU;AAAA,EACxD;AACF;","names":["path","path"]}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { P as ProviderFactoryFn, R as ResolvedProviderBackend, c as createBuiltinProviderRegistry, a as createProvider, r as resolveAndCreateProvider, b as resolveDelegatedProviderDefinition, e as resolveProviderDefinition, f as resolveProviderDefinitionEnvironments } from './contracts-DsZmZLl8.js';
|
|
2
|
+
export { e as EnvLookup, M as Message, m as Provider, P as ProviderDefinition, cd as ProviderKind, r as ProviderRequest, h as ProviderResponse, U as ProviderStreamCallbacks, f as ProviderTokenUsage, ce as ToolCall } from './ts-eval-loader-kf6J7wSc.js';
|
package/dist/provider.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { createRequire } from 'node:module'; const require = createRequire(import.meta.url);
|
|
2
|
+
import "./chunk-BV5VQLI2.js";
|
|
3
|
+
import "./chunk-JGSRUJZQ.js";
|
|
4
|
+
import {
|
|
5
|
+
createBuiltinProviderRegistry,
|
|
6
|
+
createProvider,
|
|
7
|
+
resolveAndCreateProvider,
|
|
8
|
+
resolveDelegatedProviderDefinition,
|
|
9
|
+
resolveProviderDefinition,
|
|
10
|
+
resolveProviderDefinitionEnvironments
|
|
11
|
+
} from "./chunk-6262OKXM.js";
|
|
12
|
+
import "./chunk-M7BUKBAF.js";
|
|
13
|
+
import "./chunk-7BGERE6L.js";
|
|
14
|
+
import "./chunk-PEUTJS7B.js";
|
|
15
|
+
import "./chunk-QMRVH5ZP.js";
|
|
16
|
+
export {
|
|
17
|
+
createBuiltinProviderRegistry,
|
|
18
|
+
createProvider,
|
|
19
|
+
resolveAndCreateProvider,
|
|
20
|
+
resolveDelegatedProviderDefinition,
|
|
21
|
+
resolveProviderDefinition,
|
|
22
|
+
resolveProviderDefinitionEnvironments
|
|
23
|
+
};
|
|
24
|
+
//# sourceMappingURL=provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|