@agentv/core 4.30.0-next.1 → 4.31.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-Z2BBOGE4.js → chunk-QY6BS2V4.js} +10 -27
- package/dist/{chunk-Z2BBOGE4.js.map → chunk-QY6BS2V4.js.map} +1 -1
- package/dist/evaluation/validation/index.cjs +32 -44
- package/dist/evaluation/validation/index.cjs.map +1 -1
- package/dist/evaluation/validation/index.js +32 -44
- package/dist/evaluation/validation/index.js.map +1 -1
- package/dist/index.cjs +18 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -14
- package/dist/index.d.ts +12 -14
- package/dist/index.js +11 -11
- package/dist/index.js.map +1 -1
- package/dist/{ts-eval-loader-JL5DGTJL.js → ts-eval-loader-TWWSN6OX.js} +2 -2
- package/package.json +1 -1
- /package/dist/{ts-eval-loader-JL5DGTJL.js.map → ts-eval-loader-TWWSN6OX.js.map} +0 -0
package/dist/index.d.cts
CHANGED
|
@@ -1988,7 +1988,7 @@ type ExecutionDefaults = {
|
|
|
1988
1988
|
readonly pool_workspaces?: boolean;
|
|
1989
1989
|
readonly pool_slots?: number;
|
|
1990
1990
|
};
|
|
1991
|
-
type
|
|
1991
|
+
type ResultsConfig = {
|
|
1992
1992
|
readonly repo: string;
|
|
1993
1993
|
readonly path: string;
|
|
1994
1994
|
readonly auto_push?: boolean;
|
|
@@ -2002,9 +2002,7 @@ type AgentVConfig$1 = {
|
|
|
2002
2002
|
readonly required_version?: string;
|
|
2003
2003
|
readonly eval_patterns?: readonly string[];
|
|
2004
2004
|
readonly execution?: ExecutionDefaults;
|
|
2005
|
-
readonly results?:
|
|
2006
|
-
readonly export?: ResultsExportConfig;
|
|
2007
|
-
};
|
|
2005
|
+
readonly results?: ResultsConfig;
|
|
2008
2006
|
readonly hooks?: HooksConfig;
|
|
2009
2007
|
};
|
|
2010
2008
|
/**
|
|
@@ -4198,27 +4196,27 @@ interface CheckedOutResultsRepoBranch {
|
|
|
4198
4196
|
interface PreparedResultsRepoBranch extends CheckedOutResultsRepoBranch {
|
|
4199
4197
|
readonly cleanup: () => Promise<void>;
|
|
4200
4198
|
}
|
|
4201
|
-
declare function
|
|
4199
|
+
declare function normalizeResultsConfig(config: ResultsConfig): Required<ResultsConfig>;
|
|
4202
4200
|
declare function resolveResultsRepoUrl(repo: string): string;
|
|
4203
4201
|
declare function getResultsRepoCachePaths(repo: string): ResultsRepoCachePaths;
|
|
4204
|
-
declare function ensureResultsRepoClone(config:
|
|
4205
|
-
declare function getResultsRepoStatus(config?:
|
|
4206
|
-
declare function syncResultsRepo(config:
|
|
4207
|
-
declare function checkoutResultsRepoBranch(config:
|
|
4208
|
-
declare function prepareResultsRepoBranch(config:
|
|
4202
|
+
declare function ensureResultsRepoClone(config: ResultsConfig): Promise<string>;
|
|
4203
|
+
declare function getResultsRepoStatus(config?: ResultsConfig): ResultsRepoStatus;
|
|
4204
|
+
declare function syncResultsRepo(config: ResultsConfig): Promise<ResultsRepoStatus>;
|
|
4205
|
+
declare function checkoutResultsRepoBranch(config: ResultsConfig, branchName: string): Promise<CheckedOutResultsRepoBranch>;
|
|
4206
|
+
declare function prepareResultsRepoBranch(config: ResultsConfig, branchName: string): Promise<PreparedResultsRepoBranch>;
|
|
4209
4207
|
declare function stageResultsArtifacts(params: {
|
|
4210
4208
|
readonly repoDir: string;
|
|
4211
4209
|
readonly sourceDir: string;
|
|
4212
4210
|
readonly destinationDir: string;
|
|
4213
4211
|
}): Promise<void>;
|
|
4214
|
-
declare function resolveResultsRepoRunsDir(config:
|
|
4212
|
+
declare function resolveResultsRepoRunsDir(config: ResultsConfig): string;
|
|
4215
4213
|
declare function directorySizeBytes(targetPath: string): Promise<number>;
|
|
4216
4214
|
declare function commitAndPushResultsBranch(params: {
|
|
4217
4215
|
readonly repoDir: string;
|
|
4218
4216
|
readonly branchName: string;
|
|
4219
4217
|
readonly commitMessage: string;
|
|
4220
4218
|
}): Promise<boolean>;
|
|
4221
|
-
declare function pushResultsRepoBranch(config:
|
|
4219
|
+
declare function pushResultsRepoBranch(config: ResultsConfig, branchName: string, cwd?: string): Promise<void>;
|
|
4222
4220
|
declare function createDraftResultsPr(params: {
|
|
4223
4221
|
readonly repo: string;
|
|
4224
4222
|
readonly repoDir: string;
|
|
@@ -4233,7 +4231,7 @@ declare function createDraftResultsPr(params: {
|
|
|
4233
4231
|
* Returns true if artifacts were pushed, false if no changes were detected.
|
|
4234
4232
|
*/
|
|
4235
4233
|
declare function directPushResults(params: {
|
|
4236
|
-
readonly config:
|
|
4234
|
+
readonly config: ResultsConfig;
|
|
4237
4235
|
readonly sourceDir: string;
|
|
4238
4236
|
readonly destinationPath: string;
|
|
4239
4237
|
readonly commitMessage: string;
|
|
@@ -4951,4 +4949,4 @@ type AgentKernel = {
|
|
|
4951
4949
|
};
|
|
4952
4950
|
declare function createAgentKernel(): AgentKernel;
|
|
4953
4951
|
|
|
4954
|
-
export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AgentVConfig$1 as AgentVYamlConfig, type AnthropicResolvedConfig, type ApiFormat, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, COMMON_TARGET_SETTINGS, type CacheConfig, type CheckedOutResultsRepoBranch, type ChildGraderResult, type ClaudeDiscoverOptions, type ClaudeResolvedConfig, type ClaudeSession, type CliResolvedConfig, CodeGrader, type CodeGraderConfig, type CodeGraderOptions, type CodexDiscoverOptions, type CodexSession, type CommandExecutor, type CompositeAggregatorConfig, CompositeGrader, type CompositeGraderConfig, type CompositeGraderOptions, type ConfidenceIntervalAggregation, type ContainsAllGraderConfig, type ContainsAnyGraderConfig, type ContainsGraderConfig, type Content, type ContentFile, type ContentImage, type ContentPreprocessorConfig, type ContentText, type ConversationAggregation, type ConversationMode, type ConversationTurn, type ConversationTurnInput, type CopilotCliResolvedConfig, type DiscoverOptions as CopilotDiscoverOptions, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, type CopilotSessionMeta, CostGrader, type CostGraderConfig, type CostGraderOptions, type CreateContainerOptions, DEFAULT_CATEGORY, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DEFAULT_GRADER_TEMPLATE, DEFAULT_THRESHOLD, type DependencyFailurePolicy, type DependencyResult, type DepsScanResult, DeterministicAssertionGrader, type DockerWorkspaceConfig, DockerWorkspaceProvider, type EndsWithGraderConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsGraderConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTargetRef, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type ExecInContainerOptions, type ExecResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsGrader, type ExecutionMetricsGraderConfig, type ExecutionMetricsGraderOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyGrader, type FieldAccuracyGraderConfig, type FieldAccuracyGraderOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type Grader, type GraderConfig, type GraderDispatchContext, type GraderFactory, type GraderFactoryFn, type GraderKind, GraderRegistry, type GraderResult, type IcontainsAllGraderConfig, type IcontainsAnyGraderConfig, type IcontainsGraderConfig, type InlineAssertEvaluatorConfig, type IsJsonGraderConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyGrader, type LatencyGraderConfig, type LatencyGraderOptions, LlmGrader, type LlmGraderConfig, type LlmGraderOptions, type LlmGraderPromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, PASS_THRESHOLD, type ParsedCopilotSession, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type PreparedResultsRepoBranch, type ProgressEvent, type ProjectEntry, type ProjectRegistry, type ProjectSource, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexGraderConfig, type RepoCheckout, type RepoClone, type RepoConfig, type RepoDep, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type
|
|
4952
|
+
export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AgentVConfig$1 as AgentVYamlConfig, type AnthropicResolvedConfig, type ApiFormat, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, COMMON_TARGET_SETTINGS, type CacheConfig, type CheckedOutResultsRepoBranch, type ChildGraderResult, type ClaudeDiscoverOptions, type ClaudeResolvedConfig, type ClaudeSession, type CliResolvedConfig, CodeGrader, type CodeGraderConfig, type CodeGraderOptions, type CodexDiscoverOptions, type CodexSession, type CommandExecutor, type CompositeAggregatorConfig, CompositeGrader, type CompositeGraderConfig, type CompositeGraderOptions, type ConfidenceIntervalAggregation, type ContainsAllGraderConfig, type ContainsAnyGraderConfig, type ContainsGraderConfig, type Content, type ContentFile, type ContentImage, type ContentPreprocessorConfig, type ContentText, type ConversationAggregation, type ConversationMode, type ConversationTurn, type ConversationTurnInput, type CopilotCliResolvedConfig, type DiscoverOptions as CopilotDiscoverOptions, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, type CopilotSessionMeta, CostGrader, type CostGraderConfig, type CostGraderOptions, type CreateContainerOptions, DEFAULT_CATEGORY, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DEFAULT_GRADER_TEMPLATE, DEFAULT_THRESHOLD, type DependencyFailurePolicy, type DependencyResult, type DepsScanResult, DeterministicAssertionGrader, type DockerWorkspaceConfig, DockerWorkspaceProvider, type EndsWithGraderConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsGraderConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTargetRef, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type ExecInContainerOptions, type ExecResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsGrader, type ExecutionMetricsGraderConfig, type ExecutionMetricsGraderOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyGrader, type FieldAccuracyGraderConfig, type FieldAccuracyGraderOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type Grader, type GraderConfig, type GraderDispatchContext, type GraderFactory, type GraderFactoryFn, type GraderKind, GraderRegistry, type GraderResult, type IcontainsAllGraderConfig, type IcontainsAnyGraderConfig, type IcontainsGraderConfig, type InlineAssertEvaluatorConfig, type IsJsonGraderConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyGrader, type LatencyGraderConfig, type LatencyGraderOptions, LlmGrader, type LlmGraderConfig, type LlmGraderOptions, type LlmGraderPromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, PASS_THRESHOLD, type ParsedCopilotSession, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type PreparedResultsRepoBranch, type ProgressEvent, type ProjectEntry, type ProjectRegistry, type ProjectSource, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexGraderConfig, type RepoCheckout, type RepoClone, type RepoConfig, type RepoDep, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type ResultsConfig, type ResultsRepoCachePaths, type ResultsRepoStatus, type RubricItem, type RubricsEvaluatorConfig, RunBudgetTracker, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SkillTriggerGrader, type SkillTriggerGraderConfig, type StartsWithGraderConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, type TargetHooksConfig, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageGrader, type TokenUsageGraderConfig, type TokenUsageGraderOptions, type ToolCall, type ToolTestMessage, type ToolTrajectoryExpectedItem, ToolTrajectoryGrader, type ToolTrajectoryGraderConfig, type ToolTrajectoryGraderOptions, type TraceComputeResult, type TraceSummary, type TranscriptEntry, type TranscriptJsonLine, TranscriptProvider, type TranscriptReplayEntry, type TranscriptSource, type TranspileResult, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type TsEvalResult, type TurnFailurePolicy, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceEnvConfig, type WorkspaceHookConfig, type WorkspaceHooksConfig, WorkspacePoolManager, type WorkspaceScriptConfig, addProject, assembleLlmGraderPrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, checkoutResultsRepoBranch, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, commitAndPushResultsBranch, computeTraceSummary, computeWorkspaceFingerprint, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createDraftResultsPr, createProvider, createTempWorkspace, deepEqual, defineConfig, deriveCategory, deriveProjectId, detectFormat, directPushResults, directorySizeBytes, discoverAssertions, discoverClaudeSessions, discoverCodexSessions, discoverCopilotSessions, discoverGraders, discoverProjects, discoverProviders, ensureResultsRepoClone, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractFailOnError, extractImageBlocks, extractJsonBlob, extractLastAssistantContent, extractTargetFromSuite, extractTargetRefsFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractThreshold, extractTrialsConfig, extractWorkersFromSuite, fileExists, findGitRoot, formatToolCalls, freeformEvaluationSchema, generateRubrics, getAgentvConfigDir, getAgentvHome, getOutputFilenames, getProject, getProjectsRegistryPath, getResultsRepoCachePaths, getResultsRepoStatus, getSubagentsRoot, getTextContent, getTraceStateRoot, getWorkspacePath, getWorkspacePoolRoot, getWorkspacesRoot, groupTranscriptJsonLines, initializeBaseline, isAgentSkillsFormat, isContent, isContentArray, isGraderKind, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, killAllTrackedChildren, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadProjectRegistry, loadTestById, loadTestSuite, loadTests, loadTsConfig, loadTsEvalFile, mergeExecutionMetrics, negateScore, normalizeLineEndings, normalizeResultsConfig, parseAgentSkillsEvals, parseClaudeSession, parseCodexSession, parseCopilotEvents, parseEnvOutput, parseJsonFromText, parseJsonSafe, parseYamlValue, prepareResultsRepoBranch, pushResultsRepoBranch, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, readTranscriptFile, readTranscriptJsonl, removeProject, resolveAndCreateProvider, resolveDelegatedTargetDefinition, resolveFileReference, resolveResultsRepoRunsDir, resolveResultsRepoUrl, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runBeforeSessionHook, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, saveProjectRegistry, scanRepoDeps, scoreRangeEvaluationSchema, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, stageResultsArtifacts, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, syncProject, syncProjects, syncResultsRepo, toCamelCaseDeep, toSnakeCaseDeep, toTranscriptJsonLines, tokensPerTool, touchProject, trackChild, trackedChildCount, transpileEvalYaml, transpileEvalYamlFile, trimBaselineResult };
|
package/dist/index.d.ts
CHANGED
|
@@ -1988,7 +1988,7 @@ type ExecutionDefaults = {
|
|
|
1988
1988
|
readonly pool_workspaces?: boolean;
|
|
1989
1989
|
readonly pool_slots?: number;
|
|
1990
1990
|
};
|
|
1991
|
-
type
|
|
1991
|
+
type ResultsConfig = {
|
|
1992
1992
|
readonly repo: string;
|
|
1993
1993
|
readonly path: string;
|
|
1994
1994
|
readonly auto_push?: boolean;
|
|
@@ -2002,9 +2002,7 @@ type AgentVConfig$1 = {
|
|
|
2002
2002
|
readonly required_version?: string;
|
|
2003
2003
|
readonly eval_patterns?: readonly string[];
|
|
2004
2004
|
readonly execution?: ExecutionDefaults;
|
|
2005
|
-
readonly results?:
|
|
2006
|
-
readonly export?: ResultsExportConfig;
|
|
2007
|
-
};
|
|
2005
|
+
readonly results?: ResultsConfig;
|
|
2008
2006
|
readonly hooks?: HooksConfig;
|
|
2009
2007
|
};
|
|
2010
2008
|
/**
|
|
@@ -4198,27 +4196,27 @@ interface CheckedOutResultsRepoBranch {
|
|
|
4198
4196
|
interface PreparedResultsRepoBranch extends CheckedOutResultsRepoBranch {
|
|
4199
4197
|
readonly cleanup: () => Promise<void>;
|
|
4200
4198
|
}
|
|
4201
|
-
declare function
|
|
4199
|
+
declare function normalizeResultsConfig(config: ResultsConfig): Required<ResultsConfig>;
|
|
4202
4200
|
declare function resolveResultsRepoUrl(repo: string): string;
|
|
4203
4201
|
declare function getResultsRepoCachePaths(repo: string): ResultsRepoCachePaths;
|
|
4204
|
-
declare function ensureResultsRepoClone(config:
|
|
4205
|
-
declare function getResultsRepoStatus(config?:
|
|
4206
|
-
declare function syncResultsRepo(config:
|
|
4207
|
-
declare function checkoutResultsRepoBranch(config:
|
|
4208
|
-
declare function prepareResultsRepoBranch(config:
|
|
4202
|
+
declare function ensureResultsRepoClone(config: ResultsConfig): Promise<string>;
|
|
4203
|
+
declare function getResultsRepoStatus(config?: ResultsConfig): ResultsRepoStatus;
|
|
4204
|
+
declare function syncResultsRepo(config: ResultsConfig): Promise<ResultsRepoStatus>;
|
|
4205
|
+
declare function checkoutResultsRepoBranch(config: ResultsConfig, branchName: string): Promise<CheckedOutResultsRepoBranch>;
|
|
4206
|
+
declare function prepareResultsRepoBranch(config: ResultsConfig, branchName: string): Promise<PreparedResultsRepoBranch>;
|
|
4209
4207
|
declare function stageResultsArtifacts(params: {
|
|
4210
4208
|
readonly repoDir: string;
|
|
4211
4209
|
readonly sourceDir: string;
|
|
4212
4210
|
readonly destinationDir: string;
|
|
4213
4211
|
}): Promise<void>;
|
|
4214
|
-
declare function resolveResultsRepoRunsDir(config:
|
|
4212
|
+
declare function resolveResultsRepoRunsDir(config: ResultsConfig): string;
|
|
4215
4213
|
declare function directorySizeBytes(targetPath: string): Promise<number>;
|
|
4216
4214
|
declare function commitAndPushResultsBranch(params: {
|
|
4217
4215
|
readonly repoDir: string;
|
|
4218
4216
|
readonly branchName: string;
|
|
4219
4217
|
readonly commitMessage: string;
|
|
4220
4218
|
}): Promise<boolean>;
|
|
4221
|
-
declare function pushResultsRepoBranch(config:
|
|
4219
|
+
declare function pushResultsRepoBranch(config: ResultsConfig, branchName: string, cwd?: string): Promise<void>;
|
|
4222
4220
|
declare function createDraftResultsPr(params: {
|
|
4223
4221
|
readonly repo: string;
|
|
4224
4222
|
readonly repoDir: string;
|
|
@@ -4233,7 +4231,7 @@ declare function createDraftResultsPr(params: {
|
|
|
4233
4231
|
* Returns true if artifacts were pushed, false if no changes were detected.
|
|
4234
4232
|
*/
|
|
4235
4233
|
declare function directPushResults(params: {
|
|
4236
|
-
readonly config:
|
|
4234
|
+
readonly config: ResultsConfig;
|
|
4237
4235
|
readonly sourceDir: string;
|
|
4238
4236
|
readonly destinationPath: string;
|
|
4239
4237
|
readonly commitMessage: string;
|
|
@@ -4951,4 +4949,4 @@ type AgentKernel = {
|
|
|
4951
4949
|
};
|
|
4952
4950
|
declare function createAgentKernel(): AgentKernel;
|
|
4953
4951
|
|
|
4954
|
-
export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AgentVConfig$1 as AgentVYamlConfig, type AnthropicResolvedConfig, type ApiFormat, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, COMMON_TARGET_SETTINGS, type CacheConfig, type CheckedOutResultsRepoBranch, type ChildGraderResult, type ClaudeDiscoverOptions, type ClaudeResolvedConfig, type ClaudeSession, type CliResolvedConfig, CodeGrader, type CodeGraderConfig, type CodeGraderOptions, type CodexDiscoverOptions, type CodexSession, type CommandExecutor, type CompositeAggregatorConfig, CompositeGrader, type CompositeGraderConfig, type CompositeGraderOptions, type ConfidenceIntervalAggregation, type ContainsAllGraderConfig, type ContainsAnyGraderConfig, type ContainsGraderConfig, type Content, type ContentFile, type ContentImage, type ContentPreprocessorConfig, type ContentText, type ConversationAggregation, type ConversationMode, type ConversationTurn, type ConversationTurnInput, type CopilotCliResolvedConfig, type DiscoverOptions as CopilotDiscoverOptions, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, type CopilotSessionMeta, CostGrader, type CostGraderConfig, type CostGraderOptions, type CreateContainerOptions, DEFAULT_CATEGORY, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DEFAULT_GRADER_TEMPLATE, DEFAULT_THRESHOLD, type DependencyFailurePolicy, type DependencyResult, type DepsScanResult, DeterministicAssertionGrader, type DockerWorkspaceConfig, DockerWorkspaceProvider, type EndsWithGraderConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsGraderConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTargetRef, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type ExecInContainerOptions, type ExecResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsGrader, type ExecutionMetricsGraderConfig, type ExecutionMetricsGraderOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyGrader, type FieldAccuracyGraderConfig, type FieldAccuracyGraderOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type Grader, type GraderConfig, type GraderDispatchContext, type GraderFactory, type GraderFactoryFn, type GraderKind, GraderRegistry, type GraderResult, type IcontainsAllGraderConfig, type IcontainsAnyGraderConfig, type IcontainsGraderConfig, type InlineAssertEvaluatorConfig, type IsJsonGraderConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyGrader, type LatencyGraderConfig, type LatencyGraderOptions, LlmGrader, type LlmGraderConfig, type LlmGraderOptions, type LlmGraderPromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, PASS_THRESHOLD, type ParsedCopilotSession, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type PreparedResultsRepoBranch, type ProgressEvent, type ProjectEntry, type ProjectRegistry, type ProjectSource, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexGraderConfig, type RepoCheckout, type RepoClone, type RepoConfig, type RepoDep, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type
|
|
4952
|
+
export { type AcquireWorkspaceOptions, type AgentKernel, type AgentVConfig$1 as AgentVConfig, type AgentVResolvedConfig, type AgentVConfig as AgentVTsConfig, type AgentVConfig$1 as AgentVYamlConfig, type AnthropicResolvedConfig, type ApiFormat, type ArgsMatchMode, type AssertContext, type AssertEntry, type AssertFn, type AssertResult, type AssertionEntry, type AssertionResult, type AssistantTestMessage, type AzureResolvedConfig, COMMON_TARGET_SETTINGS, type CacheConfig, type CheckedOutResultsRepoBranch, type ChildGraderResult, type ClaudeDiscoverOptions, type ClaudeResolvedConfig, type ClaudeSession, type CliResolvedConfig, CodeGrader, type CodeGraderConfig, type CodeGraderOptions, type CodexDiscoverOptions, type CodexSession, type CommandExecutor, type CompositeAggregatorConfig, CompositeGrader, type CompositeGraderConfig, type CompositeGraderOptions, type ConfidenceIntervalAggregation, type ContainsAllGraderConfig, type ContainsAnyGraderConfig, type ContainsGraderConfig, type Content, type ContentFile, type ContentImage, type ContentPreprocessorConfig, type ContentText, type ConversationAggregation, type ConversationMode, type ConversationTurn, type ConversationTurnInput, type CopilotCliResolvedConfig, type DiscoverOptions as CopilotDiscoverOptions, type CopilotLogResolvedConfig, type CopilotSdkResolvedConfig, type CopilotSession, type CopilotSessionMeta, CostGrader, type CostGraderConfig, type CostGraderOptions, type CreateContainerOptions, DEFAULT_CATEGORY, DEFAULT_EVAL_PATTERNS, DEFAULT_EXPLORATION_TOOLS, DEFAULT_GRADER_TEMPLATE, DEFAULT_THRESHOLD, type DependencyFailurePolicy, type DependencyResult, type DepsScanResult, DeterministicAssertionGrader, type DockerWorkspaceConfig, DockerWorkspaceProvider, type EndsWithGraderConfig, type EnsureSubagentsOptions, type EnsureSubagentsResult, type EnvLookup, type EqualsGraderConfig, type EvalAssertionInput, type EvalCase, type EvalConfig, type EvalMetadata, type EvalRunResult, type EvalSuiteResult, type EvalSummary, type EvalTargetRef, type EvalTest, type EvalTestInput, type EvalsJsonCase, type EvalsJsonFile, type EvaluationCache, type EvaluationContext, type EvaluationResult, type EvaluationScore, type EvaluationVerdict, type ExecInContainerOptions, type ExecResult, type ExecutionDefaults, type ExecutionError, type ExecutionMetrics, ExecutionMetricsGrader, type ExecutionMetricsGraderConfig, type ExecutionMetricsGraderOptions, type ExecutionStatus, type FailOnError, type FailureStage, FieldAccuracyGrader, type FieldAccuracyGraderConfig, type FieldAccuracyGraderOptions, type FieldAggregationType, type FieldConfig, type FieldMatchType, type GeminiResolvedConfig, type GenerateRubricsOptions, type Grader, type GraderConfig, type GraderDispatchContext, type GraderFactory, type GraderFactoryFn, type GraderKind, GraderRegistry, type GraderResult, type IcontainsAllGraderConfig, type IcontainsAnyGraderConfig, type IcontainsGraderConfig, type InlineAssertEvaluatorConfig, type IsJsonGraderConfig, type JsonObject, type JsonPrimitive, type JsonValue, LatencyGrader, type LatencyGraderConfig, type LatencyGraderOptions, LlmGrader, type LlmGraderConfig, type LlmGraderOptions, type LlmGraderPromptAssembly, type LocalPathValidationError, type MeanAggregation, type Message, type MockResolvedConfig, OTEL_BACKEND_PRESETS, type OpenAIResolvedConfig, type OpenRouterResolvedConfig, type OtelBackendPreset, type OtelExportOptions, OtelStreamingObserver, OtelTraceExporter, OtlpJsonFileExporter, type OutputMessage, PASS_THRESHOLD, type ParsedCopilotSession, type PassAtKAggregation, type PiCliResolvedConfig, type PiCodingAgentResolvedConfig, type PoolSlot, type PreparedResultsRepoBranch, type ProgressEvent, type ProjectEntry, type ProjectRegistry, type ProjectSource, type PromptInputs, type PromptScriptConfig, type Provider, type ProviderFactoryFn, type ProviderKind, ProviderRegistry, type ProviderRequest, type ProviderResponse, type ProviderStreamCallbacks, type ProviderTokenUsage, type RegexGraderConfig, type RepoCheckout, type RepoClone, type RepoConfig, type RepoDep, RepoManager, type RepoSource, type ResolvedTarget, type ResolvedWorkspaceTemplate, ResponseCache, type ResultsConfig, type ResultsRepoCachePaths, type ResultsRepoStatus, type RubricItem, type RubricsEvaluatorConfig, RunBudgetTracker, type RunEvalCaseOptions, type RunEvaluationOptions, type ScoreRange, type ScriptExecutionContext, SkillTriggerGrader, type SkillTriggerGraderConfig, type StartsWithGraderConfig, type SystemTestMessage, TEST_MESSAGE_ROLES, type TargetAccessConfig, type TargetDefinition, type TargetHooksConfig, TemplateNotDirectoryError, TemplateNotFoundError, type TestMessage, type TestMessageContent, type TestMessageRole, type TokenUsage, TokenUsageGrader, type TokenUsageGraderConfig, type TokenUsageGraderOptions, type ToolCall, type ToolTestMessage, type ToolTrajectoryExpectedItem, ToolTrajectoryGrader, type ToolTrajectoryGraderConfig, type ToolTrajectoryGraderOptions, type TraceComputeResult, type TraceSummary, type TranscriptEntry, type TranscriptJsonLine, TranscriptProvider, type TranscriptReplayEntry, type TranscriptSource, type TranspileResult, type TrialAggregation, type TrialResult, type TrialStrategy, type TrialsConfig, type TsEvalResult, type TurnFailurePolicy, type UserTestMessage, type VSCodeResolvedConfig, type WorkspaceConfig, WorkspaceCreationError, type WorkspaceEnvConfig, type WorkspaceHookConfig, type WorkspaceHooksConfig, WorkspacePoolManager, type WorkspaceScriptConfig, addProject, assembleLlmGraderPrompt, avgToolDurationMs, buildDirectoryChain, buildOutputSchema, buildPromptInputs, buildRubricOutputSchema, buildScoreRangeOutputSchema, buildSearchRoots, calculateRubricScore, captureFileChanges, checkoutResultsRepoBranch, clampScore, cleanupEvalWorkspaces, cleanupWorkspace, commitAndPushResultsBranch, computeTraceSummary, computeWorkspaceFingerprint, consumeClaudeLogEntries, consumeCodexLogEntries, consumeCopilotCliLogEntries, consumeCopilotSdkLogEntries, consumePiLogEntries, createAgentKernel, createBuiltinProviderRegistry, createBuiltinRegistry, createDraftResultsPr, createProvider, createTempWorkspace, deepEqual, defineConfig, deriveCategory, deriveProjectId, detectFormat, directPushResults, directorySizeBytes, discoverAssertions, discoverClaudeSessions, discoverCodexSessions, discoverCopilotSessions, discoverGraders, discoverProjects, discoverProviders, ensureResultsRepoClone, ensureVSCodeSubagents, evaluate, executeScript, executeWorkspaceScript, explorationRatio, extractCacheConfig, extractFailOnError, extractImageBlocks, extractJsonBlob, extractLastAssistantContent, extractTargetFromSuite, extractTargetRefsFromSuite, extractTargetsFromSuite, extractTargetsFromTestCase, extractThreshold, extractTrialsConfig, extractWorkersFromSuite, fileExists, findGitRoot, formatToolCalls, freeformEvaluationSchema, generateRubrics, getAgentvConfigDir, getAgentvHome, getOutputFilenames, getProject, getProjectsRegistryPath, getResultsRepoCachePaths, getResultsRepoStatus, getSubagentsRoot, getTextContent, getTraceStateRoot, getWorkspacePath, getWorkspacePoolRoot, getWorkspacesRoot, groupTranscriptJsonLines, initializeBaseline, isAgentSkillsFormat, isContent, isContentArray, isGraderKind, isJsonObject, isJsonValue, isNonEmptyString, isTestMessage, isTestMessageRole, killAllTrackedChildren, listTargetNames, loadConfig, loadEvalCaseById, loadEvalCases, loadEvalSuite, loadProjectRegistry, loadTestById, loadTestSuite, loadTests, loadTsConfig, loadTsEvalFile, mergeExecutionMetrics, negateScore, normalizeLineEndings, normalizeResultsConfig, parseAgentSkillsEvals, parseClaudeSession, parseCodexSession, parseCopilotEvents, parseEnvOutput, parseJsonFromText, parseJsonSafe, parseYamlValue, prepareResultsRepoBranch, pushResultsRepoBranch, readJsonFile, readTargetDefinitions, readTestSuiteMetadata, readTextFile, readTranscriptFile, readTranscriptJsonl, removeProject, resolveAndCreateProvider, resolveDelegatedTargetDefinition, resolveFileReference, resolveResultsRepoRunsDir, resolveResultsRepoUrl, resolveTargetDefinition, resolveWorkspaceTemplate, rubricEvaluationSchema, runBeforeSessionHook, runContainsAllAssertion, runContainsAnyAssertion, runContainsAssertion, runEndsWithAssertion, runEqualsAssertion, runEvalCase, runEvaluation, runIcontainsAllAssertion, runIcontainsAnyAssertion, runIcontainsAssertion, runIsJsonAssertion, runRegexAssertion, runStartsWithAssertion, saveProjectRegistry, scanRepoDeps, scoreRangeEvaluationSchema, scoreToVerdict, shouldEnableCache, shouldSkipCacheForTemperature, stageResultsArtifacts, subscribeToClaudeLogEntries, subscribeToCodexLogEntries, subscribeToCopilotCliLogEntries, subscribeToCopilotSdkLogEntries, subscribeToPiLogEntries, substituteVariables, syncProject, syncProjects, syncResultsRepo, toCamelCaseDeep, toSnakeCaseDeep, toTranscriptJsonLines, tokensPerTool, touchProject, trackChild, trackedChildCount, transpileEvalYaml, transpileEvalYamlFile, trimBaselineResult };
|
package/dist/index.js
CHANGED
|
@@ -132,7 +132,7 @@ import {
|
|
|
132
132
|
tokensPerTool,
|
|
133
133
|
trackChild,
|
|
134
134
|
trackedChildCount
|
|
135
|
-
} from "./chunk-
|
|
135
|
+
} from "./chunk-QY6BS2V4.js";
|
|
136
136
|
import {
|
|
137
137
|
COMMON_TARGET_SETTINGS,
|
|
138
138
|
TEST_MESSAGE_ROLES,
|
|
@@ -738,7 +738,7 @@ function withFriendlyGitHubAuthError(error) {
|
|
|
738
738
|
}
|
|
739
739
|
return new Error(message);
|
|
740
740
|
}
|
|
741
|
-
function
|
|
741
|
+
function normalizeResultsConfig(config) {
|
|
742
742
|
return {
|
|
743
743
|
repo: config.repo.trim(),
|
|
744
744
|
path: config.path.trim().replace(/^\/+|\/+$/g, ""),
|
|
@@ -832,7 +832,7 @@ function updateStatusFile(config, patch) {
|
|
|
832
832
|
});
|
|
833
833
|
}
|
|
834
834
|
async function ensureResultsRepoClone(config) {
|
|
835
|
-
const normalized =
|
|
835
|
+
const normalized = normalizeResultsConfig(config);
|
|
836
836
|
const cachePaths = getResultsRepoCachePaths(normalized.repo);
|
|
837
837
|
mkdirSync(cachePaths.rootDir, { recursive: true });
|
|
838
838
|
if (!existsSync(cachePaths.repoDir)) {
|
|
@@ -863,7 +863,7 @@ function getResultsRepoStatus(config) {
|
|
|
863
863
|
cache_dir: ""
|
|
864
864
|
};
|
|
865
865
|
}
|
|
866
|
-
const normalized =
|
|
866
|
+
const normalized = normalizeResultsConfig(config);
|
|
867
867
|
const cachePaths = getResultsRepoCachePaths(normalized.repo);
|
|
868
868
|
const persisted = readPersistedStatus(cachePaths.statusFile);
|
|
869
869
|
return {
|
|
@@ -879,7 +879,7 @@ function getResultsRepoStatus(config) {
|
|
|
879
879
|
};
|
|
880
880
|
}
|
|
881
881
|
async function syncResultsRepo(config) {
|
|
882
|
-
const normalized =
|
|
882
|
+
const normalized = normalizeResultsConfig(config);
|
|
883
883
|
try {
|
|
884
884
|
const repoDir = await ensureResultsRepoClone(normalized);
|
|
885
885
|
const baseBranch = await resolveDefaultBranch(repoDir);
|
|
@@ -897,7 +897,7 @@ async function syncResultsRepo(config) {
|
|
|
897
897
|
return getResultsRepoStatus(normalized);
|
|
898
898
|
}
|
|
899
899
|
async function checkoutResultsRepoBranch(config, branchName) {
|
|
900
|
-
const normalized =
|
|
900
|
+
const normalized = normalizeResultsConfig(config);
|
|
901
901
|
const repoDir = await ensureResultsRepoClone(normalized);
|
|
902
902
|
const baseBranch = await resolveDefaultBranch(repoDir);
|
|
903
903
|
await updateCacheRepo(repoDir, baseBranch);
|
|
@@ -910,7 +910,7 @@ async function checkoutResultsRepoBranch(config, branchName) {
|
|
|
910
910
|
};
|
|
911
911
|
}
|
|
912
912
|
async function prepareResultsRepoBranch(config, branchName) {
|
|
913
|
-
const normalized =
|
|
913
|
+
const normalized = normalizeResultsConfig(config);
|
|
914
914
|
const cloneDir = await ensureResultsRepoClone(normalized);
|
|
915
915
|
const baseBranch = await resolveDefaultBranch(cloneDir);
|
|
916
916
|
await updateCacheRepo(cloneDir, baseBranch);
|
|
@@ -938,7 +938,7 @@ async function stageResultsArtifacts(params) {
|
|
|
938
938
|
await cp(params.sourceDir, params.destinationDir, { recursive: true });
|
|
939
939
|
}
|
|
940
940
|
function resolveResultsRepoRunsDir(config) {
|
|
941
|
-
const normalized =
|
|
941
|
+
const normalized = normalizeResultsConfig(config);
|
|
942
942
|
return path4.join(
|
|
943
943
|
getResultsRepoCachePaths(normalized.repo).repoDir,
|
|
944
944
|
...normalized.path.split("/")
|
|
@@ -969,7 +969,7 @@ async function commitAndPushResultsBranch(params) {
|
|
|
969
969
|
return true;
|
|
970
970
|
}
|
|
971
971
|
async function pushResultsRepoBranch(config, branchName, cwd) {
|
|
972
|
-
const normalized =
|
|
972
|
+
const normalized = normalizeResultsConfig(config);
|
|
973
973
|
await runGit(["push", "-u", "origin", branchName], {
|
|
974
974
|
cwd: cwd ?? getResultsRepoCachePaths(normalized.repo).repoDir
|
|
975
975
|
});
|
|
@@ -1001,7 +1001,7 @@ async function createDraftResultsPr(params) {
|
|
|
1001
1001
|
}
|
|
1002
1002
|
var DIRECT_PUSH_MAX_RETRIES = 3;
|
|
1003
1003
|
async function directPushResults(params) {
|
|
1004
|
-
const normalized =
|
|
1004
|
+
const normalized = normalizeResultsConfig(params.config);
|
|
1005
1005
|
const repoDir = await ensureResultsRepoClone(normalized);
|
|
1006
1006
|
const baseBranch = await resolveDefaultBranch(repoDir);
|
|
1007
1007
|
await updateCacheRepo(repoDir, baseBranch);
|
|
@@ -2673,7 +2673,7 @@ export {
|
|
|
2673
2673
|
mergeExecutionMetrics,
|
|
2674
2674
|
negateScore,
|
|
2675
2675
|
normalizeLineEndings,
|
|
2676
|
-
|
|
2676
|
+
normalizeResultsConfig,
|
|
2677
2677
|
parseAgentSkillsEvals,
|
|
2678
2678
|
parseClaudeSession,
|
|
2679
2679
|
parseCodexSession,
|