@tangle-network/agent-eval 0.115.3 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/analyst/index.d.ts +16 -11
- package/dist/analyst/index.js +33 -25
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-BYHg6Irm.d.ts → analyze-runs--2x39HZ7.d.ts} +3 -3
- package/dist/{baseline-DsNteOgR.d.ts → baseline-DKq3gJpP.d.ts} +6 -3
- package/dist/belief-state/index.d.ts +6 -6
- package/dist/belief-state/index.js +1 -1
- package/dist/benchmarks/index.d.ts +12 -5
- package/dist/benchmarks/index.js +11 -10
- package/dist/builder-eval/index.d.ts +4 -4
- package/dist/builder-eval/index.js +1 -1
- package/dist/{calibration-Dz8TQV4y.d.ts → calibration-C8MTS7cw.d.ts} +2 -2
- package/dist/campaign/index.d.ts +247 -34
- package/dist/campaign/index.js +33 -13
- package/dist/chunk-3YYRZDON.js +45 -0
- package/dist/chunk-3YYRZDON.js.map +1 -0
- package/dist/{chunk-RPDDVKI7.js → chunk-4JLWXDYA.js} +2 -2
- package/dist/{chunk-WSBUZMBU.js → chunk-CCZIVI3F.js} +54 -115
- package/dist/chunk-CCZIVI3F.js.map +1 -0
- package/dist/{chunk-J6P6PK2R.js → chunk-FQNLDL4D.js} +3 -3
- package/dist/{chunk-ONM6PEAE.js → chunk-GQCZRZ7L.js} +2 -2
- package/dist/chunk-HHWE3POT.js +94 -0
- package/dist/chunk-HHWE3POT.js.map +1 -0
- package/dist/{chunk-ADYLPOSX.js → chunk-HQPHZGL6.js} +1112 -135
- package/dist/chunk-HQPHZGL6.js.map +1 -0
- package/dist/{chunk-FAOEFFRT.js → chunk-IDZTTFRR.js} +390 -78
- package/dist/chunk-IDZTTFRR.js.map +1 -0
- package/dist/{chunk-3LXTCTWL.js → chunk-JSDVRFAP.js} +2 -2
- package/dist/{chunk-MHNQWM4I.js → chunk-LQUTGLOZ.js} +5 -1
- package/dist/chunk-LQUTGLOZ.js.map +1 -0
- package/dist/{chunk-4D5RVB3W.js → chunk-LTVG32KX.js} +30 -5
- package/dist/chunk-LTVG32KX.js.map +1 -0
- package/dist/{chunk-5S5NJ63F.js → chunk-MGEHEHSN.js} +807 -15
- package/dist/chunk-MGEHEHSN.js.map +1 -0
- package/dist/{chunk-GY4SYVPJ.js → chunk-NJC7U437.js} +97 -25
- package/dist/chunk-NJC7U437.js.map +1 -0
- package/dist/{chunk-NYFUT3B3.js → chunk-ODVOOEWQ.js} +31 -10
- package/dist/chunk-ODVOOEWQ.js.map +1 -0
- package/dist/{chunk-LNQEP766.js → chunk-S2F4J57L.js} +44 -4
- package/dist/chunk-S2F4J57L.js.map +1 -0
- package/dist/chunk-VCTY3W6J.js +798 -0
- package/dist/chunk-VCTY3W6J.js.map +1 -0
- package/dist/chunk-VF3XSYTI.js +545 -0
- package/dist/chunk-VF3XSYTI.js.map +1 -0
- package/dist/{chunk-TLDB7WRY.js → chunk-YZPO4UHR.js} +28 -31
- package/dist/chunk-YZPO4UHR.js.map +1 -0
- package/dist/{chunk-KG4TD7EQ.js → chunk-ZUXV7UWZ.js} +1425 -697
- package/dist/chunk-ZUXV7UWZ.js.map +1 -0
- package/dist/cli.js +4 -2
- package/dist/cli.js.map +1 -1
- package/dist/{code-agent-session-D-g04tcy.d.ts → code-agent-session-CjZsVd19.d.ts} +1 -1
- package/dist/contract/index.d.ts +45 -31
- package/dist/contract/index.js +58 -19
- package/dist/contract/index.js.map +1 -1
- package/dist/{control-CcBiAEnn.d.ts → control-6vuGfmDH.d.ts} +5 -5
- package/dist/control.d.ts +6 -6
- package/dist/cost-ledger-DWy3XdJc.d.ts +183 -0
- package/dist/{default-registry-DltpYR5u.d.ts → default-registry-DaK8b3fv.d.ts} +2 -1
- package/dist/{emitter-BRchAAAx.d.ts → emitter-CjD7vUwv.d.ts} +2 -2
- package/dist/{failure-cluster-C48PiReX.d.ts → failure-cluster-DOAcSJ87.d.ts} +2 -2
- package/dist/{feedback-trajectory-pDcz1lQ1.d.ts → feedback-trajectory-BUnM58xL.d.ts} +3 -3
- package/dist/fuzz.d.ts +8 -16
- package/dist/fuzz.js +72 -42
- package/dist/fuzz.js.map +1 -1
- package/dist/{gepa-dne9JDPL.d.ts → gepa-eESocoDi.d.ts} +64 -12
- package/dist/hosted/index.d.ts +14 -7
- package/dist/{index-BTEpx9He.d.ts → index-PdX4VnPA.d.ts} +3 -3
- package/dist/index.d.ts +97 -55
- package/dist/index.js +343 -244
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-IwwvqZZv.d.ts → insight-report-DY4nDW9Q.d.ts} +1 -1
- package/dist/{integrity-qemeBAyx.d.ts → integrity-DqlBiLyK.d.ts} +1 -1
- package/dist/kind-factory-ClZmO25A.d.ts +171 -0
- package/dist/{llm-client-DyqEH4jH.d.ts → llm-client-qoDd18Qz.d.ts} +27 -3
- package/dist/meta-eval/index.d.ts +8 -7
- package/dist/meta-eval/index.js +1 -1
- package/dist/multishot/index.d.ts +10 -3
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +16 -6
- package/dist/pipelines/index.js +119 -23
- package/dist/pipelines/index.js.map +1 -1
- package/dist/{kind-factory-DcNg13sZ.d.ts → policy-edit-wG9uFEFm.d.ts} +114 -167
- package/dist/{pre-registration-D8h7ZxNL.d.ts → pre-registration-BWQhJ3vz.d.ts} +23 -4
- package/dist/{provenance-Bibyg1U9.d.ts → provenance-DpjwyseI.d.ts} +28 -16
- package/dist/{query-Ck190MOd.d.ts → query-CF7PG61p.d.ts} +5 -3
- package/dist/{release-report-CCtzajxP.d.ts → release-report-C8G2i5Xi.d.ts} +2 -2
- package/dist/reporting.d.ts +10 -9
- package/dist/{researcher-Dq-EtpbE.d.ts → researcher-C8XyxQsu.d.ts} +7 -7
- package/dist/rl.d.ts +17 -12
- package/dist/rl.js +2 -2
- package/dist/{rubric-predictive-validity-DYTLjGWu.d.ts → rubric-predictive-validity-p49lLVrE.d.ts} +1 -1
- package/dist/{run-campaign-UADIM77S.js → run-campaign-IM26A6PD.js} +4 -2
- package/dist/{run-record-B7RTi_ix.d.ts → run-record-BDH49H2E.d.ts} +2 -2
- package/dist/{runtime-trajectory-Dws7Kpgi.d.ts → runtime-trajectory-DGBIUt4B.d.ts} +1 -1
- package/dist/{schema-SGWcK9wa.d.ts → schema-B3Q3l9Z_.d.ts} +2 -0
- package/dist/{semantic-concept-judge-DxJmRkyJ.d.ts → semantic-concept-judge-CXnPEJbf.d.ts} +23 -5
- package/dist/{statistics-oUbOJe-S.d.ts → statistics-KUnG73jH.d.ts} +1 -1
- package/dist/{storage-Dw_f7WMt.d.ts → storage-DrX3v_5B.d.ts} +12 -1
- package/dist/{store-BsVi7ncX.d.ts → store-DGqD0Pyo.d.ts} +1 -1
- package/dist/storyboard/index.d.ts +1 -1
- package/dist/{summary-report-BJ5aNwZ1.d.ts → summary-report-C5bKFfm-.d.ts} +2 -2
- package/dist/{test-graded-scenario-mzYBKspu.d.ts → test-graded-scenario-B0ybnPY7.d.ts} +3 -3
- package/dist/traces.d.ts +19 -10
- package/dist/traces.js +16 -4
- package/dist/{types-C5gJrOVT.d.ts → types-BSw1rOUB.d.ts} +97 -38
- package/dist/{types-C7DGg5ex.d.ts → types-BkfcQnxV.d.ts} +15 -0
- package/dist/wire/index.d.ts +28 -19
- package/dist/wire/index.js +4 -2
- package/docs/design/loop-taxonomy.md +1 -2
- package/docs/distributed-driver.md +1 -1
- package/package.json +3 -3
- package/dist/chunk-4D5RVB3W.js.map +0 -1
- package/dist/chunk-5S5NJ63F.js.map +0 -1
- package/dist/chunk-ADYLPOSX.js.map +0 -1
- package/dist/chunk-FAOEFFRT.js.map +0 -1
- package/dist/chunk-GY4SYVPJ.js.map +0 -1
- package/dist/chunk-I6LVHOV3.js +0 -205
- package/dist/chunk-I6LVHOV3.js.map +0 -1
- package/dist/chunk-KG4TD7EQ.js.map +0 -1
- package/dist/chunk-LNQEP766.js.map +0 -1
- package/dist/chunk-MHNQWM4I.js.map +0 -1
- package/dist/chunk-NYFUT3B3.js.map +0 -1
- package/dist/chunk-QMXXSNC4.js +0 -761
- package/dist/chunk-QMXXSNC4.js.map +0 -1
- package/dist/chunk-TLDB7WRY.js.map +0 -1
- package/dist/chunk-WSBUZMBU.js.map +0 -1
- package/dist/cost-ledger-DuSqlw5B.d.ts +0 -113
- package/dist/policy-edit-RLn8GWof.d.ts +0 -103
- /package/dist/{chunk-RPDDVKI7.js.map → chunk-4JLWXDYA.js.map} +0 -0
- /package/dist/{chunk-J6P6PK2R.js.map → chunk-FQNLDL4D.js.map} +0 -0
- /package/dist/{chunk-ONM6PEAE.js.map → chunk-GQCZRZ7L.js.map} +0 -0
- /package/dist/{chunk-3LXTCTWL.js.map → chunk-JSDVRFAP.js.map} +0 -0
- /package/dist/{run-campaign-UADIM77S.js.map → run-campaign-IM26A6PD.js.map} +0 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,53 +1,55 @@
|
|
|
1
|
-
export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, L as LlmJsonCall, b as LlmReviewerConfig, P as ProposeFn, c as ProposeInput, d as ProposeOutput, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, k as ProposeReviewShot, R as Review, l as ReviewFn, m as ReviewInput, n as ReviewMemoryEntry, o as ReviewMemoryStore, p as RunEvidenceMetadata, V as Verification, q as VerifyFn, r as controlFailureClassFromVerification, s as controlRunToRunRecord, t as createLlmReviewer, u as evaluateActionPolicy, v as inMemoryReviewStore, w as jsonlReviewStore, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-
|
|
2
|
-
import { R as RunRecord,
|
|
3
|
-
export { g as AGENT_PROFILE_KINDS, h as AgentInterfaceProfileLike, A as AgentProfileCell,
|
|
4
|
-
import { B as BehavioralMetrics, A as RunScore, a as RunTrace, E as RunScoreWeights } from './semantic-concept-judge-
|
|
5
|
-
export { G as
|
|
6
|
-
import {
|
|
7
|
-
export {
|
|
8
|
-
export { a as AnalystHooks, A as AnalystRegistry, b as AnalystRegistryOptions, B as BudgetPolicy, D as DefaultAnalystRegistryOptions, R as RegistryRunOpts, c as buildDefaultAnalystRegistry } from './default-registry-
|
|
9
|
-
export {
|
|
1
|
+
export { A as ActionExecutionPolicy, a as ActionPolicyDecision, C as ControlRunToRunRecordOptions, L as LlmJsonCall, b as LlmReviewerConfig, P as ProposeFn, c as ProposeInput, d as ProposeOutput, e as ProposeReviewConfig, f as ProposeReviewControlAction, g as ProposeReviewControlConfig, h as ProposeReviewControlResult, i as ProposeReviewControlState, j as ProposeReviewReport, k as ProposeReviewShot, R as Review, l as ReviewFn, m as ReviewInput, n as ReviewMemoryEntry, o as ReviewMemoryStore, p as RunEvidenceMetadata, V as Verification, q as VerifyFn, r as controlFailureClassFromVerification, s as controlRunToRunRecord, t as createLlmReviewer, u as evaluateActionPolicy, v as inMemoryReviewStore, w as jsonlReviewStore, x as runProposeReview, y as runProposeReviewAsControlLoop, z as scoreFromEvals } from './control-6vuGfmDH.js';
|
|
2
|
+
import { R as RunRecord, a as RunSplitTag } from './run-record-BDH49H2E.js';
|
|
3
|
+
export { g as AGENT_PROFILE_KINDS, h as AgentInterfaceProfileLike, A as AgentProfileCell, e as AgentProfileCellInput, i as AgentProfileCellSchemaVersion, j as AgentProfileCellValidationError, k as AgentProfileDimensionValue, l as AgentProfileHarness, f as AgentProfileJson, m as AgentProfileKind, n as AgentProfileSource, o as AgentProfileSourceInput, J as JudgeScoresRecord, b as RunCostProvenance, d as RunJudgeMetadata, p as RunOutcome, q as RunRecordValidationError, c as RunTokenUsage, r as agentProfileCellHashMaterial, s as agentProfileCellKey, t as assertRunAgentProfileCell, u as buildAgentInterfaceProfileCell, v as buildAgentProfileCell, w as groupRunsByAgentProfileCell, x as isRunRecord, y as modelHasSnapshot, z as parseRunRecordSafe, B as requireAgentProfileCell, C as resolveRunCostProvenance, D as roundTripRunRecord, E as toAgentProfileJson, F as validateAgentProfileCell, G as validateRunRecord, H as verifyAgentProfileCell } from './run-record-BDH49H2E.js';
|
|
4
|
+
import { B as BehavioralMetrics, A as RunScore, a as RunTrace, E as RunScoreWeights } from './semantic-concept-judge-CXnPEJbf.js';
|
|
5
|
+
export { G as BehavioralTokenSequence, H as ConceptComplexity, J as ConceptFinding, L as ConceptSpec, M as ConceptWeightStrategy, C as CreateAnalystAiConfig, N as DEFAULT_COMPLEXITY_WEIGHTS, O as DEFAULT_RUN_SCORE_WEIGHTS, D as DEFAULT_TRACE_ANALYST_KINDS, c as DiffPolicy, F as FAILURE_MODE_KIND_SPEC, g as FindingSubject, h as FindingSubjectKind, j as FindingsDiff, k as FindingsStore, I as IMPROVEMENT_KIND_SPEC, l as KNOWLEDGE_GAP_KIND_SPEC, m as KNOWLEDGE_POISONING_KIND_SPEC, P as PersistedFinding, R as RunCritic, Q as RunCriticOptions, T as SEMANTIC_CONCEPT_JUDGE_VERSION, n as SKILL_USAGE_ANALYST, b as SemanticConceptJudgeInput, S as SemanticConceptJudgeOptions, U as SemanticConceptJudgeResult, o as SkillUsageAnalyst, V as SuboptimalCode, W as SuboptimalSignal, X as aggregateRunScore, Y as clamp01, Z as computeTraceMetrics, t as createAnalystAi, _ as createSemanticConceptJudge, u as defaultIsMaterial, v as diffFindings, $ as runSemanticConceptJudge } from './semantic-concept-judge-CXnPEJbf.js';
|
|
6
|
+
import { l as ChatRequest, p as CreateChatClientOpts } from './policy-edit-wG9uFEFm.js';
|
|
7
|
+
export { A as Analyst, a as AnalystContext, g as AnalystCost, c as AnalystFinding, i as AnalystInputKind, j as AnalystRequirements, f as AnalystRunEvent, e as AnalystRunInputs, d as AnalystRunResult, b as AnalystRunSummary, h as AnalystSeverity, k as ChatCallOpts, C as ChatClient, m as ChatResponse, n as ChatTransport, o as CliBridgeTransportOpts, D as DirectProviderTransportOpts, E as EvidenceRef, F as FindingToPolicyEditOptions, M as MockTransportOpts, q as POLICY_EDIT_AXES, r as POLICY_EDIT_CANDIDATE_RECORD_SCHEMA, s as POLICY_EDIT_TARGET_SURFACES, t as PolicyEdit, u as PolicyEditAdmission, v as PolicyEditAdmissionOptions, w as PolicyEditAxis, P as PolicyEditCandidateRecord, x as PolicyEditChange, y as PolicyEditExpectedGain, z as PolicyEditGainDirection, B as PolicyEditGainUnit, G as PolicyEditInit, H as PolicyEditRisk, I as PolicyEditSchemaVersion, J as PolicyEditSource, K as PolicyEditTarget, L as PolicyEditTargetSurface, N as PolicyEditValidationError, R as RouterTransportOpts, S as SandboxSdkTransportOpts, O as admitPolicyEdit, Q as applyPolicyEditToSurface, T as computeFindingId, U as computePolicyEditId, V as createChatClient, W as isPolicyEdit, X as makeFinding, Y as makePolicyEdit, Z as makePolicyEditCandidateRecord, _ as policyEditFromFinding, $ as policyEditsFromFindings, a0 as scorePolicyEditReadiness, a1 as validatePolicyEdit, a2 as validatePolicyEditCandidateRecord } from './policy-edit-wG9uFEFm.js';
|
|
8
|
+
export { a as AnalystHooks, A as AnalystRegistry, b as AnalystRegistryOptions, B as BudgetPolicy, D as DefaultAnalystRegistryOptions, R as RegistryRunOpts, c as buildDefaultAnalystRegistry } from './default-registry-DaK8b3fv.js';
|
|
9
|
+
export { C as CreateTraceAnalystKindOpts, a as RawAnalystFinding, c as TraceAnalystGolden, T as TraceAnalystKindSpec, d as createTraceAnalystKind, r as renderPriorFindings } from './kind-factory-ClZmO25A.js';
|
|
10
10
|
import { TCloud } from '@tangle-network/tcloud';
|
|
11
|
-
import { B as BenchmarkRunnerConfig, S as Scenario, c as BenchmarkReport, P as ProductClientConfig, C as CheckResult, T as TestResult, d as PersonaConfig, D as DriverResult, e as DriverState, b as JudgeFn, f as CollectedArtifacts, g as ScenarioResult, h as TurnMetrics, i as ScenarioFile, j as CompletionCriterion } from './types-
|
|
12
|
-
export { A as ArtifactCheck, k as ArtifactResult, E as EvalResult, F as FeedbackPattern, l as JudgeConfig, a as JudgeInput, m as JudgeRubric, J as JudgeScore, n as PersonaRigor, R as RouteMap, o as RubricDimension, p as Turn, q as TurnResult } from './types-
|
|
13
|
-
import { C as ControlSeverity, a as ControlEvalResult, b as FeedbackLabel, F as FeedbackTrajectoryStore, c as FeedbackTrajectory } from './feedback-trajectory-
|
|
14
|
-
export { d as ControlActionFailureMode, e as ControlActionOutcome, f as ControlBudget, g as ControlContext, h as ControlDecision, i as ControlRunResult, j as ControlRuntimeConfig, k as ControlRuntimeError, l as ControlStep, m as ControlStopPolicies, n as FeedbackArtifactType, o as FeedbackAttempt, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, y as FeedbackTrajectoryFilter, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, S as StopDecision, B as allCriticalPassed, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, M as objectiveEval, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, T as runAgentControlLoop, U as serializeFeedbackTrajectoriesJsonl, V as stopOnNoProgress, W as stopOnRepeatedAction, X as subjectiveEval, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-
|
|
15
|
-
import { F as FailureClass, T as ToolSpan, h as BudgetSpec, B as BudgetLedgerEntry, R as Run, L as LlmSpan, S as Span } from './schema-
|
|
16
|
-
export { A as Artifact, E as EventKind, i as FAILURE_CLASSES, G as GenericSpan, J as JudgeSpan, M as Message, c as RetrievalSpan, g as RunLayer, f as RunStatus, d as SandboxSpan, j as SpanBase, b as SpanKind, k as SpanStatus, l as TRACE_SCHEMA_VERSION, e as TraceEvent, m as isJudgeSpan, n as isLlmSpan, o as isRetrievalSpan, p as isSandboxSpan, q as isToolSpan } from './schema-
|
|
11
|
+
import { B as BenchmarkRunnerConfig, S as Scenario, c as BenchmarkReport, P as ProductClientConfig, C as CheckResult, T as TestResult, d as PersonaConfig, D as DriverResult, e as DriverState, b as JudgeFn, f as CollectedArtifacts, g as ScenarioResult, h as TurnMetrics, i as ScenarioFile, j as CompletionCriterion } from './types-BkfcQnxV.js';
|
|
12
|
+
export { A as ArtifactCheck, k as ArtifactResult, E as EvalResult, F as FeedbackPattern, l as JudgeConfig, a as JudgeInput, m as JudgeRubric, J as JudgeScore, n as PersonaRigor, R as RouteMap, o as RubricDimension, p as Turn, q as TurnResult } from './types-BkfcQnxV.js';
|
|
13
|
+
import { C as ControlSeverity, a as ControlEvalResult, b as FeedbackLabel, F as FeedbackTrajectoryStore, c as FeedbackTrajectory } from './feedback-trajectory-BUnM58xL.js';
|
|
14
|
+
export { d as ControlActionFailureMode, e as ControlActionOutcome, f as ControlBudget, g as ControlContext, h as ControlDecision, i as ControlRunResult, j as ControlRuntimeConfig, k as ControlRuntimeError, l as ControlStep, m as ControlStopPolicies, n as FeedbackArtifactType, o as FeedbackAttempt, p as FeedbackLabelKind, q as FeedbackLabelSource, r as FeedbackOptimizerRow, s as FeedbackOutcome, t as FeedbackReplayAdapter, u as FeedbackReplayResult, v as FeedbackSeverity, w as FeedbackSplitPolicy, x as FeedbackTask, y as FeedbackTrajectoryFilter, z as FileSystemFeedbackTrajectoryStore, I as InMemoryFeedbackTrajectoryStore, P as PreferenceMemoryEntry, A as ProposedSideEffect, S as StopDecision, B as allCriticalPassed, D as assignFeedbackSplit, E as controlRunToFeedbackTrajectory, G as createFeedbackTrajectory, H as feedbackTrajectoriesToDatasetScenarios, J as feedbackTrajectoriesToOptimizerRows, K as feedbackTrajectoryToDatasetScenario, L as feedbackTrajectoryToOptimizerRow, M as objectiveEval, N as parseFeedbackTrajectoriesJsonl, O as renderPreferenceMemoryMarkdown, Q as replayFeedbackTrajectories, R as replayFeedbackTrajectory, T as runAgentControlLoop, U as serializeFeedbackTrajectoriesJsonl, V as stopOnNoProgress, W as stopOnRepeatedAction, X as subjectiveEval, Y as summarizePreferenceMemory, Z as withAssignedFeedbackSplit } from './feedback-trajectory-BUnM58xL.js';
|
|
15
|
+
import { F as FailureClass, T as ToolSpan, h as BudgetSpec, B as BudgetLedgerEntry, R as Run, L as LlmSpan, S as Span } from './schema-B3Q3l9Z_.js';
|
|
16
|
+
export { A as Artifact, E as EventKind, i as FAILURE_CLASSES, G as GenericSpan, J as JudgeSpan, M as Message, c as RetrievalSpan, g as RunLayer, f as RunStatus, d as SandboxSpan, j as SpanBase, b as SpanKind, k as SpanStatus, l as TRACE_SCHEMA_VERSION, e as TraceEvent, m as isJudgeSpan, n as isLlmSpan, o as isRetrievalSpan, p as isSandboxSpan, q as isToolSpan } from './schema-B3Q3l9Z_.js';
|
|
17
|
+
import { C as CostLedger, c as CostReceiptInput, M as MaximumCharge, e as ChannelRollup } from './cost-ledger-DWy3XdJc.js';
|
|
18
|
+
export { f as CostAccountingIncompleteError, g as CostCallConflictError, h as CostCeilingReachedError, a as CostChannel, i as CostLedgerEntry, j as CostLedgerFilter, k as CostLedgerOptions, l as CostLedgerPersistence, m as CostLedgerPersistenceError, b as CostLedgerSummary, d as CostReceipt, n as CostReceiptCaptureError, o as CostReservationExceededError, p as CostResult, q as CostUsage, P as PaidCallResult, R as RunPaidCallInput, r as costForUsage, s as modelPriceKey } from './cost-ledger-DWy3XdJc.js';
|
|
17
19
|
import { A as AgentEvalError, J as JudgeError, a as ConfigError } from './errors-oeQrLqXC.js';
|
|
18
20
|
export { b as AgentEvalErrorCode, C as CaptureIntegrityError, N as NotFoundError, R as ReplayError, V as ValidationError, c as VerificationError } from './errors-oeQrLqXC.js';
|
|
19
|
-
import { c as CorrectnessChecker } from './pre-registration-
|
|
20
|
-
export { A as ArtifactCheckArtifact, e as ArtifactEventLike, f as ArtifactValidator, g as BackendIntegrityError, B as BackendIntegrityReport, h as ComparePairedArmsOptions, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, i as HypothesisResult, j as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, d as LlmJudgeOptions, M as MatchedPair, k as PairArmsOptions, m as PairArmsResult, n as PairedArmRow, P as PairedArmsComparison, o as PairedCorrectness, p as PairedMetricDelta, q as ProducedProposal, b as ProducedState, r as ProposalEventLike, s as RequirementCheck, R as RuntimeEventLike, t as SatisfiedBy, S as SignedManifest, u as SignedManifestAlgo, T as TaskGold, v as ToolCallEventLike, V as ValidationContext, w as ValidationIssue, x as ValidationResult, y as assertRealBackend, z as byteLengthRange, D as canonicalize, E as comparePairedArms, F as completionVerdict, G as composeValidators, I as containsAll, J as createLlmCorrectnessChecker, K as createTokenRecallChecker, N as evaluateHypothesis, O as extractProducedState, Q as hashJson, U as jsonHasKeys, l as llmJudge, W as pairArms, X as parseCorrectnessResponse, Y as regexMatch, Z as signManifest, _ as summarizeBackendIntegrity, $ as verifyCompletion, a0 as verifyManifest } from './pre-registration-
|
|
21
|
-
import {
|
|
22
|
-
export {
|
|
23
|
-
import {
|
|
24
|
-
export {
|
|
25
|
-
|
|
21
|
+
import { c as CorrectnessChecker } from './pre-registration-BWQhJ3vz.js';
|
|
22
|
+
export { A as ArtifactCheckArtifact, e as ArtifactEventLike, f as ArtifactValidator, g as BackendIntegrityError, B as BackendIntegrityReport, h as ComparePairedArmsOptions, C as CompletionRequirement, a as CompletionVerdict, H as HypothesisManifest, i as HypothesisResult, j as LlmCorrectnessCheckerOpts, L as LlmJudgeDimension, d as LlmJudgeOptions, M as MatchedPair, k as PairArmsOptions, m as PairArmsResult, n as PairedArmRow, P as PairedArmsComparison, o as PairedCorrectness, p as PairedMetricDelta, q as ProducedProposal, b as ProducedState, r as ProposalEventLike, s as RequirementCheck, R as RuntimeEventLike, t as SatisfiedBy, S as SignedManifest, u as SignedManifestAlgo, T as TaskGold, v as ToolCallEventLike, V as ValidationContext, w as ValidationIssue, x as ValidationResult, y as assertRealBackend, z as byteLengthRange, D as canonicalize, E as comparePairedArms, F as completionVerdict, G as composeValidators, I as containsAll, J as createLlmCorrectnessChecker, K as createTokenRecallChecker, N as evaluateHypothesis, O as extractProducedState, Q as hashJson, U as jsonHasKeys, l as llmJudge, W as pairArms, X as parseCorrectnessResponse, Y as regexMatch, Z as signManifest, _ as summarizeBackendIntegrity, $ as verifyCompletion, a0 as verifyManifest } from './pre-registration-BWQhJ3vz.js';
|
|
23
|
+
import { L as LlmCallMetadata, e as LlmUsage, a as LlmClientOptions } from './llm-client-qoDd18Qz.js';
|
|
24
|
+
export { f as LlmCallError, c as LlmCallRequest, d as LlmCallResult, g as LlmClient, h as LlmMessage, i as LlmResponseError, j as LlmRouteAssertionError, b as LlmRouteRequirements, k as assertLlmRoute, l as backoffMs, m as callLlm, n as callLlmJson, o as costReceiptFromLlm, p as costReceiptFromLlmError, q as isTransientLlmError, r as maximumChargeForLlmRequest, s as probeLlm, t as stripFencedJson } from './llm-client-qoDd18Qz.js';
|
|
25
|
+
import { T as TraceEmitter } from './emitter-CjD7vUwv.js';
|
|
26
|
+
export { R as RunCompleteHook, a as RunCompleteHookContext, S as SpanHandle, b as TraceEmitterOptions, l as llmSpanFromProvider } from './emitter-CjD7vUwv.js';
|
|
27
|
+
import { h as ReleaseConfidenceThresholds, f as ReleaseConfidenceScorecard } from './release-report-C8G2i5Xi.js';
|
|
28
|
+
export { A as ActionableSideInfo, o as AsiSeverity, B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, g as ReleaseConfidenceStatus, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-C8G2i5Xi.js';
|
|
29
|
+
export { c as CliffsMagnitude, d as CorpusAgreementOptions, e as CorpusAgreementPerDimension, C as CorpusAgreementReport, f as CorpusScoreRecord, g as EProcess, h as EProcessOptions, E as EProcessState, i as EProcessStep, M as McNemarResult, P as PairedBootstrapOptions, a as PairedBootstrapResult, j as PairedSignTestResult, k as ProportionInterval, R as RiskDifferenceResult, S as SignTestAlternative, W as WeightedCompositeInput, l as WeightedCompositeResult, b as benjaminiHochberg, m as bonferroni, n as cliffsDelta, o as cohensD, q as confidenceInterval, r as corpusInterRaterAgreement, s as corpusInterRaterAgreementFromJudgeScores, t as eProcess, u as holm, v as interRaterReliability, x as interpretCliffs, y as mannWhitneyU, z as mcnemar, A as mcnemarPower, B as mcnemarRequiredN, D as mulberry32, F as normalizeScores, p as pairedBootstrap, G as pairedMde, H as pairedRiskDifference, I as pairedSignTest, J as pairedTTest, K as partialCredit, L as passAtK, N as pearsonR, O as ranks, Q as requiredSampleSize, T as spearmanR, U as weightedComposite, V as weightedMean, w as wilcoxonSignedRank, X as wilson } from './statistics-KUnG73jH.js';
|
|
26
30
|
import { OtelExporter, OtelExportConfig } from './traces.js';
|
|
27
|
-
export { CaptureFetchContext, CaptureFetchOptions, DEFAULT_REDACTION_RULES, ExportableSpan, ExtractedUsage, FlattenOtlpOptions, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OtlpExport, OtlpFileTraceStore, OtlpFileTraceStoreOptions, OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpToRunRecordsOptions, OtlpTraceRunRecord, ProjectedOtlpSpan, REDACTION_VERSION, RedactionReport, RedactionRule, ReplayCache, ReplayCacheEntry, ReplayCacheMissError, ReplayCacheStats, ReplayFetchOptions, SPAN_KIND_ATTR_KEYS, SpanNotFoundError, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, TraceAggregate, TraceAnalystHookOptions, TraceFileMissingError, TraceInsightContext, TraceInsightFinding, TraceInsightPanelRole, TraceInsightPromptInput, TraceInsightQualityGate, TraceInsightQuestion, TraceInsightReadiness, TraceInsightSuite, TraceInsightTask, TraceNotFoundError, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, redactString, redactValue, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind } from './traces.js';
|
|
31
|
+
export { CaptureFetchContext, CaptureFetchOptions, DEFAULT_REDACTION_RULES, ExportableSpan, ExtractedUsage, FlattenOtlpOptions, INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpExport, OtlpFileTraceStore, OtlpFileTraceStoreOptions, OtlpFlatLine, OtlpResourceSpans, OtlpSpan, OtlpToRunRecordsOptions, OtlpTraceRunRecord, ProjectedOtlpSpan, REDACTION_VERSION, RedactionReport, RedactionRule, ReplayCache, ReplayCacheEntry, ReplayCacheMissError, ReplayCacheStats, ReplayFetchOptions, SPAN_KIND_ATTR_KEYS, SpanNotFoundError, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_SUBAGENT_DESCRIPTION, ToolSpanOtlpInput, TraceAggregate, TraceAnalystHookOptions, TraceFileMissingError, TraceInsightContext, TraceInsightFinding, TraceInsightPanelRole, TraceInsightPromptInput, TraceInsightQualityGate, TraceInsightQuestion, TraceInsightReadiness, TraceInsightSuite, TraceInsightTask, TraceNotFoundError, TraceStoreSource, TraceStoreToOtlpOptions, TracesToOtlpResult, applyToolSpanOtlpAttributes, asNumber, asString, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, captureFetchToRawSink, convertTraceStoresToOtlp, createOtelExporter, createOtelTracingStore, createReplayFetch, defaultTraceInsightPanel, describeTraceInsightScope, domainEvidencePattern, exportRunAsOtlp, extractOtlpAttributes, extractUsage, extractUsageFromResponse, extractUsageFromSse, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, inferDomainKeywords, inferOtlpKind, iterateRawCalls, otelRunCompleteHook, otlpToRunRecords, otlpToTraceRunRecords, planTraceInsightQuestions, projectOtlpFlatLine, readOtlpStatus, redactString, redactValue, scoreTraceInsightReadiness, stringField, tokenizeDomainWords, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceSpanKindToOpenInferenceKind } from './traces.js';
|
|
28
32
|
import { a as AnalyzeTracesInput, A as AnalyzeTracesOptions, b as AnalyzeTracesResult } from './analyst-C8HHvfJp.js';
|
|
29
33
|
export { c as AnalyzeTracesTurnSnapshot, d as analyzeTraces } from './analyst-C8HHvfJp.js';
|
|
30
34
|
import { a as TraceAnalystSpan } from './store-C1YxJDEK.js';
|
|
31
35
|
export { D as DEFAULT_TRACE_ANALYST_BUDGETS, b as DatasetOverview, E as ErrorCluster, Q as QueryTracesPage, S as SearchSpanResult, c as SearchTraceResult, d as SpanMatchRecord, e as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, T as TraceAnalysisStore, f as TraceAnalystByteBudgets, g as TraceAnalystFilters, h as TraceAnalystSpanKind, i as TraceAnalystSpanStatus, j as TraceAnalystTraceSummary, V as ViewSpansResult, k as ViewTraceOversized, l as ViewTraceResult } from './store-C1YxJDEK.js';
|
|
32
|
-
import { b as JudgeConfig, J as JudgeScore, S as Scenario$1,
|
|
33
|
-
import { A as AnalyzeRunsOptions } from './analyze-runs
|
|
34
|
-
import {
|
|
35
|
-
export {
|
|
36
|
-
import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult } from './test-graded-scenario-
|
|
37
|
-
export { D as DockerSandboxDriver, c as SandboxHarness, d as SandboxResult, e as SubprocessSandboxDriver, f as SubprocessSandboxDriverOptions, g as TestGradedRunOptions, b as TestGradedRunResult, T as TestGradedScenario, h as TestOutputParser, i as composeParsers, j as jestTestParser, p as pytestTestParser, r as runTestGradedScenario, v as vitestTestParser } from './test-graded-scenario-
|
|
38
|
-
export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-
|
|
39
|
-
export { a as aggregateLlm, b as argHash, g as groupBy, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-
|
|
36
|
+
import { b as JudgeConfig, J as JudgeScore, S as Scenario$1, G as Gate } from './types-BSw1rOUB.js';
|
|
37
|
+
import { A as AnalyzeRunsOptions } from './analyze-runs--2x39HZ7.js';
|
|
38
|
+
import { z as Objective, A as ParetoResult, p as GepaProposerConstraints, a as RunImprovementLoopResult } from './gepa-eESocoDi.js';
|
|
39
|
+
export { B as DEFAULT_RED_TEAM_CORPUS, D as Direction, b as REFERENCE_EQUIVALENCE_INPUT_LIMITS, c as REFERENCE_EQUIVALENCE_JUDGE_VERSION, n as RedTeamCase, E as RedTeamCategory, F as RedTeamFinding, H as RedTeamPayload, I as RedTeamReport, d as ReferenceEquivalenceJudgeInput, e as ReferenceEquivalenceJudgeOptions, f as ReferenceEquivalenceJudgeResult, g as ReferenceEquivalenceScenario, j as createReferenceEquivalenceJudge, J as crowdingDistance, K as dominates, L as paretoFrontier, M as paretoFrontierWithCrowding, N as redTeamDataset, Q as redTeamReport, m as runReferenceEquivalenceJudge, S as scalarScore, T as scoreRedTeamOutput, U as toolNamesForRun } from './gepa-eESocoDi.js';
|
|
40
|
+
import { S as SandboxDriver, H as HarnessConfig, a as SandboxHarnessResult } from './test-graded-scenario-B0ybnPY7.js';
|
|
41
|
+
export { D as DockerSandboxDriver, c as SandboxHarness, d as SandboxResult, e as SubprocessSandboxDriver, f as SubprocessSandboxDriverOptions, g as TestGradedRunOptions, b as TestGradedRunResult, T as TestGradedScenario, h as TestOutputParser, i as composeParsers, j as jestTestParser, p as pytestTestParser, r as runTestGradedScenario, v as vitestTestParser } from './test-graded-scenario-B0ybnPY7.js';
|
|
42
|
+
export { b as RunIntegrityError, R as RunIntegrityExpectations, c as RunIntegrityIssue, d as RunIntegrityIssueCode, a as RunIntegrityReport, e as assertRunCaptured, t as throwIfRunIncomplete } from './integrity-DqlBiLyK.js';
|
|
43
|
+
export { a as aggregateLlm, b as argHash, g as groupBy, h as hasCapturedToolArgs, j as judgeSpans, l as llmSpans, r as runFailureClass, c as runsForScenario, t as toolSpans } from './query-CF7PG61p.js';
|
|
40
44
|
export { F as FileSystemRawProviderSink, a as FileSystemRawProviderSinkOptions, I as InMemoryRawProviderSink, b as InMemoryRawProviderSinkOptions, N as NoopRawProviderSink, P as ProviderRedactor, c as RawProviderDirection, d as RawProviderEvent, R as RawProviderSink, e as RawProviderSinkFilter, f as defaultProviderRedactor, p as providerFromBaseUrl } from './raw-provider-sink-C46HDghv.js';
|
|
41
|
-
import { T as TraceStore, R as RunFilter } from './store-
|
|
42
|
-
export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, S as SpanFilter } from './store-
|
|
43
|
-
export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-
|
|
44
|
-
export { P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection, b as RuntimeTrajectoryEvidenceSummary, c as RuntimeTrajectoryHookEvent, R as RuntimeTrajectoryRecord, d as RuntimeTrajectoryRunRecord, p as parseRuntimeTrajectoryHookEvent, e as projectRuntimeTrajectoryEvidence } from './runtime-trajectory-
|
|
45
|
-
import { a as BaselineReport, b as Trajectory, T as TrajectoryStep } from './baseline-
|
|
46
|
-
export { B as BaselineOptions, M as MetricSamples, d as MetricVerdict, e as ToolStats, f as ToolUseMetrics, g as ToolUseOptions, h as buildTrajectory, i as compareToBaseline, c as computeToolUseMetrics, j as iqr, w as welchsTTest } from './baseline-
|
|
45
|
+
import { T as TraceStore, R as RunFilter } from './store-DGqD0Pyo.js';
|
|
46
|
+
export { E as EventFilter, F as FileSystemTraceStore, a as FileSystemTraceStoreOptions, I as InMemoryTraceStore, S as SpanFilter } from './store-DGqD0Pyo.js';
|
|
47
|
+
export { D as DEFAULT_FAILURE_RULES, b as FailureClassification, c as FailureContext, d as FailureRule, e as classifyFailure } from './failure-cluster-DOAcSJ87.js';
|
|
48
|
+
export { P as ProjectRuntimeTrajectoryEvidenceOptions, a as RuntimeTrajectoryEvidenceProjection, b as RuntimeTrajectoryEvidenceSummary, c as RuntimeTrajectoryHookEvent, R as RuntimeTrajectoryRecord, d as RuntimeTrajectoryRunRecord, p as parseRuntimeTrajectoryHookEvent, e as projectRuntimeTrajectoryEvidence } from './runtime-trajectory-DGBIUt4B.js';
|
|
49
|
+
import { a as BaselineReport, b as Trajectory, T as TrajectoryStep } from './baseline-DKq3gJpP.js';
|
|
50
|
+
export { B as BaselineOptions, M as MetricSamples, d as MetricVerdict, e as ToolStats, f as ToolUseMetrics, g as ToolUseOptions, h as buildTrajectory, i as compareToBaseline, c as computeToolUseMetrics, j as iqr, w as welchsTTest } from './baseline-DKq3gJpP.js';
|
|
47
51
|
import { HarnessType, AgentProfile } from '@tangle-network/agent-interface';
|
|
48
52
|
export { AgentProfile, HarnessType } from '@tangle-network/agent-interface';
|
|
49
|
-
import { b as ChannelRollup, C as CostLedger } from './cost-ledger-DuSqlw5B.js';
|
|
50
|
-
export { a as CostChannel, c as CostLedgerEntry, d as CostLedgerSummary, e as CostResult, f as CostUsage, g as costForUsage, m as modelPriceKey } from './cost-ledger-DuSqlw5B.js';
|
|
51
53
|
export { S as SeriesConvergenceOptions, a as SeriesConvergenceResult, b as analyzeSeries } from './series-convergence-D5OWMBg6.js';
|
|
52
54
|
import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
|
|
53
55
|
import { D as DatasetScenario, c as Dataset } from './dataset-NENEzRgk.js';
|
|
@@ -55,16 +57,14 @@ export { d as DatasetDifficulty, b as DatasetManifest, e as DatasetProvenance, a
|
|
|
55
57
|
export { a as CalibrationResult, c as CandidateScore, C as ContinuousAgreement, d as ContinuousAgreementOptions, b as ContinuousCalibrationResult, G as GoldenItem, P as PositionalBiasResult, S as SelfPreferenceResult, V as VerbosityBiasResult, e as calibrateJudge, f as calibrateJudgeContinuous, g as continuousAgreement, p as positionalBias, s as selfPreference, v as verbosityBias } from './judge-calibration-7C-IDmKr.js';
|
|
56
58
|
import { L as Layer, S as Severity, b as LayerResult, c as VerifyContext } from './multi-layer-verifier-BsqKuLyN.js';
|
|
57
59
|
export { F as Finding, d as LayerStatus, M as MultiLayerVerifier, a as VerificationReport, V as VerifyOptions, g as gradeSemanticStatus } from './multi-layer-verifier-BsqKuLyN.js';
|
|
58
|
-
|
|
59
|
-
export { d as
|
|
60
|
-
export {
|
|
61
|
-
export { C as CallbackResearcher, d as CallbackResearcherOptions, e as CampaignFactoryParams, f as CampaignIntegrityPolicy, g as CampaignRunContext, h as CampaignRunOutcome, i as CampaignRunner, j as CampaignScenario, k as CampaignVariant, c as EvalCampaignOptions, b as EvalCampaignResult, E as ExperimentPlan, a as ExperimentResult, l as FailedRun, F as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-Dq-EtpbE.js';
|
|
62
|
-
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, m as GateDecision, n as GateEvidence, H as HeldOutGate, o as HeldOutGateConfig, q as HeldOutGateRejectionCode, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-BJ5aNwZ1.js';
|
|
60
|
+
export { B as BENCHMARK_SPLIT_SEED, a as BenchmarkAdapter, b as BenchmarkDatasetItem, c as BenchmarkEvaluation, d as BenchmarkFamily, e as BenchmarkResponder, f as BenchmarkScenario, g as BenchmarkSource, h as BenchmarkTaskKind, i as benchmarkDeterministicSplit, j as benchmarks } from './index-PdX4VnPA.js';
|
|
61
|
+
export { C as CallbackResearcher, d as CallbackResearcherOptions, e as CampaignFactoryParams, f as CampaignIntegrityPolicy, g as CampaignRunContext, h as CampaignRunOutcome, i as CampaignRunner, j as CampaignScenario, k as CampaignVariant, c as EvalCampaignOptions, b as EvalCampaignResult, E as ExperimentPlan, a as ExperimentResult, l as FailedRun, F as FailureMode, N as NoopResearcher, R as Researcher, S as SteeringChange, r as runEvalCampaign } from './researcher-C8XyxQsu.js';
|
|
62
|
+
export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, m as GateDecision, n as GateEvidence, H as HeldOutGate, o as HeldOutGateConfig, q as HeldOutGateRejectionCode, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-C5bKFfm-.js';
|
|
63
63
|
export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
|
|
64
64
|
import '@ax-llm/ax';
|
|
65
65
|
import 'zod';
|
|
66
|
-
import './insight-report-
|
|
67
|
-
import './storage-
|
|
66
|
+
import './insight-report-DY4nDW9Q.js';
|
|
67
|
+
import './storage-DrX3v_5B.js';
|
|
68
68
|
|
|
69
69
|
/**
|
|
70
70
|
* Automated pull-request transports for the production loop.
|
|
@@ -571,6 +571,10 @@ interface AgentDriverConfig {
|
|
|
571
571
|
driverModel?: string;
|
|
572
572
|
/** System prompt context for the driver LLM to understand the product */
|
|
573
573
|
productContext?: string;
|
|
574
|
+
/** Shared account for driver-model calls. */
|
|
575
|
+
costLedger?: CostLedger;
|
|
576
|
+
/** Exact provider attempt count, required when costLedger has a cap. */
|
|
577
|
+
tcloudMaximumAttempts?: number;
|
|
574
578
|
}
|
|
575
579
|
/**
|
|
576
580
|
* AgentDriver — meta-agent that plays a persona against the real product.
|
|
@@ -584,6 +588,8 @@ declare class AgentDriver {
|
|
|
584
588
|
private client;
|
|
585
589
|
private driverModel;
|
|
586
590
|
private productContext;
|
|
591
|
+
private costLedger;
|
|
592
|
+
private tcloudMaximumAttempts?;
|
|
587
593
|
constructor(tc: TCloud, config: AgentDriverConfig);
|
|
588
594
|
/**
|
|
589
595
|
* Run a persona through the product.
|
|
@@ -648,6 +654,12 @@ interface DecideNextUserTurnOpts {
|
|
|
648
654
|
productContext?: string;
|
|
649
655
|
/** Driver LLM model. Defaults to claude-sonnet-4-6. */
|
|
650
656
|
model?: string;
|
|
657
|
+
/** Shared account for the paid driver-model call. */
|
|
658
|
+
costLedger?: CostLedger;
|
|
659
|
+
/** Attribution tags merged into the paid-call receipt. */
|
|
660
|
+
costTags?: Record<string, string>;
|
|
661
|
+
/** Exact provider attempt count, required when costLedger has a cap. */
|
|
662
|
+
tcloudMaximumAttempts?: number;
|
|
651
663
|
}
|
|
652
664
|
/**
|
|
653
665
|
* Decide the simulated user's next turn — the reactive, adversarial
|
|
@@ -665,6 +677,13 @@ interface ExecutorConfig {
|
|
|
665
677
|
model?: string;
|
|
666
678
|
/** Judges to run after execution */
|
|
667
679
|
judges: JudgeFn[];
|
|
680
|
+
/** Shared ledger for paid built-in judges. */
|
|
681
|
+
costLedger?: CostLedger;
|
|
682
|
+
costPhase?: string;
|
|
683
|
+
costTags?: Record<string, string>;
|
|
684
|
+
/** Exact maximum provider attempts configured on the supplied TCloud client. */
|
|
685
|
+
tcloudMaximumAttempts?: number;
|
|
686
|
+
signal?: AbortSignal;
|
|
668
687
|
/** Regex patterns for detecting tool/API calls in responses */
|
|
669
688
|
toolCallPatterns?: RegExp[];
|
|
670
689
|
/** Block delimiter pattern (default: :::type\n...\n:::) */
|
|
@@ -865,6 +884,10 @@ declare class CrossFamilyError extends Error {
|
|
|
865
884
|
*/
|
|
866
885
|
declare function assertCrossFamily(models: string[], opts?: AssertCrossFamilyOptions): JudgeFamily[];
|
|
867
886
|
|
|
887
|
+
type JudgeParseErrorOptions = {
|
|
888
|
+
cause?: unknown;
|
|
889
|
+
llmCall?: LlmCallMetadata;
|
|
890
|
+
};
|
|
868
891
|
/**
|
|
869
892
|
* A judge's LLM response could not be parsed into scored dimensions.
|
|
870
893
|
* Thrown instead of fabricating a `{ dimension: 'parse_error', score: 0 }`
|
|
@@ -877,9 +900,9 @@ declare class JudgeParseError extends JudgeError {
|
|
|
877
900
|
readonly judgeName: string;
|
|
878
901
|
/** The raw (truncated) model response that failed to parse. */
|
|
879
902
|
readonly raw: string;
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
903
|
+
/** Paid-call metadata remains available even when the verdict is unusable. */
|
|
904
|
+
readonly llmCall?: LlmCallMetadata;
|
|
905
|
+
constructor(judgeName: string, raw: string, options?: JudgeParseErrorOptions);
|
|
883
906
|
}
|
|
884
907
|
/**
|
|
885
908
|
* Create a domain expert judge with a configurable domain.
|
|
@@ -1728,9 +1751,9 @@ declare function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVa
|
|
|
1728
1751
|
*
|
|
1729
1752
|
* Fail-loud: a judge that errored or returned malformed output is recorded in
|
|
1730
1753
|
* `failedJudges` with `perDimension: null`, never folded into a zero. If EVERY
|
|
1731
|
-
* judge failed the reducer throws — a silent zero here would corrupt the
|
|
1732
|
-
* number. A failed judge still burned tokens, so its `costUsd` is still summed.
|
|
1754
|
+
* judge failed the reducer throws — a silent zero here would corrupt the result.
|
|
1733
1755
|
*/
|
|
1756
|
+
|
|
1734
1757
|
/** One judge's verdict. `perDimension: null` ⇒ that judge call failed (threw or
|
|
1735
1758
|
* returned malformed output) — recorded, never folded into a zero. */
|
|
1736
1759
|
interface JudgeVerdict<D extends string = string> {
|
|
@@ -1740,8 +1763,10 @@ interface JudgeVerdict<D extends string = string> {
|
|
|
1740
1763
|
perDimension: Record<D, number> | null;
|
|
1741
1764
|
/** Optional one-line rationale; the first non-empty one becomes the aggregate's. */
|
|
1742
1765
|
rationale?: string;
|
|
1743
|
-
/** Optional reported cost
|
|
1766
|
+
/** Optional reported cost, committed by the caller's CostLedger. */
|
|
1744
1767
|
costUsd?: number;
|
|
1768
|
+
/** Optional provider usage, committed by the caller's CostLedger. */
|
|
1769
|
+
usage?: LlmUsage;
|
|
1745
1770
|
/** Optional per-dimension reasoning/evidence. Carried through to
|
|
1746
1771
|
* `EnsembleAggregate.verdicts` verbatim — never folded into the math. */
|
|
1747
1772
|
detail?: Partial<Record<D, {
|
|
@@ -1761,8 +1786,6 @@ interface EnsembleAggregate<D extends string = string> {
|
|
|
1761
1786
|
maxDisagreement: number;
|
|
1762
1787
|
/** Models whose verdict was null (failed). */
|
|
1763
1788
|
failedJudges: string[];
|
|
1764
|
-
/** Sum of `costUsd` over ALL verdicts, failed included. */
|
|
1765
|
-
costUsd: number;
|
|
1766
1789
|
/** First non-empty survivor rationale, or `'llm-judge'`. */
|
|
1767
1790
|
rationale: string;
|
|
1768
1791
|
/** The input verdicts, verbatim — drill-down to raw scores, `detail`
|
|
@@ -1880,6 +1903,8 @@ interface EnsembleJudgeOptions<D extends string> {
|
|
|
1880
1903
|
/** Judge model ids — one `scoreWith` call per entry. List a model twice to
|
|
1881
1904
|
* sample it twice (votes are suffix-keyed `model#2` so none overwrite). */
|
|
1882
1905
|
models: string[];
|
|
1906
|
+
/** Explicit scoring revision for closure state not visible in the static panel config. */
|
|
1907
|
+
judgeVersion?: string;
|
|
1883
1908
|
/**
|
|
1884
1909
|
* Score the artifact with one model. Throw (or reject) on failure — the
|
|
1885
1910
|
* panel records that model as a failed judge; it is never folded into a
|
|
@@ -1888,7 +1913,14 @@ interface EnsembleJudgeOptions<D extends string> {
|
|
|
1888
1913
|
scoreWith: (model: string, input: {
|
|
1889
1914
|
artifact: unknown;
|
|
1890
1915
|
scenario?: unknown;
|
|
1916
|
+
signal: AbortSignal;
|
|
1891
1917
|
}) => Promise<JudgeVerdict<D>>;
|
|
1918
|
+
/** Recover usage from a failed provider response when its error retains one. */
|
|
1919
|
+
receiptFromError?: (error: Error, model: string) => CostReceiptInput | undefined;
|
|
1920
|
+
/** Used by direct score calls; campaigns supply their run ledger in score(). */
|
|
1921
|
+
costLedger?: CostLedger;
|
|
1922
|
+
/** Required per model when the shared ledger has a dollar cap. */
|
|
1923
|
+
maximumCharge?: MaximumCharge | ((model: string) => MaximumCharge);
|
|
1892
1924
|
/**
|
|
1893
1925
|
* Per-model retry policy, applied via `withJudgeRetry`. The panel's
|
|
1894
1926
|
* `models` list drives the fan-out, so `retry.models` (the fallback
|
|
@@ -4874,10 +4906,14 @@ interface IntentMatchResult {
|
|
|
4874
4906
|
interface IntentMatchOptions {
|
|
4875
4907
|
model?: string;
|
|
4876
4908
|
timeoutMs?: number;
|
|
4909
|
+
maxTokens?: number;
|
|
4877
4910
|
maxSourceChars?: number;
|
|
4878
4911
|
maxPerFileChars?: number;
|
|
4879
4912
|
maxHtmlChars?: number;
|
|
4880
4913
|
llm?: LlmClientOptions;
|
|
4914
|
+
costLedger?: CostLedger;
|
|
4915
|
+
costPhase?: string;
|
|
4916
|
+
signal?: AbortSignal;
|
|
4881
4917
|
}
|
|
4882
4918
|
/**
|
|
4883
4919
|
* Run the intent-match judge. Soft-fails to available=false on error.
|
|
@@ -5317,8 +5353,14 @@ interface CreateDefaultReviewerOptions {
|
|
|
5317
5353
|
model: string;
|
|
5318
5354
|
/** Per-call timeout. Default 300s. */
|
|
5319
5355
|
timeoutMs?: number;
|
|
5356
|
+
/** Provider-enforced output limit. Default 4000. */
|
|
5357
|
+
maxTokens?: number;
|
|
5320
5358
|
/** LlmClient transport config (baseUrl, apiKey, authHeader, etc.). */
|
|
5321
5359
|
llm?: LlmClientOptions;
|
|
5360
|
+
/** Shared run spend account. */
|
|
5361
|
+
costLedger?: CostLedger;
|
|
5362
|
+
costPhase?: string;
|
|
5363
|
+
signal?: AbortSignal;
|
|
5322
5364
|
/**
|
|
5323
5365
|
* Override the prompt builder. Default: `buildReviewerPrompt`.
|
|
5324
5366
|
* Consumers with different reviewer voices pass their own.
|
|
@@ -6881,4 +6923,4 @@ type CachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1> = JudgeCo
|
|
|
6881
6923
|
*/
|
|
6882
6924
|
declare function cachedJudge<TArtifact, TScenario extends Scenario$1 = Scenario$1>(judge: JudgeConfig<TArtifact, TScenario>, store: VerdictCacheStore, options: CachedJudgeOptions): CachedJudge<TArtifact, TScenario>;
|
|
6883
6925
|
|
|
6884
|
-
export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgentProfileRuntimeReceipt, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, type ClusterBootstrapInterval, type ClusterSignFlipAlternative, type ClusterSignFlipResult, type ClusteredBinaryCluster, type ClusteredMatchedPair, type ClusteredPairedBinaryOptions, type ClusteredPairedBinaryResult, type ClusteredPairedBinaryStatistics, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ControlEvalResult, ControlSeverity, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, type CostReport, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, type DataAcquisitionPlan, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, type HarnessAdapter, HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, type HeadroomClass, type HeadroomInput, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmClientOptions, LlmSpan, LockedJsonlAppender, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatcherResult, type MeasurementPolicy, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, ParetoResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, ProductClient, ProductClientConfig, type ProfileAxisSpec, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, RunTrace, type RuntimeResolution, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type ScoreKnowledgeReadinessOptions, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, Span, type SpanPredicate, type SplitGoldOptions, type SteeringBundle, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, type TaskHeadroom, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type UserQuestion, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCapabilityHeadroom, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, bisect, blendHeldout, blockingKnowledgeEval, buildAgreementJudge, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, capabilityHeadroom, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, clusteredPairedBinary, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, harnessAxisOf, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, mergeSteeringBundle, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, index as profile, promptBisect, proposeSynthesisTargets, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runRecordToProductBenchmarkRecord, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scoreKnowledgeReadiness, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, selectHarnessVariant, sentenceReorderMutator, splitGold, statusAdvanced, summarizeHarnessResults, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, userQuestionsForKnowledgeGaps, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
|
|
6926
|
+
export { ATTESTATION_ALGORITHM, type ActiveLearningOptions, type AdapterRun, AgentDriver, type AgentDriverConfig, AgentEvalError, type AgentProfileRuntimeReceipt, type AgreementResult, type AlignmentOp, AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult, type AntiSlopConfig, type AntiSlopIssue, type AntiSlopReport, type AssertCapabilityHeadroomOptions, type AssertCrossFamilyOptions, type AssertSingleBackendOptions, type AttestationProvenance, type AttestationVerification, type AttestedReport, type AutoPrClient, AxGepaSteeringOptimizer, type AxSteeringOptimizerConfig, type BackendDescriptor, BaselineReport, BehaviorAssertion, BehavioralMetrics, BenchmarkReport, BenchmarkRunner, BenchmarkRunnerConfig, type BisectOptions, type BisectResult, type BisectStep, type BlendWeights, BudgetBreachError, BudgetGuard, BudgetLedgerEntry, BudgetSpec, type BuildAgreementJudgeOptions, CODING_HARNESSES, type CachedJudge, type CachedJudgeOptions, CallExpectation, type CanaryAlert, type CanaryKind, type CanaryLeak, type CanaryOptions, type CanaryReport, type CanarySeverity, type CandidateComparison, type CandidateScenario, type CapabilityHeadroomOptions, type CapabilityHeadroomResult, type CausalAttributionReport, type CellVerdict, ChannelRollup, ChatRequest, CheckResult, type ClusterBootstrapInterval, type ClusterSignFlipAlternative, type ClusterSignFlipResult, type ClusteredBinaryCluster, type ClusteredMatchedPair, type ClusteredPairedBinaryOptions, type ClusteredPairedBinaryResult, type ClusteredPairedBinaryStatistics, CollectedArtifacts, type CommandRunner, type CompareLabels, CompletionCriterion, ConfigError, type ContinuityCheck, type ContinuityCheckResult, type ContinuityReport, type ContinuitySnapshotPair, type ContractCheckResult, type ContractJudgeOptions, type ContractMetric, type ContractReport, type ContractRule, type ContractRuleKind, type ContractSpan, type ContractVerdict, type ContractViolation, ControlEvalResult, ControlSeverity, ConvergenceTracker, CorrectnessChecker, type CostEntry, CostLedger, CostReceiptInput, type CostReport, type CostSummary, CostTracker, type CounterfactualContext, type CounterfactualMutation, type CounterfactualResult, type CounterfactualRunner, CreateChatClientOpts, type CreateDefaultReviewerOptions, type CreateExperimentInput, type CreateSandboxPoolOpts, CrossFamilyError, type CrossTraceDiff, type CrossTraceDiffOptions, DEFAULT_AGENT_SLOS, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, type DataAcquisitionPlan, Dataset, DatasetScenario, type DecideNextUserTurnOpts, DefaultVerdict, type DeployFamily, type DeployGateLayerInput, type DeployRunResult, type DeployRunner, type DescriptionLengthCandidate, type DescriptionLengthConfig, type DescriptionLengthDecision, type DescriptionLengthEvidence, DescriptionLengthGate, type DescriptionLengthRejectionCode, type DetectorEvent, type DetectorSeverity, type DetectorSignal, type DiffScorecardOptions, type DirEntry, type DiscoverPersonasOptions, type DiscoveredPersona, DriverResult, DriverState, DualAgentBench, type DualAgentBenchConfig, type DualAgentReport, type DualAgentRound, type DualAgentScenario, type DualAgentScenarioResult, ERROR_COUNT_PATTERNS, type EnsembleAggregate, type EnsembleJudgeOptions, type ErrorCountPattern, type ErrorStreakOptions, type EvalToolDef, EvalTraceStore, type EvolutionRound, type ExecutorConfig, type Expectation, type Experiment, type ExperimentProvenance, type ExperimentRep, type ExperimentStats, type ExperimentStore, ExperimentTracker, type ExperimentTrackerOptions, type ExperimentVerdict, type ExportedRewardModel, type ExtractOptions, type ExtractResult, type FactorContribution, type FactorialCell, FailureClass, FeedbackLabel, FeedbackTrajectory, FeedbackTrajectoryStore, type FieldAgreementSpec, type FieldDestination, type FileChange, type FlowAction, type FlowLayerEnv, type FlowLayerFactoryInput, type FlowRunner, type FlowRunnerStepResult, type FlowSpec, type FlowStep, type GhCliClientOptions, type GoldScenario, type GoldSplit, type GoldenSeverity, type GoldenSpec, HARNESS_NATIVE_MODEL, type HarnessAdapter, HarnessConfig, type HarnessExperimentConfig, type HarnessExperimentResult, type HarnessIntervention, type HarnessRunRequest, type HarnessRunResult, type HarnessScenario, type HarnessSelection, type HarnessVariant, type HarnessVariantReport, type HeadroomClass, type HeadroomInput, type HeldOutPartition, type HiddenCriteriaGrader, type HiddenGradeResult, type HiddenLeak, HoldoutAuditor, type HttpGithubClientOptions, INTENT_MATCH_JUDGE_VERSION, type ImageData, type ImprovementThresholds, type ImprovementVerdictResult, InMemoryWorkspaceInspector, type InferenceScorer, type InspectorContext, type IntentMatchInput, type IntentMatchOptions, type IntentMatchResult, type InteractionContribution, JudgeError, type JudgeFamily, type JudgeFleetOptions, JudgeFn, JudgeParseError, type JudgeReplayResult, type JudgeRetryOutcome, type JudgeRetryPolicy, JudgeRunner, type JudgeScoreInput, type JudgeVerdict, type KeywordConceptSpec, type KeywordCoverageFinding, type KeywordCoverageOptions, type KeywordCoverageResult, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, type LangfuseEnvelope, type LangfuseGeneration, type LangfuseScore, Layer, LayerResult, type LeaderboardOptions, type LeaderboardRow, type LiveProofArtifact, type LiveProofConfig, type LiveProofContext, type LiveProofResult, LlmCallMetadata, LlmClientOptions, LlmSpan, LlmUsage, LockedJsonlAppender, MODEL_PRICING, type MakeEvalToolsConfig, type MatchResult, type MatcherResult, MaximumCharge, type MeasurementPolicy, type MergeOptions, MetricsCollector, type ModelCostRollup, type ModelPreflight, type ModelSeats, ModelsUnreachableError, type MuffledFinder, type MuffledFinding, type MultiToolchainLayerConfig, type Mutator, Mutex, type NoLeakOptions, type NoProgressOptions, Objective, type Oracle, type OracleObservation, type OracleReport, type OracleResult, type OrthogonalityInput, type OrthogonalityResult, OtelExportConfig, OtelExporter, type OtelPipelineHandle, type OtelPipelineOptions, PairwiseSteeringOptimizer, type ParaphraseRobustnessScenarioInput, type ParaphraseRobustnessScenarioResult, ParetoResult, type ParseStudentLabel, type PartitionHeldOutOptions, PersonaConfig, type Playbook, type PlaybookEntry, type PoolSlot, type PrReviewAuditCase, type PrReviewBenchmarkSummary, type PrReviewComment, type PrReviewMatchedFinding, type PrReviewOutcome, type PrReviewReferenceFinding, type PrReviewScore, type PrReviewScoreWeights, type PrReviewSeverity, type PrReviewSource, type PreflightModelsOptions, type PreflightOutcome, type ProductBenchmarkArm, type ProductBenchmarkArtifactPaths, type ProductBenchmarkBudgets, type ProductBenchmarkExportOptions, type ProductBenchmarkExportResult, type ProductBenchmarkManifest, type ProductBenchmarkProfileRef, type ProductBenchmarkRecord, type ProductBenchmarkRepoRef, type ProductBenchmarkRunInput, type ProductBenchmarkScenario, type ProductBenchmarkSingleRunExportOptions, type ProductBenchmarkSplit, type ProductBenchmarkSubstrateVersions, type ProductBenchmarkValidationReport, ProductClient, ProductClientConfig, type ProfileAxisSpec, type PromptHandle, PromptRegistry, type ProposeAutomatedPullRequestInput, type ProposeAutomatedPullRequestResult, type ProvenanceReader, type RecordRunsOptions, type ReferenceMatchResult, type ReferenceReplayAdapter, type ReferenceReplayAdapterFn, type ReferenceReplayAdapterLike, type ReferenceReplayAggregate, type ReferenceReplayCandidate, type ReferenceReplayCase, type ReferenceReplayCaseRun, type ReferenceReplayExecutionScenario, type ReferenceReplayItem, type ReferenceReplayMatch, type ReferenceReplayMatchStrategy, type ReferenceReplayMatcher, type ReferenceReplayPromotionDecision, type ReferenceReplayPromotionPolicy, type ReferenceReplayRun, type ReferenceReplayRunContext, type ReferenceReplayRunOptions, type ReferenceReplayRunStore, type ReferenceReplayScenario, type ReferenceReplayScenarioScore, type ReferenceReplayScore, type ReferenceReplayScoreOptions, type ReferenceReplaySplit, type ReferenceReplaySplitComparison, type ReferenceReplaySteeringRowsOptions, type ReflectionContext, type ReflectionProposal, ReleaseConfidenceScorecard, ReleaseConfidenceThresholds, type RenderStudentPrompt, type RepeatedActionOptions, type RepoRef, type ReviewerMemoryEntry, type ReviewerOutput, type ReviewerPromptInput, type ReviewerSoftFailDefaults, type ReviewerVerificationSummary, type RobustnessResult, type RoutedField, Run, type RunCommandInput, type RunCommandResult, type RunDistillationOptions, type RunDistillationResult, RunFilter, RunRecord, type RunRecordBackend, type RunRecordFilter, RunScore, RunScoreWeights, RunSplitTag, RunTrace, type RuntimeResolution, SandboxDriver, SandboxHarnessResult, type SandboxJudgeKind, type SandboxJudgeResult, type SandboxJudgeSpec, type SandboxPool, type ScanOptions, Scenario, type ScenarioCost, ScenarioFile, ScenarioRegistry, ScenarioResult, type ScoreKnowledgeReadinessOptions, type Scorecard, type ScorecardCell, type ScorecardCellDiff, type ScorecardDiff, type ScorecardEntry, type ScorecardLogLine, type ScoredTarget, type SeatName, type SeatPresetName, SeatUnsetError, type SelfPlayOptions, type SelfPlayProposer, type SelfPlayScorer, type SerializedRegex, Severity, type SingleBackendDivergence, SingleBackendError, type SingleBackendField, type SingleBackendReport, type Slo, type SloCheckResult, type SloComparator, type SloReport, type SloSeverity, type SlopCategory, type SlotFactory, Span, type SpanPredicate, type SplitGoldOptions, type SteeringBundle, type SteeringDelta, type SteeringOptimizationResult, type SteeringOptimizationRow, type SteeringOptimizationSelector, type SteeringOptimizerBackend, type SteeringOptimizerConfig, type SteeringRolePrompt, type StepAttribution, type StreamingDetector, type SynthesisReason, type SynthesisTarget, type TaskHeadroom, TestResult, type TextMatcher, type ThresholdContract, TokenCounter, type TokenSpec, type ToolMatcher, ToolSpan, TraceAnalystSpan, type TraceContract, TraceContractBuilder, TraceEmitter, TraceStore, type TracedAnalystOptions, type TracedJudgeOptions, Trajectory, TrajectoryStep, type TreatmentClass, type TreatmentGate, type TreatmentGateInput, type TreatmentGateOptions, type TrialTrace, TurnMetrics, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, type UiFinding, type UiFindingScreenshot, type UiFindingSeverity, type UiLens, type UserQuestion, type VerdictCacheStats, type VerdictCacheStore, VerifyContext, type VisualDiffOptions, type VisualDiffResult, type ViteDeployRunnerInput, type WorkerDriverContext, type WorkflowTopology, type WorkspaceAssertion, type WorkspaceAssertionResult, type WorkspaceInspector, type WorkspaceSnapshot, type WranglerDeployRunnerInput, acquisitionPlansForKnowledgeGaps, adversarialJudge, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregatePrReviewScore, analyzeAntiSlop, appendScorecard, assertCapabilityHeadroom, assertCrossFamily, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertSingleBackend, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, bisect, blendHeldout, blockingKnowledgeEval, buildAgreementJudge, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildWorkerDriverSystemPrompt, cachedJudge, canaryLeakView, canonicalJson, capabilityHeadroom, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, classifyTreatment, clusteredPairedBinary, codeExecutionJudge, coherenceJudge, collectionPreserved, commentsForSource, commitBisect, compareReferenceReplay, compilerJudge, computeExperimentStats, contentHash, contractJudge, costReport, createAntiSlopJudge, createCustomJudge, createDefaultReviewer, createDomainExpertJudge, createIntentMatchJudge, createSandboxPool, crossTraceDiff, dataDescriptionBits, decideNextUserTurn, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultJudges, defaultParseStudentLabel, defaultReferenceReplayMatcher, defaultRenderStudentPrompt, deployGateLayer, diffScorecard, discoverPersonas, distillPlaybook, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateContract, evaluateOracles, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, extractAssetUrls, extractErrorCount, fieldAgreement, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, harnessAxisOf, hashContent, hashToUnit, hiddenGrade, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryRunRecordBackend, inMemoryVerdictCache, isHiddenDestination, isModelPriced, isOtelConfigured, jsonShape, jsonlReferenceReplayStore, jsonlRunRecordBackend, judgeFamily, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, loadGoldScenarios, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, matchGoldens, matchSpan, mergeLayerResults, mergeSteeringBundle, modelDescriptionBits, multiToolchainLayer, noProgressDetector, notBlocked, observeAll, paraphraseRobustness, paraphraseRobustnessScenarios, parseGoldJsonl, parseReflectionResponse, partitionHeldOut, passOrthogonality, pixelDeltaRatio, politenessPrefixMutator, preflightModels, printDriverSummary, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, index as profile, promptBisect, proposeSynthesisTargets, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatches, renderMarkdownReport, renderPlaybookMarkdown, renderSteeringText, repeatedActionDetector, replayScorerOverCorpus, replayTraceThroughJudge, resolveModelPricing, resolveSeat, routeFields, rowCount, rowWhere, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runDistillation, runE2EWorkflow, runExpectations, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runRecordToProductBenchmarkRecord, runReferenceReplay, runScore, runSelfPlay, scanForMuffledGates, scoreContinuity, scoreKnowledgeReadiness, scorePrReviewComments, scorePrReviewSource, scoreReferenceReplay, seatPresets, securityJudge, selectHarnessVariant, sentenceReorderMutator, splitGold, statusAdvanced, summarizeHarnessResults, summarizePrReviewBenchmark, testJudge, textInSnapshot, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, traceContract, traceJudge, traceJudgeEnsemble, tracedAnalyzeTraces, typoMutator, urlContains, userQuestionsForKnowledgeGaps, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, verifyAttestation, visualDiff, viteDeployRunner, weightedRecall, whitespaceCollapseMutator, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner };
|