@tangle-network/agent-eval 0.140.1 → 0.141.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/analyst/index.d.ts +74 -6
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +6 -6
- package/dist/{benchmark-command-Bo_hkhQf.js → benchmark-command-D8xXNVVh.js} +2239 -1820
- package/dist/benchmark-command-D8xXNVVh.js.map +1 -0
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-HwoBE32G.js → benchmarks-Ce8GdJbc.js} +4 -4
- package/dist/{benchmarks-HwoBE32G.js.map → benchmarks-Ce8GdJbc.js.map} +1 -1
- package/dist/campaign/index.js +3 -3
- package/dist/{campaign-BzjYNYVZ.js → campaign-COFTVy3Q.js} +3 -3
- package/dist/{campaign-BzjYNYVZ.js.map → campaign-COFTVy3Q.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/contract/index.d.ts.map +1 -1
- package/dist/contract/index.js +56 -13
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-BOiBXNFh.js → default-registry-BupFSXSN.js} +108 -20
- package/dist/default-registry-BupFSXSN.js.map +1 -0
- package/dist/default-registry-Ci7wAAR8.d.ts.map +1 -1
- package/dist/{dspy-rlm-engine-Bkak4nzo.js → dspy-rlm-engine-Bw8Zz9CX.js} +7 -4
- package/dist/{dspy-rlm-engine-Bkak4nzo.js.map → dspy-rlm-engine-Bw8Zz9CX.js.map} +1 -1
- package/dist/{extract-usage-C5vMw-0R.js → extract-usage-7l1Xq5ti.js} +2 -2
- package/dist/{extract-usage-C5vMw-0R.js.map → extract-usage-7l1Xq5ti.js.map} +1 -1
- package/dist/index.d.ts +68 -31
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +122 -46
- package/dist/index.js.map +1 -1
- package/dist/{kind-factory-DB7nIs35.js → kind-factory-Bvwe3pup.js} +11 -1
- package/dist/kind-factory-Bvwe3pup.js.map +1 -0
- package/dist/multishot/index.d.ts +29 -6
- package/dist/multishot/index.d.ts.map +1 -1
- package/dist/multishot/index.js +53 -3
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/replay-BqTgoioO.d.ts.map +1 -1
- package/dist/{replay-k2MsOmv5.js → replay-DjUfTrHD.js} +40 -48
- package/dist/replay-DjUfTrHD.js.map +1 -0
- package/dist/{single-run-lock-CiQThJxB.js → single-run-lock-BcvPQY57.js} +65 -15
- package/dist/single-run-lock-BcvPQY57.js.map +1 -0
- package/dist/{skill-usage-CVVnoIx-.d.ts → skill-usage-CNZTk_LT.d.ts} +8 -4
- package/dist/{skill-usage-CVVnoIx-.d.ts.map → skill-usage-CNZTk_LT.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-CSBQ8Qma.js → skillopt-optimization-method-DV6c4ooy.js} +2 -2
- package/dist/{skillopt-optimization-method-CSBQ8Qma.js.map → skillopt-optimization-method-DV6c4ooy.js.map} +1 -1
- package/dist/{store-otlp-vRByAR6h.js → store-otlp-D4I90_vR.js} +2 -2
- package/dist/{store-otlp-vRByAR6h.js.map → store-otlp-D4I90_vR.js.map} +1 -1
- package/dist/traces.js +4 -4
- package/docs/adapters-observability.md +35 -21
- package/docs/concepts.md +1 -0
- package/docs/distributed-driver.md +7 -3
- package/docs/feature-guide.md +1 -0
- package/docs/insight-report.md +1 -1
- package/docs/trace-analysis.md +11 -0
- package/docs/wire-protocol.md +70 -2
- package/package.json +5 -3
- package/dist/benchmark-command-Bo_hkhQf.js.map +0 -1
- package/dist/default-registry-BOiBXNFh.js.map +0 -1
- package/dist/kind-factory-DB7nIs35.js.map +0 -1
- package/dist/replay-k2MsOmv5.js.map +0 -1
- package/dist/single-run-lock-CiQThJxB.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -6,18 +6,18 @@ import { a as estimateTokens, i as estimateCost, n as MetricsCollector, o as isM
|
|
|
6
6
|
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-DMFxsLKr.js";
|
|
7
7
|
import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-BQd7mzyT.js";
|
|
8
8
|
import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-B3WXSH5Y.js";
|
|
9
|
-
import { C as createChatClient, S as computeTraceMetrics, _ as CONTROL_INTEGRITY_ANALYST, a as analystFindingDigest, c as completedAnalystReviewQuality, d as validateAnalystReviewDecisions, f as DEFAULT_TRACE_ANALYST_KINDS, g as FAILURE_MODE_KIND_SPEC, h as IMPROVEMENT_KIND_SPEC, i as assertExactRegistryRunOpts, l as readAnalystReview, m as KNOWLEDGE_GAP_KIND_SPEC, n as AnalystRegistry, o as analystRunDigest, p as KNOWLEDGE_POISONING_KIND_SPEC, r as ExactAnalystRunExecutionError, s as assertUniqueFindingIds, t as buildDefaultAnalystRegistry, u as snapshotAnalystRun, v as ControlIntegrityAnalyst, y as emitControlIntegrityFindings } from "./default-registry-
|
|
9
|
+
import { C as createChatClient, S as computeTraceMetrics, _ as CONTROL_INTEGRITY_ANALYST, a as analystFindingDigest, c as completedAnalystReviewQuality, d as validateAnalystReviewDecisions, f as DEFAULT_TRACE_ANALYST_KINDS, g as FAILURE_MODE_KIND_SPEC, h as IMPROVEMENT_KIND_SPEC, i as assertExactRegistryRunOpts, l as readAnalystReview, m as KNOWLEDGE_GAP_KIND_SPEC, n as AnalystRegistry, o as analystRunDigest, p as KNOWLEDGE_POISONING_KIND_SPEC, r as ExactAnalystRunExecutionError, s as assertUniqueFindingIds, t as buildDefaultAnalystRegistry, u as snapshotAnalystRun, v as ControlIntegrityAnalyst, y as emitControlIntegrityFindings } from "./default-registry-BupFSXSN.js";
|
|
10
10
|
import { INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OUTPUT_VALUE, RUN_COST_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, applyLlmSpanOtlpAttributes, asNumber, contextInputTokens, firstNumberAttr } from "./trace-attributes.js";
|
|
11
|
-
import { C as TraceNotFoundError, G as resolveTraceAnalystLimits, J as extractOtlpAttributes, K as asString, Q as readOtlpStatus, S as TraceFileTooLargeError, W as DEFAULT_TRACE_ANALYST_LIMITS, X as inferOtlpKind, Y as firstStringAttr, Z as projectOtlpFlatLine, _ as TraceAnalysisLimitError, b as TraceFileMalformedError, c as traceAnalystFunctionGroup, et as stringField, g as SpanNotFoundError, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst, it as traceSpanKindToOpenInferenceKind, l as createBoundedTraceAnalysisStore, m as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, n as renderPriorFindings, nt as classifyOtlpSpanRole, o as TRACE_ANALYST_TOOL_NAMESPACE, p as DEFAULT_TRACE_ANALYST_BUDGETS, r as renderUpstreamFindings, rt as isOtlpModelCall, s as buildTraceAnalysisToolDescriptors, t as createTraceAnalyst, tt as applyToolSpanOtlpAttributes, v as TraceAnalysisStoreContractError, x as TraceFileMissingError, y as TraceAnalysisValidationError } from "./kind-factory-
|
|
11
|
+
import { C as TraceNotFoundError, G as resolveTraceAnalystLimits, J as extractOtlpAttributes, K as asString, Q as readOtlpStatus, S as TraceFileTooLargeError, W as DEFAULT_TRACE_ANALYST_LIMITS, X as inferOtlpKind, Y as firstStringAttr, Z as projectOtlpFlatLine, _ as TraceAnalysisLimitError, b as TraceFileMalformedError, c as traceAnalystFunctionGroup, et as stringField, g as SpanNotFoundError, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst, it as traceSpanKindToOpenInferenceKind, l as createBoundedTraceAnalysisStore, m as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, n as renderPriorFindings, nt as classifyOtlpSpanRole, o as TRACE_ANALYST_TOOL_NAMESPACE, p as DEFAULT_TRACE_ANALYST_BUDGETS, r as renderUpstreamFindings, rt as isOtlpModelCall, s as buildTraceAnalysisToolDescriptors, t as createTraceAnalyst, tt as applyToolSpanOtlpAttributes, v as TraceAnalysisStoreContractError, x as TraceFileMissingError, y as TraceAnalysisValidationError } from "./kind-factory-Bvwe3pup.js";
|
|
12
12
|
import { a as computeFindingId, o as makeFinding, s as makeProposalFinding } from "./usage-receipt-CgxMEBZq.js";
|
|
13
13
|
import { c as SUPERVISOR_RUN_INTEGRITY_SCHEMA, n as supervisorRunRolloutLines, t as analyzeSupervisorRunIntegrity } from "./integrity-hruKe6oe.js";
|
|
14
14
|
import { c as assertMintedLines, f as isRolloutLine, i as ROLLOUT_SCHEMA, m as validateRolloutLine, p as isTrainableSplit, s as assertMinted, u as assertRolloutLine } from "./schema-C6DW4ZHR.js";
|
|
15
15
|
import { a as scoreOrigin, n as observedScore, o as trainingReward, r as observedSplitScore, s as trainingScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
16
16
|
import { a as clamp01, i as aggregateRunScore, r as DEFAULT_RUN_SCORE_WEIGHTS } from "./proposal-findings-2GIUo1et.js";
|
|
17
17
|
import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, h as defineTraceAnalyst, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-DJQtFr95.js";
|
|
18
|
-
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-
|
|
18
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-Bw8Zz9CX.js";
|
|
19
19
|
import { f as Mutex, p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
|
|
20
|
-
import { At as pairedDeltaTest, Ct as paretoFrontier, D as runCanaries, Dt as decidePairedPromotion, Ft as summarizeBackendIntegrity, G as buildReflectionPrompt, It as JudgeParseError, K as parseReflectionResponse, Mt as assertRealAgentReceipts, Nt as assertRealBackend, Ot as pairedDecisionShape, Pt as summarizeAgentReceiptIntegrity, St as dominates, Tt as scalarScore, W as DEFAULT_MUTATION_PRIMITIVES, _t as canonicalJson, at as createReferenceEquivalenceJudge, bt as inMemoryVerdictCache, ct as redTeamDataset, dt as toolNamesForRun, ft as Dataset, gt as cachedJudge, ht as llmJudge, it as REFERENCE_EQUIVALENCE_JUDGE_VERSION, jt as BackendIntegrityError, kt as minimumPairsForPairedDeltaTest, lt as redTeamReport, mt as hashScenarios, ot as runReferenceEquivalenceJudge, pt as HoldoutLockedError, rt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, st as DEFAULT_RED_TEAM_CORPUS, ut as scoreRedTeamOutput, vt as contentHash, wt as paretoFrontierWithCrowding, xt as crowdingDistance, yt as fileVerdictCache, z as surfaceContentHash } from "./skillopt-optimization-method-
|
|
20
|
+
import { At as pairedDeltaTest, Ct as paretoFrontier, D as runCanaries, Dt as decidePairedPromotion, Ft as summarizeBackendIntegrity, G as buildReflectionPrompt, It as JudgeParseError, K as parseReflectionResponse, Mt as assertRealAgentReceipts, Nt as assertRealBackend, Ot as pairedDecisionShape, Pt as summarizeAgentReceiptIntegrity, St as dominates, Tt as scalarScore, W as DEFAULT_MUTATION_PRIMITIVES, _t as canonicalJson, at as createReferenceEquivalenceJudge, bt as inMemoryVerdictCache, ct as redTeamDataset, dt as toolNamesForRun, ft as Dataset, gt as cachedJudge, ht as llmJudge, it as REFERENCE_EQUIVALENCE_JUDGE_VERSION, jt as BackendIntegrityError, kt as minimumPairsForPairedDeltaTest, lt as redTeamReport, mt as hashScenarios, ot as runReferenceEquivalenceJudge, pt as HoldoutLockedError, rt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, st as DEFAULT_RED_TEAM_CORPUS, ut as scoreRedTeamOutput, vt as contentHash, wt as paretoFrontierWithCrowding, xt as crowdingDistance, yt as fileVerdictCache, z as surfaceContentHash } from "./skillopt-optimization-method-DV6c4ooy.js";
|
|
21
21
|
import { $ as selfPreference, A as pairedRiskDifference, B as requiredSampleSize, C as mulberry32, D as pairedCohensDz, E as pairedBootstrap, F as partialCredit, G as wilson, H as weightedComposite, I as passAtK, J as normalCdf, K as studentTCdf, L as pearsonR, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, P as pairedTTest, Q as positionalBias, R as ranks, S as mcnemarRequiredN, T as pairedBinaryScale, U as weightedMean, V as spearmanR, W as wilcoxonSignedRank, X as calibrateJudgeContinuous, Y as calibrateJudge, Z as continuousAgreement, _ as interpretCliffs, a as MANN_WHITNEY_EXACT_MAX_WORK, b as mcnemar, c as bonferroni, d as confidenceInterval, et as verbosityBias, f as corpusInterRaterAgreement, g as interRaterReliability, h as holm, i as MANN_WHITNEY_EXACT_MAX_STATES, j as pairedRiskDifferenceExact, k as pairedMde, l as cliffsDelta, m as eProcess, n as DECISION_PAIRED_DELTA_STATISTIC, o as WILCOXON_EXACT_MAX_N, p as corpusInterRaterAgreementFromJudgeScores, r as DEFAULT_PERMUTATIONS, s as benjaminiHochberg, t as BOOTSTRAP_GATE_MIN_N, u as cohensD, v as isBinaryOutcomeVector, w as normalizeScores, x as mcnemarPower, y as mannWhitneyU, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
|
|
22
22
|
import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
|
|
23
23
|
import { n as llmSpanFromProvider, t as TraceEmitter } from "./emitter-CPBAhxum.js";
|
|
@@ -29,10 +29,10 @@ import { a as toHarborTrajectories, i as relabelImportedSplit, n as HARBOR_IMPOR
|
|
|
29
29
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
30
30
|
import { n as unmintableReasons, t as mintRolloutRows } from "./mint-Ctwk079K.js";
|
|
31
31
|
import { C as SUPERVISOR_RUN_SCHEMA, T as showMeasured, _ as readClaudeCodeSupervisorRun, b as rollupSupervisorRuns, c as writeSupervisorRunReport, d as renderSupervisorRunHeadline, f as renderSupervisorRunMarkdown, g as claudeCodeSupervisorRunReader, t as analyzeSupervisorRun, v as analyzeSupervisorRunSources, w as isUnavailable } from "./supervisor-run-BEns49Nx.js";
|
|
32
|
-
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-
|
|
33
|
-
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-
|
|
34
|
-
import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-
|
|
35
|
-
import { B as agentProfileModelId, G as createLlmCorrectnessChecker, H as harnessAxisOf, I as CODING_HARNESSES, J as verifyCompletion, K as createTokenRecallChecker, L as HARNESS_NATIVE_MODEL, R as agentProfileHash, U as extractProducedState, V as expandProfileAxes, W as completionVerdict, q as parseCorrectnessResponse, z as agentProfileId } from "./campaign-
|
|
32
|
+
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-DjUfTrHD.js";
|
|
33
|
+
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-D4I90_vR.js";
|
|
34
|
+
import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-7l1Xq5ti.js";
|
|
35
|
+
import { B as agentProfileModelId, G as createLlmCorrectnessChecker, H as harnessAxisOf, I as CODING_HARNESSES, J as verifyCompletion, K as createTokenRecallChecker, L as HARNESS_NATIVE_MODEL, R as agentProfileHash, U as extractProducedState, V as expandProfileAxes, W as completionVerdict, q as parseCorrectnessResponse, z as agentProfileId } from "./campaign-COFTVy3Q.js";
|
|
36
36
|
import { n as iqr, r as welchsTTest, t as compareToBaseline } from "./baseline-C-GocmIW.js";
|
|
37
37
|
import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
|
|
38
38
|
import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-BScZvqMV.js";
|
|
@@ -42,7 +42,7 @@ import { a as InMemoryTraceStore, i as FileSystemTraceStore, n as assertRunCaptu
|
|
|
42
42
|
import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
43
43
|
import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "./tool-use-metrics-DEGMKycK.js";
|
|
44
44
|
import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
|
|
45
|
-
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-
|
|
45
|
+
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-Ce8GdJbc.js";
|
|
46
46
|
import { t as runEvalCampaign } from "./eval-campaign-YdkpWWoT.js";
|
|
47
47
|
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
|
|
48
48
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
@@ -831,19 +831,42 @@ function assertCapabilityHeadroom(result, opts = {}) {
|
|
|
831
831
|
}
|
|
832
832
|
}
|
|
833
833
|
//#endregion
|
|
834
|
+
//#region src/deprecation.ts
|
|
835
|
+
/**
|
|
836
|
+
* One-shot deprecation warnings. A deprecated surface warns the FIRST time it
|
|
837
|
+
* is used in a process — loud enough to be seen, quiet enough not to flood a
|
|
838
|
+
* matrix run that constructs it per cell.
|
|
839
|
+
*/
|
|
840
|
+
const warned = /* @__PURE__ */ new Set();
|
|
841
|
+
/** Emit `message` on console.warn once per process per `key`. */
|
|
842
|
+
function warnDeprecatedOnce(key, message) {
|
|
843
|
+
if (warned.has(key)) return;
|
|
844
|
+
warned.add(key);
|
|
845
|
+
console.warn(`[agent-eval deprecation] ${message}`);
|
|
846
|
+
}
|
|
847
|
+
/** Test seam: forget prior warnings so once-semantics can be asserted. */
|
|
848
|
+
function resetDeprecationWarnings() {
|
|
849
|
+
warned.clear();
|
|
850
|
+
}
|
|
851
|
+
//#endregion
|
|
834
852
|
//#region src/client.ts
|
|
853
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
835
854
|
/**
|
|
836
855
|
* ProductClient — configurable HTTP client for exercising any agent's APIs.
|
|
837
856
|
*
|
|
838
857
|
* Routes are config, not hardcoded. Each agent provides its own RouteMap.
|
|
858
|
+
*
|
|
859
|
+
* @deprecated A one-product REST client does not belong in the generic
|
|
860
|
+
* substrate. Moves to the product repo in the next major
|
|
861
|
+
* (tangle-network/agent-runtime#694); vendor it if you construct it directly.
|
|
839
862
|
*/
|
|
840
|
-
const DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
841
863
|
var ProductClient = class {
|
|
842
864
|
baseUrl;
|
|
843
865
|
routes;
|
|
844
866
|
cookies = "";
|
|
845
867
|
timeoutMs;
|
|
846
868
|
constructor(config) {
|
|
869
|
+
warnDeprecatedOnce("ProductClient", "ProductClient is deprecated and moves to the product repo in the next major (tangle-network/agent-runtime#694).");
|
|
847
870
|
this.baseUrl = config.baseUrl.replace(/\/+$/, "");
|
|
848
871
|
this.routes = config.routes;
|
|
849
872
|
this.timeoutMs = config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
@@ -1373,6 +1396,11 @@ const RIGOR_STANCE = {
|
|
|
1373
1396
|
* Uses a driver LLM (Claude/GPT-4o) to decide what to say each turn.
|
|
1374
1397
|
* Not scripted — the driver gets the current product state and decides
|
|
1375
1398
|
* the next realistic user message.
|
|
1399
|
+
*
|
|
1400
|
+
* @deprecated A one-product driver welded to `ProductClient` does not belong
|
|
1401
|
+
* in the generic substrate. Moves to the product repo in the next major
|
|
1402
|
+
* (tangle-network/agent-runtime#694). A driver is an `AgentProfile` on a
|
|
1403
|
+
* graph edge; use the agent-graph path once it lands, or vendor this class.
|
|
1376
1404
|
*/
|
|
1377
1405
|
var AgentDriver = class {
|
|
1378
1406
|
chat;
|
|
@@ -1381,6 +1409,7 @@ var AgentDriver = class {
|
|
|
1381
1409
|
productContext;
|
|
1382
1410
|
costLedger;
|
|
1383
1411
|
constructor(chat, config) {
|
|
1412
|
+
warnDeprecatedOnce("AgentDriver", "AgentDriver is deprecated and moves to the product repo in the next major (tangle-network/agent-runtime#694).");
|
|
1384
1413
|
this.chat = chat;
|
|
1385
1414
|
this.client = config.client;
|
|
1386
1415
|
this.driverModel = config.driverModel ?? "claude-sonnet-4-6";
|
|
@@ -1499,8 +1528,13 @@ function describeCompletion(persona, state) {
|
|
|
1499
1528
|
* signs off (DONE) only when a real practitioner would act on the work
|
|
1500
1529
|
* unmodified. Pure function of persona, product state, and product context
|
|
1501
1530
|
* — exported so harness authors can inspect and regression-test it.
|
|
1531
|
+
*
|
|
1532
|
+
* @deprecated A role expressed as a code function can never be optimized.
|
|
1533
|
+
* Removal is a major; until then, treat this output as SEED data for a
|
|
1534
|
+
* registry-backed directive prompt (tangle-network/agent-runtime#694).
|
|
1502
1535
|
*/
|
|
1503
1536
|
function buildDriverSystemPrompt(persona, state, productContext = "") {
|
|
1537
|
+
warnDeprecatedOnce("buildDriverSystemPrompt", "buildDriverSystemPrompt is deprecated: roles belong in registry-backed prompt data, not code (tangle-network/agent-runtime#694).");
|
|
1504
1538
|
const rigor = persona.rigor ?? "demanding";
|
|
1505
1539
|
const expertise = persona.expertise ? ` You are ${persona.expertise}.` : "";
|
|
1506
1540
|
const pressure = persona.pressurePoints && persona.pressurePoints.length > 0 ? `\nA competent ${persona.role} here MUST get the agent to address each of:\n${persona.pressurePoints.map((p) => ` - ${p}`).join("\n")}\nDo NOT hand these to the agent. Probe whether it surfaces them itself. If it misses one, press on exactly that gap until it delivers or demonstrably fails.\n` : "";
|
|
@@ -1529,48 +1563,18 @@ Sign-off: respond with exactly "DONE" only when a ${persona.role} would act on t
|
|
|
1529
1563
|
Output ONLY your next message to the agent — in character, first person, no meta-commentary, no stage directions.`;
|
|
1530
1564
|
}
|
|
1531
1565
|
/**
|
|
1532
|
-
* Build the WORKER-DRIVER system prompt — the harness-aware sibling of
|
|
1533
|
-
* `buildDriverSystemPrompt`. Where that one role-plays a demanding *user* of a
|
|
1534
|
-
* product, this one is a meta-agent that DRIVES a capable coding *worker* to
|
|
1535
|
-
* complete a goal: it writes rich, high-signal instructions that direct the
|
|
1536
|
-
* worker to exploit its harness's full power (parallelize, sub-agents, run-to-
|
|
1537
|
-
* completion, tools/MCP), matched to what THAT harness can actually do. The
|
|
1538
|
-
* load-bearing contract: the driver never writes a thin steer — every message
|
|
1539
|
-
* is the dense, specific directive a world-class engineering lead would write.
|
|
1540
|
-
* Pure function; exported so harness authors can inspect and regression-test it.
|
|
1541
|
-
*/
|
|
1542
|
-
function buildWorkerDriverSystemPrompt(ctx) {
|
|
1543
|
-
const harness = ctx.harness ?? "a coding agent";
|
|
1544
|
-
const brief = ctx.harnessBrief ? `\nThis worker's harness (${harness}) can:\n${ctx.harnessBrief}\nDrive it to USE these — and never ask for a capability it lacks.\n` : `\nThe worker runs in ${harness}. Drive it to exploit whatever its harness offers — parallel tool calls, sub-agents, run-to-completion — and never ask for a capability it lacks.\n`;
|
|
1545
|
-
const progress = ctx.progress ? `\nWhat the worker has done so far:\n${ctx.progress}\n` : "\nThe worker has not started yet — your first instruction sets the whole plan.\n";
|
|
1546
|
-
const context = ctx.context ? `\nContext:\n${ctx.context}\n` : "";
|
|
1547
|
-
return `You are a DRIVER: a meta-agent whose entire job is to drive a capable coding worker to ACHIEVE A GOAL by writing it precise, high-signal instructions. You do not do the work yourself — you direct the worker to do it, brilliantly, and you hold it to a standard it would not hold itself to.
|
|
1548
|
-
|
|
1549
|
-
The goal: ${ctx.goal}
|
|
1550
|
-
${context}${brief}${progress}
|
|
1551
|
-
THE BAR — non-negotiable. Every instruction you write is dense, specific, and complete. You write the message a world-class engineering lead writes to a strong report: the exact next objective, the concrete sub-steps, what to do in parallel vs in sequence, what to verify and how, what "done" looks like, and the failure modes to avoid. A thin steer — "try again", "fix the issues", a single vague sentence — is a failure on YOUR part, not the worker's. Out-drive a human power-user.
|
|
1552
|
-
|
|
1553
|
-
HOW to drive (each turn):
|
|
1554
|
-
1. Read what the worker actually did (its trace/state above), not what it claims. Find the real gap between here and the goal.
|
|
1555
|
-
2. Decide the next objective — the largest correct step the worker can take now.
|
|
1556
|
-
3. Decompose it: name the sub-tasks that can run IN PARALLEL (independent files/checks/searches) and tell the worker to fan them out (sub-agents / parallel tool calls where its harness allows); name what must run in SEQUENCE and why.
|
|
1557
|
-
4. Make it exploit the harness: drive it to run to completion under its own autonomy, spawn sub-agents for separable work, use its tools/MCP, and not stop at the first plausible stopping point.
|
|
1558
|
-
5. Specify verification: the exact command / test / check that proves the step landed, and tell it to run that and report the result — never accept "done" without the check.
|
|
1559
|
-
6. Name the traps: the specific failure modes for THIS step (editing the wrong file, a check that passes vacuously, scope creep) and forbid them.
|
|
1560
|
-
7. As the task gets harder, decompose MORE, not less — more parallel branches, deeper sub-agent trees, tighter verification.
|
|
1561
|
-
|
|
1562
|
-
COMPLETION: drive toward the goal's real acceptance check. Do not declare done — the deliverable's checker does. If the worker claims it is done, drive it to PROVE it with the check; if the check fails, drive the fix.
|
|
1563
|
-
|
|
1564
|
-
Output ONLY your next instruction to the worker — direct, detailed, actionable, in the first person as the driver. No meta-commentary, no preamble.`;
|
|
1565
|
-
}
|
|
1566
|
-
/**
|
|
1567
1566
|
* Decide the simulated user's next turn — the reactive, adversarial
|
|
1568
1567
|
* turn-generation core of `AgentDriver`, exposed standalone so an in-process
|
|
1569
1568
|
* eval harness can drive multi-shot conversations without the `ProductClient`
|
|
1570
1569
|
* workspace machinery. Returns the next user message, or the literal "DONE"
|
|
1571
1570
|
* when the simulated professional would sign off.
|
|
1571
|
+
*
|
|
1572
|
+
* @deprecated The persona-driver loop becomes a 2-node agent graph (driver
|
|
1573
|
+
* profile + delegates edge); removal is a major
|
|
1574
|
+
* (tangle-network/agent-runtime#694).
|
|
1572
1575
|
*/
|
|
1573
1576
|
async function decideNextUserTurn(chat, opts) {
|
|
1577
|
+
warnDeprecatedOnce("decideNextUserTurn", "decideNextUserTurn is deprecated: the persona-driver loop becomes a 2-node agent graph (tangle-network/agent-runtime#694).");
|
|
1574
1578
|
const { persona, state, history, productContext = "", model = "claude-sonnet-4-6" } = opts;
|
|
1575
1579
|
const lastResponse = history.length > 0 ? history[history.length - 1].content.slice(0, 2e3) : "(no conversation yet — this is the first message)";
|
|
1576
1580
|
const recentHistory = history.slice(-6).map((h) => `${h.role}: ${h.content.slice(0, 500)}`).join("\n\n");
|
|
@@ -3139,6 +3143,78 @@ function classifyTreatment(_record, gate) {
|
|
|
3139
3143
|
return gate.gated ? "treatment-not-applied" : "measurable";
|
|
3140
3144
|
}
|
|
3141
3145
|
//#endregion
|
|
3146
|
+
//#region src/worker-driver-seed.ts
|
|
3147
|
+
/**
|
|
3148
|
+
* Worker-driver seed DATA — the knowledge that used to be hardcoded inside
|
|
3149
|
+
* `buildWorkerDriverSystemPrompt` (deleted; zero callers), re-expressed as
|
|
3150
|
+
* plain data so a driver PROFILE can seed its directive prompt from it and an
|
|
3151
|
+
* optimizer (GEPA/DSPy) can then improve on it. A role expressed as a code
|
|
3152
|
+
* function can never improve; a role expressed as versionable prompt data can
|
|
3153
|
+
* (tangle-network/agent-runtime#694).
|
|
3154
|
+
*
|
|
3155
|
+
* Two exports:
|
|
3156
|
+
* - `WORKER_DRIVER_DOCTRINE` — the "never write a thin steer" driving
|
|
3157
|
+
* contract: the bar for every instruction a driver sends a worker.
|
|
3158
|
+
* - `HARNESS_BRIEFS` — per-harness capability + caveat briefs the driver
|
|
3159
|
+
* weaves into its instructions so it drives the worker to exploit what
|
|
3160
|
+
* THAT harness can actually do, and never asks for a capability it lacks.
|
|
3161
|
+
*
|
|
3162
|
+
* Both are seeds, not law: a consumer copies them into its directive prompt
|
|
3163
|
+
* registry (e.g. `prompts/<surface>/v0.md`) and optimizes from there.
|
|
3164
|
+
*/
|
|
3165
|
+
/**
|
|
3166
|
+
* The driving contract a worker-driver directive seeds from. Every message the
|
|
3167
|
+
* driver writes is held to this bar; a thin steer is the driver's failure, not
|
|
3168
|
+
* the worker's.
|
|
3169
|
+
*/
|
|
3170
|
+
const WORKER_DRIVER_DOCTRINE = `You are a DRIVER: a meta-agent whose entire job is to drive a capable coding worker to ACHIEVE A GOAL by writing it precise, high-signal instructions. You do not do the work yourself — you direct the worker to do it, brilliantly, and you hold it to a standard it would not hold itself to.
|
|
3171
|
+
|
|
3172
|
+
THE BAR — non-negotiable. Every instruction you write is dense, specific, and complete. You write the message a world-class engineering lead writes to a strong report: the exact next objective, the concrete sub-steps, what to do in parallel vs in sequence, what to verify and how, what "done" looks like, and the failure modes to avoid. A thin steer — "try again", "fix the issues", a single vague sentence — is a failure on YOUR part, not the worker's. Out-drive a human power-user.
|
|
3173
|
+
|
|
3174
|
+
HOW to drive (each turn):
|
|
3175
|
+
1. Read what the worker actually did (its trace/state), not what it claims. Find the real gap between here and the goal.
|
|
3176
|
+
2. Decide the next objective — the largest correct step the worker can take now.
|
|
3177
|
+
3. Decompose it: name the sub-tasks that can run IN PARALLEL (independent files/checks/searches) and tell the worker to fan them out (sub-agents / parallel tool calls where its harness allows); name what must run in SEQUENCE and why.
|
|
3178
|
+
4. Make it exploit the harness: drive it to run to completion under its own autonomy, spawn sub-agents for separable work, use its tools/MCP, and not stop at the first plausible stopping point.
|
|
3179
|
+
5. Specify verification: the exact command / test / check that proves the step landed, and tell it to run that and report the result — never accept "done" without the check.
|
|
3180
|
+
6. Name the traps: the specific failure modes for THIS step (editing the wrong file, a check that passes vacuously, scope creep) and forbid them.
|
|
3181
|
+
7. As the task gets harder, decompose MORE, not less — more parallel branches, deeper sub-agent trees, tighter verification.
|
|
3182
|
+
|
|
3183
|
+
COMPLETION: drive toward the goal's real acceptance check. Do not declare done — the deliverable's checker does. If the worker claims it is done, drive it to PROVE it with the check; if the check fails, drive the fix.
|
|
3184
|
+
|
|
3185
|
+
Output ONLY your next instruction to the worker — direct, detailed, actionable, in the first person as the driver. No meta-commentary, no preamble.`;
|
|
3186
|
+
/**
|
|
3187
|
+
* Capability + caveat briefs per harness, keyed by harness id. Free text so
|
|
3188
|
+
* the substrate stays decoupled from any runtime harness type — the keys are
|
|
3189
|
+
* the conventional harness ids, but the record accepts any string so a
|
|
3190
|
+
* consumer can extend it with its own.
|
|
3191
|
+
*/
|
|
3192
|
+
const HARNESS_BRIEFS = {
|
|
3193
|
+
"claude-code": [
|
|
3194
|
+
"- parallel tool calls in a single turn (batch independent reads/greps/commands)",
|
|
3195
|
+
"- parallel Task sub-agents (~10 concurrent) for separable investigation or bulk work; sub-agents cannot spawn sub-agents",
|
|
3196
|
+
"- native WebSearch/WebFetch plus MCP servers when configured",
|
|
3197
|
+
"- skills, hooks, and slash commands when installed in the workspace",
|
|
3198
|
+
"- runs to completion under its own autonomy; drive it NOT to stop at the first plausible stopping point"
|
|
3199
|
+
].join("\n"),
|
|
3200
|
+
codex: [
|
|
3201
|
+
"- long-horizon autonomous runs (strong run-to-completion; suited to hours-long missions)",
|
|
3202
|
+
"- sandboxed exec with full shell access; MCP servers when configured",
|
|
3203
|
+
"- no native sub-agent fan-out: express parallelism as explicit backgrounded commands or sequential decomposition",
|
|
3204
|
+
"- structured output via --output-schema when a machine-parsed result is needed"
|
|
3205
|
+
].join("\n"),
|
|
3206
|
+
opencode: [
|
|
3207
|
+
"- router-backed: any model the router serves; model choice is a live lever",
|
|
3208
|
+
"- tool calls and MCP servers when configured",
|
|
3209
|
+
"- no native sub-agent tree; decompose into sequential objectives with explicit verification between them"
|
|
3210
|
+
].join("\n"),
|
|
3211
|
+
"router-tools": [
|
|
3212
|
+
"- a plain tool loop over router chat completions: no autonomy, no sub-agents, no workspace",
|
|
3213
|
+
"- every capability must be driven explicitly, one tool call at a time",
|
|
3214
|
+
"- keep objectives small and verification immediate; never assume it will continue on its own"
|
|
3215
|
+
].join("\n")
|
|
3216
|
+
};
|
|
3217
|
+
//#endregion
|
|
3142
3218
|
//#region src/anti-slop.ts
|
|
3143
3219
|
const DEFAULT_HEDGES = [
|
|
3144
3220
|
/\bi\s+could\s+be\s+wrong\b/i,
|
|
@@ -12387,6 +12463,6 @@ function assertProductBenchmarkRun(runDir) {
|
|
|
12387
12463
|
return report;
|
|
12388
12464
|
}
|
|
12389
12465
|
//#endregion
|
|
12390
|
-
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_LIMITS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExactAnalystRunExecutionError, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertExactRegistryRunOpts, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createDspyRlmTraceEngine, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalyst, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, defineTraceAnalyst, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, resolveTraceAnalystLimits, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runTraceAnalyst, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12466
|
+
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_LIMITS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExactAnalystRunExecutionError, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_BRIEFS, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, WORKER_DRIVER_DOCTRINE, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertExactRegistryRunOpts, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createDspyRlmTraceEngine, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalyst, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, defineTraceAnalyst, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resetDeprecationWarnings, resolveModelPricing, resolveSeat, resolveTraceAnalystLimits, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runTraceAnalyst, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, warnDeprecatedOnce, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12391
12467
|
|
|
12392
12468
|
//# sourceMappingURL=index.js.map
|