@tangle-network/agent-eval 0.138.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -1
- package/dist/analyst/index.d.ts +41 -94
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +9 -24
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-D8dkki-J.js → benchmark-CYtcIF2V.js} +2 -2
- package/dist/{benchmark-D8dkki-J.js.map → benchmark-CYtcIF2V.js.map} +1 -1
- package/dist/{benchmark-DlQgU_XI.d.ts → benchmark-DDVdWcwA.d.ts} +3 -3
- package/dist/{benchmark-DlQgU_XI.d.ts.map → benchmark-DDVdWcwA.d.ts.map} +1 -1
- package/dist/{benchmark-command-CMqVqReF.js → benchmark-command-BKfjOBJ5.js} +243 -38
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-BJ_xK5rQ.js → benchmarks-zxhy1QV3.js} +4 -4
- package/dist/{benchmarks-BJ_xK5rQ.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-BIBS-NHV.js → campaign-DrS6_hLd.js} +10 -9
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{client-BwPKohkJ.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-BwPKohkJ.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-B4-IMYcS.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +8 -7
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-CHDLA0Ss.js → cost-ledger-CZ9diLxY.js} +7 -7
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-B1D3COAc.d.ts → cost-ledger-DKgyIWRj.d.ts} +5 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/{default-registry-lp5R0lve.js → default-registry-BgJJItGr.js} +57 -1532
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-9MozgKL7.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-9MozgKL7.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/{exact-types-Dpw2LeHA.d.ts → exact-types-MaaFcllV.d.ts} +2 -2
- package/dist/{exact-types-Dpw2LeHA.d.ts.map → exact-types-MaaFcllV.d.ts.map} +1 -1
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-CS391dOE.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-CS391dOE.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-CoNep7rl.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -3
- package/dist/{feedback-trajectory-CoNep7rl.d.ts.map → feedback-trajectory-BJUWOkJM.d.ts.map} +1 -1
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-D0cxAdaV.d.ts → index-BTm_P9aC.d.ts} +11 -11
- package/dist/{index-D0cxAdaV.d.ts.map → index-BTm_P9aC.d.ts.map} +1 -1
- package/dist/{index-B2-IxCMB.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-B2-IxCMB.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-sMN_hI4E.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-sMN_hI4E.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/{index-CjVYlVBK.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-CjVYlVBK.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -56
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -176
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-CXd8VBDR.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-CXd8VBDR.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-B-MLFz0I.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-B-MLFz0I.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-C0Yx1I14.js → ledger-core-Dxz0Rkwa.js} +3 -85
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-Cj3c7PEm.js → llm-client-bkztEfIx.js} +2 -2
- package/dist/{llm-client-Cj3c7PEm.js.map → llm-client-bkztEfIx.js.map} +1 -1
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{release-report-CoyvyLBs.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CoyvyLBs.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-DbIYwso6.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Cb-4Vf0k.js → replay-SA4OB7O7.js} +48 -137
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-BCeOEjtR.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-BCeOEjtR.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-sE2l_NV6.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-sE2l_NV6.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-w2klGv1u.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-w2klGv1u.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-CbE0A8Xg.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-CbE0A8Xg.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-DwHMk1Ai.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-DwHMk1Ai.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-DYXDPZW0.js → semantic-concept-judge-BuIJ9IfB.js} +43 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-DLEvyW2z.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-DLEvyW2z.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-Bv3G4VkA.d.ts → skill-usage-B-BFS8M2.d.ts} +54 -39
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-CjKMZy0d.js → skillopt-optimization-method-BbGnCC53.js} +18 -802
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-CzfnA8O-.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-mf70aXKp.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-mf70aXKp.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-BKinV4yD.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-BKinV4yD.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -5
- package/dist/{types-zFYez3PK.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-zFYez3PK.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-BtJhn8v6.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-BtJhn8v6.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-5q2T25iW.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-5q2T25iW.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +170 -484
- package/package.json +1 -2
- package/dist/analyze-runs-CPYxfPWT.d.ts +0 -72
- package/dist/analyze-runs-CPYxfPWT.d.ts.map +0 -1
- package/dist/benchmark-command-CMqVqReF.js.map +0 -1
- package/dist/campaign-BIBS-NHV.js.map +0 -1
- package/dist/completion-verifier-B4-IMYcS.d.ts.map +0 -1
- package/dist/cost-ledger-B1D3COAc.d.ts.map +0 -1
- package/dist/cost-ledger-CHDLA0Ss.js.map +0 -1
- package/dist/default-registry-PUhIVRWz.d.ts +0 -215
- package/dist/default-registry-PUhIVRWz.d.ts.map +0 -1
- package/dist/default-registry-lp5R0lve.js.map +0 -1
- package/dist/ledger-core-C0Yx1I14.js.map +0 -1
- package/dist/registry-C4yJTza7.d.ts +0 -178
- package/dist/registry-C4yJTza7.d.ts.map +0 -1
- package/dist/replay-Cb-4Vf0k.js.map +0 -1
- package/dist/replay-DbIYwso6.d.ts.map +0 -1
- package/dist/semantic-concept-judge-DYXDPZW0.js.map +0 -1
- package/dist/single-run-lock-D_bS5xhj.js +0 -318
- package/dist/single-run-lock-D_bS5xhj.js.map +0 -1
- package/dist/skill-usage-Bv3G4VkA.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-CjKMZy0d.js.map +0 -1
- package/dist/skillopt-optimization-method-CzfnA8O-.d.ts.map +0 -1
- package/dist/store-otlp-BenKynPE.js +0 -1688
- package/dist/store-otlp-BenKynPE.js.map +0 -1
- package/dist/tools-DZGdROtG.js +0 -255
- package/dist/tools-DZGdROtG.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -2,22 +2,22 @@ import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
|
2
2
|
import { _ as observeAll, a as jsonlReviewStore, c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, g as noProgressDetector, h as errorStreakDetector, i as inMemoryReviewStore, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, r as createLlmReviewer, s as controlRunToRunRecord, t as controlFailureClassFromVerification, u as objectiveEval, v as repeatedActionDetector, y as evaluateActionPolicy } from "./propose-review-control-DLXz4FCX.js";
|
|
3
3
|
import { a as LimitExceededError, c as ValidationError, i as JudgeError, l as VerificationError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ReplayError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
|
|
4
4
|
import { _ as verifyManifest, a as assertRunAgentProfileCell, c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, g as signManifest, h as hashJson, i as agentProfileCellKey, l as requireAgentProfileCell, m as evaluateHypothesis, n as AgentProfileCellValidationError, o as buildAgentInterfaceProfileCell, p as canonicalize, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-CbfBm2g6.js";
|
|
5
|
-
import { C as renderUpstreamFindings, G as createChatClient, K as createAnalystAi, S as renderPriorFindings, W as computeTraceMetrics, a as analystFindingDigest, b as emitControlIntegrityFindings, c as completedAnalystReviewQuality, d as validateAnalystReviewDecisions, f as DEFAULT_TRACE_ANALYST_KINDS, g as FAILURE_MODE_KIND_SPEC, h as IMPROVEMENT_KIND_SPEC, i as assertExactRegistryRunOpts, l as readAnalystReview, m as KNOWLEDGE_GAP_KIND_SPEC, n as AnalystRegistry, o as analystRunDigest, p as KNOWLEDGE_POISONING_KIND_SPEC, r as ExactAnalystRunExecutionError, s as assertUniqueFindingIds, t as buildDefaultAnalystRegistry, u as snapshotAnalystRun, v as CONTROL_INTEGRITY_ANALYST, x as createTraceAnalystKind, y as ControlIntegrityAnalyst } from "./default-registry-lp5R0lve.js";
|
|
6
5
|
import { a as estimateTokens, i as estimateCost, n as MetricsCollector, o as isModelPriced, r as TokenCounter, s as resolveModelPricing, t as MODEL_PRICING } from "./metrics-C9YY1OcL.js";
|
|
7
|
-
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-
|
|
6
|
+
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-CZ9diLxY.js";
|
|
8
7
|
import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-BQd7mzyT.js";
|
|
9
|
-
import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-
|
|
8
|
+
import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-bkztEfIx.js";
|
|
9
|
+
import { C as createChatClient, S as computeTraceMetrics, _ as CONTROL_INTEGRITY_ANALYST, a as analystFindingDigest, c as completedAnalystReviewQuality, d as validateAnalystReviewDecisions, f as DEFAULT_TRACE_ANALYST_KINDS, g as FAILURE_MODE_KIND_SPEC, h as IMPROVEMENT_KIND_SPEC, i as assertExactRegistryRunOpts, l as readAnalystReview, m as KNOWLEDGE_GAP_KIND_SPEC, n as AnalystRegistry, o as analystRunDigest, p as KNOWLEDGE_POISONING_KIND_SPEC, r as ExactAnalystRunExecutionError, s as assertUniqueFindingIds, t as buildDefaultAnalystRegistry, u as snapshotAnalystRun, v as ControlIntegrityAnalyst, y as emitControlIntegrityFindings } from "./default-registry-BgJJItGr.js";
|
|
10
10
|
import { INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OUTPUT_VALUE, RUN_COST_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, applyLlmSpanOtlpAttributes, asNumber, contextInputTokens, firstNumberAttr } from "./trace-attributes.js";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import { f as Mutex, p as mapConcurrent } from "./ledger-core-C0Yx1I14.js";
|
|
14
|
-
import { i as traceAnalystFunctionGroup, n as buildTraceAnalysisToolDescriptors, r as buildTraceAnalystTools, t as TRACE_ANALYST_TOOL_NAMESPACE } from "./tools-DZGdROtG.js";
|
|
11
|
+
import { C as TraceNotFoundError, G as resolveTraceAnalystLimits, J as extractOtlpAttributes, K as asString, Q as readOtlpStatus, S as TraceFileTooLargeError, W as DEFAULT_TRACE_ANALYST_LIMITS, X as inferOtlpKind, Y as firstStringAttr, Z as projectOtlpFlatLine, _ as TraceAnalysisLimitError, b as TraceFileMalformedError, c as traceAnalystFunctionGroup, et as stringField, g as SpanNotFoundError, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst, it as traceSpanKindToOpenInferenceKind, l as createBoundedTraceAnalysisStore, m as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, n as renderPriorFindings, nt as classifyOtlpSpanRole, o as TRACE_ANALYST_TOOL_NAMESPACE, p as DEFAULT_TRACE_ANALYST_BUDGETS, r as renderUpstreamFindings, rt as isOtlpModelCall, s as buildTraceAnalysisToolDescriptors, t as createTraceAnalyst, tt as applyToolSpanOtlpAttributes, v as TraceAnalysisStoreContractError, x as TraceFileMissingError, y as TraceAnalysisValidationError } from "./kind-factory-CFxA0JQX.js";
|
|
12
|
+
import { a as computeFindingId, o as makeFinding, s as makeProposalFinding } from "./usage-receipt-CgxMEBZq.js";
|
|
15
13
|
import { c as SUPERVISOR_RUN_INTEGRITY_SCHEMA, n as supervisorRunRolloutLines, t as analyzeSupervisorRunIntegrity } from "./integrity-CCXTftiL.js";
|
|
16
14
|
import { c as assertMintedLines, f as isRolloutLine, i as ROLLOUT_SCHEMA, m as validateRolloutLine, p as isTrainableSplit, s as assertMinted, u as assertRolloutLine } from "./schema-C6DW4ZHR.js";
|
|
17
15
|
import { a as scoreOrigin, n as observedScore, o as trainingReward, r as observedSplitScore, s as trainingScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
18
16
|
import { a as clamp01, i as aggregateRunScore, r as DEFAULT_RUN_SCORE_WEIGHTS } from "./proposal-findings-2GIUo1et.js";
|
|
19
|
-
import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-
|
|
20
|
-
import {
|
|
17
|
+
import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, h as defineTraceAnalyst, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-BuIJ9IfB.js";
|
|
18
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DTkVyDX-.js";
|
|
19
|
+
import { f as Mutex, p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
|
|
20
|
+
import { At as pairedDeltaTest, Ct as paretoFrontier, D as runCanaries, Dt as decidePairedPromotion, Ft as summarizeBackendIntegrity, G as buildReflectionPrompt, It as JudgeParseError, K as parseReflectionResponse, Mt as assertRealAgentReceipts, Nt as assertRealBackend, Ot as pairedDecisionShape, Pt as summarizeAgentReceiptIntegrity, St as dominates, Tt as scalarScore, W as DEFAULT_MUTATION_PRIMITIVES, _t as canonicalJson, at as createReferenceEquivalenceJudge, bt as inMemoryVerdictCache, ct as redTeamDataset, dt as toolNamesForRun, ft as Dataset, gt as cachedJudge, ht as llmJudge, it as REFERENCE_EQUIVALENCE_JUDGE_VERSION, jt as BackendIntegrityError, kt as minimumPairsForPairedDeltaTest, lt as redTeamReport, mt as hashScenarios, ot as runReferenceEquivalenceJudge, pt as HoldoutLockedError, rt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, st as DEFAULT_RED_TEAM_CORPUS, ut as scoreRedTeamOutput, vt as contentHash, wt as paretoFrontierWithCrowding, xt as crowdingDistance, yt as fileVerdictCache, z as surfaceContentHash } from "./skillopt-optimization-method-BbGnCC53.js";
|
|
21
21
|
import { $ as selfPreference, A as pairedRiskDifference, B as requiredSampleSize, C as mulberry32, D as pairedCohensDz, E as pairedBootstrap, F as partialCredit, G as wilson, H as weightedComposite, I as passAtK, J as normalCdf, K as studentTCdf, L as pearsonR, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, P as pairedTTest, Q as positionalBias, R as ranks, S as mcnemarRequiredN, T as pairedBinaryScale, U as weightedMean, V as spearmanR, W as wilcoxonSignedRank, X as calibrateJudgeContinuous, Y as calibrateJudge, Z as continuousAgreement, _ as interpretCliffs, a as MANN_WHITNEY_EXACT_MAX_WORK, b as mcnemar, c as bonferroni, d as confidenceInterval, et as verbosityBias, f as corpusInterRaterAgreement, g as interRaterReliability, h as holm, i as MANN_WHITNEY_EXACT_MAX_STATES, j as pairedRiskDifferenceExact, k as pairedMde, l as cliffsDelta, m as eProcess, n as DECISION_PAIRED_DELTA_STATISTIC, o as WILCOXON_EXACT_MAX_N, p as corpusInterRaterAgreementFromJudgeScores, r as DEFAULT_PERMUTATIONS, s as benjaminiHochberg, t as BOOTSTRAP_GATE_MIN_N, u as cohensD, v as isBinaryOutcomeVector, w as normalizeScores, x as mcnemarPower, y as mannWhitneyU, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
|
|
22
22
|
import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
|
|
23
23
|
import { n as llmSpanFromProvider, t as TraceEmitter } from "./emitter-CPBAhxum.js";
|
|
@@ -25,13 +25,14 @@ import { a as isRetrievalSpan, i as isLlmSpan, n as TRACE_SCHEMA_VERSION, o as i
|
|
|
25
25
|
import { a as roundTripRunRecord, i as parseRunRecordSafe, n as isRunRecord, o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord, t as RunRecordValidationError } from "./run-record-vRgqWmJw.js";
|
|
26
26
|
import { a as evaluateReleaseConfidence, i as assertReleaseConfidence, n as bootstrapCi, r as judgeReplayGate, t as renderReleaseReport } from "./release-report-B5XPBvAU.js";
|
|
27
27
|
import { i as toRewardRows, r as toJsonl, s as toSftRows } from "./exporters-q9iL-2Jf.js";
|
|
28
|
-
import { a as toHarborTrajectories, i as relabelImportedSplit, n as HARBOR_IMPORT_GAP, o as toHarborTrajectory, r as fromHarborTrajectory, t as ATIF_SCHEMA_VERSION } from "./rollout-
|
|
28
|
+
import { a as toHarborTrajectories, i as relabelImportedSplit, n as HARBOR_IMPORT_GAP, o as toHarborTrajectory, r as fromHarborTrajectory, t as ATIF_SCHEMA_VERSION } from "./rollout-8nj3mYvx.js";
|
|
29
29
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
30
30
|
import { n as unmintableReasons, t as mintRolloutRows } from "./mint-Ctwk079K.js";
|
|
31
31
|
import { C as SUPERVISOR_RUN_SCHEMA, T as showMeasured, _ as readClaudeCodeSupervisorRun, b as rollupSupervisorRuns, c as writeSupervisorRunReport, d as renderSupervisorRunHeadline, f as renderSupervisorRunMarkdown, g as claudeCodeSupervisorRunReader, t as analyzeSupervisorRun, v as analyzeSupervisorRunSources, w as isUnavailable } from "./supervisor-run-B2EWUmQY.js";
|
|
32
|
-
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
32
|
+
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-SA4OB7O7.js";
|
|
33
|
+
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-DX4fGIcf.js";
|
|
34
|
+
import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-DZs601Va.js";
|
|
35
|
+
import { B as agentProfileModelId, G as createLlmCorrectnessChecker, H as harnessAxisOf, I as CODING_HARNESSES, J as verifyCompletion, K as createTokenRecallChecker, L as HARNESS_NATIVE_MODEL, R as agentProfileHash, U as extractProducedState, V as expandProfileAxes, W as completionVerdict, q as parseCorrectnessResponse, z as agentProfileId } from "./campaign-DrS6_hLd.js";
|
|
35
36
|
import { n as iqr, r as welchsTTest, t as compareToBaseline } from "./baseline-C-GocmIW.js";
|
|
36
37
|
import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
|
|
37
38
|
import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-BScZvqMV.js";
|
|
@@ -41,16 +42,15 @@ import { a as InMemoryTraceStore, i as FileSystemTraceStore, n as assertRunCaptu
|
|
|
41
42
|
import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
42
43
|
import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "./tool-use-metrics-DEGMKycK.js";
|
|
43
44
|
import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
|
|
44
|
-
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-
|
|
45
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
45
|
+
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-zxhy1QV3.js";
|
|
46
|
+
import { t as runEvalCampaign } from "./eval-campaign-BmptJj50.js";
|
|
46
47
|
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
|
|
47
|
-
import { AxGEPA, AxSignature, ax } from "@ax-llm/ax";
|
|
48
48
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
49
49
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
50
50
|
import { createHash } from "node:crypto";
|
|
51
|
+
import { execSync, spawnSync } from "node:child_process";
|
|
51
52
|
import { readFile } from "node:fs/promises";
|
|
52
53
|
import { cpus } from "node:os";
|
|
53
|
-
import { execSync, spawnSync } from "node:child_process";
|
|
54
54
|
import { gzipSync } from "node:zlib";
|
|
55
55
|
//#region src/auto-pr.ts
|
|
56
56
|
/**
|
|
@@ -4265,80 +4265,21 @@ function renderSteeringText(bundle) {
|
|
|
4265
4265
|
}
|
|
4266
4266
|
//#endregion
|
|
4267
4267
|
//#region src/steering-optimizer.ts
|
|
4268
|
+
/**
|
|
4269
|
+
* Rank already-evaluated steering variants.
|
|
4270
|
+
*
|
|
4271
|
+
* Model-backed candidate generation belongs to the official GEPA campaign
|
|
4272
|
+
* integration. This class only performs the deterministic selection it owns.
|
|
4273
|
+
*/
|
|
4268
4274
|
var PairwiseSteeringOptimizer = class {
|
|
4269
4275
|
optimize(rows, config = {}) {
|
|
4270
|
-
const
|
|
4271
|
-
if (
|
|
4276
|
+
const rankings = rankRows(rows, config.weights);
|
|
4277
|
+
if (rankings.length === 0) throw new Error("no steering optimization rows");
|
|
4272
4278
|
return {
|
|
4273
4279
|
backend: "pairwise",
|
|
4274
|
-
recommendedVariantId:
|
|
4280
|
+
recommendedVariantId: rankings[0].variantId,
|
|
4275
4281
|
rationale: `Highest observed mean aggregate across ${rows.length} scored run(s).`,
|
|
4276
|
-
rankings
|
|
4277
|
-
};
|
|
4278
|
-
}
|
|
4279
|
-
};
|
|
4280
|
-
var AxGepaSteeringOptimizer = class {
|
|
4281
|
-
config;
|
|
4282
|
-
constructor(config) {
|
|
4283
|
-
this.config = config;
|
|
4284
|
-
}
|
|
4285
|
-
async optimize(rows) {
|
|
4286
|
-
const fallback = new PairwiseSteeringOptimizer().optimize(rows, this.config);
|
|
4287
|
-
const minScenarioWinners = this.config.minScenarioWinners ?? 6;
|
|
4288
|
-
const variantIds = [...new Set(rows.map((row) => row.variantId))];
|
|
4289
|
-
const byScenario = collapseScenarioWinners(rows, this.config.weights);
|
|
4290
|
-
if (variantIds.length < 2 || byScenario.length < minScenarioWinners) return {
|
|
4291
|
-
...fallback,
|
|
4292
|
-
backend: "ax-gepa",
|
|
4293
|
-
skipped: true,
|
|
4294
|
-
rationale: `AxGEPA skipped: need >=2 variants and >=${minScenarioWinners} scenario winners, got ${variantIds.length} variant(s) and ${byScenario.length} scenario winner(s).`
|
|
4295
|
-
};
|
|
4296
|
-
const signature = `task:string, split:string, seedPreview:string -> variantId:class "${variantIds.join(", ")}", rationale:string`;
|
|
4297
|
-
const selector = ax(AxSignature.from(signature), { description: "Choose the best steering bundle variant for an autopilot task." });
|
|
4298
|
-
const shuffled = seededShuffle(byScenario, signature);
|
|
4299
|
-
const splitIndex = Math.max(1, Math.floor(shuffled.length * .8));
|
|
4300
|
-
const train = shuffled.slice(0, splitIndex);
|
|
4301
|
-
const validation = shuffled.slice(splitIndex);
|
|
4302
|
-
if (!validation.length) return {
|
|
4303
|
-
...fallback,
|
|
4304
|
-
backend: "ax-gepa",
|
|
4305
|
-
skipped: true,
|
|
4306
|
-
rationale: "AxGEPA skipped: no validation examples after split."
|
|
4307
|
-
};
|
|
4308
|
-
const studentAI = createAxService(this.config.provider, this.config.apiKey, this.config.model);
|
|
4309
|
-
const optimizer = new AxGEPA({
|
|
4310
|
-
studentAI,
|
|
4311
|
-
teacherAI: createAxService(this.config.provider, this.config.apiKey, this.config.teacherModel ?? this.config.model),
|
|
4312
|
-
numTrials: 8,
|
|
4313
|
-
minibatch: true,
|
|
4314
|
-
minibatchSize: 4,
|
|
4315
|
-
earlyStoppingTrials: 3,
|
|
4316
|
-
sampleCount: 1
|
|
4317
|
-
});
|
|
4318
|
-
const metric = ({ prediction, example }) => stringField$1(prediction, "variantId") === stringField$1(example, "variantId") ? 1 : 0;
|
|
4319
|
-
const compiled = await optimizer.compile(selector, train, metric, {
|
|
4320
|
-
validationExamples: validation,
|
|
4321
|
-
maxMetricCalls: 64
|
|
4322
|
-
});
|
|
4323
|
-
if (compiled.optimizedProgram !== void 0) selector.applyOptimization(compiled.optimizedProgram);
|
|
4324
|
-
const selectVariant = async (row) => {
|
|
4325
|
-
const prediction = await selector.forward(studentAI, row);
|
|
4326
|
-
return {
|
|
4327
|
-
variantId: String(prediction.variantId),
|
|
4328
|
-
rationale: String(prediction.rationale ?? "")
|
|
4329
|
-
};
|
|
4330
|
-
};
|
|
4331
|
-
return {
|
|
4332
|
-
...fallback,
|
|
4333
|
-
backend: "ax-gepa",
|
|
4334
|
-
rationale: `AxGEPA trained a variant selector from ${byScenario.length} scored scenario winner(s); default winner remains ${fallback.recommendedVariantId}.`,
|
|
4335
|
-
selector: {
|
|
4336
|
-
backend: "ax-gepa",
|
|
4337
|
-
signature,
|
|
4338
|
-
labels: variantIds,
|
|
4339
|
-
rationale: compiled.bestScore !== void 0 ? `bestScore=${compiled.bestScore}` : void 0
|
|
4340
|
-
},
|
|
4341
|
-
selectVariant
|
|
4282
|
+
rankings
|
|
4342
4283
|
};
|
|
4343
4284
|
}
|
|
4344
4285
|
};
|
|
@@ -4355,64 +4296,6 @@ function rankRows(rows, weights) {
|
|
|
4355
4296
|
runs: values.length
|
|
4356
4297
|
})).sort((a, b) => b.mean - a.mean);
|
|
4357
4298
|
}
|
|
4358
|
-
function collapseScenarioWinners(rows, weights) {
|
|
4359
|
-
const byScenario = /* @__PURE__ */ new Map();
|
|
4360
|
-
for (const row of rows) {
|
|
4361
|
-
const bucket = byScenario.get(row.scenarioId) ?? [];
|
|
4362
|
-
bucket.push(row);
|
|
4363
|
-
byScenario.set(row.scenarioId, bucket);
|
|
4364
|
-
}
|
|
4365
|
-
return [...byScenario.entries()].map(([scenarioId, scenarioRows]) => {
|
|
4366
|
-
const best = scenarioRows.map((row) => ({
|
|
4367
|
-
row,
|
|
4368
|
-
aggregate: aggregateRunScore(row.score, weights)
|
|
4369
|
-
})).sort((a, b) => b.aggregate - a.aggregate)[0];
|
|
4370
|
-
return {
|
|
4371
|
-
task: String(best.row.metadata?.task ?? best.row.metadata?.seed_preview ?? scenarioId),
|
|
4372
|
-
split: String(best.row.metadata?.split ?? "train"),
|
|
4373
|
-
seedPreview: String(best.row.metadata?.seed_preview ?? ""),
|
|
4374
|
-
variantId: best.row.variantId
|
|
4375
|
-
};
|
|
4376
|
-
});
|
|
4377
|
-
}
|
|
4378
|
-
function createAxService(provider, apiKey, model) {
|
|
4379
|
-
return createAnalystAi({
|
|
4380
|
-
provider,
|
|
4381
|
-
apiKey,
|
|
4382
|
-
model
|
|
4383
|
-
});
|
|
4384
|
-
}
|
|
4385
|
-
function stringField$1(value, field) {
|
|
4386
|
-
if (!value || typeof value !== "object") return void 0;
|
|
4387
|
-
const candidate = value[field];
|
|
4388
|
-
return typeof candidate === "string" ? candidate : void 0;
|
|
4389
|
-
}
|
|
4390
|
-
function seededShuffle(items, seed) {
|
|
4391
|
-
const rng = mulberry32$1(hashString(seed));
|
|
4392
|
-
const out = [...items];
|
|
4393
|
-
for (let i = out.length - 1; i > 0; i--) {
|
|
4394
|
-
const j = Math.floor(rng() * (i + 1));
|
|
4395
|
-
[out[i], out[j]] = [out[j], out[i]];
|
|
4396
|
-
}
|
|
4397
|
-
return out;
|
|
4398
|
-
}
|
|
4399
|
-
function hashString(value) {
|
|
4400
|
-
let h = 2166136261;
|
|
4401
|
-
for (let i = 0; i < value.length; i++) {
|
|
4402
|
-
h ^= value.charCodeAt(i);
|
|
4403
|
-
h = Math.imul(h, 16777619);
|
|
4404
|
-
}
|
|
4405
|
-
return h >>> 0;
|
|
4406
|
-
}
|
|
4407
|
-
function mulberry32$1(seed) {
|
|
4408
|
-
let a = seed >>> 0;
|
|
4409
|
-
return () => {
|
|
4410
|
-
a = a + 1831565813 | 0;
|
|
4411
|
-
let t = Math.imul(a ^ a >>> 15, 1 | a);
|
|
4412
|
-
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
4413
|
-
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
4414
|
-
};
|
|
4415
|
-
}
|
|
4416
4299
|
//#endregion
|
|
4417
4300
|
//#region src/workspace-inspector.ts
|
|
4418
4301
|
var InMemoryWorkspaceInspector = class {
|
|
@@ -11316,56 +11199,48 @@ function sectionHash(section) {
|
|
|
11316
11199
|
}
|
|
11317
11200
|
//#endregion
|
|
11318
11201
|
//#region src/traced-analyst.ts
|
|
11319
|
-
/**
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
async function tracedAnalyzeTraces(input, options, traceOpts) {
|
|
11323
|
-
const parentSpan = await traceOpts.emitter.span({
|
|
11202
|
+
/** Run a recursive trace investigation and record its engine steps. */
|
|
11203
|
+
async function tracedAnalyzeTraces(input, options, traceOptions) {
|
|
11204
|
+
const parentSpan = await traceOptions.emitter.span({
|
|
11324
11205
|
kind: "custom",
|
|
11325
11206
|
name: "analyst:analyze-traces",
|
|
11326
|
-
parentSpanId:
|
|
11207
|
+
parentSpanId: traceOptions.parentSpanId,
|
|
11327
11208
|
attributes: {
|
|
11209
|
+
"analyst.engine": options.engine.id,
|
|
11328
11210
|
"analyst.question_length": input.question.length,
|
|
11329
|
-
"analyst.
|
|
11330
|
-
"analyst.
|
|
11211
|
+
"analyst.max_iterations": options.limits?.maxIterations ?? 12,
|
|
11212
|
+
"analyst.max_llm_calls": options.limits?.maxLlmCalls ?? 8,
|
|
11331
11213
|
"eval.phase": "analyst"
|
|
11332
11214
|
}
|
|
11333
11215
|
});
|
|
11334
|
-
|
|
11335
|
-
|
|
11336
|
-
|
|
11337
|
-
|
|
11338
|
-
|
|
11216
|
+
try {
|
|
11217
|
+
const result = await analyzeTraces(input, options);
|
|
11218
|
+
for (const [index, step] of result.trajectory.entries()) {
|
|
11219
|
+
const encoded = JSON.stringify(step);
|
|
11220
|
+
await (await traceOptions.emitter.span({
|
|
11339
11221
|
kind: "custom",
|
|
11340
|
-
name: `analyst:
|
|
11222
|
+
name: `analyst:step-${index + 1}`,
|
|
11341
11223
|
parentSpanId: parentSpan.span.spanId,
|
|
11342
11224
|
attributes: {
|
|
11343
|
-
"analyst.
|
|
11344
|
-
"analyst.
|
|
11345
|
-
"analyst.is_error": turn.isError,
|
|
11346
|
-
"analyst.code_length": turn.code.length,
|
|
11347
|
-
"analyst.output_length": turn.output.length,
|
|
11225
|
+
"analyst.step": index + 1,
|
|
11226
|
+
"analyst.step_bytes": Buffer.byteLength(encoded),
|
|
11348
11227
|
"eval.phase": "analyst"
|
|
11349
11228
|
}
|
|
11350
|
-
});
|
|
11351
|
-
if (turn.isError) await turnSpan.fail("Turn produced an error");
|
|
11352
|
-
else await turnSpan.end();
|
|
11353
|
-
if (originalOnTurn) await originalOnTurn(turn);
|
|
11229
|
+
})).end();
|
|
11354
11230
|
}
|
|
11355
|
-
};
|
|
11356
|
-
try {
|
|
11357
|
-
const result = await analyzeTraces(input, wrappedOptions);
|
|
11358
11231
|
await parentSpan.end({ attributes: {
|
|
11359
|
-
"analyst.
|
|
11360
|
-
"analyst.
|
|
11232
|
+
"analyst.engine": options.engine.id,
|
|
11233
|
+
"analyst.model_calls": result.modelCalls,
|
|
11234
|
+
"analyst.tool_calls": result.toolCalls,
|
|
11235
|
+
"analyst.step_count": result.trajectory.length,
|
|
11361
11236
|
"analyst.finding_count": result.findings.length,
|
|
11362
11237
|
"analyst.answer_length": result.answer.length,
|
|
11363
11238
|
"eval.phase": "analyst"
|
|
11364
11239
|
} });
|
|
11365
11240
|
return result;
|
|
11366
|
-
} catch (
|
|
11367
|
-
await parentSpan.fail(
|
|
11368
|
-
throw
|
|
11241
|
+
} catch (error) {
|
|
11242
|
+
await parentSpan.fail(error instanceof Error ? error : String(error));
|
|
11243
|
+
throw error;
|
|
11369
11244
|
}
|
|
11370
11245
|
}
|
|
11371
11246
|
//#endregion
|
|
@@ -12512,6 +12387,6 @@ function assertProductBenchmarkRun(runDir) {
|
|
|
12512
12387
|
return report;
|
|
12513
12388
|
}
|
|
12514
12389
|
//#endregion
|
|
12515
|
-
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, AxGepaSteeringOptimizer, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExactAnalystRunExecutionError, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertExactRegistryRunOpts, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAnalystAi, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalystKind, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12390
|
+
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_LIMITS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExactAnalystRunExecutionError, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertExactRegistryRunOpts, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createDspyRlmTraceEngine, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalyst, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, defineTraceAnalyst, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, resolveTraceAnalystLimits, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runTraceAnalyst, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12516
12391
|
|
|
12517
12392
|
//# sourceMappingURL=index.js.map
|