@tangle-network/agent-eval 0.137.0 → 0.139.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +78 -0
- package/README.md +34 -0
- package/dist/analyst/index.d.ts +485 -104
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +10 -607
- package/dist/analyst/index.js.map +1 -1
- package/dist/{benchmark-YDrpumqB.js → benchmark-CYtcIF2V.js} +299 -159
- package/dist/benchmark-CYtcIF2V.js.map +1 -0
- package/dist/{benchmark-CHX4orG7.d.ts → benchmark-DDVdWcwA.d.ts} +67 -15
- package/dist/benchmark-DDVdWcwA.d.ts.map +1 -0
- package/dist/benchmark-command-BKfjOBJ5.js +4537 -0
- package/dist/benchmark-command-BKfjOBJ5.js.map +1 -0
- package/dist/benchmarks/index.d.ts +1 -1
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-DCLkQOmc.js → benchmarks-zxhy1QV3.js} +4 -3
- package/dist/{benchmarks-DCLkQOmc.js.map → benchmarks-zxhy1QV3.js.map} +1 -1
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +4 -3
- package/dist/{campaign-lgObcHFC.js → campaign-DrS6_hLd.js} +19 -11
- package/dist/campaign-DrS6_hLd.js.map +1 -0
- package/dist/canonical-D011XM8r.js +86 -0
- package/dist/canonical-D011XM8r.js.map +1 -0
- package/dist/cli.js +10 -3
- package/dist/cli.js.map +1 -1
- package/dist/{client-C8L6h6Wf.d.ts → client-BohnDFBq.d.ts} +4 -4
- package/dist/{client-C8L6h6Wf.d.ts.map → client-BohnDFBq.d.ts.map} +1 -1
- package/dist/{completion-verifier-DSyRNVzU.d.ts → completion-verifier-IPoP4fQO.d.ts} +178 -4
- package/dist/completion-verifier-IPoP4fQO.d.ts.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +9 -8
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +2 -2
- package/dist/{cost-ledger-D-5_-dhi.js → cost-ledger-CZ9diLxY.js} +90 -45
- package/dist/cost-ledger-CZ9diLxY.js.map +1 -0
- package/dist/{cost-ledger-D2o6JOrL.d.ts → cost-ledger-DKgyIWRj.d.ts} +6 -2
- package/dist/cost-ledger-DKgyIWRj.d.ts.map +1 -0
- package/dist/default-registry-B8vf7Rmf.d.ts +118 -0
- package/dist/default-registry-B8vf7Rmf.d.ts.map +1 -0
- package/dist/default-registry-BgJJItGr.js +2364 -0
- package/dist/default-registry-BgJJItGr.js.map +1 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js +344 -0
- package/dist/dspy-rlm-engine-DTkVyDX-.js.map +1 -0
- package/dist/{eval-campaign-CHqfLnff.js → eval-campaign-BmptJj50.js} +2 -2
- package/dist/{eval-campaign-CHqfLnff.js.map → eval-campaign-BmptJj50.js.map} +1 -1
- package/dist/exact-types-MaaFcllV.d.ts +234 -0
- package/dist/exact-types-MaaFcllV.d.ts.map +1 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts +32 -0
- package/dist/external-optimizer-contracts-BrxY2Sli.d.ts.map +1 -0
- package/dist/{extract-usage-p-56bh8q.js → extract-usage-DZs601Va.js} +2 -2
- package/dist/{extract-usage-p-56bh8q.js.map → extract-usage-DZs601Va.js.map} +1 -1
- package/dist/{feedback-trajectory-N_F0PwHz.d.ts → feedback-trajectory-BJUWOkJM.d.ts} +3 -2
- package/dist/feedback-trajectory-BJUWOkJM.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -1
- package/dist/fuzz.js +1 -1
- package/dist/{hf-dataset-DBJXXoY1.js → hf-dataset-XggBupCr.js} +2 -2
- package/dist/{hf-dataset-DBJXXoY1.js.map → hf-dataset-XggBupCr.js.map} +1 -1
- package/dist/hosted/index.d.ts +3 -3
- package/dist/{index-C-Pr4OWg.d.ts → index-BTm_P9aC.d.ts} +12 -11
- package/dist/index-BTm_P9aC.d.ts.map +1 -0
- package/dist/{index-U3RHOShi.d.ts → index-CWOPCJiw.d.ts} +2 -2
- package/dist/{index-U3RHOShi.d.ts.map → index-CWOPCJiw.d.ts.map} +1 -1
- package/dist/{index-DRNl6g_N.d.ts → index-CtR1xh4V.d.ts} +3 -3
- package/dist/{index-DRNl6g_N.d.ts.map → index-CtR1xh4V.d.ts.map} +1 -1
- package/dist/index-DEb46kc6.d.ts.map +1 -1
- package/dist/{index-BnP1QJUv.d.ts → index-_66rVpwN.d.ts} +5 -5
- package/dist/{index-BnP1QJUv.d.ts.map → index-_66rVpwN.d.ts.map} +1 -1
- package/dist/index.d.ts +35 -55
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +55 -514
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-B9ooYH_g.d.ts → insight-report-Bu5Wi9tG.d.ts} +4 -4
- package/dist/{insight-report-B9ooYH_g.d.ts.map → insight-report-Bu5Wi9tG.d.ts.map} +1 -1
- package/dist/{integrity-CKxosZ5Z.d.ts → integrity-COTh3DTH.d.ts} +2 -2
- package/dist/{integrity-CKxosZ5Z.d.ts.map → integrity-COTh3DTH.d.ts.map} +1 -1
- package/dist/kind-factory-CFxA0JQX.js +2133 -0
- package/dist/kind-factory-CFxA0JQX.js.map +1 -0
- package/dist/ledger-core/index.js +2 -1
- package/dist/{ledger-core-t6sItivm.js → ledger-core-Dxz0Rkwa.js} +210 -99
- package/dist/ledger-core-Dxz0Rkwa.js.map +1 -0
- package/dist/{llm-client-DKB25jV8.js → llm-client-bkztEfIx.js} +5 -5
- package/dist/llm-client-bkztEfIx.js.map +1 -0
- package/dist/meta-eval/index.d.ts +2 -2
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/{proposal-findings-DCawte-y.js → proposal-findings-2GIUo1et.js} +2 -68
- package/dist/proposal-findings-2GIUo1et.js.map +1 -0
- package/dist/{release-report-CofgVNZt.d.ts → release-report-fZarvIm-.d.ts} +3 -3
- package/dist/{release-report-CofgVNZt.d.ts.map → release-report-fZarvIm-.d.ts.map} +1 -1
- package/dist/{replay-K8FaC0CB.d.ts → replay-DjG4IG60.d.ts} +34 -143
- package/dist/replay-DjG4IG60.d.ts.map +1 -0
- package/dist/{replay-Bju0T8Ls.js → replay-SA4OB7O7.js} +48 -136
- package/dist/replay-SA4OB7O7.js.map +1 -0
- package/dist/reporting.d.ts +4 -4
- package/dist/{researcher-Da0Wj-bt.d.ts → researcher-BxhtGfKa.d.ts} +5 -5
- package/dist/{researcher-Da0Wj-bt.d.ts.map → researcher-BxhtGfKa.d.ts.map} +1 -1
- package/dist/{reward-hacking-CQ3hTCO3.d.ts → reward-hacking-CqSLiV51.d.ts} +2 -2
- package/dist/{reward-hacking-CQ3hTCO3.d.ts.map → reward-hacking-CqSLiV51.d.ts.map} +1 -1
- package/dist/rl.d.ts +5 -5
- package/dist/rl.js +1 -1
- package/dist/rollout/index.d.ts +1 -1
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-DQFl0UXA.js → rollout-8nj3mYvx.js} +2 -2
- package/dist/{rollout-DQFl0UXA.js.map → rollout-8nj3mYvx.js.map} +1 -1
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts → rubric-predictive-validity-DQBQj6uV.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-C4sztLR3.d.ts.map → rubric-predictive-validity-DQBQj6uV.d.ts.map} +1 -1
- package/dist/{run-evidence-BDIircdA.d.ts → run-evidence-C4RcRQT5.d.ts} +3 -3
- package/dist/{run-evidence-BDIircdA.d.ts.map → run-evidence-C4RcRQT5.d.ts.map} +1 -1
- package/dist/{run-record-BPCa2rQ8.d.ts → run-record-CztDMXVF.d.ts} +2 -2
- package/dist/{run-record-BPCa2rQ8.d.ts.map → run-record-CztDMXVF.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-Bz64IckK.js → semantic-concept-judge-BuIJ9IfB.js} +49 -6
- package/dist/semantic-concept-judge-BuIJ9IfB.js.map +1 -0
- package/dist/{server-KjXZZUDX.js → server-DaCpLfi0.js} +3 -3
- package/dist/{server-KjXZZUDX.js.map → server-DaCpLfi0.js.map} +1 -1
- package/dist/single-run-lock-BTTtPZ9N.js +989 -0
- package/dist/single-run-lock-BTTtPZ9N.js.map +1 -0
- package/dist/{skill-usage-CFDLLlhF.d.ts → skill-usage-B-BFS8M2.d.ts} +65 -40
- package/dist/skill-usage-B-BFS8M2.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-f4o9sUT4.js → skillopt-optimization-method-BbGnCC53.js} +20 -979
- package/dist/skillopt-optimization-method-BbGnCC53.js.map +1 -0
- package/dist/{skillopt-optimization-method-BpbnlvAZ.d.ts → skillopt-optimization-method-_s0Tub7Y.d.ts} +11 -39
- package/dist/skillopt-optimization-method-_s0Tub7Y.d.ts.map +1 -0
- package/dist/{statistics-_7P642CN.d.ts → statistics-B5d0Zd-z.d.ts} +2 -2
- package/dist/{statistics-_7P642CN.d.ts.map → statistics-B5d0Zd-z.d.ts.map} +1 -1
- package/dist/store-otlp-DX4fGIcf.js +757 -0
- package/dist/store-otlp-DX4fGIcf.js.map +1 -0
- package/dist/{summary-report-DHipz9Kx.d.ts → summary-report-Cg7BifAM.d.ts} +3 -3
- package/dist/{summary-report-DHipz9Kx.d.ts.map → summary-report-Cg7BifAM.d.ts.map} +1 -1
- package/dist/tool-groups-CdYq22lX.d.ts +258 -0
- package/dist/tool-groups-CdYq22lX.d.ts.map +1 -0
- package/dist/traces.d.ts +7 -6
- package/dist/traces.js +5 -4
- package/dist/{types-CTGbIm57.d.ts → types-BBFNHxSK.d.ts} +5 -5
- package/dist/{types-CTGbIm57.d.ts.map → types-BBFNHxSK.d.ts.map} +1 -1
- package/dist/{types-CKswbJGO.d.ts → types-DoEYskCd.d.ts} +5 -5
- package/dist/{types-CKswbJGO.d.ts.map → types-DoEYskCd.d.ts.map} +1 -1
- package/dist/{types-CTvKfr5F.d.ts → types-uPrS6mD-.d.ts} +2 -2
- package/dist/{types-CTvKfr5F.d.ts.map → types-uPrS6mD-.d.ts.map} +1 -1
- package/dist/usage-receipt-CgxMEBZq.js +134 -0
- package/dist/usage-receipt-CgxMEBZq.js.map +1 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/wire/index.js +1 -1
- package/docs/trace-analysis.md +191 -385
- package/package.json +5 -4
- package/dist/analyze-runs-PVtnfjvA.d.ts +0 -72
- package/dist/analyze-runs-PVtnfjvA.d.ts.map +0 -1
- package/dist/benchmark-CHX4orG7.d.ts.map +0 -1
- package/dist/benchmark-YDrpumqB.js.map +0 -1
- package/dist/campaign-lgObcHFC.js.map +0 -1
- package/dist/completion-verifier-DSyRNVzU.d.ts.map +0 -1
- package/dist/concurrency-MUjT7VjM.js +0 -109
- package/dist/concurrency-MUjT7VjM.js.map +0 -1
- package/dist/cost-ledger-D-5_-dhi.js.map +0 -1
- package/dist/cost-ledger-D2o6JOrL.d.ts.map +0 -1
- package/dist/default-registry-CLXbRt0f.js +0 -2594
- package/dist/default-registry-CLXbRt0f.js.map +0 -1
- package/dist/default-registry-Dc5D_Loc.d.ts +0 -202
- package/dist/default-registry-Dc5D_Loc.d.ts.map +0 -1
- package/dist/feedback-trajectory-N_F0PwHz.d.ts.map +0 -1
- package/dist/index-C-Pr4OWg.d.ts.map +0 -1
- package/dist/ledger-core-t6sItivm.js.map +0 -1
- package/dist/llm-client-DKB25jV8.js.map +0 -1
- package/dist/proposal-findings-DCawte-y.js.map +0 -1
- package/dist/registry-BdM7SuTr.d.ts +0 -124
- package/dist/registry-BdM7SuTr.d.ts.map +0 -1
- package/dist/replay-Bju0T8Ls.js.map +0 -1
- package/dist/replay-K8FaC0CB.d.ts.map +0 -1
- package/dist/semantic-concept-judge-Bz64IckK.js.map +0 -1
- package/dist/skill-usage-CFDLLlhF.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-BpbnlvAZ.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-f4o9sUT4.js.map +0 -1
- package/dist/tools-DZk2Jn64.js +0 -1876
- package/dist/tools-DZk2Jn64.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -2,35 +2,37 @@ import { t as __exportAll } from "./rolldown-runtime-8H4AJuhK.js";
|
|
|
2
2
|
import { _ as observeAll, a as jsonlReviewStore, c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, g as noProgressDetector, h as errorStreakDetector, i as inMemoryReviewStore, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, r as createLlmReviewer, s as controlRunToRunRecord, t as controlFailureClassFromVerification, u as objectiveEval, v as repeatedActionDetector, y as evaluateActionPolicy } from "./propose-review-control-DLXz4FCX.js";
|
|
3
3
|
import { a as LimitExceededError, c as ValidationError, i as JudgeError, l as VerificationError, n as CaptureIntegrityError, o as NotFoundError, r as ConfigError, s as ReplayError, t as AgentEvalError } from "./errors-D-LKuDhb.js";
|
|
4
4
|
import { _ as verifyManifest, a as assertRunAgentProfileCell, c as groupRunsByAgentProfileCell, d as validateAgentProfileCell, f as verifyAgentProfileCell, g as signManifest, h as hashJson, i as agentProfileCellKey, l as requireAgentProfileCell, m as evaluateHypothesis, n as AgentProfileCellValidationError, o as buildAgentInterfaceProfileCell, p as canonicalize, r as agentProfileCellHashMaterial, s as buildAgentProfileCell, t as AGENT_PROFILE_KINDS, u as toAgentProfileJson } from "./agent-profile-cell-CbfBm2g6.js";
|
|
5
|
-
import { L as computeTraceMetrics, R as createChatClient, a as KNOWLEDGE_GAP_KIND_SPEC, d as emitControlIntegrityFindings, f as createTraceAnalystKind, i as KNOWLEDGE_POISONING_KIND_SPEC, l as CONTROL_INTEGRITY_ANALYST, m as renderUpstreamFindings, n as AnalystRegistry, o as IMPROVEMENT_KIND_SPEC, p as renderPriorFindings, r as DEFAULT_TRACE_ANALYST_KINDS, s as FAILURE_MODE_KIND_SPEC, t as buildDefaultAnalystRegistry, u as ControlIntegrityAnalyst, z as createAnalystAi } from "./default-registry-CLXbRt0f.js";
|
|
6
5
|
import { a as estimateTokens, i as estimateCost, n as MetricsCollector, o as isModelPriced, r as TokenCounter, s as resolveModelPricing, t as MODEL_PRICING } from "./metrics-C9YY1OcL.js";
|
|
7
|
-
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-
|
|
6
|
+
import { a as CostLedgerPersistenceError, c as costForTokenPricing, i as CostLedger, l as costForUsage, n as CostCallConflictError, o as CostReceiptCaptureError, r as CostCeilingReachedError, s as CostReservationExceededError, t as CostAccountingIncompleteError, u as modelPriceKey } from "./cost-ledger-CZ9diLxY.js";
|
|
8
7
|
import { a as providerFromBaseUrl, i as defaultProviderRedactor, n as InMemoryRawProviderSink, r as NoopRawProviderSink, t as FileSystemRawProviderSink } from "./raw-provider-sink-BQd7mzyT.js";
|
|
9
|
-
import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-
|
|
8
|
+
import { a as assertLlmRoute, c as callLlmJson, d as isTransientLlmError, f as maximumChargeForLlmRequest, i as LlmRouteAssertionError, l as costReceiptFromLlm, m as stripFencedJson, n as LlmClient, o as backoffMs, p as probeLlm, r as LlmResponseError, s as callLlm, t as LlmCallError, u as costReceiptFromLlmError } from "./llm-client-bkztEfIx.js";
|
|
9
|
+
import { C as createChatClient, S as computeTraceMetrics, _ as CONTROL_INTEGRITY_ANALYST, a as analystFindingDigest, c as completedAnalystReviewQuality, d as validateAnalystReviewDecisions, f as DEFAULT_TRACE_ANALYST_KINDS, g as FAILURE_MODE_KIND_SPEC, h as IMPROVEMENT_KIND_SPEC, i as assertExactRegistryRunOpts, l as readAnalystReview, m as KNOWLEDGE_GAP_KIND_SPEC, n as AnalystRegistry, o as analystRunDigest, p as KNOWLEDGE_POISONING_KIND_SPEC, r as ExactAnalystRunExecutionError, s as assertUniqueFindingIds, t as buildDefaultAnalystRegistry, u as snapshotAnalystRun, v as ControlIntegrityAnalyst, y as emitControlIntegrityFindings } from "./default-registry-BgJJItGr.js";
|
|
10
10
|
import { INPUT_VALUE, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, OPENINFERENCE_SPAN_KIND, OUTPUT_VALUE, RUN_COST_ATTR_KEYS, SPAN_KIND_ATTR_KEYS, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, applyLlmSpanOtlpAttributes, asNumber, contextInputTokens, firstNumberAttr } from "./trace-attributes.js";
|
|
11
|
-
import {
|
|
12
|
-
import { a as
|
|
11
|
+
import { C as TraceNotFoundError, G as resolveTraceAnalystLimits, J as extractOtlpAttributes, K as asString, Q as readOtlpStatus, S as TraceFileTooLargeError, W as DEFAULT_TRACE_ANALYST_LIMITS, X as inferOtlpKind, Y as firstStringAttr, Z as projectOtlpFlatLine, _ as TraceAnalysisLimitError, b as TraceFileMalformedError, c as traceAnalystFunctionGroup, et as stringField, g as SpanNotFoundError, h as TRACE_ANALYSIS_LIMITS, i as runTraceAnalyst, it as traceSpanKindToOpenInferenceKind, l as createBoundedTraceAnalysisStore, m as TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, n as renderPriorFindings, nt as classifyOtlpSpanRole, o as TRACE_ANALYST_TOOL_NAMESPACE, p as DEFAULT_TRACE_ANALYST_BUDGETS, r as renderUpstreamFindings, rt as isOtlpModelCall, s as buildTraceAnalysisToolDescriptors, t as createTraceAnalyst, tt as applyToolSpanOtlpAttributes, v as TraceAnalysisStoreContractError, x as TraceFileMissingError, y as TraceAnalysisValidationError } from "./kind-factory-CFxA0JQX.js";
|
|
12
|
+
import { a as computeFindingId, o as makeFinding, s as makeProposalFinding } from "./usage-receipt-CgxMEBZq.js";
|
|
13
13
|
import { c as SUPERVISOR_RUN_INTEGRITY_SCHEMA, n as supervisorRunRolloutLines, t as analyzeSupervisorRunIntegrity } from "./integrity-CCXTftiL.js";
|
|
14
14
|
import { c as assertMintedLines, f as isRolloutLine, i as ROLLOUT_SCHEMA, m as validateRolloutLine, p as isTrainableSplit, s as assertMinted, u as assertRolloutLine } from "./schema-C6DW4ZHR.js";
|
|
15
15
|
import { a as scoreOrigin, n as observedScore, o as trainingReward, r as observedSplitScore, s as trainingScore, t as isRealnessGated } from "./reward-nw2xZGZG.js";
|
|
16
|
-
import {
|
|
17
|
-
import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-
|
|
18
|
-
import {
|
|
16
|
+
import { a as clamp01, i as aggregateRunScore, r as DEFAULT_RUN_SCORE_WEIGHTS } from "./proposal-findings-2GIUo1et.js";
|
|
17
|
+
import { a as RunCritic, d as defaultIsMaterial, f as diffFindings, h as defineTraceAnalyst, i as runSemanticConceptJudge, n as SEMANTIC_CONCEPT_JUDGE_VERSION, o as SKILL_USAGE_ANALYST, p as LockedJsonlAppender, r as createSemanticConceptJudge, s as SkillUsageAnalyst, t as DEFAULT_COMPLEXITY_WEIGHTS, u as FindingsStore } from "./semantic-concept-judge-BuIJ9IfB.js";
|
|
18
|
+
import { t as createDspyRlmTraceEngine } from "./dspy-rlm-engine-DTkVyDX-.js";
|
|
19
|
+
import { f as Mutex, p as mapConcurrent } from "./ledger-core-Dxz0Rkwa.js";
|
|
20
|
+
import { At as pairedDeltaTest, Ct as paretoFrontier, D as runCanaries, Dt as decidePairedPromotion, Ft as summarizeBackendIntegrity, G as buildReflectionPrompt, It as JudgeParseError, K as parseReflectionResponse, Mt as assertRealAgentReceipts, Nt as assertRealBackend, Ot as pairedDecisionShape, Pt as summarizeAgentReceiptIntegrity, St as dominates, Tt as scalarScore, W as DEFAULT_MUTATION_PRIMITIVES, _t as canonicalJson, at as createReferenceEquivalenceJudge, bt as inMemoryVerdictCache, ct as redTeamDataset, dt as toolNamesForRun, ft as Dataset, gt as cachedJudge, ht as llmJudge, it as REFERENCE_EQUIVALENCE_JUDGE_VERSION, jt as BackendIntegrityError, kt as minimumPairsForPairedDeltaTest, lt as redTeamReport, mt as hashScenarios, ot as runReferenceEquivalenceJudge, pt as HoldoutLockedError, rt as REFERENCE_EQUIVALENCE_INPUT_LIMITS, st as DEFAULT_RED_TEAM_CORPUS, ut as scoreRedTeamOutput, vt as contentHash, wt as paretoFrontierWithCrowding, xt as crowdingDistance, yt as fileVerdictCache, z as surfaceContentHash } from "./skillopt-optimization-method-BbGnCC53.js";
|
|
19
21
|
import { $ as selfPreference, A as pairedRiskDifference, B as requiredSampleSize, C as mulberry32, D as pairedCohensDz, E as pairedBootstrap, F as partialCredit, G as wilson, H as weightedComposite, I as passAtK, J as normalCdf, K as studentTCdf, L as pearsonR, M as pairedRiskDifferenceScore, N as pairedSignTest, O as pairedDeltaTieFraction, P as pairedTTest, Q as positionalBias, R as ranks, S as mcnemarRequiredN, T as pairedBinaryScale, U as weightedMean, V as spearmanR, W as wilcoxonSignedRank, X as calibrateJudgeContinuous, Y as calibrateJudge, Z as continuousAgreement, _ as interpretCliffs, a as MANN_WHITNEY_EXACT_MAX_WORK, b as mcnemar, c as bonferroni, d as confidenceInterval, et as verbosityBias, f as corpusInterRaterAgreement, g as interRaterReliability, h as holm, i as MANN_WHITNEY_EXACT_MAX_STATES, j as pairedRiskDifferenceExact, k as pairedMde, l as cliffsDelta, m as eProcess, n as DECISION_PAIRED_DELTA_STATISTIC, o as WILCOXON_EXACT_MAX_N, p as corpusInterRaterAgreementFromJudgeScores, r as DEFAULT_PERMUTATIONS, s as benjaminiHochberg, t as BOOTSTRAP_GATE_MIN_N, u as cohensD, v as isBinaryOutcomeVector, w as normalizeScores, x as mcnemarPower, y as mannWhitneyU, z as requiredPairedSampleSize } from "./statistics-ByxzSiOM.js";
|
|
20
22
|
import { n as pairArms, r as pairRunRecords, t as comparePairedArms } from "./paired-arms-iZ08VFMN.js";
|
|
21
23
|
import { n as llmSpanFromProvider, t as TraceEmitter } from "./emitter-CPBAhxum.js";
|
|
22
|
-
import { h as hashCanonical, m as canonicalString } from "./ledger-core-t6sItivm.js";
|
|
23
24
|
import { a as isRetrievalSpan, i as isLlmSpan, n as TRACE_SCHEMA_VERSION, o as isSandboxSpan, r as isJudgeSpan, s as isToolSpan, t as FAILURE_CLASSES } from "./schema-CRhEY1SO.js";
|
|
24
25
|
import { a as roundTripRunRecord, i as parseRunRecordSafe, n as isRunRecord, o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord, t as RunRecordValidationError } from "./run-record-vRgqWmJw.js";
|
|
25
26
|
import { a as evaluateReleaseConfidence, i as assertReleaseConfidence, n as bootstrapCi, r as judgeReplayGate, t as renderReleaseReport } from "./release-report-B5XPBvAU.js";
|
|
26
27
|
import { i as toRewardRows, r as toJsonl, s as toSftRows } from "./exporters-q9iL-2Jf.js";
|
|
27
|
-
import { a as toHarborTrajectories, i as relabelImportedSplit, n as HARBOR_IMPORT_GAP, o as toHarborTrajectory, r as fromHarborTrajectory, t as ATIF_SCHEMA_VERSION } from "./rollout-
|
|
28
|
+
import { a as toHarborTrajectories, i as relabelImportedSplit, n as HARBOR_IMPORT_GAP, o as toHarborTrajectory, r as fromHarborTrajectory, t as ATIF_SCHEMA_VERSION } from "./rollout-8nj3mYvx.js";
|
|
28
29
|
import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
|
|
29
30
|
import { n as unmintableReasons, t as mintRolloutRows } from "./mint-Ctwk079K.js";
|
|
30
31
|
import { C as SUPERVISOR_RUN_SCHEMA, T as showMeasured, _ as readClaudeCodeSupervisorRun, b as rollupSupervisorRuns, c as writeSupervisorRunReport, d as renderSupervisorRunHeadline, f as renderSupervisorRunMarkdown, g as claudeCodeSupervisorRunReader, t as analyzeSupervisorRun, v as analyzeSupervisorRunSources, w as isUnavailable } from "./supervisor-run-B2EWUmQY.js";
|
|
31
|
-
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-
|
|
32
|
-
import {
|
|
33
|
-
import {
|
|
32
|
+
import { A as TRACE_ANALYST_ACTOR_DESCRIPTION, C as domainEvidencePattern, D as tokenizeDomainWords, E as scoreTraceInsightReadiness, O as traceAnalystOnRunComplete, S as describeTraceInsightScope, T as planTraceInsightQuestions, _ as otlpToTraceRunRecords, a as convertTraceStoresToOtlp, b as buildTraceInsightPrompt, c as otelRunCompleteHook, d as captureFetchToRawSink, f as ToolTraceMissingError, g as otlpToRunRecords, h as otlpRowsToTraceRunRecords, i as iterateRawCalls, j as TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, k as analyzeTraces, l as OTEL_AGENT_EVAL_SCOPE, m as otlpRowsToRunRecords, n as ReplayCacheMissError, o as createOtelExporter, p as toolSpansToTraceAnalysisStore, r as createReplayFetch, s as createOtelTracingStore, t as ReplayCache, u as exportRunAsOtlp, v as flattenOtlpExportToNdjson, w as inferDomainKeywords, x as defaultTraceInsightPanel, y as buildTraceInsightContext } from "./replay-SA4OB7O7.js";
|
|
33
|
+
import { i as otlpTextToTraceAnalysisStore, n as OtlpFileTraceStore } from "./store-otlp-DX4fGIcf.js";
|
|
34
|
+
import { n as extractUsageFromResponse, r as extractUsageFromSse, t as extractUsage } from "./extract-usage-DZs601Va.js";
|
|
35
|
+
import { B as agentProfileModelId, G as createLlmCorrectnessChecker, H as harnessAxisOf, I as CODING_HARNESSES, J as verifyCompletion, K as createTokenRecallChecker, L as HARNESS_NATIVE_MODEL, R as agentProfileHash, U as extractProducedState, V as expandProfileAxes, W as completionVerdict, q as parseCorrectnessResponse, z as agentProfileId } from "./campaign-DrS6_hLd.js";
|
|
34
36
|
import { n as iqr, r as welchsTTest, t as compareToBaseline } from "./baseline-C-GocmIW.js";
|
|
35
37
|
import { a as judgeSpans, c as runsForScenario, i as hasCapturedToolArgs, l as toolSpans, n as argHash, o as llmSpans, r as groupBy, s as runFailureClass, t as aggregateLlm } from "./query-Di7eEQ79.js";
|
|
36
38
|
import { a as checkBehavioralCanary, i as canaryLeakView, o as checkCanaries, r as HoldoutAuditor, s as runBehavioralCanaries, t as analyzeRuns } from "./analyze-runs-BScZvqMV.js";
|
|
@@ -40,16 +42,15 @@ import { a as InMemoryTraceStore, i as FileSystemTraceStore, n as assertRunCaptu
|
|
|
40
42
|
import { i as redactValue, n as REDACTION_VERSION, r as redactString, t as DEFAULT_REDACTION_RULES } from "./redact-7Aq1ukl-.js";
|
|
41
43
|
import { n as DEFAULT_RULES, r as classifyFailure, t as computeToolUseMetrics } from "./tool-use-metrics-DEGMKycK.js";
|
|
42
44
|
import { t as analyzeSeries } from "./series-convergence-CjO2QdRW.js";
|
|
43
|
-
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-
|
|
44
|
-
import { t as runEvalCampaign } from "./eval-campaign-
|
|
45
|
+
import { _ as deterministicSplit, g as BENCHMARK_SPLIT_SEED, t as benchmarks_exports } from "./benchmarks-zxhy1QV3.js";
|
|
46
|
+
import { t as runEvalCampaign } from "./eval-campaign-BmptJj50.js";
|
|
45
47
|
import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
|
|
46
|
-
import { AxGEPA, AxSignature, ax } from "@ax-llm/ax";
|
|
47
48
|
import { accessSync, appendFileSync, constants, cpSync, existsSync, mkdirSync, promises, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
|
|
48
49
|
import { basename, delimiter, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
49
50
|
import { createHash } from "node:crypto";
|
|
51
|
+
import { execSync, spawnSync } from "node:child_process";
|
|
50
52
|
import { readFile } from "node:fs/promises";
|
|
51
53
|
import { cpus } from "node:os";
|
|
52
|
-
import { execSync, spawnSync } from "node:child_process";
|
|
53
54
|
import { gzipSync } from "node:zlib";
|
|
54
55
|
//#region src/auto-pr.ts
|
|
55
56
|
/**
|
|
@@ -1603,343 +1604,6 @@ async function decideNextUserTurn(chat, opts) {
|
|
|
1603
1604
|
return paid.value.content.trim();
|
|
1604
1605
|
}
|
|
1605
1606
|
//#endregion
|
|
1606
|
-
//#region src/feedback-trajectory-review.ts
|
|
1607
|
-
/** Bind an analyst finding's complete canonical JSON content to a stable digest. */
|
|
1608
|
-
function analystFindingDigest(finding) {
|
|
1609
|
-
return hashCanonical(snapshotAnalystFinding(finding, "analyst finding"));
|
|
1610
|
-
}
|
|
1611
|
-
/** Bind the complete analyst result to one immutable review target. */
|
|
1612
|
-
function analystRunDigest(run) {
|
|
1613
|
-
return hashCanonical(snapshotAnalystRun(run, "analyst run"));
|
|
1614
|
-
}
|
|
1615
|
-
function snapshotAnalystRun(value, context = "analyst run") {
|
|
1616
|
-
let snapshot;
|
|
1617
|
-
try {
|
|
1618
|
-
snapshot = JSON.parse(canonicalString(value));
|
|
1619
|
-
} catch (cause) {
|
|
1620
|
-
throw new TypeError(`${context} must have a canonical JSON representation`, { cause });
|
|
1621
|
-
}
|
|
1622
|
-
if (!isRecord$1(snapshot)) throw new TypeError(`${context} must be an object`);
|
|
1623
|
-
assertOnlyKeys(snapshot, [
|
|
1624
|
-
"run_id",
|
|
1625
|
-
"correlation_id",
|
|
1626
|
-
"started_at",
|
|
1627
|
-
"ended_at",
|
|
1628
|
-
"findings",
|
|
1629
|
-
"per_analyst",
|
|
1630
|
-
"total_cost_usd",
|
|
1631
|
-
"total_cost_provenance"
|
|
1632
|
-
], context);
|
|
1633
|
-
requiredString(snapshot.run_id, `${context} run_id`);
|
|
1634
|
-
requiredString(snapshot.correlation_id, `${context} correlation_id`);
|
|
1635
|
-
canonicalTimestamp(snapshot.started_at, `${context} started_at`);
|
|
1636
|
-
canonicalTimestamp(snapshot.ended_at, `${context} ended_at`);
|
|
1637
|
-
snapshot.findings = snapshotAnalystFindings(snapshot.findings, `${context} findings`);
|
|
1638
|
-
if (!Array.isArray(snapshot.per_analyst)) throw new TypeError(`${context} per_analyst must be an array`);
|
|
1639
|
-
for (const [index, summary] of snapshot.per_analyst.entries()) {
|
|
1640
|
-
if (!isRecord$1(summary)) throw new TypeError(`${context} per_analyst ${index} must be an object`);
|
|
1641
|
-
requiredString(summary.analyst_id, `${context} per_analyst ${index} analyst_id`);
|
|
1642
|
-
}
|
|
1643
|
-
if (typeof snapshot.total_cost_usd !== "number" || !Number.isFinite(snapshot.total_cost_usd) || snapshot.total_cost_usd < 0) throw new TypeError(`${context} total_cost_usd must be a finite non-negative number`);
|
|
1644
|
-
if (snapshot.total_cost_provenance !== void 0 && !isRecord$1(snapshot.total_cost_provenance)) throw new TypeError(`${context} total_cost_provenance must be an object`);
|
|
1645
|
-
return snapshot;
|
|
1646
|
-
}
|
|
1647
|
-
function snapshotAnalystFindings(value, context = "analyst run findings") {
|
|
1648
|
-
if (!Array.isArray(value)) throw new TypeError(`${context} must be an array`);
|
|
1649
|
-
const findings = value.map((finding, index) => snapshotAnalystFinding(finding, `${context} finding ${index}`));
|
|
1650
|
-
assertUniqueFindingIds(findings.map((finding) => finding.finding_id));
|
|
1651
|
-
return findings;
|
|
1652
|
-
}
|
|
1653
|
-
function readAnalystReview(trajectory) {
|
|
1654
|
-
const analystAttempts = trajectory.attempts.filter((attempt) => isRecord$1(attempt.artifact) && attempt.artifact.type === "analyst-run");
|
|
1655
|
-
const analysis = isRecord$1(trajectory.metadata?.analysis) ? trajectory.metadata.analysis : void 0;
|
|
1656
|
-
if (analystAttempts.length === 0) {
|
|
1657
|
-
if (analysis?.kind === "analyst-run") throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing its archived run`);
|
|
1658
|
-
return;
|
|
1659
|
-
}
|
|
1660
|
-
if (analystAttempts.length !== 1) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" must contain exactly one archived run`);
|
|
1661
|
-
if (analysis?.kind !== "analyst-run") throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing review state`);
|
|
1662
|
-
const artifact = analystAttempts[0].artifact;
|
|
1663
|
-
if (!isRecord$1(artifact)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" has an invalid archived run`);
|
|
1664
|
-
const runId = requiredString(artifact.analystRunId, `analyst trajectory "${trajectory.id}" run id`);
|
|
1665
|
-
if (analysis.runId !== runId) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" run identity does not match its review state`);
|
|
1666
|
-
const artifactRunDigest = requiredDigest(artifact.runDigest, `analyst trajectory "${trajectory.id}" archived run digest`);
|
|
1667
|
-
const storedRunDigest = requiredDigest(analysis.runDigest, `analyst trajectory "${trajectory.id}" review run digest`);
|
|
1668
|
-
if (artifactRunDigest !== storedRunDigest) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" run digest does not match its review state`);
|
|
1669
|
-
const findings = snapshotAnalystFindings(artifact.findings, `analyst trajectory "${trajectory.id}"`);
|
|
1670
|
-
const findingIds = findings.map((finding) => finding.finding_id);
|
|
1671
|
-
const analystIds = stringArray(artifact.analystIds, `analyst trajectory "${trajectory.id}" analyst ids`);
|
|
1672
|
-
const attemptMetadata = analystAttempts[0].metadata;
|
|
1673
|
-
if (!isRecord$1(attemptMetadata)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" is missing archived run metadata`);
|
|
1674
|
-
const archivedRun = snapshotAnalystRun({
|
|
1675
|
-
run_id: runId,
|
|
1676
|
-
correlation_id: artifact.correlationId,
|
|
1677
|
-
started_at: analysis.startedAt,
|
|
1678
|
-
ended_at: analysis.endedAt,
|
|
1679
|
-
findings,
|
|
1680
|
-
per_analyst: attemptMetadata.perAnalyst,
|
|
1681
|
-
total_cost_usd: analysis.knownCostUsd,
|
|
1682
|
-
...analysis.costProvenance === void 0 ? {} : { total_cost_provenance: analysis.costProvenance }
|
|
1683
|
-
}, `analyst trajectory "${trajectory.id}" archived run`);
|
|
1684
|
-
const knownAnalystIds = new Set(analystIds);
|
|
1685
|
-
for (const [index, finding] of findings.entries()) if (!knownAnalystIds.has(finding.analyst_id)) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" omits generating analyst "${finding.analyst_id}" at finding ${index}`);
|
|
1686
|
-
const reviewDecisions = validateAnalystReviewDecisions({
|
|
1687
|
-
runId,
|
|
1688
|
-
runDigest: storedRunDigest,
|
|
1689
|
-
findings,
|
|
1690
|
-
analystIds,
|
|
1691
|
-
decisions: analysis.reviewDecisions,
|
|
1692
|
-
requireComplete: true
|
|
1693
|
-
});
|
|
1694
|
-
const expectedRunDigest = analystRunDigest(archivedRun);
|
|
1695
|
-
if (storedRunDigest !== expectedRunDigest) throw new TypeError(`feedbackTrajectoryToOptimizerRow: analyst trajectory "${trajectory.id}" archived run digest mismatch`);
|
|
1696
|
-
return {
|
|
1697
|
-
runId,
|
|
1698
|
-
runDigest: expectedRunDigest,
|
|
1699
|
-
findings,
|
|
1700
|
-
findingIds,
|
|
1701
|
-
analystIds,
|
|
1702
|
-
reviewDecisions
|
|
1703
|
-
};
|
|
1704
|
-
}
|
|
1705
|
-
function completedAnalystReviewQuality(review) {
|
|
1706
|
-
const findingDecisions = review.reviewDecisions.filter((decision) => decision.verdict !== "completeness_assessed");
|
|
1707
|
-
const completeness = review.reviewDecisions.filter((decision) => decision.verdict === "completeness_assessed");
|
|
1708
|
-
if (completeness.length !== 1) throw new TypeError("feedbackTrajectoryToOptimizerRow: analyst run requires exactly one independent completeness_assessed decision");
|
|
1709
|
-
const confirmed = findingDecisions.filter((decision) => decision.verdict === "confirmed").length;
|
|
1710
|
-
const rejected = findingDecisions.length - confirmed;
|
|
1711
|
-
const emitted = review.findingIds.length;
|
|
1712
|
-
const missed = completeness[0].missedIssues.length;
|
|
1713
|
-
const precision = emitted === 0 ? 1 : confirmed / emitted;
|
|
1714
|
-
const recallDenominator = confirmed + missed;
|
|
1715
|
-
const recall = recallDenominator === 0 ? 1 : confirmed / recallDenominator;
|
|
1716
|
-
return {
|
|
1717
|
-
precision,
|
|
1718
|
-
recall,
|
|
1719
|
-
f1: precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall),
|
|
1720
|
-
counts: {
|
|
1721
|
-
emitted,
|
|
1722
|
-
confirmed,
|
|
1723
|
-
rejected,
|
|
1724
|
-
missed
|
|
1725
|
-
}
|
|
1726
|
-
};
|
|
1727
|
-
}
|
|
1728
|
-
function validateAnalystReviewDecisions(input) {
|
|
1729
|
-
if (!Array.isArray(input.decisions)) throw new TypeError("analyst review decisions must be an array");
|
|
1730
|
-
const findings = snapshotAnalystFindings(input.findings);
|
|
1731
|
-
const expectedRunDigest = requiredDigest(input.runDigest, "analyst review run digest");
|
|
1732
|
-
const findingsById = new Map(findings.map((finding) => [finding.finding_id, finding]));
|
|
1733
|
-
const generatingAnalystIds = new Set(input.analystIds);
|
|
1734
|
-
const seenFindingIds = /* @__PURE__ */ new Set();
|
|
1735
|
-
let completenessCount = 0;
|
|
1736
|
-
const decisions = input.decisions.map((value, index) => {
|
|
1737
|
-
if (!isRecord$1(value)) throw new TypeError(`analyst review decision ${index} must be an object`);
|
|
1738
|
-
const source = requiredString(value.source, `analyst review decision ${index} source`);
|
|
1739
|
-
if (!isAnalystReviewSource(source)) throw new TypeError(`analyst review decision ${index} source must be user, judge, environment, metric, or policy`);
|
|
1740
|
-
const reviewerId = requiredString(value.reviewerId, `analyst review decision ${index} reviewerId`);
|
|
1741
|
-
if (generatingAnalystIds.has(reviewerId)) throw new TypeError(`analyst review decision ${index} reviewerId must differ from the generating analyst`);
|
|
1742
|
-
const reviewId = requiredString(value.reviewId, `analyst review decision ${index} reviewId`);
|
|
1743
|
-
if (reviewId === input.runId) throw new TypeError(`analyst review decision ${index} reviewId must identify an independent review`);
|
|
1744
|
-
const reason = requiredString(value.reason, `analyst review decision ${index} reason`);
|
|
1745
|
-
const decidedAt = canonicalTimestamp(value.decidedAt, `analyst review decision ${index} decidedAt`);
|
|
1746
|
-
const runDigest = requiredDigest(value.runDigest, `analyst review decision ${index} runDigest`);
|
|
1747
|
-
if (runDigest !== expectedRunDigest) throw new TypeError(`analyst review decision ${index} run digest mismatch`);
|
|
1748
|
-
if (value.verdict === "completeness_assessed") {
|
|
1749
|
-
assertOnlyKeys(value, [
|
|
1750
|
-
"runDigest",
|
|
1751
|
-
"verdict",
|
|
1752
|
-
"missedIssues",
|
|
1753
|
-
"source",
|
|
1754
|
-
"reviewerId",
|
|
1755
|
-
"reviewId",
|
|
1756
|
-
"reason",
|
|
1757
|
-
"decidedAt"
|
|
1758
|
-
], `analyst review decision ${index}`);
|
|
1759
|
-
completenessCount += 1;
|
|
1760
|
-
if (completenessCount > 1) throw new TypeError("duplicate completeness_assessed analyst review decision");
|
|
1761
|
-
return {
|
|
1762
|
-
runDigest,
|
|
1763
|
-
verdict: "completeness_assessed",
|
|
1764
|
-
missedIssues: validateMissedIssues(value.missedIssues, findingsById, `analyst review decision ${index}`),
|
|
1765
|
-
source,
|
|
1766
|
-
reviewerId,
|
|
1767
|
-
reviewId,
|
|
1768
|
-
reason,
|
|
1769
|
-
decidedAt
|
|
1770
|
-
};
|
|
1771
|
-
}
|
|
1772
|
-
if (value.verdict !== "confirmed" && value.verdict !== "rejected") throw new TypeError(`analyst review decision ${index} verdict must be confirmed, rejected, or completeness_assessed`);
|
|
1773
|
-
assertOnlyKeys(value, [
|
|
1774
|
-
"runDigest",
|
|
1775
|
-
"findingId",
|
|
1776
|
-
"findingDigest",
|
|
1777
|
-
"verdict",
|
|
1778
|
-
"source",
|
|
1779
|
-
"reviewerId",
|
|
1780
|
-
"reviewId",
|
|
1781
|
-
"reason",
|
|
1782
|
-
"decidedAt"
|
|
1783
|
-
], `analyst review decision ${index}`);
|
|
1784
|
-
const findingId = requiredString(value.findingId, `analyst review decision ${index} findingId`);
|
|
1785
|
-
const finding = findingsById.get(findingId);
|
|
1786
|
-
if (!finding) throw new TypeError(`analyst review decision references unknown finding id "${findingId}"`);
|
|
1787
|
-
if (seenFindingIds.has(findingId)) throw new TypeError(`duplicate analyst review decision for finding id "${findingId}"`);
|
|
1788
|
-
seenFindingIds.add(findingId);
|
|
1789
|
-
const findingDigest = requiredString(value.findingDigest, `analyst review decision ${index} findingDigest`);
|
|
1790
|
-
const expectedDigest = analystFindingDigest(finding);
|
|
1791
|
-
if (findingDigest !== expectedDigest) throw new TypeError(`analyst review decision ${index} digest mismatch for finding id "${findingId}"`);
|
|
1792
|
-
return {
|
|
1793
|
-
runDigest,
|
|
1794
|
-
findingId,
|
|
1795
|
-
findingDigest: expectedDigest,
|
|
1796
|
-
verdict: value.verdict,
|
|
1797
|
-
source,
|
|
1798
|
-
reviewerId,
|
|
1799
|
-
reviewId,
|
|
1800
|
-
reason,
|
|
1801
|
-
decidedAt
|
|
1802
|
-
};
|
|
1803
|
-
});
|
|
1804
|
-
if (input.requireComplete) {
|
|
1805
|
-
const missing = findings.map((finding) => finding.finding_id).filter((findingId) => !seenFindingIds.has(findingId));
|
|
1806
|
-
if (missing.length > 0) throw new TypeError(`feedbackTrajectoryToOptimizerRow: missing independent decisions for finding ids: ${missing.join(", ")}`);
|
|
1807
|
-
if (completenessCount !== 1) throw new TypeError("feedbackTrajectoryToOptimizerRow: analyst run requires exactly one independent completeness_assessed decision");
|
|
1808
|
-
}
|
|
1809
|
-
return decisions;
|
|
1810
|
-
}
|
|
1811
|
-
function assertUniqueFindingIds(findingIds) {
|
|
1812
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1813
|
-
for (const findingId of findingIds) {
|
|
1814
|
-
if (findingId.trim().length === 0) throw new TypeError("analyst finding id must not be empty");
|
|
1815
|
-
if (seen.has(findingId)) throw new TypeError(`analyst run contains duplicate finding id "${findingId}"`);
|
|
1816
|
-
seen.add(findingId);
|
|
1817
|
-
}
|
|
1818
|
-
}
|
|
1819
|
-
function snapshotAnalystFinding(value, context) {
|
|
1820
|
-
let snapshot;
|
|
1821
|
-
try {
|
|
1822
|
-
snapshot = JSON.parse(canonicalString(value));
|
|
1823
|
-
} catch (cause) {
|
|
1824
|
-
throw new TypeError(`${context} must have a canonical JSON representation`, { cause });
|
|
1825
|
-
}
|
|
1826
|
-
assertAnalystFinding(snapshot, context);
|
|
1827
|
-
return snapshot;
|
|
1828
|
-
}
|
|
1829
|
-
function assertAnalystFinding(value, context) {
|
|
1830
|
-
if (!isRecord$1(value)) throw new TypeError(`${context} must be an object`);
|
|
1831
|
-
assertOnlyKeys(value, [
|
|
1832
|
-
"schema_version",
|
|
1833
|
-
"finding_id",
|
|
1834
|
-
"analyst_id",
|
|
1835
|
-
"produced_at",
|
|
1836
|
-
"severity",
|
|
1837
|
-
"area",
|
|
1838
|
-
"claim",
|
|
1839
|
-
"rationale",
|
|
1840
|
-
"evidence_refs",
|
|
1841
|
-
"recommended_action",
|
|
1842
|
-
"validation_plan",
|
|
1843
|
-
"confidence",
|
|
1844
|
-
"subject",
|
|
1845
|
-
"derived_from_judge",
|
|
1846
|
-
"metadata"
|
|
1847
|
-
], context);
|
|
1848
|
-
if (value.schema_version !== "1.0.0") throw new TypeError(`${context} schema_version must be "1.0.0"`);
|
|
1849
|
-
requiredString(value.finding_id, `${context} finding_id`);
|
|
1850
|
-
requiredString(value.analyst_id, `${context} analyst_id`);
|
|
1851
|
-
canonicalTimestamp(value.produced_at, `${context} produced_at`);
|
|
1852
|
-
if (value.severity !== "critical" && value.severity !== "high" && value.severity !== "medium" && value.severity !== "low" && value.severity !== "info") throw new TypeError(`${context} severity is invalid`);
|
|
1853
|
-
requiredString(value.area, `${context} area`);
|
|
1854
|
-
requiredString(value.claim, `${context} claim`);
|
|
1855
|
-
optionalString$1(value.rationale, `${context} rationale`);
|
|
1856
|
-
value.evidence_refs = validateEvidenceRefs(value.evidence_refs, `${context} evidence_refs`);
|
|
1857
|
-
optionalString$1(value.recommended_action, `${context} recommended_action`);
|
|
1858
|
-
optionalString$1(value.validation_plan, `${context} validation_plan`);
|
|
1859
|
-
if (typeof value.confidence !== "number" || !Number.isFinite(value.confidence) || value.confidence < 0 || value.confidence > 1) throw new TypeError(`${context} confidence must be a finite number from 0 through 1`);
|
|
1860
|
-
optionalString$1(value.subject, `${context} subject`);
|
|
1861
|
-
if (value.derived_from_judge !== void 0 && typeof value.derived_from_judge !== "boolean") throw new TypeError(`${context} derived_from_judge must be a boolean`);
|
|
1862
|
-
if (value.metadata !== void 0 && !isRecord$1(value.metadata)) throw new TypeError(`${context} metadata must be an object`);
|
|
1863
|
-
}
|
|
1864
|
-
function validateMissedIssues(value, findingsById, context) {
|
|
1865
|
-
if (!Array.isArray(value)) throw new TypeError(`${context} missedIssues must be an array`);
|
|
1866
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1867
|
-
return value.map((issue, index) => {
|
|
1868
|
-
const issueContext = `${context} missedIssues ${index}`;
|
|
1869
|
-
if (!isRecord$1(issue)) throw new TypeError(`${issueContext} must be an object`);
|
|
1870
|
-
assertOnlyKeys(issue, [
|
|
1871
|
-
"id",
|
|
1872
|
-
"reason",
|
|
1873
|
-
"evidence"
|
|
1874
|
-
], issueContext);
|
|
1875
|
-
const id = requiredString(issue.id, `${issueContext} id`);
|
|
1876
|
-
if (findingsById.has(id)) throw new TypeError(`${issueContext} id "${id}" is already an emitted finding id`);
|
|
1877
|
-
if (seen.has(id)) throw new TypeError(`duplicate missed issue id "${id}"`);
|
|
1878
|
-
seen.add(id);
|
|
1879
|
-
return {
|
|
1880
|
-
id,
|
|
1881
|
-
reason: requiredString(issue.reason, `${issueContext} reason`),
|
|
1882
|
-
...issue.evidence === void 0 ? {} : { evidence: validateEvidenceRefs(issue.evidence, `${issueContext} evidence`) }
|
|
1883
|
-
};
|
|
1884
|
-
});
|
|
1885
|
-
}
|
|
1886
|
-
function validateEvidenceRefs(value, context) {
|
|
1887
|
-
if (!Array.isArray(value)) throw new TypeError(`${context} must be an array`);
|
|
1888
|
-
return value.map((evidence, index) => {
|
|
1889
|
-
const evidenceContext = `${context} ${index}`;
|
|
1890
|
-
if (!isRecord$1(evidence)) throw new TypeError(`${evidenceContext} must be an object`);
|
|
1891
|
-
assertOnlyKeys(evidence, [
|
|
1892
|
-
"kind",
|
|
1893
|
-
"uri",
|
|
1894
|
-
"excerpt"
|
|
1895
|
-
], evidenceContext);
|
|
1896
|
-
if (evidence.kind !== "span" && evidence.kind !== "event" && evidence.kind !== "artifact" && evidence.kind !== "finding" && evidence.kind !== "metric") throw new TypeError(`${evidenceContext} kind is invalid`);
|
|
1897
|
-
const uri = requiredString(evidence.uri, `${evidenceContext} uri`);
|
|
1898
|
-
const excerpt = evidence.excerpt;
|
|
1899
|
-
optionalString$1(excerpt, `${evidenceContext} excerpt`);
|
|
1900
|
-
return {
|
|
1901
|
-
kind: evidence.kind,
|
|
1902
|
-
uri,
|
|
1903
|
-
...excerpt === void 0 ? {} : { excerpt }
|
|
1904
|
-
};
|
|
1905
|
-
});
|
|
1906
|
-
}
|
|
1907
|
-
function assertOnlyKeys(value, allowed, name) {
|
|
1908
|
-
const allowedKeys = new Set(allowed);
|
|
1909
|
-
const unexpected = Object.keys(value).filter((key) => !allowedKeys.has(key));
|
|
1910
|
-
if (unexpected.length > 0) throw new TypeError(`${name} contains unknown fields: ${unexpected.sort().join(", ")}`);
|
|
1911
|
-
}
|
|
1912
|
-
function stringArray(value, name) {
|
|
1913
|
-
if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) throw new TypeError(`${name} must be an array of strings`);
|
|
1914
|
-
const strings = value.map((item) => requiredString(item, name));
|
|
1915
|
-
if (new Set(strings).size !== strings.length) throw new TypeError(`${name} must contain unique values`);
|
|
1916
|
-
return strings;
|
|
1917
|
-
}
|
|
1918
|
-
function requiredString(value, name) {
|
|
1919
|
-
if (typeof value !== "string" || value.trim().length === 0) throw new TypeError(`${name} must be a non-empty string`);
|
|
1920
|
-
return value;
|
|
1921
|
-
}
|
|
1922
|
-
function requiredDigest(value, name) {
|
|
1923
|
-
const digest = requiredString(value, name);
|
|
1924
|
-
if (!/^sha256:[a-f0-9]{64}$/.test(digest)) throw new TypeError(`${name} must be a sha256 digest`);
|
|
1925
|
-
return digest;
|
|
1926
|
-
}
|
|
1927
|
-
function optionalString$1(value, name) {
|
|
1928
|
-
if (value !== void 0 && typeof value !== "string") throw new TypeError(`${name} must be a string`);
|
|
1929
|
-
}
|
|
1930
|
-
function canonicalTimestamp(value, name) {
|
|
1931
|
-
const timestamp = requiredString(value, name);
|
|
1932
|
-
const parsed = new Date(timestamp);
|
|
1933
|
-
if (Number.isNaN(parsed.valueOf()) || parsed.toISOString() !== timestamp) throw new TypeError(`${name} must be a canonical ISO 8601 UTC timestamp`);
|
|
1934
|
-
return timestamp;
|
|
1935
|
-
}
|
|
1936
|
-
function isAnalystReviewSource(value) {
|
|
1937
|
-
return value === "user" || value === "judge" || value === "environment" || value === "metric" || value === "policy";
|
|
1938
|
-
}
|
|
1939
|
-
function isRecord$1(value) {
|
|
1940
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1941
|
-
}
|
|
1942
|
-
//#endregion
|
|
1943
1607
|
//#region src/feedback-trajectory.ts
|
|
1944
1608
|
const DEFAULT_SPLIT_POLICY = {
|
|
1945
1609
|
trainPct: 70,
|
|
@@ -2106,7 +1770,9 @@ function analystRunToFeedbackTrajectory(run, options) {
|
|
|
2106
1770
|
runDigest,
|
|
2107
1771
|
correlationId: archivedRun.correlation_id,
|
|
2108
1772
|
analystIds,
|
|
2109
|
-
findings
|
|
1773
|
+
findings,
|
|
1774
|
+
..."execution_plan" in archivedRun ? { executionPlan: archivedRun.execution_plan } : {},
|
|
1775
|
+
..."completion" in archivedRun ? { completion: archivedRun.completion } : {}
|
|
2110
1776
|
},
|
|
2111
1777
|
createdAt,
|
|
2112
1778
|
metadata: {
|
|
@@ -4599,80 +4265,21 @@ function renderSteeringText(bundle) {
|
|
|
4599
4265
|
}
|
|
4600
4266
|
//#endregion
|
|
4601
4267
|
//#region src/steering-optimizer.ts
|
|
4268
|
+
/**
|
|
4269
|
+
* Rank already-evaluated steering variants.
|
|
4270
|
+
*
|
|
4271
|
+
* Model-backed candidate generation belongs to the official GEPA campaign
|
|
4272
|
+
* integration. This class only performs the deterministic selection it owns.
|
|
4273
|
+
*/
|
|
4602
4274
|
var PairwiseSteeringOptimizer = class {
|
|
4603
4275
|
optimize(rows, config = {}) {
|
|
4604
|
-
const
|
|
4605
|
-
if (
|
|
4276
|
+
const rankings = rankRows(rows, config.weights);
|
|
4277
|
+
if (rankings.length === 0) throw new Error("no steering optimization rows");
|
|
4606
4278
|
return {
|
|
4607
4279
|
backend: "pairwise",
|
|
4608
|
-
recommendedVariantId:
|
|
4280
|
+
recommendedVariantId: rankings[0].variantId,
|
|
4609
4281
|
rationale: `Highest observed mean aggregate across ${rows.length} scored run(s).`,
|
|
4610
|
-
rankings
|
|
4611
|
-
};
|
|
4612
|
-
}
|
|
4613
|
-
};
|
|
4614
|
-
var AxGepaSteeringOptimizer = class {
|
|
4615
|
-
config;
|
|
4616
|
-
constructor(config) {
|
|
4617
|
-
this.config = config;
|
|
4618
|
-
}
|
|
4619
|
-
async optimize(rows) {
|
|
4620
|
-
const fallback = new PairwiseSteeringOptimizer().optimize(rows, this.config);
|
|
4621
|
-
const minScenarioWinners = this.config.minScenarioWinners ?? 6;
|
|
4622
|
-
const variantIds = [...new Set(rows.map((row) => row.variantId))];
|
|
4623
|
-
const byScenario = collapseScenarioWinners(rows, this.config.weights);
|
|
4624
|
-
if (variantIds.length < 2 || byScenario.length < minScenarioWinners) return {
|
|
4625
|
-
...fallback,
|
|
4626
|
-
backend: "ax-gepa",
|
|
4627
|
-
skipped: true,
|
|
4628
|
-
rationale: `AxGEPA skipped: need >=2 variants and >=${minScenarioWinners} scenario winners, got ${variantIds.length} variant(s) and ${byScenario.length} scenario winner(s).`
|
|
4629
|
-
};
|
|
4630
|
-
const signature = `task:string, split:string, seedPreview:string -> variantId:class "${variantIds.join(", ")}", rationale:string`;
|
|
4631
|
-
const selector = ax(AxSignature.from(signature), { description: "Choose the best steering bundle variant for an autopilot task." });
|
|
4632
|
-
const shuffled = seededShuffle(byScenario, signature);
|
|
4633
|
-
const splitIndex = Math.max(1, Math.floor(shuffled.length * .8));
|
|
4634
|
-
const train = shuffled.slice(0, splitIndex);
|
|
4635
|
-
const validation = shuffled.slice(splitIndex);
|
|
4636
|
-
if (!validation.length) return {
|
|
4637
|
-
...fallback,
|
|
4638
|
-
backend: "ax-gepa",
|
|
4639
|
-
skipped: true,
|
|
4640
|
-
rationale: "AxGEPA skipped: no validation examples after split."
|
|
4641
|
-
};
|
|
4642
|
-
const studentAI = createAxService(this.config.provider, this.config.apiKey, this.config.model);
|
|
4643
|
-
const optimizer = new AxGEPA({
|
|
4644
|
-
studentAI,
|
|
4645
|
-
teacherAI: createAxService(this.config.provider, this.config.apiKey, this.config.teacherModel ?? this.config.model),
|
|
4646
|
-
numTrials: 8,
|
|
4647
|
-
minibatch: true,
|
|
4648
|
-
minibatchSize: 4,
|
|
4649
|
-
earlyStoppingTrials: 3,
|
|
4650
|
-
sampleCount: 1
|
|
4651
|
-
});
|
|
4652
|
-
const metric = ({ prediction, example }) => stringField$1(prediction, "variantId") === stringField$1(example, "variantId") ? 1 : 0;
|
|
4653
|
-
const compiled = await optimizer.compile(selector, train, metric, {
|
|
4654
|
-
validationExamples: validation,
|
|
4655
|
-
maxMetricCalls: 64
|
|
4656
|
-
});
|
|
4657
|
-
if (compiled.optimizedProgram !== void 0) selector.applyOptimization(compiled.optimizedProgram);
|
|
4658
|
-
const selectVariant = async (row) => {
|
|
4659
|
-
const prediction = await selector.forward(studentAI, row);
|
|
4660
|
-
return {
|
|
4661
|
-
variantId: String(prediction.variantId),
|
|
4662
|
-
rationale: String(prediction.rationale ?? "")
|
|
4663
|
-
};
|
|
4664
|
-
};
|
|
4665
|
-
return {
|
|
4666
|
-
...fallback,
|
|
4667
|
-
backend: "ax-gepa",
|
|
4668
|
-
rationale: `AxGEPA trained a variant selector from ${byScenario.length} scored scenario winner(s); default winner remains ${fallback.recommendedVariantId}.`,
|
|
4669
|
-
selector: {
|
|
4670
|
-
backend: "ax-gepa",
|
|
4671
|
-
signature,
|
|
4672
|
-
labels: variantIds,
|
|
4673
|
-
rationale: compiled.bestScore !== void 0 ? `bestScore=${compiled.bestScore}` : void 0
|
|
4674
|
-
},
|
|
4675
|
-
selectVariant
|
|
4282
|
+
rankings
|
|
4676
4283
|
};
|
|
4677
4284
|
}
|
|
4678
4285
|
};
|
|
@@ -4689,64 +4296,6 @@ function rankRows(rows, weights) {
|
|
|
4689
4296
|
runs: values.length
|
|
4690
4297
|
})).sort((a, b) => b.mean - a.mean);
|
|
4691
4298
|
}
|
|
4692
|
-
function collapseScenarioWinners(rows, weights) {
|
|
4693
|
-
const byScenario = /* @__PURE__ */ new Map();
|
|
4694
|
-
for (const row of rows) {
|
|
4695
|
-
const bucket = byScenario.get(row.scenarioId) ?? [];
|
|
4696
|
-
bucket.push(row);
|
|
4697
|
-
byScenario.set(row.scenarioId, bucket);
|
|
4698
|
-
}
|
|
4699
|
-
return [...byScenario.entries()].map(([scenarioId, scenarioRows]) => {
|
|
4700
|
-
const best = scenarioRows.map((row) => ({
|
|
4701
|
-
row,
|
|
4702
|
-
aggregate: aggregateRunScore(row.score, weights)
|
|
4703
|
-
})).sort((a, b) => b.aggregate - a.aggregate)[0];
|
|
4704
|
-
return {
|
|
4705
|
-
task: String(best.row.metadata?.task ?? best.row.metadata?.seed_preview ?? scenarioId),
|
|
4706
|
-
split: String(best.row.metadata?.split ?? "train"),
|
|
4707
|
-
seedPreview: String(best.row.metadata?.seed_preview ?? ""),
|
|
4708
|
-
variantId: best.row.variantId
|
|
4709
|
-
};
|
|
4710
|
-
});
|
|
4711
|
-
}
|
|
4712
|
-
function createAxService(provider, apiKey, model) {
|
|
4713
|
-
return createAnalystAi({
|
|
4714
|
-
provider,
|
|
4715
|
-
apiKey,
|
|
4716
|
-
model
|
|
4717
|
-
});
|
|
4718
|
-
}
|
|
4719
|
-
function stringField$1(value, field) {
|
|
4720
|
-
if (!value || typeof value !== "object") return void 0;
|
|
4721
|
-
const candidate = value[field];
|
|
4722
|
-
return typeof candidate === "string" ? candidate : void 0;
|
|
4723
|
-
}
|
|
4724
|
-
function seededShuffle(items, seed) {
|
|
4725
|
-
const rng = mulberry32$1(hashString(seed));
|
|
4726
|
-
const out = [...items];
|
|
4727
|
-
for (let i = out.length - 1; i > 0; i--) {
|
|
4728
|
-
const j = Math.floor(rng() * (i + 1));
|
|
4729
|
-
[out[i], out[j]] = [out[j], out[i]];
|
|
4730
|
-
}
|
|
4731
|
-
return out;
|
|
4732
|
-
}
|
|
4733
|
-
function hashString(value) {
|
|
4734
|
-
let h = 2166136261;
|
|
4735
|
-
for (let i = 0; i < value.length; i++) {
|
|
4736
|
-
h ^= value.charCodeAt(i);
|
|
4737
|
-
h = Math.imul(h, 16777619);
|
|
4738
|
-
}
|
|
4739
|
-
return h >>> 0;
|
|
4740
|
-
}
|
|
4741
|
-
function mulberry32$1(seed) {
|
|
4742
|
-
let a = seed >>> 0;
|
|
4743
|
-
return () => {
|
|
4744
|
-
a = a + 1831565813 | 0;
|
|
4745
|
-
let t = Math.imul(a ^ a >>> 15, 1 | a);
|
|
4746
|
-
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
4747
|
-
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
4748
|
-
};
|
|
4749
|
-
}
|
|
4750
4299
|
//#endregion
|
|
4751
4300
|
//#region src/workspace-inspector.ts
|
|
4752
4301
|
var InMemoryWorkspaceInspector = class {
|
|
@@ -11650,56 +11199,48 @@ function sectionHash(section) {
|
|
|
11650
11199
|
}
|
|
11651
11200
|
//#endregion
|
|
11652
11201
|
//#region src/traced-analyst.ts
|
|
11653
|
-
/**
|
|
11654
|
-
|
|
11655
|
-
|
|
11656
|
-
async function tracedAnalyzeTraces(input, options, traceOpts) {
|
|
11657
|
-
const parentSpan = await traceOpts.emitter.span({
|
|
11202
|
+
/** Run a recursive trace investigation and record its engine steps. */
|
|
11203
|
+
async function tracedAnalyzeTraces(input, options, traceOptions) {
|
|
11204
|
+
const parentSpan = await traceOptions.emitter.span({
|
|
11658
11205
|
kind: "custom",
|
|
11659
11206
|
name: "analyst:analyze-traces",
|
|
11660
|
-
parentSpanId:
|
|
11207
|
+
parentSpanId: traceOptions.parentSpanId,
|
|
11661
11208
|
attributes: {
|
|
11209
|
+
"analyst.engine": options.engine.id,
|
|
11662
11210
|
"analyst.question_length": input.question.length,
|
|
11663
|
-
"analyst.
|
|
11664
|
-
"analyst.
|
|
11211
|
+
"analyst.max_iterations": options.limits?.maxIterations ?? 12,
|
|
11212
|
+
"analyst.max_llm_calls": options.limits?.maxLlmCalls ?? 8,
|
|
11665
11213
|
"eval.phase": "analyst"
|
|
11666
11214
|
}
|
|
11667
11215
|
});
|
|
11668
|
-
|
|
11669
|
-
|
|
11670
|
-
|
|
11671
|
-
|
|
11672
|
-
|
|
11216
|
+
try {
|
|
11217
|
+
const result = await analyzeTraces(input, options);
|
|
11218
|
+
for (const [index, step] of result.trajectory.entries()) {
|
|
11219
|
+
const encoded = JSON.stringify(step);
|
|
11220
|
+
await (await traceOptions.emitter.span({
|
|
11673
11221
|
kind: "custom",
|
|
11674
|
-
name: `analyst:
|
|
11222
|
+
name: `analyst:step-${index + 1}`,
|
|
11675
11223
|
parentSpanId: parentSpan.span.spanId,
|
|
11676
11224
|
attributes: {
|
|
11677
|
-
"analyst.
|
|
11678
|
-
"analyst.
|
|
11679
|
-
"analyst.is_error": turn.isError,
|
|
11680
|
-
"analyst.code_length": turn.code.length,
|
|
11681
|
-
"analyst.output_length": turn.output.length,
|
|
11225
|
+
"analyst.step": index + 1,
|
|
11226
|
+
"analyst.step_bytes": Buffer.byteLength(encoded),
|
|
11682
11227
|
"eval.phase": "analyst"
|
|
11683
11228
|
}
|
|
11684
|
-
});
|
|
11685
|
-
if (turn.isError) await turnSpan.fail("Turn produced an error");
|
|
11686
|
-
else await turnSpan.end();
|
|
11687
|
-
if (originalOnTurn) await originalOnTurn(turn);
|
|
11229
|
+
})).end();
|
|
11688
11230
|
}
|
|
11689
|
-
};
|
|
11690
|
-
try {
|
|
11691
|
-
const result = await analyzeTraces(input, wrappedOptions);
|
|
11692
11231
|
await parentSpan.end({ attributes: {
|
|
11693
|
-
"analyst.
|
|
11694
|
-
"analyst.
|
|
11232
|
+
"analyst.engine": options.engine.id,
|
|
11233
|
+
"analyst.model_calls": result.modelCalls,
|
|
11234
|
+
"analyst.tool_calls": result.toolCalls,
|
|
11235
|
+
"analyst.step_count": result.trajectory.length,
|
|
11695
11236
|
"analyst.finding_count": result.findings.length,
|
|
11696
11237
|
"analyst.answer_length": result.answer.length,
|
|
11697
11238
|
"eval.phase": "analyst"
|
|
11698
11239
|
} });
|
|
11699
11240
|
return result;
|
|
11700
|
-
} catch (
|
|
11701
|
-
await parentSpan.fail(
|
|
11702
|
-
throw
|
|
11241
|
+
} catch (error) {
|
|
11242
|
+
await parentSpan.fail(error instanceof Error ? error : String(error));
|
|
11243
|
+
throw error;
|
|
11703
11244
|
}
|
|
11704
11245
|
}
|
|
11705
11246
|
//#endregion
|
|
@@ -12846,6 +12387,6 @@ function assertProductBenchmarkRun(runDir) {
|
|
|
12846
12387
|
return report;
|
|
12847
12388
|
}
|
|
12848
12389
|
//#endregion
|
|
12849
|
-
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, AxGepaSteeringOptimizer, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceAnalystTools, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAnalystAi, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalystKind, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12390
|
+
export { AGENT_PROFILE_KINDS, ATIF_SCHEMA_VERSION, ATTESTATION_ALGORITHM, AgentDriver, AgentEvalError, AgentProfileCellValidationError, AnalystRegistry, BENCHMARK_SPLIT_SEED, BOOTSTRAP_GATE_MIN_N, BackendIntegrityError, BenchmarkRunner, BudgetBreachError, BudgetGuard, CODING_HARNESSES, CONTROL_INTEGRITY_ANALYST, CallbackResearcher, CaptureIntegrityError, ConfigError, ControlIntegrityAnalyst, ConvergenceTracker, CostAccountingIncompleteError, CostCallConflictError, CostCeilingReachedError, CostLedger, CostLedgerPersistenceError, CostReceiptCaptureError, CostReservationExceededError, CostTracker, CrossFamilyError, DECISION_PAIRED_DELTA_STATISTIC, DEFAULT_AGENT_SLOS, DEFAULT_COMPLEXITY_WEIGHTS, DEFAULT_RULES as DEFAULT_FAILURE_RULES, DEFAULT_FINDERS, DEFAULT_HARNESS_OBJECTIVES, DEFAULT_MUTATION_PRIMITIVES, DEFAULT_MUTATORS, DEFAULT_PERMUTATIONS, DEFAULT_PR_REVIEW_SCORE_WEIGHTS, DEFAULT_REDACTION_RULES, DEFAULT_RED_TEAM_CORPUS, DEFAULT_RUN_SCORE_WEIGHTS, DEFAULT_SEVERITY_WEIGHTS, DEFAULT_TRACE_ANALYST_BUDGETS, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_LIMITS, Dataset, DescriptionLengthGate, DockerSandboxDriver, DualAgentBench, ERROR_COUNT_PATTERNS, EvalTraceStore, ExactAnalystRunExecutionError, ExperimentTracker, FAILURE_CLASSES, FAILURE_MODE_KIND_SPEC, FileSystemFeedbackTrajectoryStore, FileSystemRawProviderSink, FileSystemTraceStore, FindingsStore, HARBOR_IMPORT_GAP, HARNESS_NATIVE_MODEL, HeldOutGate, HoldoutAuditor, HoldoutLockedError, IMPROVEMENT_KIND_SPEC, INPUT_VALUE, INTENT_MATCH_JUDGE_VERSION, InMemoryFeedbackTrajectoryStore, InMemoryRawProviderSink, InMemoryTraceStore, InMemoryWorkspaceInspector, JudgeError, JudgeParseError, JudgeRunner, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LLM_CACHED_TOKENS, LLM_CACHED_TOKEN_ATTR_KEYS, LLM_CACHE_WRITE_TOKENS, LLM_CACHE_WRITE_TOKEN_ATTR_KEYS, LLM_CONTEXT_TOKENS, LLM_COST_ATTR_KEYS, LLM_COST_USD, LLM_INPUT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_MODEL_ATTR_KEYS, LLM_MODEL_NAME, LLM_OUTPUT_TOKENS, LLM_OUTPUT_TOKEN_ATTR_KEYS, LLM_REASONING_TOKENS, LLM_REASONING_TOKEN_ATTR_KEYS, LimitExceededError, LlmCallError, LlmClient, LlmResponseError, LlmRouteAssertionError, LockedJsonlAppender, MANN_WHITNEY_EXACT_MAX_STATES, MANN_WHITNEY_EXACT_MAX_WORK, MODEL_PRICING, MetricsCollector, ModelsUnreachableError, MultiLayerVerifier, Mutex, NoopRawProviderSink, NoopResearcher, NotFoundError, OPENINFERENCE_SPAN_KIND, OTEL_AGENT_EVAL_SCOPE, OUTPUT_VALUE, OtlpFileTraceStore, PairwiseSteeringOptimizer, ProductClient, PromptRegistry, REDACTION_VERSION, REFERENCE_EQUIVALENCE_INPUT_LIMITS, REFERENCE_EQUIVALENCE_JUDGE_VERSION, RESEARCH_REPORT_HARD_PAIR_FLOOR, ROLLOUT_SCHEMA, RUN_COST_ATTR_KEYS, ReplayCache, ReplayCacheMissError, ReplayError, RunCritic, RunIntegrityError, RunRecordValidationError, SEMANTIC_CONCEPT_JUDGE_VERSION, SKILL_USAGE_ANALYST, SPAN_KIND_ATTR_KEYS, SUPERVISOR_RUN_INTEGRITY_SCHEMA, SUPERVISOR_RUN_SCHEMA, SandboxHarness, ScenarioRegistry, SeatUnsetError, SingleBackendError, SkillUsageAnalyst, SpanNotFoundError, SubprocessSandboxDriver, TOOL_ARGS_CAPTURED, TOOL_LATENCY_MS, TOOL_NAME, TOOL_NAME_ATTR_KEYS, TRACE_ANALYSIS_LIMITS, TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION, TRACE_ANALYST_TOOL_NAMESPACE, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX, TRACE_SCHEMA_VERSION, TokenCounter, ToolTraceMissingError, TraceAnalysisLimitError, TraceAnalysisStoreContractError, TraceAnalysisValidationError, TraceContractBuilder, TraceEmitter, TraceFileMalformedError, TraceFileMissingError, TraceFileTooLargeError, TraceNotFoundError, UI_FINDING_SEVERITIES, UI_LENSES, UNIVERSAL_FINDERS, ValidationError, VerificationError, WILCOXON_EXACT_MAX_N, acquisitionPlansForKnowledgeGaps, agentProfileCellHashMaterial, agentProfileCellKey, agentProfileHash, agentProfileId, agentProfileModelId, agentVisibleFields, aggregateJudgeVerdicts, aggregateLlm, aggregatePrReviewScore, aggregateRunScore, allCriticalPassed, analystFindingDigest, analystRunDigest, analystRunToFeedbackTrajectory, analystRunToReviewRequests, analyzeAntiSlop, analyzeSeries, analyzeSupervisorRun, analyzeSupervisorRunIntegrity, analyzeSupervisorRunSources, analyzeTraces, appendScorecard, applyLlmSpanOtlpAttributes, applyToolSpanOtlpAttributes, argHash, asNumber, asString, assertCapabilityHeadroom, assertCrossFamily, assertExactRegistryRunOpts, assertLlmRoute, assertMinted, assertMintedLines, assertModelsServed, assertNoHiddenLeak, assertProductBenchmarkRun, assertRealAgentReceipts, assertRealBackend, assertReleaseConfidence, assertRolloutLine, assertRunAgentProfileCell, assertRunCaptured, assertSingleBackend, assignFeedbackSplit, assignHeldOutTag, attachCostToReport, attest, attributeCounterfactuals, backoffMs, deterministicSplit as benchmarkDeterministicSplit, benchmarks_exports as benchmarks, benjaminiHochberg, bisect, blendHeldout, blockingKnowledgeEval, bonferroni, bootstrapCi, buildAgentInterfaceProfileCell, buildAgentProfileCell, buildDefaultAnalystRegistry, buildDriverSystemPrompt, buildProductBenchmarkManifest, buildReflectionPrompt, buildReviewerPrompt, buildTraceAnalysisToolDescriptors, buildTraceInsightContext, buildTraceInsightPrompt, buildTrajectory, buildWorkerDriverSystemPrompt, byteLengthRange, cachedJudge, calibrateJudge, calibrateJudgeContinuous, callLlm, callLlmJson, canaryLeakView, canonicalJson, canonicalize, capabilityHeadroom, captureFetchToRawSink, causalAttribution, checkBehavioralCanary, checkCanaries, checkSlos, checkTraceContracts, clamp01, classifyFailure, classifyOtlpSpanRole, classifyTreatment, claudeCodeSupervisorRunReader, cliffsDelta, clusteredPairedBinary, cohensD, collectionPreserved, commentsForSource, commitBisect, comparePairedArms, compareReferenceReplay, compareToBaseline, compilerJudge, completionVerdict, composeParsers, composeValidators, computeExperimentStats, computeFindingId, computeToolUseMetrics, computeTraceMetrics, confidenceInterval, containsAll, contentHash, contextInputTokens, continuousAgreement, contractJudge, controlFailureClassFromVerification, controlRunToFeedbackTrajectory, controlRunToRunRecord, convertTraceStoresToOtlp, corpusInterRaterAgreement, corpusInterRaterAgreementFromJudgeScores, costForTokenPricing, costForUsage, costReceiptFromLlm, costReceiptFromLlmError, costReport, createAntiSlopJudge, createBoundedTraceAnalysisStore, createChatClient, createDefaultReviewer, createDspyRlmTraceEngine, createFeedbackTrajectory, createIntentMatchJudge, createLlmCorrectnessChecker, createLlmReviewer, createOtelExporter, createOtelTracingStore, createReferenceEquivalenceJudge, createReplayFetch, createSandboxPool, createSemanticConceptJudge, createTokenRecallChecker, createTraceAnalyst, crossTraceDiff, crowdingDistance, dataDescriptionBits, decideNextUserTurn, decidePairedPromotion, decideReferenceReplayPromotion, decideReferenceReplayRunPromotion, defaultBlendWeights, defaultIsMaterial, defaultProviderRedactor, defaultReferenceReplayMatcher, defaultTraceInsightPanel, defineTraceAnalyst, deployGateLayer, describeTraceInsightScope, diffFindings, diffScorecard, discoverPersonas, distillPlaybook, domainEvidencePattern, dominates, eProcess, emitControlIntegrityFindings, ensembleJudge, errorStreakDetector, estimateCost, estimateTokens, evaluateActionPolicy, evaluateContract, evaluateHypothesis, evaluateInterimReleaseConfidence, evaluateOracles, evaluateReleaseConfidence, evaluateTraceContract, executeScenario, expandProfileAxes, expectAgent, exportProductBenchmark, exportProductBenchmarkRuns, exportRewardModel, exportRunAsOtlp, extractAssetUrls, extractErrorCount, extractOtlpAttributes, extractProducedState, extractUsage, extractUsageFromResponse, extractUsageFromSse, feedbackTrajectoriesToDatasetScenarios, feedbackTrajectoriesToOptimizerRows, feedbackTrajectoryToDatasetScenario, feedbackTrajectoryToOptimizerRow, fileContains, fileExists, fileExperimentStore, fileVerdictCache, findAutoMatchNoExpectation, findConstructorCwdDropped, findFallbackToPass, findLiteralTruePass, findProductBenchmarkArtifacts, findSkipCountsAsPass, firstNumberAttr, firstStringAttr, flattenOtlpExportToNdjson, flowLayer, fnv1a32, formatBenchmarkReport, formatDriverReport, formatFindings, formatScorecardDiff, fromHarborTrajectory, gainHistogram, gateTreatmentApplied, gateTreatmentFromMetrics, gateTreatmentFromSpans, gateTreatmentFromToolSpans, ghCliClient, gitProvenanceReader, precision as goldenPrecision, gradeOnHidden, gradeSemanticStatus, groupBy, groupRunsByAgentProfileCell, harnessAxisOf, hasCapturedToolArgs, hashContent, hashJson, hashScenarios, hashToUnit, hiddenGrade, holm, htmlContainsElement, httpGithubClient, improvementVerdict, inMemoryExperimentStore, inMemoryReferenceReplayStore, inMemoryReviewStore, inMemoryRunRecordBackend, inMemoryVerdictCache, inferDomainKeywords, inferOtlpKind, interRaterReliability, interpretCliffs, iqr, isBinaryOutcomeVector, isHiddenDestination, isJudgeSpan, isLlmSpan, isModelPriced, isOtelConfigured, isOtlpModelCall, isRealnessGated, isRetrievalSpan, isRolloutLine, isRunRecord, isSandboxSpan, isToolSpan, isTrainableSplit, isTransientLlmError, isUnavailable, iterateRawCalls, jestTestParser, jsonHasKeys, jsonShape, jsonlReferenceReplayStore, jsonlReviewStore, jsonlRunRecordBackend, judgeFamily, judgeReplayGate, judgeSpans, keyPreserved, knowledgeReadinessTracePayload, leaderboard, linterJudge, llmJudge, llmSpanFromProvider, llmSpans, loadScorecard, loadScorerFromGrader, localCommandRunner, lowercaseMutator, makeEvalTools, makeFinding, makeProposalFinding, mannWhitneyU, mapConcurrent, matchGoldens, matchSpan, maximumChargeForLlmRequest, mcnemar, mcnemarPower, mcnemarRequiredN, mergeLayerResults, mergeSteeringBundle, minimumPairsForPairedDeltaTest, mintRolloutRows, modelDescriptionBits, modelHasSnapshot, modelPriceKey, mulberry32, multiToolchainLayer, noProgressDetector, normalCdf, normalizeScores, notBlocked, objectiveEval, observeAll, observedScore, observedSplitScore, otelRunCompleteHook, otlpRowsToRunRecords, otlpRowsToTraceRunRecords, otlpTextToTraceAnalysisStore, otlpToRunRecords, otlpToTraceRunRecords, pairArms, pairRunRecords, pairedBinaryScale, pairedBootstrap, pairedCohensDz, pairedDecisionShape, pairedDeltaTest, pairedDeltaTieFraction, pairedEvalueSequence, pairedMde, pairedRiskDifference, pairedRiskDifferenceExact, pairedRiskDifferenceScore, pairedSignTest, pairedTTest, paraphraseRobustness, paraphraseRobustnessScenarios, paretoChart, paretoFrontier, paretoFrontierWithCrowding, parseCorrectnessResponse, parseFeedbackTrajectoriesJsonl, parseReflectionResponse, parseRunRecordSafe, parseRuntimeTrajectoryHookEvent, partialCredit, partitionHeldOut, passAtK, passOrthogonality, pearsonR, pixelDeltaRatio, planTraceInsightQuestions, politenessPrefixMutator, positionalBias, preflightModels, printDriverSummary, probeLlm, productBenchmarkIntegrityFailures, productBenchmarkMutableSurfaces, productBenchmarkRepoIdentity, productBenchmarkSplits, profile_exports as profile, projectOtlpFlatLine, projectRuntimeTrajectoryEvidence, promptBisect, proposeSynthesisTargets, providerFromBaseUrl, pytestTestParser, ranks, readClaudeCodeSupervisorRun, readOtlpStatus, readProductBenchmarkManifest, readProductBenchmarkRecords, recordRuns, recordRunsToScorecard, redTeamDataset, redTeamReport, redactString, redactValue, referenceReplayRunsToSteeringRows, referenceReplayScenarioToRunScore, regexMatch, regexMatches, relabelImportedSplit, renderMarkdownReport, renderPlaybookMarkdown, renderPreferenceMemoryMarkdown, renderPriorFindings, renderReleaseReport, renderSteeringText, renderSupervisorRunHeadline, renderSupervisorRunMarkdown, renderUpstreamFindings, repeatedActionDetector, replayFeedbackTrajectories, replayFeedbackTrajectory, replayScorerOverCorpus, replayTraceThroughJudge, requireAgentProfileCell, requiredPairedSampleSize, requiredSampleSize, researchReport, resolveModelPricing, resolveSeat, resolveTraceAnalystLimits, rollupSupervisorRuns, roundTripRunRecord, routeFields, rowCount, rowWhere, runAgentControlLoop, runAssertions, runBehavioralCanaries, runCanaries, runCounterfactual, runE2EWorkflow, runEvalCampaign, runExpectations, runFailureClass, runHarnessExperiment, runIntentMatchJudge, runJudgeFleet, runKeywordCoverageJudge, runKeywordCoverageJudgeUrl, runLiveProof, runProposeReview, runProposeReviewAsControlLoop, runRecordToProductBenchmarkRecord, runReferenceEquivalenceJudge, runReferenceReplay, runScore, runSelfPlay, runSemanticConceptJudge, runTaskScore, runTestGradedScenario, runTraceAnalyst, runsForScenario, scalarScore, scanForMuffledGates, scoreContinuity, scoreFromEvals, scoreKnowledgeReadiness, scoreOrigin, scorePrReviewComments, scorePrReviewSource, scoreRedTeamOutput, scoreReferenceReplay, scoreTraceInsightReadiness, seatPresets, securityJudge, selectHarnessVariant, selfPreference, sentenceReorderMutator, serializeFeedbackTrajectoriesJsonl, showMeasured, signManifest, spearmanR, statusAdvanced, stopOnNoProgress, stopOnRepeatedAction, stringField, stripFencedJson, studentTCdf, subjectiveEval, summarizeAgentReceiptIntegrity, summarizeBackendIntegrity, summarizeHarnessResults, summarizePrReviewBenchmark, summarizePreferenceMemory, summaryTable, supervisorRunRolloutLines, testJudge, textInSnapshot, throwIfRunIncomplete, toAgentProfileJson, toHarborTrajectories, toHarborTrajectory, toJsonl, toLangfuseEnvelope, toOpenAiTool, toPrometheusText, toRewardRows, toSftRows, tokenizeDomainWords, toolNamesForRun, toolSpans, toolSpansToTraceAnalysisStore, traceAnalystFunctionGroup, traceAnalystOnRunComplete, traceContract, traceJudge, traceJudgeEnsemble, traceSpanKindToOpenInferenceKind, tracedAnalyzeTraces, trainingReward, trainingScore, typoMutator, unmintableReasons, urlContains, userQuestionsForKnowledgeGaps, validateAgentProfileCell, validateProductBenchmarkManifest, validateProductBenchmarkRecord, validateProductBenchmarkRun, validateRolloutLine, validateRunRecord, verbosityBias, verifyAgentProfileCell, verifyAttestation, verifyCompletion, verifyManifest, visualDiff, viteDeployRunner, vitestTestParser, weightedComposite, weightedMean, weightedRecall, welchsTTest, whitespaceCollapseMutator, wilcoxonSignedRank, wilson, withAssignedFeedbackSplit, withHeldoutBlend, withJudgeRetry, withOtelPipeline, wranglerDeployRunner, writeSupervisorRunReport };
|
|
12850
12391
|
|
|
12851
12392
|
//# sourceMappingURL=index.js.map
|