@tangle-network/agent-eval 0.220.2 → 0.221.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts → agent-profile-Bc7ADJY4.d.ts} +2 -2
- package/dist/{agent-profile-D01Io2Xw.d.ts.map → agent-profile-Bc7ADJY4.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +77 -78
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +204 -25
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-DqPJaJ4X.js → analyze-runs-CHmsykNJ.js} +4 -4
- package/dist/{analyze-runs-DqPJaJ4X.js.map → analyze-runs-CHmsykNJ.js.map} +1 -1
- package/dist/{analyze-runs-BWAHBDAw.d.ts → analyze-runs-D7In4c3z.d.ts} +2 -2
- package/dist/{analyze-runs-BWAHBDAw.d.ts.map → analyze-runs-D7In4c3z.d.ts.map} +1 -1
- package/dist/{benchmark-command-Dx0E4a6z.js → benchmark-command-8XQfpUK3.js} +13 -11
- package/dist/benchmark-command-8XQfpUK3.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/calibration-CTwxpJ-t.js +197 -0
- package/dist/calibration-CTwxpJ-t.js.map +1 -0
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-BVPC6ODQ.js → campaign-Bt6rbWIU.js} +9 -8
- package/dist/{campaign-BVPC6ODQ.js.map → campaign-Bt6rbWIU.js.map} +1 -1
- package/dist/cli.js +1 -1
- package/dist/{completion-verifier-Cigap8CG.js → completion-verifier-YwoeujPl.js} +2 -2
- package/dist/{completion-verifier-Cigap8CG.js.map → completion-verifier-YwoeujPl.js.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +7 -7
- package/dist/decision-point-Co3u3W8k.js +831 -0
- package/dist/decision-point-Co3u3W8k.js.map +1 -0
- package/dist/default-registry-BMHqNR6_.d.ts +257 -0
- package/dist/default-registry-BMHqNR6_.d.ts.map +1 -0
- package/dist/{default-registry-ovrIBGmH.js → default-registry-BbFCkqvB.js} +607 -14
- package/dist/default-registry-BbFCkqvB.js.map +1 -0
- package/dist/{define-agent-eval-C2nFhHPu.d.ts → define-agent-eval-Bsoi2GUV.d.ts} +7 -7
- package/dist/{define-agent-eval-C2nFhHPu.d.ts.map → define-agent-eval-Bsoi2GUV.d.ts.map} +1 -1
- package/dist/{define-agent-eval-UQY3VbJ2.js → define-agent-eval-CJhECJNL.js} +4 -4
- package/dist/{define-agent-eval-UQY3VbJ2.js.map → define-agent-eval-CJhECJNL.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js → dspy-rlm-engine-BTy7u0jQ.js} +3 -3
- package/dist/{dspy-rlm-engine-Bn_Mt_LW.js.map → dspy-rlm-engine-BTy7u0jQ.js.map} +1 -1
- package/dist/{engine-BpGGw6cd.d.ts → engine-BF95CWzH.d.ts} +2 -2
- package/dist/{engine-BpGGw6cd.d.ts.map → engine-BF95CWzH.d.ts.map} +1 -1
- package/dist/{eval-campaign-BUMIxKTE.js → eval-campaign-DoX5vz3E.js} +2 -2
- package/dist/{eval-campaign-BUMIxKTE.js.map → eval-campaign-DoX5vz3E.js.map} +1 -1
- package/dist/{evaluation-CBdQt_kr.d.ts → evaluation-D1PGzkPR.d.ts} +3 -3
- package/dist/{evaluation-CBdQt_kr.d.ts.map → evaluation-D1PGzkPR.d.ts.map} +1 -1
- package/dist/evaluation-wdGyZ7jV.js +147 -0
- package/dist/evaluation-wdGyZ7jV.js.map +1 -0
- package/dist/evaluation.d.ts +1 -1
- package/dist/evaluation.js +1 -136
- package/dist/{exact-types-BYFmYq44.d.ts → exact-types-CVn1QxFO.d.ts} +2 -2
- package/dist/{exact-types-BYFmYq44.d.ts.map → exact-types-CVn1QxFO.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +3 -3
- package/dist/experiment/index.js +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js → external-optimizer-process-E5yY2SYS.js} +2 -2
- package/dist/{external-optimizer-process-wOLfzsCC.js.map → external-optimizer-process-E5yY2SYS.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-D670Oxf2.js → external-optimizer-subprocess-DA02c_L6.js} +12 -19
- package/dist/{external-optimizer-subprocess-D670Oxf2.js.map → external-optimizer-subprocess-DA02c_L6.js.map} +1 -1
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts → feedback-trajectory-BZhVCWuz.d.ts} +3 -3
- package/dist/{feedback-trajectory-DhB9WzYg.d.ts.map → feedback-trajectory-BZhVCWuz.d.ts.map} +1 -1
- package/dist/finding-composition-8n0fFgVI.js +131 -0
- package/dist/finding-composition-8n0fFgVI.js.map +1 -0
- package/dist/{findings-store-CvNo712L.js → findings-store-DRiFdoid.js} +4 -4
- package/dist/{findings-store-CvNo712L.js.map → findings-store-DRiFdoid.js.map} +1 -1
- package/dist/{index-CVFStrQD.d.ts → index-BJZ7s-sv.d.ts} +7 -58
- package/dist/index-BJZ7s-sv.d.ts.map +1 -0
- package/dist/{index-Q6DvdcwI.d.ts → index-BvX3CFNS.d.ts} +6 -6
- package/dist/{index-Q6DvdcwI.d.ts.map → index-BvX3CFNS.d.ts.map} +1 -1
- package/dist/index-DWd1gUwR.d.ts +1086 -0
- package/dist/index-DWd1gUwR.d.ts.map +1 -0
- package/dist/{index-DAXfpVf7.d.ts → index-y7i8l6Y_.d.ts} +8 -8
- package/dist/{index-DAXfpVf7.d.ts.map → index-y7i8l6Y_.d.ts.map} +1 -1
- package/dist/index.d.ts +18 -17
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -15
- package/dist/{kind-factory-BhxbtUsQ.js → kind-factory-Dl0PmNQL.js} +2 -2
- package/dist/{kind-factory-BhxbtUsQ.js.map → kind-factory-Dl0PmNQL.js.map} +1 -1
- package/dist/{llm-judge-Cc8vDJh_.js → llm-judge-3uAeRawE.js} +4 -4
- package/dist/{llm-judge-Cc8vDJh_.js.map → llm-judge-3uAeRawE.js.map} +1 -1
- package/dist/{matrix-Cf-AGxGh.d.ts → matrix-BXo3PSqM.d.ts} +2 -2
- package/dist/{matrix-Cf-AGxGh.d.ts.map → matrix-BXo3PSqM.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +3 -638
- package/dist/meta-eval/index.js +3 -193
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/outcome-observations-FLmRpgYa.js +78 -0
- package/dist/outcome-observations-FLmRpgYa.js.map +1 -0
- package/dist/{paired-arms-Ct4LTKc1.js → paired-arms-B2ttLc0_.js} +108 -31
- package/dist/paired-arms-B2ttLc0_.js.map +1 -0
- package/dist/{power-and-mde-N5nU4_Ly.js → power-and-mde-CQX-0weG.js} +2 -2
- package/dist/{power-and-mde-N5nU4_Ly.js.map → power-and-mde-CQX-0weG.js.map} +1 -1
- package/dist/{prime-protocol-CckHBxGu.d.ts → prime-protocol-BEaqBoQE.d.ts} +2 -2
- package/dist/{prime-protocol-CckHBxGu.d.ts.map → prime-protocol-BEaqBoQE.d.ts.map} +1 -1
- package/dist/{proposal-findings-E1aSC_cU.js → proposal-findings-D0_zkQ2l.js} +2 -2
- package/dist/{proposal-findings-E1aSC_cU.js.map → proposal-findings-D0_zkQ2l.js.map} +1 -1
- package/dist/{registry-B_yG-1jh.d.ts → registry-DDKaUrmM.d.ts} +3 -3
- package/dist/{registry-B_yG-1jh.d.ts.map → registry-DDKaUrmM.d.ts.map} +1 -1
- package/dist/reporting.d.ts +1 -1
- package/dist/reporting.js +3 -3
- package/dist/rl.d.ts +4 -5
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +5 -4
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-noY7uE03.d.ts → rubric-predictive-validity-Cd5SCtAf.d.ts} +58 -3
- package/dist/rubric-predictive-validity-Cd5SCtAf.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-D3_oI72N.js → rubric-predictive-validity-CeKW2GQN.js} +3 -77
- package/dist/rubric-predictive-validity-CeKW2GQN.js.map +1 -0
- package/dist/search/index.d.ts +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts → search-kernel-C6__V0IS.d.ts} +2 -2
- package/dist/{search-kernel-Dloxd6gs.d.ts.map → search-kernel-C6__V0IS.d.ts.map} +1 -1
- package/dist/{skillopt-optimization-method-DLKqdhcR.js → skillopt-optimization-method-D21dH_Bh.js} +4 -4
- package/dist/{skillopt-optimization-method-DLKqdhcR.js.map → skillopt-optimization-method-D21dH_Bh.js.map} +1 -1
- package/dist/{statistical-heldout-3z3I-kJN.d.ts → statistical-heldout-pD1Yah-0.d.ts} +58 -8
- package/dist/statistical-heldout-pD1Yah-0.d.ts.map +1 -0
- package/dist/{store-otlp-dyWE3wAG.js → store-otlp-BVZfR168.js} +2 -2
- package/dist/{store-otlp-dyWE3wAG.js.map → store-otlp-BVZfR168.js.map} +1 -1
- package/dist/{store-tool-spans-DIUgZbzu.js → store-tool-spans-CjYiF4fF.js} +3 -3
- package/dist/{store-tool-spans-DIUgZbzu.js.map → store-tool-spans-CjYiF4fF.js.map} +1 -1
- package/dist/{store-tool-spans-DPwXHQkH.d.ts → store-tool-spans-DfnwWMFR.d.ts} +4 -4
- package/dist/{store-tool-spans-DPwXHQkH.d.ts.map → store-tool-spans-DfnwWMFR.d.ts.map} +1 -1
- package/dist/{summary-report-BsXJoYX5.js → summary-report-DHDWxe6p.js} +3 -3
- package/dist/{summary-report-BsXJoYX5.js.map → summary-report-DHDWxe6p.js.map} +1 -1
- package/dist/systemone-D3MrW63C.d.ts +400 -0
- package/dist/systemone-D3MrW63C.d.ts.map +1 -0
- package/dist/{systemone-protocol-DKoi1NdK.d.ts → systemone-protocol-Bl2EI2KQ.d.ts} +4 -2
- package/dist/{systemone-protocol-DKoi1NdK.d.ts.map → systemone-protocol-Bl2EI2KQ.d.ts.map} +1 -1
- package/dist/systemone-protocol.d.ts +2 -2
- package/dist/systemone-protocol.js +7 -7
- package/dist/systemone-protocol.js.map +1 -1
- package/dist/{systemone-review-CUBMuTLi.d.ts → systemone-review-C8DRiT_-.d.ts} +3 -3
- package/dist/{systemone-review-CUBMuTLi.d.ts.map → systemone-review-C8DRiT_-.d.ts.map} +1 -1
- package/dist/{systemone-review-DzdOJ8I6.js → systemone-review-DFGtIUcf.js} +2 -2
- package/dist/{systemone-review-DzdOJ8I6.js.map → systemone-review-DFGtIUcf.js.map} +1 -1
- package/dist/systemone.d.ts +5 -5
- package/dist/systemone.js +5 -343
- package/dist/{tool-groups-LISbBWCg.d.ts → tool-groups-DK1bu5TD.d.ts} +3 -3
- package/dist/tool-groups-DK1bu5TD.d.ts.map +1 -0
- package/dist/{trace-questions-C42fYEE4.d.ts → trace-questions-D_OKYZ6u.d.ts} +3 -3
- package/dist/{trace-questions-C42fYEE4.d.ts.map → trace-questions-D_OKYZ6u.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +3 -3
- package/dist/{types-DqFMNTgK.js → types-CnWmj8NW.js} +19 -6
- package/dist/types-CnWmj8NW.js.map +1 -0
- package/dist/{types-CHfn1zg5.d.ts → types-DDKfH-N6.d.ts} +2 -2
- package/dist/{types-CHfn1zg5.d.ts.map → types-DDKfH-N6.d.ts.map} +1 -1
- package/dist/{types-8yQzNm8u.d.ts → types-DGImvHIr.d.ts} +150 -4
- package/dist/types-DGImvHIr.d.ts.map +1 -0
- package/dist/{types-D8GXK9Z5.d.ts → types-IERw_HxR.d.ts} +2 -2
- package/dist/{types-D8GXK9Z5.d.ts.map → types-IERw_HxR.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/docs/behavioral-trace-questions.md +99 -0
- package/docs/experiment.md +27 -0
- package/docs/finding-layers.md +54 -1
- package/docs/systemone.md +107 -0
- package/package.json +1 -1
- package/dist/benchmark-command-Dx0E4a6z.js.map +0 -1
- package/dist/default-registry-Cwg_QN3d.d.ts +0 -70
- package/dist/default-registry-Cwg_QN3d.d.ts.map +0 -1
- package/dist/default-registry-ovrIBGmH.js.map +0 -1
- package/dist/evaluation.js.map +0 -1
- package/dist/index-CVFStrQD.d.ts.map +0 -1
- package/dist/judge-gate-Dc_mXQsL.d.ts +0 -453
- package/dist/judge-gate-Dc_mXQsL.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/outcome-store-CNt4iZ67.d.ts +0 -58
- package/dist/outcome-store-CNt4iZ67.d.ts.map +0 -1
- package/dist/paired-arms-Ct4LTKc1.js.map +0 -1
- package/dist/rubric-predictive-validity-D3_oI72N.js.map +0 -1
- package/dist/rubric-predictive-validity-noY7uE03.d.ts.map +0 -1
- package/dist/statistical-heldout-3z3I-kJN.d.ts.map +0 -1
- package/dist/systemone-DVA7t76V.d.ts +0 -135
- package/dist/systemone-DVA7t76V.d.ts.map +0 -1
- package/dist/systemone.js.map +0 -1
- package/dist/tool-groups-LISbBWCg.d.ts.map +0 -1
- package/dist/types-8yQzNm8u.d.ts.map +0 -1
- package/dist/types-DqFMNTgK.js.map +0 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-CnWmj8NW.js","names":[],"sources":["../src/analyst/types.ts"],"sourcesContent":["/**\n * Analyst contract — the missing orchestration layer over agent-eval's\n * existing analyzers (analyzeTraces, MultiLayerVerifier,\n * SemanticConceptJudge, JudgeFn, ...).\n *\n * Each existing primitive returns its own output shape. The Analyst\n * contract is the single envelope every primitive lifts into, so a\n * registry can run N analysts against a run and a single renderer can\n * compose findings without knowing which analyzer produced them.\n *\n * The contract is intentionally domain-agnostic: nothing here knows\n * about code, voice, RAG, or any particular agent stack. Analysts\n * declare what INPUT KIND they need (a trace store, an artifact dir,\n * a RunRecord, a JudgeInput, or `custom`), and the registry routes\n * the matching input from `AnalystRunInputs`.\n */\n\nimport { createHash } from 'node:crypto'\nimport type { CostLedgerHandle, CostProvenance } from '../cost-ledger'\nimport type { RunRecord, RunTokenUsage } from '../run-record'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { JudgeInput } from '../types'\nimport type { ChatClient } from './chat-client'\nimport {\n type FindingConditions,\n type FindingTreatment,\n findingCompositionKey,\n mergeFindingComposition,\n} from './finding-composition'\nimport type {\n FindingEvidenceUnit,\n FindingLayer,\n FindingMeasure,\n FindingMethod,\n FindingScope,\n} from './finding-scope'\n\n/**\n * Unified envelope every analyst emits. Schema-versioned so renderers\n * and time-series diffs survive future field additions.\n *\n * 1.1.0 adds what a finding is about and what it was measured on: `layer`,\n * `scope`, `method`, `measure`, `confounders`, `valid_for`, `claim_id` and\n * `evidence_units`, with optional `treatment` and `conditions` composition. They are all optional, and a 1.0.0 finding carries none of\n * them; `readFindingScope` reads such a finding as layer and n unknown.\n * `parseAnalystFinding` (finding-scope.ts) is the one validator for both.\n */\nexport interface AnalystFinding {\n schema_version: AnalystFindingSchemaVersion\n /**\n * Stable hash over identity-defining fields (analyst_id + canonical\n * claim + area + optional subject). Two findings from two runs that\n * \"are the same finding\" share this id — that's what `diffFindings`\n * uses to compute appeared/disappeared sets across runs.\n */\n finding_id: string\n analyst_id: string\n produced_at: string\n severity: AnalystSeverity\n /**\n * Coarse classification. Renderers group by this. Free-form so\n * domain-specific analysts can introduce categories without a\n * schema change ('agent-reasoning', 'verification', 'cost',\n * 'tool-use', 'safety', 'latency', 'data-quality', ...).\n */\n area: string\n claim: string\n rationale?: string\n evidence_refs: EvidenceRef[]\n recommended_action?: string\n validation_plan?: string\n /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */\n confidence: number\n /**\n * Optional subject the finding is about — leaf id, agent id, request\n * id. Included in finding_id when present so per-subject findings\n * diff cleanly across runs.\n */\n subject?: string\n /** True when this finding was lifted from a judge result rather than observed\n * directly in a trace or artifact. Descriptive only: proposal access is\n * controlled by `ProposalFinding.proposal_origin`. */\n derived_from_judge?: boolean\n /** Analyst-private extras; renderers ignore unless they know the analyst. */\n metadata?: Record<string, unknown>\n /** 1.1.0: the level the claim is about, from one event up to the whole lab. */\n layer?: FindingLayer\n /** 1.1.0: the population the claim was measured on. */\n scope?: FindingScope\n /** 1.1.0: how the claim was produced, and the judge's calibration when judged. */\n method?: FindingMethod\n /** 1.1.0: the number behind the claim, when it has one. */\n measure?: FindingMeasure\n /** 1.1.0: known differences between the compared units besides the treatment. */\n confounders?: string[]\n /** 1.1.0: the conditions the claim was measured under, in words. */\n valid_for?: string\n /**\n * 1.1.0: stable key for the claim across subjects and layers. `promote`\n * groups children by it; a finding without one never lifts.\n */\n claim_id?: string\n /**\n * 1.1.0: the measured units with their lineage. When present, `scope.n` and\n * `scope.population` must match them and no unit may sit above `layer`.\n */\n evidence_units?: FindingEvidenceUnit[]\n /** Deliberately assigned bundles; every bundle may touch several treatment layers. */\n treatment?: FindingTreatment[]\n /** Observed condition snapshots with population and observation-time provenance. */\n conditions?: FindingConditions[]\n}\n\n/** Every finding schema this package reads. Writers emit 1.1.0 only when a 1.1.0 field is set. */\nconst ANALYST_FINDING_SCHEMA_VERSIONS = ['1.0.0', '1.1.0'] as const\n\nexport type AnalystFindingSchemaVersion = (typeof ANALYST_FINDING_SCHEMA_VERSIONS)[number]\n\n/** Finding severity. `AnalystSeverity` derives from this array, so the type\n * and the schema that validates a finding cannot name different levels. */\nexport const ANALYST_SEVERITIES = ['critical', 'high', 'medium', 'low', 'info'] as const\n\nexport type AnalystSeverity = (typeof ANALYST_SEVERITIES)[number]\n\n/** Data sources that candidate generation may intentionally learn from. */\nexport type ProposalFindingOrigin = 'search' | 'production'\n\n/** A finding explicitly admitted as candidate-generation input. */\nexport type ProposalFinding = AnalystFinding & {\n readonly proposal_origin: ProposalFindingOrigin\n}\n\nexport interface EvidenceRef {\n /**\n * Where the evidence lives. `span` and `event` refer to OTLP trace\n * elements; `artifact` to a file inside the run's artifact tree;\n * `finding` to another AnalystFinding (cross-analyst chaining);\n * `metric` to a named scalar reading the renderer knows how to read.\n */\n kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric'\n uri: string\n excerpt?: string\n}\n\n// ── Analyst contract ─────────────────────────────────────────────────\n\n/**\n * The discriminator the registry uses to pass the right input.\n * `custom` is the escape hatch — analysts that need something else\n * (e.g. an embedding cache, a partner SDK handle) read it from\n * `AnalystRunInputs.custom[<analyst id>]`.\n */\nexport type AnalystInputKind =\n | 'trace-store'\n | 'artifact-dir'\n | 'run-record'\n | 'judge-input'\n | 'custom'\n\nexport interface AnalystCost {\n /** `deterministic` analysts MUST NOT call the LLM. */\n kind: 'deterministic' | 'llm'\n /** Optional declared upper bound; the registry can enforce a budget. */\n est_usd_per_run?: number\n /** Models the analyst expects to use (informational). */\n models?: string[]\n /** Maximum post-cancellation wait for provider usage. Model analysts default to 5 seconds. */\n settlement_timeout_ms?: number\n}\n\nexport interface AnalystRequirements {\n /** Min number of shots / samples the analyst needs to produce signal. */\n min_shots?: number\n /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */\n capabilities?: string[]\n}\n\n/**\n * What's passed to every analyst call. The registry resolves which\n * field the analyst's `inputKind` selects and asserts it's present.\n */\nexport interface AnalystRunInputs {\n traceStore?: TraceAnalysisStore\n artifactDir?: string\n runRecord?: RunRecord\n judgeInput?: JudgeInput\n /** Keyed by analyst id; populated by callers that registered custom analysts. */\n custom?: Record<string, unknown>\n}\n\nexport interface AnalystContext {\n runId: string\n /** Stable correlation id so logs from a single registry.run() share a tag. */\n correlationId: string\n /** Enforced wall-clock deadline (epoch ms). */\n deadlineMs?: number\n /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */\n budgetUsd?: number\n /** Shared paid-call account when the analyst runs inside a larger campaign. */\n costLedger?: CostLedgerHandle\n /** Attribution phase used when writing to the shared paid-call account. */\n costPhase?: string\n /**\n * Shared chat client. Analysts that call an LLM go through this so\n * the operator picks transport (sandbox-sdk | router | cli-bridge |\n * direct-provider | mock) at the registry boundary without touching\n * analyst code.\n */\n chat?: ChatClient\n /**\n * Findings from a prior run the operator wants the analyst to see as\n * retrieval context. Kinds that take advantage of cross-run memory\n * (failure-mode \"I saw this cluster last run\", knowledge-gap \"the wiki\n * page I asked for is still missing\") render these into the actor's\n * working set. Filtering is the operator's job: pass the slice that\n * matches the analyst's id, or pass everything and let the kind\n * filter. Empty / absent means no cross-run context.\n */\n priorFindings?: ReadonlyArray<AnalystFinding>\n /**\n * Findings emitted by analysts that completed earlier in this registry run.\n * This is separate from `priorFindings`: upstream findings are dependency\n * context for the current pass, while prior findings are cross-run memory.\n * The registry populates this only when `RegistryRunOpts.chainFindings` is on.\n */\n upstreamFindings?: ReadonlyArray<AnalystFinding>\n /**\n * Report metered work independently of findings. This keeps an empty finding\n * set from erasing token/cost telemetry. Multiple receipts are accumulated.\n */\n recordUsage?: (receipt: AnalystUsageReceipt) => void\n /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */\n tags?: Record<string, string>\n /** Logger callback — analysts SHOULD prefer this over console.* for testability. */\n log?: (msg: string, fields?: Record<string, unknown>) => void\n /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */\n signal?: AbortSignal\n /**\n * Optional live-execution port. A runtime that owns a sandbox or checkout\n * fills it so an analyst can execute a bounded probe against the run's\n * produced state instead of reasoning about it from the trace alone. This\n * package defines only the port: no field here reaches for an agent loop,\n * and an absent probe means the analyst works from recorded evidence.\n */\n probe?: ExecutionProbe\n}\n\n// ── Live-execution port ─────────────────────────────────────────────\n\n/** One bounded command an analyst asks the probe to run. */\nexport interface ExecutionProbeRequest {\n command: string\n /** Working directory inside the probed environment. */\n cwd?: string\n /** Hard wall-clock deadline for this one execution. */\n timeoutMs: number\n /** Bytes of combined output retained; the prober truncates beyond it. */\n maxOutputBytes?: number\n signal?: AbortSignal\n}\n\n/**\n * Typed outcome of one probe execution. `succeeded: false` is a PROBE failure\n * (the environment could not run the command); a command that ran and exited\n * non-zero is a successful observation with a non-zero `exitCode`.\n */\nexport type ExecutionProbeOutcome =\n | {\n succeeded: true\n exitCode: number\n stdout: string\n stderr: string\n durationMs: number\n /** True when output was cut at `maxOutputBytes`. */\n truncated: boolean\n }\n | { succeeded: false; error: { class: string; message: string } }\n\n/**\n * The seam a runtime fills to let analysts observe produced state live.\n * Implementations own sandboxing, credentials, and cleanup; analysts only\n * submit bounded requests and read typed outcomes.\n */\nexport interface ExecutionProbe {\n /** One plain sentence naming what is being probed (e.g. a sandbox id). */\n readonly description: string\n execute(request: ExecutionProbeRequest): Promise<ExecutionProbeOutcome>\n}\n\n/**\n * The minimal contract. Concrete analysts can refine `TInput` so\n * implementations stay type-safe (e.g. a trace analyst's `TInput` is\n * `TraceAnalysisStore`); the registry passes the right field from\n * `AnalystRunInputs` based on `inputKind`.\n */\nexport interface Analyst<TInput = unknown> {\n /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */\n readonly id: string\n /** Human-readable. One sentence. */\n readonly description: string\n readonly inputKind: AnalystInputKind\n readonly cost: AnalystCost\n readonly requires?: AnalystRequirements\n /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */\n readonly version: string\n analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>\n}\n\n/** Metered work performed by one analyst call. */\nexport interface AnalystUsageReceipt {\n /** Number of model-usage records observed at the provider boundary. */\n calls: number | null\n /** Null when the provider did not return token accounting. */\n tokens: RunTokenUsage | null\n /** Observed, estimated, or explicitly uncaptured dollar cost. */\n cost: CostProvenance\n /** Known lower bound when one or more calls have uncaptured cost. */\n knownCostUsd?: number\n /**\n * Token counts the provider reported on only one side. Present exactly when\n * `tokens` is null and at least one side WAS reported: `RunTokenUsage` has no\n * nullable side, so a one-sided count cannot live in `tokens` without writing\n * a zero nobody measured. Read it as a lower bound, never as a total — the\n * field exists so a null `tokens` cannot hide a real count.\n */\n partialTokens?: { input: number | null; output: number | null }\n /**\n * True when the token counts were DERIVED by the transport (from character\n * lengths, say) rather than measured by the model provider. `cost.kind` is\n * `estimated` both for a rate estimate over exact tokens and for one over\n * derived tokens; this is the field that separates them.\n */\n tokensEstimated?: boolean\n}\n\n// ── finding_id stability ─────────────────────────────────────────────\n\n/**\n * Compute the stable finding_id from the identity-defining fields.\n * Legacy identity hashes {analyst_id, area, subject, normalized claim}. Supplied\n * composition adds normalized treatment/fixed-condition identity; covariate provenance\n * remains on the record without making every observation time a different claim.\n * Analysts that emit findings whose claim text varies per run (timestamps,\n * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,\n * or (b) move the variable part into `rationale`/`metadata` and keep the\n * `claim` static.\n */\nexport function computeFindingId(input: {\n analyst_id: string\n area: string\n subject?: string\n claim: string\n /** Override the claim for hashing — use when the displayed claim has run-specific bits. */\n id_basis?: string\n treatment?: FindingTreatment[]\n conditions?: FindingConditions[]\n}): string {\n const composition = findingCompositionKey(input)\n const basis = JSON.stringify({\n a: input.analyst_id,\n r: input.area,\n s: input.subject ?? '',\n c: normalizeClaim(input.id_basis ?? input.claim),\n ...(composition ? { composition } : {}),\n })\n return `f_${createHash('sha256').update(basis).digest('hex').slice(0, 20)}`\n}\n\nfunction normalizeClaim(c: string): string {\n // Lowercase, collapse whitespace, strip trailing punctuation. Goal:\n // \"Leaf X failed install\" and \"Leaf X failed install.\" hash the same.\n return c\n .toLowerCase()\n .replace(/\\s+/g, ' ')\n .replace(/[.!?;:,]+$/g, '')\n .trim()\n}\n\n/** Fields that exist only in schema 1.1.0. */\nexport const FINDING_SCOPE_FIELDS = [\n 'layer',\n 'scope',\n 'method',\n 'measure',\n 'confounders',\n 'valid_for',\n 'claim_id',\n 'evidence_units',\n 'treatment',\n 'conditions',\n] as const satisfies ReadonlyArray<keyof AnalystFinding>\n\n/**\n * Convenience factory: produce a fully-formed AnalystFinding with the\n * id computed automatically. Analyst code stays terse.\n */\nexport function makeFinding(\n init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n id_basis?: string\n produced_at?: string\n },\n): AnalystFinding {\n const { id_basis, produced_at, ...fields } = init\n const rest = { ...fields, ...mergeFindingComposition([fields]) }\n return {\n schema_version: FINDING_SCOPE_FIELDS.some((field) => rest[field] !== undefined)\n ? '1.1.0'\n : '1.0.0',\n finding_id: computeFindingId({\n analyst_id: rest.analyst_id,\n area: rest.area,\n subject: rest.subject,\n claim: rest.claim,\n id_basis,\n treatment: rest.treatment,\n conditions: rest.conditions,\n }),\n produced_at: produced_at ?? new Date().toISOString(),\n ...rest,\n }\n}\n\n/** Build a finding whose source is explicitly allowed during candidate generation. */\nexport function makeProposalFinding(\n init: Omit<ProposalFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n id_basis?: string\n produced_at?: string\n },\n): ProposalFinding {\n const { proposal_origin, ...finding } = init\n return { ...makeFinding(finding), proposal_origin }\n}\n\n// ── Registry result envelope ────────────────────────────────────────\n\nexport interface AnalystRunSummary {\n analyst_id: string\n status: 'ok' | 'skipped' | 'failed'\n /** Why skipped — missing input, budget exceeded, capability unmet. */\n reason?: string\n findings_count: number\n latency_ms: number\n /** Additive model usage and cost provenance for this analyst. */\n usage: AnalystUsageReceipt\n /** When `status='failed'`: the error class + message, never the full stack. */\n error?: { class: string; message: string }\n}\n\nexport interface AnalystRunResult {\n run_id: string\n correlation_id: string\n started_at: string\n ended_at: string\n findings: AnalystFinding[]\n per_analyst: AnalystRunSummary[]\n /** Total LLM cost in USD across all analysts in this registry.run(). */\n total_cost_usd: number\n /**\n * Provenance for `total_cost_usd`. When uncaptured, the numeric field is only\n * the known subtotal and must not be treated as the run's total spend.\n */\n total_cost_provenance?: CostProvenance\n}\n\n// ── Streaming event envelope ────────────────────────────────────────\n\n/**\n * Events emitted by `AnalystRegistry.runStream(...)` in real time as\n * the registry executes. UIs subscribe via `for await (const ev of\n * registry.runStream(...))`; `registry.run(...)` is a thin collector\n * over the same stream, so the two surfaces share their invariants.\n *\n * Per-finding events are intentionally omitted — analyzers are batch\n * operations (a recursive engine returns the full `findings:json[]` at the\n * end of the responder), so streaming inside one analyst would only\n * emit partial JSON consumers can't render. The kind-completion event\n * is the right granularity; subscribers wanting per-finding rendering\n * iterate `event.findings` themselves.\n */\nexport type AnalystRunEvent =\n | {\n type: 'run-started'\n run_id: string\n correlation_id: string\n started_at: string\n /** The ordered list of analyst ids the registry will run. */\n analyst_ids: ReadonlyArray<string>\n }\n | {\n type: 'analyst-skipped'\n summary: AnalystRunSummary\n }\n | {\n type: 'analyst-started'\n analyst_id: string\n started_at: string\n }\n | {\n type: 'analyst-completed'\n /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */\n summary: AnalystRunSummary\n findings: ReadonlyArray<AnalystFinding>\n }\n | {\n type: 'run-completed'\n result: AnalystRunResult\n }\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAwHA,MAAa,qBAAqB;CAAC;CAAY;CAAQ;CAAU;CAAO;AAAM;;;;;;;;;;;AAmO9E,SAAgB,iBAAiB,OAStB;CACT,MAAM,cAAc,sBAAsB,KAAK;CAC/C,MAAM,QAAQ,KAAK,UAAU;EAC3B,GAAG,MAAM;EACT,GAAG,MAAM;EACT,GAAG,MAAM,WAAW;EACpB,GAAG,eAAe,MAAM,YAAY,MAAM,KAAK;EAC/C,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;CACvC,CAAC;CACD,OAAO,KAAK,WAAW,QAAQ,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,GAAG,EAAE;AAC1E;AAEA,SAAS,eAAe,GAAmB;CAGzC,OAAO,EACJ,YAAY,CAAC,CACb,QAAQ,QAAQ,GAAG,CAAC,CACpB,QAAQ,eAAe,EAAE,CAAC,CAC1B,KAAK;AACV;;AAGA,MAAa,uBAAuB;CAClC;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;;;;AAMA,SAAgB,YACd,MAIgB;CAChB,MAAM,EAAE,UAAU,aAAa,GAAG,WAAW;CAC7C,MAAM,OAAO;EAAE,GAAG;EAAQ,GAAG,wBAAwB,CAAC,MAAM,CAAC;CAAE;CAC/D,OAAO;EACL,gBAAgB,qBAAqB,MAAM,UAAU,KAAK,WAAW,KAAA,CAAS,IAC1E,UACA;EACJ,YAAY,iBAAiB;GAC3B,YAAY,KAAK;GACjB,MAAM,KAAK;GACX,SAAS,KAAK;GACd,OAAO,KAAK;GACZ;GACA,WAAW,KAAK;GAChB,YAAY,KAAK;EACnB,CAAC;EACD,aAAa,gCAAe,IAAI,KAAK,EAAA,CAAE,YAAY;EACnD,GAAG;CACL;AACF;;AAGA,SAAgB,oBACd,MAIiB;CACjB,MAAM,EAAE,iBAAiB,GAAG,YAAY;CACxC,OAAO;EAAE,GAAG,YAAY,OAAO;EAAG;CAAgB;AACpD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { g as PaidCallResult, i as CostChannel, l as CostLedgerSummary, o as CostLedgerHandle, r as CostBreakdown, u as CostProvenance, y as RunPaidCallInput } from "./cost-ledger-C1W3ICmP.js";
|
|
2
2
|
import { d as RunTokenUsage, s as RunSearchCoordinates } from "./run-record-C3awiXPf.js";
|
|
3
|
-
import { v as ProposalFinding } from "./types-
|
|
3
|
+
import { v as ProposalFinding } from "./types-DGImvHIr.js";
|
|
4
4
|
import { j as LlmCallMetadata } from "./types-eMhdacVp.js";
|
|
5
5
|
import { t as SeriesDistribution } from "./descriptive-B2iPaT9J.js";
|
|
6
6
|
import { A as SearchLedgerHash, F as SearchNodeRegisteredEvent, G as SearchSplit, I as SearchNodeStatus, L as SearchOpenedEvent, N as SearchNodeDecision, O as SearchLedgerEntry, P as SearchNodeRef, Q as SearchTaskOutcome, R as SearchOperationKind, U as SearchReservation, Z as SearchTask, b as SearchClosedEvent, d as SearchCellCancelledEvent, i as SearchArtifactRef, l as SearchCandidateSurface, o as SearchAudit, p as SearchCellStage, t as NodeEstimate, w as SearchEdgeRecordedEvent, z as SearchOperationRecordedEvent } from "./search-ledger-types-p2xKfiDJ.js";
|
|
@@ -1017,4 +1017,4 @@ interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scena
|
|
|
1017
1017
|
}
|
|
1018
1018
|
//#endregion
|
|
1019
1019
|
export { SearchCompletion as $, LabeledScenarioWrite as A, ScoredSurfaceOutcome as B, JudgeDimension as C, LabeledScenarioSampleArgs as D, LabeledScenarioRecord as E, ProposeContext as F, SearchProposerView as G, SurfaceProposer as H, ProposedCandidate as I, searchProposerView as J, SearchSummaryOptions as K, RedactionStatus as L, OptimizerConfig as M, ParetoParent as N, LabeledScenarioSource as O, ProposalTrackContext as P, SearchCell as Q, Scenario as R, JudgeConfig as S, LabelTrust as T, TraceSpan as U, SessionScript as V, isProposedCandidate as W, SearchStateView as X, SearchState as Y, searchUnitScores as Z, GateDecision as _, CampaignResult as a, SearchUnitScore as at, GenerationRecord as b, CampaignTraceWriter as c, searchTaskSetDigest as ct, DispatchContext as d, SearchDecisionRecord as et, DispatchFn as f, GateContribution as g, GateContext as h, CampaignCostMeter as i, SearchSpend as it, MutableSurface as j, LabeledScenarioStore as k, CodeSurface as l, GateCheckStatus as m, CampaignArtifactWriter as n, SearchOperation as nt, CampaignScenarioIdentity as o, searchCellId as ot, Gate as p, renderSearchSummary as q, CampaignCellResult as r, SearchScoredCell as rt, CampaignTokenUsage as s, searchNodeId as st, CampaignAggregates as t, SearchNode as tt, ComponentSurface as u, GateResult as v, JudgeScore as w, JudgeAggregate as x, GenerationCandidate as y, ScenarioAggregate as z };
|
|
1020
|
-
//# sourceMappingURL=types-
|
|
1020
|
+
//# sourceMappingURL=types-DDKfH-N6.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-DDKfH-N6.d.ts","names":[],"sources":["../src/campaign/search-state-identity.ts","../src/campaign/search-state-types.ts","../src/campaign/search-state-indexes.ts","../src/campaign/search-state-view.ts","../src/campaign/search-state-projection.ts","../src/campaign/search-summary.ts","../src/campaign/types.ts"],"mappings":";;;;;;;;;;iBAOgB,aAAa,kBAAkB;;iBAK/B,aACd,kBACA,gBACA,gBACA,OAAO,aACP;;iBAMc,oBAAoB,gBAAgB,eAAe;;;UCHlD;EACf;EACA,UAAU;EACV,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA;;;UAIe;EACf;EACA;;EAEA;EACA;;UAGe;EACf;;EAEA;EACA,gBAAgB;EAChB,UAAU;EACV,mBAAmB;;;EAGnB;;EAEA;;;;EAIA,kBAAkB;EAClB;EACA;EACA,QAAQ;EACR;;EAEA;EACA,oBAAoB;EACpB;EACA,OAAO;EACP;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,OAAO;EACP;EACA,OAAO;EACP;EACA,aAAa;EACb;EACA;EACA,SAAS;;EAET;;EAEA;EACA,WAAW;;EAEX;;EAEA;EACA;EACA;EACA;;UAGe;EACf;EACA,eAAe;EACf,aAAa;EACb;;EAEA,SAAS;;;EAGT,WAAW;;;EAGX,oBAAoB;;EAEpB;;EAEA;;UAGe;EACf;EACA;;UAGe;EACf,OAAO;EACP;EACA;EACA;EACA;EACA,SAAS;EACT,YAAY;EACZ;EACA;EACA;EACA,WAAW;EACX;EACA;EACA;EACA,OAAO;;;;cCvII;WACF,OAAK;IAA4B,OAAA;IAAqB;;WACtD,OAAK,YAAA;WACL;WACA,cAAY;WACZ,OAAK,YAAA;WACL;WACA,OAAK,YAAA;WACL,YAAU,YAAA;EAEnB,KAAK,iBAAiB;EAKtB;EAIA,QAAQ;EAIR,gBAAgB;EAIhB,KAAK,iBAAiB;EAItB;EAIA,KAAK,iBAAiB;EAKtB,QAAQ;EAKR,UAAU,sBAAsB;EAKhC,YAAY,gBAAgB,OAAO,cAAc;;;;;;;;;;iBChCnC,iBAAiB,gBAAgB,qBAAqB;;;;;;cAyBzD;WACF;WACA,QAAQ;WACR,QAAQ;WACR;IAAQ;IAAkB,WAAW;;;WAErC;WACA,OAAO;WACP,YAAY;;;;;;;WAOZ;IAAU;IAA4B;;mBAC9B;mBACA;EAEjB,YAAY,OAAO,aAAa;;MAiB5B;EAIJ,KAAK,iBAAiB;EAItB,QAAQ;;EAKR;;EAKA,SAAS;EAKT,gBAAgB;EAIhB,KAAK,iBAAiB;;EAKtB,SAAS;EAKT,KAAK,iBAAiB;;EAKtB,MAAM;IAAU;MAAyB;EAKzC,UAAU,sBAAsB;;;EAMhC,YAAY,gBAAgB,OAAO,cAAc;;EAKjD,WAAW,gBAAgB,OAAO,cAAc;;EAKhD,QAAQ,aAAa,uBAAuB;UAmCpC;;;;;;;;;cCnJG,uBAAuB,gBAAgB,mBAAmB;WAC5D;UACD;UACA;UACA;UACA;UACA;UACA;UACA;;mBAES;;UAET;mBACS;mBACA;;WAER,OAAK;EAEd,YAAY;EAiCZ,MAAM,OAAO,mBAAmB;EA6BhC,YAAY;;EAKZ,KAAK;UAIG;UAgCA;UA6CA;UA2BA;UA8BA;UA8CA;UAsHA;UAwEA;UAyEA;UAiBA;UAmDA;UAaA;;;UA2EA;;EASR;;;EAmBA;IACE,QAAQ;IACR,QAAQ;IACR;MAAQ;MAAkB,WAAW;;IACrC;IACA,OAAO;IACP;MAAU;MAA4B;;;;;;;;UCjuBzB;WACN;WACA;;;EAGT,YAAY,0BAA0B;;EAEtC,WAAW,0BAA0B;;;iBAIvB,mBAAmB,OAAO,kBAAkB;UAW3C;;;;;;;;EAQf,OAAO;;EAEP;;;;;;;;;;;iBAYc,oBAAoB,OAAO,iBAAiB,SAAS;;;;;UCrCpD;EACf;EACA;EACA;;;;;EAKA;;;UAIe,iCAAiC,KAAK;EACrD;;;;;UAMe;EACf;;EAEA;EACA;EACA;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;EACX,MAAM;;EAEN;;EAEA;;;;;;;;EAQA;;;;KAKU,WAAW,kBAAkB,UAAU,cACjD,UAAU,WACV,KAAK,oBACF,QAAQ;;;;UAOI,cAAc,WAAW;EACxC;EACA;EACA;;EAEA;;;EAGA,sBAAsB,UAAU,WAAW,sBAAsB,UAAU,cAAc;;UAK1E;;EAEf;;EAEA;;;;;;;;;UAUe,YAAY,WAAW,kBAAkB,WAAW;EACnE;EACA,YAAY;;;;EAIZ;;;EAGA,MAAM;IACJ,UAAU;IACV,UAAU;IACV,QAAQ;;IAER,aAAa;IACb;IACA,WAAW;MACT,aAAa,QAAQ;EACzB,aAAa,UAAU;;;;;;;;;;;UAYR;EACf,YAAY;EACZ;EACA;;EAEA,UAAU;;;;;;;EAOV;;;;EAIA,eAAe,eAAe;IAAQ;IAAe;;;;;EAIrD;;;EAGA;;EAEA;;EAEA,WAAW,eAAe;;;;;;;UAUX;WACN;;;WAGA;;WAEA;;WAEA;;WAEA;;WAEA;;WAEA;;;WAGA;aACE;aACA;aACA;;;WAGF;;;UAIM;WACN;WACA,YAAY,SAAS;;;;;;;;;;;KAYpB,0BAA0B,mBAAmB;;;;;;;UAQxC;EACf,SAAS;;EAET;;;;EAIA;;;;;;EAMA,cAAc,SAAS;;;;iBAKT,oBACd,OAAO,iBAAiB,oBACvB,SAAS;;;;;;;;;UAkBK;EACf,SAAS;EACT;;;EAGA,YAAY;;;EAGZ;;EAEA;EACA;EACA;;;UAIe;EACf;EACA;EACA;EACA;EACA;;;;;UAMe;;;EAGf;;EAEA;EACA;EACA;EACA,YAAY;;;;;EAKZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;EAC1E;IACE;IACA;;;;;UAMa,eAAe,YAAY;;;;WAIjC,gBAAgB;;;;;WAKhB;;;;WAIA,UAAU;IAAgB;IAAgB,UAAU;;;;WAGpD,QAAQ;;;;WAIR;;;WAGA,SAAS,cAAc;WACvB,UAAU,cAAc;;WAExB;WACA;WACA,QAAQ;;WAER,QAAQ;;;WAGR,kBAAkB;;;WAGlB,mBAAmB;;;;WAInB,gBAAgB;;;;WAIhB;;;;;;;WAOA,gBAAgB,cAAc;;WAE9B,aAAa;WACb;;;;;;;;;;;;;;UAeM,gBAAgB,YAAY;EAC3C;;;;;EAKA,QAAQ,KAAK,eAAe,aAAa,QAAQ,MAAM,iBAAiB;;;EAGxE,QAAQ;IAAQ,SAAS,cAAc;;IAAwB;IAAe;;;UAGtE;EACR;EACA;EACA,mBAAmB,qBAAqB;;UAGzB,wBAAwB;EACvC,UAAU;;;KAMA;;KAGA;UAEK;EACf;EACA,QAAQ;EACR;;UAGe,YAAY,WAAW,kBAAkB;EACxD,oBAAoB,YAAY;EAChC,oBAAoB,YAAY;;EAEhC,aAAa,YAAY,eAAe;;;;;EAKxC,sBAAsB,YAAY,eAAe;;;;;;;EAOjD,yBAAyB,YAAY,eAAe;;;EAGpD,uBAAuB,YAAY;EACnC,WAAW;EACX;IAAQ;IAAmB;;;EAE3B,aAAa;EACb;EACA,QAAQ;;UAGO;EACf,UAAU;EACV;EACA,mBAAmB;EACnB;;;UAIe,KAAK,qBAAqB,kBAAkB,WAAW;EACtE;EACA,OAAO,KAAK,YAAY,WAAW,aAAa,QAAQ;;;;UAOzC;EACf,KAAK,cAAc,aAAa,0BAA0B;EAC1D,SAAS;;UAGM;EACf,IAAI,aAAa;EACjB,aAAa,aAAa;;;;;UAMX;EACf,MAAM,cAAc,kBAAkB,aAAa;EACnD,UAAU,cAAc,iBAAiB;;;;;;KAO/B,qBAAqB;;;;;UAMhB;;EAEf,YAAY,GACV,OAAO,KAAK,iBAAiB;IAC3B,UAAU;MAEX,QAAQ,eAAe;;;;;KAQhB;KAOA;;;;;;;;;;;;;;;KAgBA;;;;UAgBK,qBAAqB,kBAAkB,WAAW,UAAU;EAC3E,UAAU;EACV,UAAU;EACV,aAAa,eAAe;EAC5B,QAAQ;EACR;EACA;EACA,iBAAiB;;;;;EAKjB,aAAa;;EAEb;;UAGe,sBAAsB,kBAAkB,WAAW,UAAU,6BACpE,qBAAqB,WAAW;;EAExC;;;EAGA;;UAGe;EACf;;EAEA;;;;EAIA;EACA;IACE;IACA,SAAS,wBAAwB;IACjC;IACA;;;;;IAKA,WAAW;;;UAIE;EACf,QAAQ,OAAO,uBAAuB;EACtC,OAAO,MAAM,4BAA4B,QAAQ;EACjD,QAAQ;IACN;IACA;IACA,UAAU;;;IAGV,SAAS,OAAO;;;UAMH,mBAAmB;;EAElC,gBAAgB;;;EAGhB;EACA;;EAEA;IAAY;IAAsB;;EAClC;EACA;EACA;EACA,UAAU;EACV,aAAa,eAAe;;;EAG5B;;EAEA,gBAAgB;;EAEhB;;;EAGA,YAAY;;;EAGZ;;;EAGA;EACA;EACA;EACA;;;;EAIA;;EAEA;;EAEA;EACA;;UAGe;EACf;EACA;EACA;EACA;;;;;;EAMA,cAAc;;UAGC;EACf;EACA;EACA;;;EAGA,cAAc;;UAGC;EACf;EACA,YAAY;EACZ;;;;;;UAOe;EACf;;;;EAIA;;;;EAIA;;;;EAIA;IACE;IACA;IACA,iBAAiB;MAAQ;MAAgB;;;;;EAI3C,YAAY;;;;;;;;;;EAUZ,WAAW;IAAQ;IAAoB;IAAmB;IAAgB;;;;EAG1E;;;;EAIA;;;;EAIA,cAAc,SAAS;;UAGR;EACf,SAAS,eAAe;EACxB,YAAY,eAAe;;EAE3B,MAAM;;EAEN;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe,eAAe,qBAAqB,kBAAkB,WAAW;;EAEhF;;EAEA;EACA;;EAEA;EACA;EACA;EACA;EACA,OAAO,MAAM,mBAAmB;EAChC,YAAY;EACZ;IACE,aAAa;IACb;;EAEF,OAAO;EACP;EACA;EACA,iBAAiB;;;EAGjB,WAAW,MAAM,2BAA2B,KAAK"}
|
|
@@ -287,6 +287,79 @@ interface BoundedTraceAnalysisStoreOptions {
|
|
|
287
287
|
/** Apply the public validation, cancellation, not-found, and size rules to any adapter. */
|
|
288
288
|
declare function createBoundedTraceAnalysisStore(source: TraceAnalysisStore, options?: BoundedTraceAnalysisStoreOptions): TraceAnalysisStore;
|
|
289
289
|
//#endregion
|
|
290
|
+
//#region src/analyst/finding-composition.d.ts
|
|
291
|
+
/** What was deliberately changed; this is separate from a finding's event-to-meta scope. */
|
|
292
|
+
declare const TREATMENT_LAYERS: readonly ['Task', 'Model', 'Budget', 'Team', 'Roles', 'Tools', 'Memory', 'Process', 'Oversight'];
|
|
293
|
+
type TreatmentLayer = (typeof TREATMENT_LAYERS)[number];
|
|
294
|
+
declare const findingTreatmentSchema: z.ZodObject<{
|
|
295
|
+
id: z.ZodString;
|
|
296
|
+
definition_digest: z.ZodString;
|
|
297
|
+
definition_refs: z.ZodArray<z.ZodObject<{
|
|
298
|
+
kind: z.ZodEnum<{
|
|
299
|
+
artifact: "artifact";
|
|
300
|
+
event: "event";
|
|
301
|
+
finding: "finding";
|
|
302
|
+
metric: "metric";
|
|
303
|
+
span: "span";
|
|
304
|
+
}>;
|
|
305
|
+
uri: z.ZodString;
|
|
306
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
307
|
+
}, z.core.$strict>>;
|
|
308
|
+
layers: z.ZodArray<z.ZodEnum<{
|
|
309
|
+
Budget: "Budget";
|
|
310
|
+
Memory: "Memory";
|
|
311
|
+
Model: "Model";
|
|
312
|
+
Oversight: "Oversight";
|
|
313
|
+
Process: "Process";
|
|
314
|
+
Roles: "Roles";
|
|
315
|
+
Task: "Task";
|
|
316
|
+
Team: "Team";
|
|
317
|
+
Tools: "Tools";
|
|
318
|
+
}>>;
|
|
319
|
+
}, z.core.$strict>;
|
|
320
|
+
/** Required nullable values distinguish an unavailable observation from observed absence. */
|
|
321
|
+
declare const findingConditionsSchema: z.ZodObject<{
|
|
322
|
+
id: z.ZodString;
|
|
323
|
+
population: z.ZodArray<z.ZodString>;
|
|
324
|
+
observed_at: z.ZodString;
|
|
325
|
+
evidence_refs: z.ZodArray<z.ZodObject<{
|
|
326
|
+
kind: z.ZodEnum<{
|
|
327
|
+
artifact: "artifact";
|
|
328
|
+
event: "event";
|
|
329
|
+
finding: "finding";
|
|
330
|
+
metric: "metric";
|
|
331
|
+
span: "span";
|
|
332
|
+
}>;
|
|
333
|
+
uri: z.ZodString;
|
|
334
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
335
|
+
}, z.core.$strict>>;
|
|
336
|
+
runtime_version: z.ZodNullable<z.ZodString>;
|
|
337
|
+
lab_commit: z.ZodNullable<z.ZodString>;
|
|
338
|
+
model_version: z.ZodNullable<z.ZodString>;
|
|
339
|
+
provider_version: z.ZodNullable<z.ZodString>;
|
|
340
|
+
seat: z.ZodNullable<z.ZodString>;
|
|
341
|
+
host_load: z.ZodNullable<z.ZodNumber>;
|
|
342
|
+
started_at: z.ZodNullable<z.ZodString>;
|
|
343
|
+
operator_steers: z.ZodNullable<z.ZodArray<z.ZodObject<{
|
|
344
|
+
id: z.ZodString;
|
|
345
|
+
at: z.ZodNullable<z.ZodString>;
|
|
346
|
+
planned: z.ZodNullable<z.ZodBoolean>;
|
|
347
|
+
evidence_ref: z.ZodObject<{
|
|
348
|
+
kind: z.ZodEnum<{
|
|
349
|
+
artifact: "artifact";
|
|
350
|
+
event: "event";
|
|
351
|
+
finding: "finding";
|
|
352
|
+
metric: "metric";
|
|
353
|
+
span: "span";
|
|
354
|
+
}>;
|
|
355
|
+
uri: z.ZodString;
|
|
356
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
357
|
+
}, z.core.$strict>;
|
|
358
|
+
}, z.core.$strict>>>;
|
|
359
|
+
}, z.core.$strict>;
|
|
360
|
+
type FindingTreatment = z.infer<typeof findingTreatmentSchema>;
|
|
361
|
+
type FindingConditions = z.infer<typeof findingConditionsSchema>;
|
|
362
|
+
//#endregion
|
|
290
363
|
//#region src/analyst/finding-scope.d.ts
|
|
291
364
|
/**
|
|
292
365
|
* The layer ladder, lowest first. `problem` is what discovery-lab calls a
|
|
@@ -444,6 +517,71 @@ declare const ProposalFindingSchema: z.ZodObject<{
|
|
|
444
517
|
}>;
|
|
445
518
|
derived_from: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
446
519
|
}, z.core.$strict>>>;
|
|
520
|
+
treatment: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
521
|
+
id: z.ZodString;
|
|
522
|
+
definition_digest: z.ZodString;
|
|
523
|
+
definition_refs: z.ZodArray<z.ZodObject<{
|
|
524
|
+
kind: z.ZodEnum<{
|
|
525
|
+
artifact: "artifact";
|
|
526
|
+
event: "event";
|
|
527
|
+
finding: "finding";
|
|
528
|
+
metric: "metric";
|
|
529
|
+
span: "span";
|
|
530
|
+
}>;
|
|
531
|
+
uri: z.ZodString;
|
|
532
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
533
|
+
}, z.core.$strict>>;
|
|
534
|
+
layers: z.ZodArray<z.ZodEnum<{
|
|
535
|
+
Budget: "Budget";
|
|
536
|
+
Memory: "Memory";
|
|
537
|
+
Model: "Model";
|
|
538
|
+
Oversight: "Oversight";
|
|
539
|
+
Process: "Process";
|
|
540
|
+
Roles: "Roles";
|
|
541
|
+
Task: "Task";
|
|
542
|
+
Team: "Team";
|
|
543
|
+
Tools: "Tools";
|
|
544
|
+
}>>;
|
|
545
|
+
}, z.core.$strict>>>;
|
|
546
|
+
conditions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
547
|
+
id: z.ZodString;
|
|
548
|
+
population: z.ZodArray<z.ZodString>;
|
|
549
|
+
observed_at: z.ZodString;
|
|
550
|
+
evidence_refs: z.ZodArray<z.ZodObject<{
|
|
551
|
+
kind: z.ZodEnum<{
|
|
552
|
+
artifact: "artifact";
|
|
553
|
+
event: "event";
|
|
554
|
+
finding: "finding";
|
|
555
|
+
metric: "metric";
|
|
556
|
+
span: "span";
|
|
557
|
+
}>;
|
|
558
|
+
uri: z.ZodString;
|
|
559
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
560
|
+
}, z.core.$strict>>;
|
|
561
|
+
runtime_version: z.ZodNullable<z.ZodString>;
|
|
562
|
+
lab_commit: z.ZodNullable<z.ZodString>;
|
|
563
|
+
model_version: z.ZodNullable<z.ZodString>;
|
|
564
|
+
provider_version: z.ZodNullable<z.ZodString>;
|
|
565
|
+
seat: z.ZodNullable<z.ZodString>;
|
|
566
|
+
host_load: z.ZodNullable<z.ZodNumber>;
|
|
567
|
+
started_at: z.ZodNullable<z.ZodString>;
|
|
568
|
+
operator_steers: z.ZodNullable<z.ZodArray<z.ZodObject<{
|
|
569
|
+
id: z.ZodString;
|
|
570
|
+
at: z.ZodNullable<z.ZodString>;
|
|
571
|
+
planned: z.ZodNullable<z.ZodBoolean>;
|
|
572
|
+
evidence_ref: z.ZodObject<{
|
|
573
|
+
kind: z.ZodEnum<{
|
|
574
|
+
artifact: "artifact";
|
|
575
|
+
event: "event";
|
|
576
|
+
finding: "finding";
|
|
577
|
+
metric: "metric";
|
|
578
|
+
span: "span";
|
|
579
|
+
}>;
|
|
580
|
+
uri: z.ZodString;
|
|
581
|
+
excerpt: z.ZodOptional<z.ZodString>;
|
|
582
|
+
}, z.core.$strict>;
|
|
583
|
+
}, z.core.$strict>>>;
|
|
584
|
+
}, z.core.$strict>>>;
|
|
447
585
|
proposal_origin: z.ZodEnum<{
|
|
448
586
|
production: "production";
|
|
449
587
|
search: "search";
|
|
@@ -497,7 +635,7 @@ declare function independentUnitCount(units: ReadonlyArray<FindingEvidenceUnit>)
|
|
|
497
635
|
*
|
|
498
636
|
* 1.1.0 adds what a finding is about and what it was measured on: `layer`,
|
|
499
637
|
* `scope`, `method`, `measure`, `confounders`, `valid_for`, `claim_id` and
|
|
500
|
-
* `evidence_units
|
|
638
|
+
* `evidence_units`, with optional `treatment` and `conditions` composition. They are all optional, and a 1.0.0 finding carries none of
|
|
501
639
|
* them; `readFindingScope` reads such a finding as layer and n unknown.
|
|
502
640
|
* `parseAnalystFinding` (finding-scope.ts) is the one validator for both.
|
|
503
641
|
*/
|
|
@@ -561,6 +699,10 @@ interface AnalystFinding {
|
|
|
561
699
|
* `scope.population` must match them and no unit may sit above `layer`.
|
|
562
700
|
*/
|
|
563
701
|
evidence_units?: FindingEvidenceUnit[];
|
|
702
|
+
/** Deliberately assigned bundles; every bundle may touch several treatment layers. */
|
|
703
|
+
treatment?: FindingTreatment[];
|
|
704
|
+
/** Observed condition snapshots with population and observation-time provenance. */
|
|
705
|
+
conditions?: FindingConditions[];
|
|
564
706
|
}
|
|
565
707
|
/** Every finding schema this package reads. Writers emit 1.1.0 only when a 1.1.0 field is set. */
|
|
566
708
|
declare const ANALYST_FINDING_SCHEMA_VERSIONS: readonly ['1.0.0', '1.1.0'];
|
|
@@ -767,7 +909,9 @@ interface AnalystUsageReceipt {
|
|
|
767
909
|
}
|
|
768
910
|
/**
|
|
769
911
|
* Compute the stable finding_id from the identity-defining fields.
|
|
770
|
-
*
|
|
912
|
+
* Legacy identity hashes {analyst_id, area, subject, normalized claim}. Supplied
|
|
913
|
+
* composition adds normalized treatment/fixed-condition identity; covariate provenance
|
|
914
|
+
* remains on the record without making every observation time a different claim.
|
|
771
915
|
* Analysts that emit findings whose claim text varies per run (timestamps,
|
|
772
916
|
* counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,
|
|
773
917
|
* or (b) move the variable part into `rationale`/`metadata` and keep the
|
|
@@ -780,6 +924,8 @@ declare function computeFindingId(input: {
|
|
|
780
924
|
claim: string;
|
|
781
925
|
/** Override the claim for hashing — use when the displayed claim has run-specific bits. */
|
|
782
926
|
id_basis?: string;
|
|
927
|
+
treatment?: FindingTreatment[];
|
|
928
|
+
conditions?: FindingConditions[];
|
|
783
929
|
}): string;
|
|
784
930
|
/**
|
|
785
931
|
* Convenience factory: produce a fully-formed AnalystFinding with the
|
|
@@ -861,5 +1007,5 @@ type AnalystRunEvent = {
|
|
|
861
1007
|
result: AnalystRunResult;
|
|
862
1008
|
};
|
|
863
1009
|
//#endregion
|
|
864
|
-
export {
|
|
865
|
-
//# sourceMappingURL=types-
|
|
1010
|
+
export { QueryTracesPage as $, FindingReaderLabel as A, TREATMENT_LAYERS as B, FINDING_LAYERS as C, FindingMeasure as D, FindingLayer as E, lineageRoots as F, ReadSpanSourceResult as G, createBoundedTraceAnalysisStore as H, parseAnalystFinding as I, TraceAnalysisStore as J, SpanSourceReader as K, readFindingScope as L, FindingScopeReading as M, ProposalFindingSchema as N, FindingMethod as O, independentUnitCount as P, ErrorCluster as Q, FindingConditions as R, makeProposalFinding as S, FindingEvidenceUnit as T, BoundedTraceAnalysisStoreOptions as U, TreatmentLayer as V, ReadSpanSourceInput as W, DEFAULT_TRACE_ANALYST_BUDGETS as X, TraceAnalysisStoreContext as Y, DatasetOverview as Z, ExecutionProbeRequest as _, AnalystFindingSchemaVersion as a, TraceAnalystSpan as at, computeFindingId as b, AnalystRunEvent as c, TraceAnalystTraceSummary as ct, AnalystRunSummary as d, ViewTraceResult as dt, SearchSpanResult as et, AnalystSeverity as f, ExecutionProbeOutcome as g, ExecutionProbe as h, AnalystFinding as i, TraceAnalystFilters as it, FindingScope as j, FindingParseOutcome as k, AnalystRunInputs as l, ViewSpansResult as lt, EvidenceRef as m, AnalystContext as n, SpanMatchRecord as nt, AnalystInputKind as o, TraceAnalystSpanKind as ot, AnalystUsageReceipt as p, TRACE_ANALYSIS_LIMITS as q, AnalystCost as r, TraceAnalystByteBudgets as rt, AnalystRequirements as s, TraceAnalystSpanStatus as st, Analyst as t, SearchTraceResult as tt, AnalystRunResult as u, ViewTraceOversized as ut, ProposalFinding as v, FindingCalibration as w, makeFinding as x, ProposalFindingOrigin as y, FindingTreatment as z };
|
|
1011
|
+
//# sourceMappingURL=types-DGImvHIr.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types-DGImvHIr.d.ts","names":[],"sources":["../src/trace-analyst/types.ts","../src/trace-analyst/store-contract.ts","../src/trace-analyst/store-boundary.ts","../src/analyst/finding-composition.ts","../src/analyst/finding-scope.ts","../src/analyst/types.ts"],"mappings":";;;;;;;;;;;;KAsBY,uBAAuB;KAEvB;;;;UAKK;EACf;EACA;EACA;EACA;EACA,MAAM;EACN;EACA;EACA;EACA,QAAQ;EACR;EACA;EACA;EACA;EACA;;;EAGA,YAAY;;UAGG;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA;;;;;;;UAQe;;EAEf;;EAEA;;EAEA;;EAEA;EACA;EACA;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;EACA;IAAU;IAAqB;;;;;;EAK/B,gBAAgB;EAChB;IAAc;IAAkB;;;UAGjB;EACf,QAAQ;EACR;EACA;;;;;UAMe;EACf;EACA,QAAQ;EACR,YAAY;;UAGG;EACf;;EAEA,gBAAgB;;EAEhB;EACA;;UAGe;EACf;EACA,OAAO;;EAEP;;EAEA;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,WAAW;;;EAGX;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN;;UAGe;EACf;EACA;EACA,MAAM;EACN;;;UAIe;;;EAGf;;;EAGA;;;EAGA;;;EAGA;;cAGW,+BAA+B;;;cClM/B;WACX;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;UAGe;EACf,SAAS;;;UAIM;EACf;EACA;EACA;EACA;EACA;EACA;;KAGU;EAEN;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;IACE;IACA;IACA;IACA;IACA;;;;KAKI,oBACV,OAAO,qBACP,UAAU,8BACP,QAAQ;;;;;;;;UASI;;EAEf,iBAAiB;EAEjB,SAAS,kBAAkB,UAAU,4BAA4B;EAEjE,SACE;IAAS;IAAkB;KAC3B,UAAU,4BACT;EAEH,YACE,UAAU,qBACV,UAAU,4BACT,QAAQ;EAEX,YACE;IAAS,UAAU;IAAqB;IAAe;KACvD,UAAU,4BACT,QAAQ;EAEX,YAAY,UAAU,qBAAqB,UAAU,4BAA4B;EAEjF,UACE;IACE;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,UACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,YACE;IACE;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;EAEX,WACE;IACE;IACA;IACA;IACA;KAEF,UAAU,4BACT,QAAQ;;UAGI;EACf,UAAU,QAAQ;;;;;iBC1GJ,gCACd,QAAQ,oBACR,UAAS,mCACR;;;;cC7BU;KAWD,yBAAyB;cAWxB,wBAAsB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAUxB,EAAA,KAAA;;cAGE,yBAAuB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BzB,EAAA,KAAA;KAEC,mBAAmB,EAAE,aAAa;KAClC,oBAAoB,EAAE,aAAa;;;;;;;;cCjClC;KAaD,uBAAuB;UAOlB;;;;;;;;EAQf;;EAEA;;EAEA;IAAU;IAAc;;;EAExB;;UAGe;;EAEf;;EAEA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,cAAc;;UAGC;EACf;EACA;EACA;EACA;;EAEA;;;;;;;;UASe;EACf;EACA,OAAO;EACP;;;cA6JW,uBAAqB,EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAGN,EAAA,KAAA;KAQhB;EACN;EAAiB,OAAO;;EACxB;EAAkB;IAAS;IAAc;;;;iBAG/B,oBAAoB,iBAAiB;;;;;UAiBpC;EACf,OAAO;EACP;;EAEA;EACA;EACA,QAAQ;;;;;;;;KASE;iBAEI,iBAAiB,SAAS,iBAAiB;;;;;;iBAkC3C,aAAa,OAAO,cAAc,uBAAuB;;iBA0BzD,qBAAqB,OAAO,cAAc;;;;;;;;;;;;;UCzUzC;EACf,gBAAgB;;;;;;;EAOhB;EACA;EACA;EACA,UAAU;;;;;;;EAOV;EACA;EACA;EACA,eAAe;EACf;EACA;;EAEA;;;;;;EAMA;;;;EAIA;;EAEA,WAAW;;EAEX,QAAQ;;EAER,QAAQ;;EAER,SAAS;;EAET,UAAU;;EAEV;;EAEA;;;;;EAKA;;;;;EAKA,iBAAiB;;EAEjB,YAAY;;EAEZ,aAAa;;;cAIT;KAEM,sCAAsC;;;cAIrC;KAED,0BAA0B;;KAG1B;;KAGA,kBAAkB;WACnB,iBAAiB;;UAGX;;;;;;;EAOf;EACA;EACA;;;;;;;;KAWU;UAOK;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;;;;;UAOe;EACf,aAAa;EACb;EACA,YAAY;EACZ,aAAa;;EAEb,SAAS;;UAGM;EACf;;EAEA;;EAEA;;EAEA;;EAEA,aAAa;;EAEb;;;;;;;EAOA,OAAO;;;;;;;;;;EAUP,gBAAgB,cAAc;;;;;;;EAO9B,mBAAmB,cAAc;;;;;EAKjC,eAAe,SAAS;;EAExB,OAAO;;EAEP,OAAO,aAAa,SAAS;;EAE7B,SAAS;;;;;;;;EAQT,QAAQ;;;UAMO;EACf;;EAEA;;EAEA;;EAEA;EACA,SAAS;;;;;;;KAQC;EAEN;EACA;EACA;EACA;EACA;;EAEA;;EAEA;EAAkB;IAAS;IAAe;;;;;;;;UAO/B;;WAEN;EACT,QAAQ,SAAS,wBAAwB,QAAQ;;;;;;;;UASlC,QAAQ;;WAEd;;WAEA;WACA,WAAW;WACX,MAAM;WACN,WAAW;;WAEX;EACT,QAAQ,OAAO,QAAQ,KAAK,iBAAiB,QAAQ;;;UAItC;;EAEf;;EAEA,QAAQ;;EAER,MAAM;;EAEN;;;;;;;;EAQA;IAAkB;IAAsB;;;;;;;;EAOxC;;;;;;;;;;;;iBAec,iBAAiB;EAC/B;EACA;EACA;EACA;;EAEA;EACA,YAAY;EACZ,aAAa;;;;;;iBAyCC,YACd,MAAM,KAAK;EACT;EACA;IAED;;iBAsBa,oBACd,MAAM,KAAK;EACT;EACA;IAED;UAOc;EACf;EACA;;EAEA;EACA;EACA;;EAEA,OAAO;;EAEP;IAAU;IAAe;;;UAGV;EACf;EACA;EACA;EACA;EACA,UAAU;EACV,aAAa;;EAEb;;;;;EAKA,wBAAwB;;;;;;;;;;;;;;;KAkBd;EAEN;EACA;EACA;EACA;;EAEA,aAAa;;EAGb;EACA,SAAS;;EAGT;EACA;EACA;;EAGA;;EAEA,SAAS;EACT,UAAU,cAAc;;EAGxB;EACA,QAAQ"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { c as RunSplitTag } from "./run-record-C3awiXPf.js";
|
|
2
|
-
import { R as Scenario, d as DispatchContext } from "./types-
|
|
2
|
+
import { R as Scenario, d as DispatchContext } from "./types-DDKfH-N6.js";
|
|
3
3
|
//#region src/benchmarks/types.d.ts
|
|
4
4
|
type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
|
|
5
5
|
type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
|
|
@@ -99,4 +99,4 @@ declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
|
|
|
99
99
|
declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
|
|
100
100
|
//#endregion
|
|
101
101
|
export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
|
|
102
|
-
//# sourceMappingURL=types-
|
|
102
|
+
//# sourceMappingURL=types-IERw_HxR.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types-
|
|
1
|
+
{"version":3,"file":"types-IERw_HxR.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC"}
|
package/dist/wire/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { m as CustomTokenPricing, o as CostLedgerHandle } from "../cost-ledger-C1W3ICmP.js";
|
|
2
2
|
import { x as ChatClient } from "../types-eMhdacVp.js";
|
|
3
3
|
import { s as TraceStore } from "../store-B3nWRy3B.js";
|
|
4
|
-
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-
|
|
4
|
+
import { m as FeedbackTrajectoryStore } from "../feedback-trajectory-BZhVCWuz.js";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { ServerType } from "@hono/node-server";
|
|
7
7
|
import { Hono } from "hono";
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Bounded behavioral trace questions
|
|
2
|
+
|
|
3
|
+
`behavioralAnalyst` on `@tangle-network/agent-eval/analyst` retains its call-free arithmetic
|
|
4
|
+
pass. Its optional `traceQuestions` field adds explicitly selected event, turn and agent
|
|
5
|
+
windows. Callers supply a model, evaluator, its explicit `ExactExecutionComponentIdentity`, the existing findings ledger and a named,
|
|
6
|
+
versioned window extraction policy. The option refuses excess windows or bytes; it never
|
|
7
|
+
silently samples or truncates input. The default limits and fixed questions live in
|
|
8
|
+
`src/analyst/behavioral-trace-validation.ts` and `behavioral-trace-questions.ts`.
|
|
9
|
+
|
|
10
|
+
Each window includes a nonempty bounded `events` list of `{ id, content, evidenceRefs }`.
|
|
11
|
+
IDs are unique within the window; `unresolved` is reserved. `content` is the sanitized event
|
|
12
|
+
text or JSON actually available to the judge. References must resolve retained immutable
|
|
13
|
+
source artifacts; the caller owns their retention and faithful extraction. The stored content
|
|
14
|
+
digest binds the supplied sanitized event, but does not certify an external artifact's availability.
|
|
15
|
+
The window byte cap covers these events as well as the rest of the window. `maxRequestBytes`
|
|
16
|
+
also bounds the canonical native body, including all questions and retained mapping, before
|
|
17
|
+
any dispatch. Neither cap truncates input.
|
|
18
|
+
|
|
19
|
+
Each window names a source trace in the analyzed store, a subject, source references,
|
|
20
|
+
immutable timestamps and capture limitations. The caller owns extracting and sanitizing
|
|
21
|
+
the actual source content and resolving the references; membership in the trace store
|
|
22
|
+
does not certify that an arbitrary supplied excerpt faithfully represents that trace.
|
|
23
|
+
Native capture gaps stay explicit. Root-stream attempt numbers do not establish turns.
|
|
24
|
+
Selection policy settings belong in `windowPolicy.config`, so changing extraction changes
|
|
25
|
+
calibration identity. Never include labels, rewards, expected outputs or test answers in
|
|
26
|
+
`state`; independent gold travels only to the calibration reader.
|
|
27
|
+
|
|
28
|
+
The fixed battery has nine observations per window, including negative, unsure and
|
|
29
|
+
unavailable answers. Eight semantic questions and eight ancillary supporting-event choices share one native
|
|
30
|
+
SystemOne request via `decisionPoint`, following DomainMatch's finite source-choice pattern.
|
|
31
|
+
Each support choice has the supplied event IDs plus an explicit `unresolved` option. It is
|
|
32
|
+
an independent classification of relevant evidence, not a generated rationale, causal grounding,
|
|
33
|
+
or a guarantee that another answer is justified. Unresolved support does not relabel a semantic
|
|
34
|
+
Yes/No or change its existing alarm eligibility. Ancillary policies are explicitly record-only;
|
|
35
|
+
they have no independent routing authority or calibration certificate.
|
|
36
|
+
|
|
37
|
+
The exact ordered `sourceMap` (window and trace IDs, event IDs, sanitized-content digests and
|
|
38
|
+
references) and its digest live in the persisted decision definition's support-question
|
|
39
|
+
instructions of `evidence.unverified-completion`; other support questions reference that
|
|
40
|
+
map owner and digest rather than copying the map. Replay reconstructs that map from the durable record and returns it in each
|
|
41
|
+
observation's `metadata.behavioral_trace_question.support`, including selected IDs, unresolved,
|
|
42
|
+
unavailable and probabilities. Source event content remains in the supplied state, so retain
|
|
43
|
+
the referenced artifacts to inspect the selected source. State/input digests alone are not
|
|
44
|
+
source retention. Changing IDs, content, references or map order under the same invocation
|
|
45
|
+
conflicts before dispatch. The versioned request contract invalidates old calibration certificates.
|
|
46
|
+
|
|
47
|
+
Exact tool repetition is computed from tool names and canonical arguments
|
|
48
|
+
on distinct physical attempt IDs. Each attempt also requires an explicit `eventId` in the
|
|
49
|
+
window. Computed positive support returns the matching attempts' source event IDs; a computed
|
|
50
|
+
absence or incomplete capture has unresolved support rather than an invented event. Duplicate capture rows are deduplicated; conflicting
|
|
51
|
+
payloads for one attempt are refused. An incomplete tool capture can establish observed
|
|
52
|
+
repetition, but cannot establish its absence. Repetition alone makes no claim about waste.
|
|
53
|
+
The computed observation has no fabricated model identity, probabilities or paid receipt.
|
|
54
|
+
|
|
55
|
+
Decision intents, native records and question observations share the supplied findings
|
|
56
|
+
journal. A repeated analysis with the same run, explicit durable `analysisId`, configuration and windows
|
|
57
|
+
replays without another inference. Changing those inputs under the same invocation is a
|
|
58
|
+
conflict; use a new `analysisId` for an intentional new analysis. Observation times come from the
|
|
59
|
+
source window, so replays do not conflict merely because the wall clock advanced.
|
|
60
|
+
A concurrent reader of an unresolved intent cannot finalize observations. Registry logging correlation IDs may change across restarts without spending again.
|
|
61
|
+
Cancellation and deadlines suppress delivery while retaining paid work and durable records.
|
|
62
|
+
The existing analyst adapter supplies one shared budget and forwards every provider receipt;
|
|
63
|
+
missing usage remains unknown. A replay reports zero new physical calls, not a free model
|
|
64
|
+
receipt for a previous call.
|
|
65
|
+
|
|
66
|
+
## Independent calibration and alarms
|
|
67
|
+
|
|
68
|
+
`calibrateBehavioralTraceQuestions` joins independent `{questionId, windowId, label, source}`
|
|
69
|
+
gold rows to the emitted findings. It returns all nine per-question rows with exact n,
|
|
70
|
+
source counts, missing/unsure labels, missing predictions, abstentions, confusion counts,
|
|
71
|
+
agreement and Cohen's kappa from the existing statistics owner. Its `disagreements` list
|
|
72
|
+
shows every wrong, unsure or missing prediction with the selected event IDs/unresolved and
|
|
73
|
+
complete scoped map; historical findings without captured support return null. It never
|
|
74
|
+
invents pointers for old decisions. Semantic question digests stay independent of the event-ID
|
|
75
|
+
set, so this reader aggregates multiple windows/maps. Generic decision-point `reliability()`
|
|
76
|
+
instead groups the exact definition, which includes the map and can yield one-map samples;
|
|
77
|
+
that output is not a cross-map probability calibration or threshold-tuning claim. Gold unsure/null labels
|
|
78
|
+
are exclusions, never No; missing predictions remain disagreements. Empty and degenerate
|
|
79
|
+
samples have null kappa. A missing prediction or one-class gold set cannot issue a routing
|
|
80
|
+
certificate. The minimum kappa is owned by `DECISION_POINT_MIN_KAPPA` on `/systemone`.
|
|
81
|
+
|
|
82
|
+
The evaluator identity covers transport, normalization, receipt interpretation, pricing and spending policy; it must change when those semantics change. It is captured once and bound in exact execution and calibration identities.
|
|
83
|
+
|
|
84
|
+
Certificates bind the exact question, option order, implementation version, evaluator identity, layer,
|
|
85
|
+
window policy, requested model and served model. Computed certificates bind the extractor
|
|
86
|
+
and policy with a null served model. Store their validation evidence in observation metadata;
|
|
87
|
+
`method.calibration` remains exclusive to judged findings. A label file is evidence only
|
|
88
|
+
for its declared scope: AgentRx root-cause categories are not labels for every question,
|
|
89
|
+
turn or agent, and an absent failure category is not a negative label.
|
|
90
|
+
|
|
91
|
+
Install returned certificates explicitly in `traceQuestions.calibrations` by layer and
|
|
92
|
+
question. Uncalibrated, mismatched, uncertain or failed judgments cannot alarm. A qualifying
|
|
93
|
+
positive recommends only `coordinator-alarm` for reversible investigation. This surface
|
|
94
|
+
has no pass/fail acceptance authority and performs no coordinator action itself.
|
|
95
|
+
Integration fixture labels test wiring only; they are never scientific calibration.
|
|
96
|
+
|
|
97
|
+
Before enabling real alarms, retain independent human/compatible AgentRx labels and the
|
|
98
|
+
per-question report. Diagnostic development anchors are not a sealed holdout or prevalence
|
|
99
|
+
estimate. Keep unseen calibration evidence for judging a changed rubric or extraction policy.
|
package/docs/experiment.md
CHANGED
|
@@ -302,3 +302,30 @@ separation or equivalence. Their numeric point estimates remain `0` for compatib
|
|
|
302
302
|
this is a placeholder, not a measured zero. Retain the trial count with Wilson
|
|
303
303
|
results, and use the returned `n` for paired results, before displaying an estimate.
|
|
304
304
|
Nonempty-sample estimators retain their existing semantics and limitations.
|
|
305
|
+
|
|
306
|
+
### As-assigned and as-formed arm populations
|
|
307
|
+
|
|
308
|
+
Supply `assignments` to the existing `comparePairedArms(rows, options)` reader when the
|
|
309
|
+
expected seed population is known. Each roster entry names `arm`, `pairKey`, optional
|
|
310
|
+
`repKey` and optional treatment array. Build this roster independently of completed run
|
|
311
|
+
records; a roster inferred from successful runs cannot reveal missing assignments.
|
|
312
|
+
|
|
313
|
+
Observation rows retain `pass` and `metrics` independently of their optional
|
|
314
|
+
`formation: { formed, reason? }` verdict. The execution owner defines formation; this
|
|
315
|
+
reader consumes its boolean or explicit null. Null or absent outcomes/metrics are unknown,
|
|
316
|
+
while false and zero are measurements. A team-not-formed record can still have an observed
|
|
317
|
+
acceptance result and cost, and those stay in the as-assigned summary.
|
|
318
|
+
|
|
319
|
+
With a roster, `armPopulations` reports each arm's assignments, missing observations,
|
|
320
|
+
formed/not-formed/unknown counts and both `asAssigned` and `asFormed` summaries. Only an
|
|
321
|
+
explicit true formation verdict enters as-formed. Each summary reports all its units,
|
|
322
|
+
passes, failures, missing outcomes and metric-specific observed/missing denominators.
|
|
323
|
+
`passRate` is null when any outcome is missing; metric sums/means are null when no reading
|
|
324
|
+
exists. This prevents an absent result from becoming a measured failure or zero cost.
|
|
325
|
+
|
|
326
|
+
The same left join supplies matched-pair statistics: missing seeds carry identity-only
|
|
327
|
+
placeholder rows, so pair coverage includes declared seed identities while correctness
|
|
328
|
+
and metric statistics still require actual observations on both sides. Duplicate
|
|
329
|
+
assignments, results outside the roster and corrupt metrics fail loudly. Without a roster,
|
|
330
|
+
the existing observed-pair behavior remains and `armPopulations` is absent: the assignment
|
|
331
|
+
population is unknown, never inferred from completed rows.
|
package/docs/finding-layers.md
CHANGED
|
@@ -65,5 +65,58 @@ Event, turn, agent, team and meta findings are never placed on the ladder.
|
|
|
65
65
|
## Analysts
|
|
66
66
|
|
|
67
67
|
- `team-shape` (computed, `team`) reads a Runtime spawn journal and coordination log: `teamShapeInputFromSources` takes them from `readRuntimeSupervisorRun`, and the same parser reads the structure-only reductions `extract.py` writes. E (command) edges come from the id path (`R:s1:s0` is a child of `R:s1`), G (governance) edges from the journal holding each `settled` row plus each delivered steer's sender, and C (communication) edges from coordination rows naming a receiver, sent by `from`/`by` or else the log owner's digest. Classes, version 1, checked in order: single, governed swarm, fleet, partition, `hive depth N`, star with sub-leads, flat star (`root integrates`, `root does not integrate` or `integration unknown`, from the caller's optional `integrations`). The 50% and 20% thresholds are provisional and carried in `method.version`. Every class check is reported as yes, no or `not observable from this record`; an owner digest is not an agent id, so lateral C and steer governance are usually not observable, and the record never reads them as absent. The four recorded Magnetar runs under `tests/fixtures/team-shape`, with the lead's gold labels, are the acceptance fixtures.
|
|
68
|
-
- `failure-modes` (judged, `agent`, `team` or `run`) asks fixed-choice questions through `systemOneAnalyst`: a MAST-adapted failure mode, delegation-brief completeness and steer handling. `createFailureModesAnalyst` refuses a rubric whose recorded calibration κ is below `FAILURE_MODES_MIN_KAPPA
|
|
68
|
+
- `failure-modes` (judged, `agent`, `team` or `run`) asks fixed-choice questions through `systemOneAnalyst`: a MAST-adapted failure mode, delegation-brief completeness and steer handling. `createFailureModesAnalyst` refuses a rubric whose recorded calibration κ is below the source-owned `FAILURE_MODES_MIN_KAPPA`. `loadFailureModeGold` reads public AgentRx annotations through `loadPublicBenchmarkRows` plus a caller's gold file; `calibrateFailureModes` scores the judge's labels against that gold. The full AgentRx dataset is external; [recorded boundary fixtures](../tests/fixtures/agentrx-actor-boundary/README.md) retain only the cases needed to check admission and historical receipt preservation.
|
|
69
69
|
- `rollup` (computed) pools run findings per play with a lineage-clustered bootstrap, then lets `promote` decide what climbs. Each lifted finding gets the pooled interval over every run beneath it and its observation and cluster counts.
|
|
70
|
+
|
|
71
|
+
### Failure-mode gold admission
|
|
72
|
+
|
|
73
|
+
[`loadFailureModeGold`](../src/analyst/failure-modes.ts) owns AgentRx's projection onto agent/team root-cause labels. It reads the annotated root failure, including its actor; it does not infer an assistant fault from a user's behavior. User-origin underspecification has no counterpart in the agent's `fail-to-ask-for-clarification` label. The source owns the recognized agent roles; unfamiliar or missing actors cannot silently become agents.
|
|
74
|
+
|
|
75
|
+
A successful load reports `eligibility`, `agentRxMapping`, eligible `gold`, and `unmapped` rows. The existing `unmapped` field retains each source id/category and adds its root failure id, step, actor, and a typed reason: `unsupported-root-actor`, `unknown-root-actor`, or `category-without-counterpart`. `agentRxMapping` records the mapping version and `agent-or-team-root-cause` target, or is null for caller-only gold. Consumers must carry that provenance and the actual exclusion reasons into a **new** calibration receipt, rather than treating every exclusion as an unmapped category.
|
|
76
|
+
|
|
77
|
+
An all-excluded, otherwise valid AgentRx file returns `eligibility: 'unknown'`, no gold labels, and the exclusion evidence. That is a parsed source without calibration evidence, not a successful empty calibration or a measured absence of failures. Invalid input still returns a failed outcome. Caller-supplied gold remains the caller's assertion and is not certified as independent by this loader.
|
|
78
|
+
|
|
79
|
+
Caller gold ids must be disjoint from **all** AgentRx source ids, including excluded rows. A caller cannot reintroduce an excluded source id as eligible gold; overlapping ids are invalid. Caller ids with surrounding whitespace are invalid, so whitespace cannot disguise an overlapping source id. Valid caller ids are preserved exactly. Caller-only gold and disjoint caller ids remain supported.
|
|
80
|
+
|
|
81
|
+
Existing receipt rows, source hashes, labels and verdicts are historical evidence. Loading a new mapping never regrades, relabels or filters a stored receipt. A new mapping, selection policy, question or input representation requires a new calibration configuration with its rule fixed before judging.
|
|
82
|
+
|
|
83
|
+
### Calibration and runtime contract
|
|
84
|
+
|
|
85
|
+
The maintained [`FAILURE_MODES_QUESTIONS`](../src/analyst/failure-modes.ts) object is the one question/criteria owner. `createFailureModesAnalyst` currently sends all its questions; calibration of `failure_mode` alone does not qualify the other outputs.
|
|
86
|
+
|
|
87
|
+
| Output | Target and required evidence |
|
|
88
|
+
| --- | --- |
|
|
89
|
+
| `failure_mode` | The causal agent/team failure explaining the observed shortfall, rather than an arbitrary downstream symptom or a user-origin annotation. AgentRx gold targets the annotated root. Inputs need the applicable task/policy, observed actions and tool results, actor identity, and an independently established outcome where available. A user ambiguity alone does not establish an agent's failure to clarify. |
|
|
90
|
+
| `delegation_brief` | The actual delivered briefs and whether they include the goal, constraints, expected output and check; not an inferred brief from child creation. |
|
|
91
|
+
| `steer_handling` | The delivered correction and the agent's subsequent behavior; lack of a correction record is not proof that none arrived. |
|
|
92
|
+
|
|
93
|
+
Before the next calibration, the consuming owner must use this full shared question object on the same declared target and evidence representation as runtime, and report independent gold, paired counts and reliability **per output**. Unknown or missing required observations stay outside the judge's measured population with an explicit reason. `none`, `no-delegation` and `no-steer` are observed classifications, not substitutes for unavailable evidence. Merely requesting all questions does not create independent labels for them.
|
|
94
|
+
|
|
95
|
+
This is the required consumer migration, not a claim that it has shipped: Lab's existing calibration asks only `failure_mode` over public trajectories, while its runtime reads coordination/spawn logs and asks all outputs. Lab owns reconciling those inputs and recording the target, mapping version, selection/exclusions, questions, model/served identity and source hashes in its next receipt. An applicable task/policy may be supplied as observation context; expected benchmark actions/outputs, root annotations, category rationales and other gold must remain outside the judged state. No old receipt is promoted by this source correction.
|
|
96
|
+
|
|
97
|
+
## Treatment and recorded conditions
|
|
98
|
+
|
|
99
|
+
Optional `treatment` and `conditions` arrays on a 1.1.0 finding preserve run composition.
|
|
100
|
+
A treatment is a named bundle with an immutable definition digest, source references and
|
|
101
|
+
all applicable `TREATMENT_LAYERS` tags. These input tags are separate from `FindingLayer`,
|
|
102
|
+
which says how far a claim holds. Arms and findings may carry several bundles.
|
|
103
|
+
|
|
104
|
+
Each conditions snapshot names its measured population, observation time and evidence.
|
|
105
|
+
Runtime/Lab/model/provider versions, seat, host load, start time and operator steers are
|
|
106
|
+
explicit nullable readings. Null means unavailable; an empty observed steer list means
|
|
107
|
+
none were observed. A snapshot ID identifies an immutable record; rollup and promotion reject conflicting content for the same ID anywhere in their input, before grouping. Capture conditions from
|
|
108
|
+
execution evidence; do not ask a judge to invent them or infer formation from an assignment.
|
|
109
|
+
The public types are on `@tangle-network/agent-eval/analyst`.
|
|
110
|
+
|
|
111
|
+
The finding identity includes the treatment definitions and fixed environment conditions
|
|
112
|
+
when composition is supplied. The finding factory normalizes unordered composition arrays, so reordering bundles, layer tags or condition snapshots preserves both identity and ledger replay payload.
|
|
113
|
+
Absent legacy composition and explicitly empty arrays remain distinguishable. Existing
|
|
114
|
+
findings without composition retain their old identity and remain readable.
|
|
115
|
+
|
|
116
|
+
`rollupFindings` and `promote` use the same composition identity for grouping and output
|
|
117
|
+
IDs, including the underlying-run re-pooling after promotion. Different bundles, fixed
|
|
118
|
+
versions, seats or unplanned/unknown steering states do not silently pool. Varying load,
|
|
119
|
+
observation time and source references retain their population-linked provenance without
|
|
120
|
+
splitting an otherwise matching group. Matching recorded conditions do not establish
|
|
121
|
+
that every possible confounder was observed; experiment owners still enforce declared
|
|
122
|
+
factors and interleaving before execution.
|