@tangle-network/agent-eval 0.215.1 → 0.216.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-DmzL8KaJ.d.ts → agent-profile-CIA2Atnt.d.ts} +3 -3
- package/dist/{agent-profile-DmzL8KaJ.d.ts.map → agent-profile-CIA2Atnt.d.ts.map} +1 -1
- package/dist/agreement-irr-DsTURxdr.js +186 -0
- package/dist/agreement-irr-DsTURxdr.js.map +1 -0
- package/dist/analyst/index.d.ts +485 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +1543 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-BhFLOztS.d.ts → analyze-runs-CSDAh8A7.d.ts} +2 -2
- package/dist/{analyze-runs-BhFLOztS.d.ts.map → analyze-runs-CSDAh8A7.d.ts.map} +1 -1
- package/dist/{benchmark-command-CgpGi7vY.js → benchmark-command-CByl9dxG.js} +16 -32
- package/dist/benchmark-command-CByl9dxG.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +6 -6
- package/dist/campaign/index.js +5 -5
- package/dist/{campaign-CpnsCgfz.js → campaign-BQ-Jujuj.js} +6 -6
- package/dist/{campaign-CpnsCgfz.js.map → campaign-BQ-Jujuj.js.map} +1 -1
- package/dist/cli.js +2 -2
- package/dist/cluster-bootstrap-B3JywQQn.js +66 -0
- package/dist/cluster-bootstrap-B3JywQQn.js.map +1 -0
- package/dist/contract/index.d.ts +9 -9
- package/dist/contract/index.js +6 -6
- package/dist/{default-registry-B4B7GMvI.d.ts → default-registry-BrWh62zf.d.ts} +6 -6
- package/dist/{default-registry-B4B7GMvI.d.ts.map → default-registry-BrWh62zf.d.ts.map} +1 -1
- package/dist/{default-registry-KP4-Tolz.js → default-registry-Cn1LQ2Ve.js} +741 -756
- package/dist/default-registry-Cn1LQ2Ve.js.map +1 -0
- package/dist/{define-agent-eval-DfXvNho5.d.ts → define-agent-eval-CEWkX5wZ.d.ts} +7 -7
- package/dist/{define-agent-eval-DfXvNho5.d.ts.map → define-agent-eval-CEWkX5wZ.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BdfnqXeP.js → define-agent-eval-Ch3MWX6k.js} +3 -3
- package/dist/{define-agent-eval-BdfnqXeP.js.map → define-agent-eval-Ch3MWX6k.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +2 -2
- package/dist/{dspy-rlm-engine-DZfHM7kT.js → dspy-rlm-engine-D-KX_e64.js} +3 -3
- package/dist/{dspy-rlm-engine-DZfHM7kT.js.map → dspy-rlm-engine-D-KX_e64.js.map} +1 -1
- package/dist/{engine-C37HS2kr.d.ts → engine-C_zNWfM8.d.ts} +4 -4
- package/dist/{engine-C37HS2kr.d.ts.map → engine-C_zNWfM8.d.ts.map} +1 -1
- package/dist/evaluation-Cn2dRZIz.d.ts +75 -0
- package/dist/evaluation-Cn2dRZIz.d.ts.map +1 -0
- package/dist/evaluation.d.ts +2 -74
- package/dist/{exact-types-Dlb73wN4.d.ts → exact-types-CD0lf5Qd.d.ts} +2 -2
- package/dist/{exact-types-Dlb73wN4.d.ts.map → exact-types-CD0lf5Qd.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +1 -1
- package/dist/{external-optimizer-process-GngofQML.js → external-optimizer-process-nnsY5D_4.js} +2 -2
- package/dist/{external-optimizer-process-GngofQML.js.map → external-optimizer-process-nnsY5D_4.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BUxifb8g.js → external-optimizer-subprocess-DcerlZSG.js} +261 -2
- package/dist/external-optimizer-subprocess-DcerlZSG.js.map +1 -0
- package/dist/{feedback-trajectory-BLN_6fEs.d.ts → feedback-trajectory-DkaDj0dj.d.ts} +3 -3
- package/dist/{feedback-trajectory-BLN_6fEs.d.ts.map → feedback-trajectory-DkaDj0dj.d.ts.map} +1 -1
- package/dist/{index-B7m2uzJL.d.ts → index-DlH0Y9gI.d.ts} +8 -8
- package/dist/{index-B7m2uzJL.d.ts.map → index-DlH0Y9gI.d.ts.map} +1 -1
- package/dist/{index-B3WBm953.d.ts → index-NvGBiO8H.d.ts} +6 -6
- package/dist/{index-B3WBm953.d.ts.map → index-NvGBiO8H.d.ts.map} +1 -1
- package/dist/{index-tYLUvk_w.d.ts → index-oQ4aHziK.d.ts} +7 -7
- package/dist/{index-tYLUvk_w.d.ts.map → index-oQ4aHziK.d.ts.map} +1 -1
- package/dist/index.d.ts +15 -15
- package/dist/index.js +12 -11
- package/dist/index.js.map +1 -1
- package/dist/{kind-factory-AaEhUADH.js → kind-factory-CbZ-6tvu.js} +2 -2
- package/dist/{kind-factory-AaEhUADH.js.map → kind-factory-CbZ-6tvu.js.map} +1 -1
- package/dist/{llm-judge-Cd5s5Ei6.js → llm-judge-PXV7Xljl.js} +3 -3
- package/dist/{llm-judge-Cd5s5Ei6.js.map → llm-judge-PXV7Xljl.js.map} +1 -1
- package/dist/{matrix-DXVAJMLD.d.ts → matrix-CHMiebFx.d.ts} +2 -2
- package/dist/{matrix-DXVAJMLD.d.ts.map → matrix-CHMiebFx.d.ts.map} +1 -1
- package/dist/meta-eval/index.js +1 -62
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +1 -1
- package/dist/{prime-protocol-BHia9vDz.d.ts → prime-protocol-CEE4BR9N.d.ts} +2 -2
- package/dist/{prime-protocol-BHia9vDz.d.ts.map → prime-protocol-CEE4BR9N.d.ts.map} +1 -1
- package/dist/{produced-state-BGp4BX8t.js → produced-state-ZCtDBxAE.js} +2 -2
- package/dist/{produced-state-BGp4BX8t.js.map → produced-state-ZCtDBxAE.js.map} +1 -1
- package/dist/{proposal-findings-CXblsskH.js → proposal-findings-BZiMTzAF.js} +2 -36
- package/dist/proposal-findings-BZiMTzAF.js.map +1 -0
- package/dist/{registry-CMYIaDgS.d.ts → registry-JYfPxcVz.d.ts} +9 -11
- package/dist/registry-JYfPxcVz.d.ts.map +1 -0
- package/dist/{report-command-CtIIsQKP.js → report-command-y8H6RSX8.js} +23 -7
- package/dist/report-command-y8H6RSX8.js.map +1 -0
- package/dist/rl.d.ts +3 -3
- package/dist/rl.js +1 -1
- package/dist/search/index.d.ts +2 -2
- package/dist/{search-kernel-BqyBZ9Wm.d.ts → search-kernel-CpcKH5Ih.d.ts} +2 -2
- package/dist/{search-kernel-BqyBZ9Wm.d.ts.map → search-kernel-CpcKH5Ih.d.ts.map} +1 -1
- package/dist/{semantic-concept-judge-BPqbwIVs.js → semantic-concept-judge-zQBG9hHU.js} +3 -3
- package/dist/{semantic-concept-judge-BPqbwIVs.js.map → semantic-concept-judge-zQBG9hHU.js.map} +1 -1
- package/dist/{skillopt-optimization-method-Dcepr6QL.js → skillopt-optimization-method-CCtzralq.js} +4 -4
- package/dist/{skillopt-optimization-method-Dcepr6QL.js.map → skillopt-optimization-method-CCtzralq.js.map} +1 -1
- package/dist/{statistical-heldout-CJTj2EoS.d.ts → statistical-heldout-BeR4bOdg.d.ts} +2 -2
- package/dist/{statistical-heldout-CJTj2EoS.d.ts.map → statistical-heldout-BeR4bOdg.d.ts.map} +1 -1
- package/dist/{store-otlp-DWVhUgEj.js → store-otlp-DtB4Slsr.js} +2 -2
- package/dist/{store-otlp-DWVhUgEj.js.map → store-otlp-DtB4Slsr.js.map} +1 -1
- package/dist/{store-tool-spans-BNzls3OD.d.ts → store-tool-spans-DR1SrCLK.d.ts} +4 -4
- package/dist/{store-tool-spans-BNzls3OD.d.ts.map → store-tool-spans-DR1SrCLK.d.ts.map} +1 -1
- package/dist/{store-tool-spans-BNICrhsb.js → store-tool-spans-fZIigrpP.js} +3 -3
- package/dist/{store-tool-spans-BNICrhsb.js.map → store-tool-spans-fZIigrpP.js.map} +1 -1
- package/dist/supervisor-run/index.d.ts +1 -1
- package/dist/supervisor-run/index.d.ts.map +1 -1
- package/dist/supervisor-run/index.js +1 -1
- package/dist/systemone-CpTLAlqb.d.ts +135 -0
- package/dist/systemone-CpTLAlqb.d.ts.map +1 -0
- package/dist/{systemone-review-DTClwgmE.js → systemone-review-BZWfrtq1.js} +2 -2
- package/dist/{systemone-review-DTClwgmE.js.map → systemone-review-BZWfrtq1.js.map} +1 -1
- package/dist/{systemone-review-C4dZgrgN.d.ts → systemone-review-B_yo1xsd.d.ts} +2 -2
- package/dist/{systemone-review-C4dZgrgN.d.ts.map → systemone-review-B_yo1xsd.d.ts.map} +1 -1
- package/dist/systemone.d.ts +4 -134
- package/dist/systemone.js +1 -1
- package/dist/terminal-record-BHocwFw1.js.map +1 -1
- package/dist/{tool-groups-BKRrtqy4.d.ts → tool-groups-mcxOXVlb.d.ts} +3 -3
- package/dist/tool-groups-mcxOXVlb.d.ts.map +1 -0
- package/dist/{tool-waste-BYt5rdt8.js → tool-waste-D52PqU9Q.js} +3 -156
- package/dist/tool-waste-D52PqU9Q.js.map +1 -0
- package/dist/{trace-questions-CLx8vt6t.d.ts → trace-questions-DG7nqLQ7.d.ts} +3 -3
- package/dist/{trace-questions-CLx8vt6t.d.ts.map → trace-questions-DG7nqLQ7.d.ts.map} +1 -1
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +3 -3
- package/dist/{types-CGEmVYOk.d.ts → types-B4C5nFw6.d.ts} +2 -2
- package/dist/{types-CGEmVYOk.d.ts.map → types-B4C5nFw6.d.ts.map} +1 -1
- package/dist/{types-DjOMwGY_.d.ts → types-BqkVmHfv.d.ts} +7 -1
- package/dist/types-BqkVmHfv.d.ts.map +1 -0
- package/dist/{types-DinHtRJp.d.ts → types-CfZMIzB_.d.ts} +2 -2
- package/dist/{types-DinHtRJp.d.ts.map → types-CfZMIzB_.d.ts.map} +1 -1
- package/dist/{types-CW5b90JH.js → types-IXD2HJzu.js} +16 -3
- package/dist/types-IXD2HJzu.js.map +1 -0
- package/dist/{types-DHb5TI36.d.ts → types-oKRhQAde.d.ts} +344 -3
- package/dist/types-oKRhQAde.d.ts.map +1 -0
- package/dist/wire/index.d.ts +1 -1
- package/docs/concepts.md +1 -0
- package/docs/finding-layers.md +69 -0
- package/package.json +1 -1
- package/dist/benchmark-command-CgpGi7vY.js.map +0 -1
- package/dist/default-registry-KP4-Tolz.js.map +0 -1
- package/dist/evaluation.d.ts.map +0 -1
- package/dist/external-optimizer-subprocess-BUxifb8g.js.map +0 -1
- package/dist/proposal-findings-CXblsskH.js.map +0 -1
- package/dist/registry-CMYIaDgS.d.ts.map +0 -1
- package/dist/report-command-CtIIsQKP.js.map +0 -1
- package/dist/systemone.d.ts.map +0 -1
- package/dist/tool-groups-BKRrtqy4.d.ts.map +0 -1
- package/dist/tool-waste-BYt5rdt8.js.map +0 -1
- package/dist/types-CW5b90JH.js.map +0 -1
- package/dist/types-DHb5TI36.d.ts.map +0 -1
- package/dist/types-DjOMwGY_.d.ts.map +0 -1
package/dist/analyst/index.js
CHANGED
|
@@ -1,18 +1,26 @@
|
|
|
1
|
+
import { i as nominalAgreement } from "../agreement-irr-DsTURxdr.js";
|
|
1
2
|
import { i as CostLedger } from "../cost-ledger-D2FnJPBF.js";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
3
|
+
import { t as FileLedgerJournal } from "../journal-DvKCvjty.js";
|
|
4
|
+
import { a as merkleRoot } from "../ledger-core-B2hahzrt.js";
|
|
5
|
+
import { A as parseAnalystFinding, C as ProposalFindingSchema, D as lineageRoots, E as independentUnitCount, M as spanningWindow, O as mergeEvidenceUnits, S as FINDING_LAYERS, T as findingLayerRank, j as readFindingScope, k as mostSevere, w as UNKNOWN_LINEAGE, x as AnalystFindingSchema } from "../external-optimizer-subprocess-DcerlZSG.js";
|
|
6
|
+
import { a as makeFinding, i as computeFindingId, o as makeProposalFinding, r as FINDING_SCOPE_FIELDS, t as ANALYST_FINDING_SCHEMA_VERSIONS } from "../types-IXD2HJzu.js";
|
|
7
|
+
import { n as isProposalFinding, t as assertProposalFindings } from "../proposal-findings-BZiMTzAF.js";
|
|
8
|
+
import { B as FINDING_SUBJECT_SYNTAX, F as parseRawFinding, G as DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, H as findingSubjectGrammarPromptFor, I as parseTraceSpanEvidenceUri, K as resolveTraceAnalystLimits, L as coerceJson, M as RAW_FINDING_SCHEMA_PROMPT, N as RawAnalystFindingSchema, P as evidenceRefsFromRawFinding, R as stripCodeFences, U as parseFindingSubject, V as KIND_EXPECTED_SUBJECTS, W as renderFindingSubject, i as buildTraceToolsForGroup, n as renderPriorFindings, r as runTraceAnalyst, t as createTraceAnalyst, z as FINDING_SUBJECT_KINDS } from "../kind-factory-CbZ-6tvu.js";
|
|
4
9
|
import { i as validateUsageSettlementTimeout, n as settleUsageReceiptFromCostLedger } from "../usage-receipt-Cx3xNeyB.js";
|
|
5
|
-
import { a as KNOWLEDGE_POISONING_KIND_SPEC, c as FAILURE_MODE_KIND_SPEC, d as emitControlIntegrityFindings, f as behavioralAnalyst, i as DEFAULT_TRACE_ANALYST_KINDS, l as CONTROL_INTEGRITY_ANALYST, n as AnalystRegistry, o as KNOWLEDGE_GAP_KIND_SPEC, p as deriveEfficiencyFindings, r as ExactAnalystRunExecutionError, s as IMPROVEMENT_KIND_SPEC, t as buildDefaultAnalystRegistry, u as ControlIntegrityAnalyst } from "../default-registry-
|
|
6
|
-
import {
|
|
7
|
-
import { t as
|
|
8
|
-
import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-BPqbwIVs.js";
|
|
10
|
+
import { a as KNOWLEDGE_POISONING_KIND_SPEC, c as FAILURE_MODE_KIND_SPEC, d as emitControlIntegrityFindings, f as behavioralAnalyst, i as DEFAULT_TRACE_ANALYST_KINDS, l as CONTROL_INTEGRITY_ANALYST, n as AnalystRegistry, o as KNOWLEDGE_GAP_KIND_SPEC, p as deriveEfficiencyFindings, r as ExactAnalystRunExecutionError, s as IMPROVEMENT_KIND_SPEC, t as buildDefaultAnalystRegistry, u as ControlIntegrityAnalyst } from "../default-registry-Cn1LQ2Ve.js";
|
|
11
|
+
import { t as createDspyRlmTraceEngine } from "../dspy-rlm-engine-D-KX_e64.js";
|
|
12
|
+
import { a as diffFindings, i as defaultIsMaterial, n as runSemanticConceptJudge, o as createChatTraceEngine, r as FindingsStore, t as SEMANTIC_CONCEPT_JUDGE_VERSION } from "../semantic-concept-judge-zQBG9hHU.js";
|
|
9
13
|
import { t as createChatClient } from "../chat-client-DthPOYGd.js";
|
|
14
|
+
import { systemOneAnalyst } from "../systemone.js";
|
|
10
15
|
import { a as scoreAnalystFindings, i as summarizeAnalystBenchmarkRunner, n as runAnalystBenchmark, r as traceStoreEvidenceResolver, t as registryBenchmarkRunner } from "../benchmark-BywPeAUo.js";
|
|
11
|
-
import { $ as ANALYST_BENCHMARK_MANIFEST_FILE, A as readAnalystInstructionsOverride, B as parseVerificationOutcome, C as emptyPublicBenchmarkRunner, D as summarizeCodeTraceCalibration, E as renderCodeTraceCalibrationMarkdown, F as publicBenchmarkRlmInstructions, G as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, H as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, I as publicBenchmarkSystemPrompt, J as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, L as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, M as MAX_INCORRECT_BLOCKS, N as MAX_INCORRECT_BLOCK_STEPS, O as analystInstructionsOverrideFromText, P as publicBenchmarkProtocolSha256, Q as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, R as appendVerificationArtifactsToOtlp, S as adaptPublicBenchmarkFindings, T as readAnalystBenchmarkArtifact, U as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, V as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, W as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, X as analystBenchmarkImplementationDigest, Y as analystBenchmarkDependencyLockDigest, Z as ANALYST_BENCHMARK_COST_LEDGER_FILE, _ as publicDirectAnalystDefinition, a as primeCodeTraceAnalystDefinition, at as codeTraceBenchCase, b as analystDefinitionAsymmetries, c as preparePublicAnalystBenchmark, ct as agentRxPredictionsToFindings, d as selectPublicBenchmarkRows, dt as normalizeBenchmarkLabel, et as ANALYST_BENCHMARK_OBSERVATIONS_FILE, f as createPublicBenchmarkRlmRunner, g as createPublicBenchmarkDirectRunner, h as runReplVariableAnalystDefinition, i as primeAnalystProtocolSha256, it as summarizeAgentRxCalibration, j as CODE_TRACE_BENCH_ANALYST_PROMPT, k as effectiveAnalystProtocolSha256, l as publicBenchmarkDistributions, lt as normalizeAgentRxCategory, m as rlmEngineLimits, n as renderAnalystBenchmarkMarkdown, nt as AGENT_RX_UPSTREAM_REVISION, o as runInlineAnalystDefinition, ot as codeTracerPredictionsToFindings, p as publicRlmAnalystDefinition, q as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, r as createPrimeBenchmarkRunner, rt as renderAgentRxCalibrationMarkdown, s as loadPublicBenchmarkRows, st as agentRxBenchmarkCase, t as runAnalystBenchmarkCommand, tt as compareAnalystRunners, u as publicBenchmarkSelectionReport, ut as roundAgentRxStep, v as runChunkedAnalystDefinition, w as expandCodeTraceFailureBlocks, x as analystDefinitionProtocolSha256, y as AnalystExpressivenessError, z as loadCodeTraceVerificationArtifacts } from "../benchmark-command-
|
|
16
|
+
import { $ as ANALYST_BENCHMARK_MANIFEST_FILE, A as readAnalystInstructionsOverride, B as parseVerificationOutcome, C as emptyPublicBenchmarkRunner, D as summarizeCodeTraceCalibration, E as renderCodeTraceCalibrationMarkdown, F as publicBenchmarkRlmInstructions, G as ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, H as ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, I as publicBenchmarkSystemPrompt, J as ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, K as ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, L as DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, M as MAX_INCORRECT_BLOCKS, N as MAX_INCORRECT_BLOCK_STEPS, O as analystInstructionsOverrideFromText, P as publicBenchmarkProtocolSha256, Q as ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, R as appendVerificationArtifactsToOtlp, S as adaptPublicBenchmarkFindings, T as readAnalystBenchmarkArtifact, U as ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, V as ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, W as ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, X as analystBenchmarkImplementationDigest, Y as analystBenchmarkDependencyLockDigest, Z as ANALYST_BENCHMARK_COST_LEDGER_FILE, _ as publicDirectAnalystDefinition, a as primeCodeTraceAnalystDefinition, at as codeTraceBenchCase, b as analystDefinitionAsymmetries, c as preparePublicAnalystBenchmark, ct as agentRxPredictionsToFindings, d as selectPublicBenchmarkRows, dt as normalizeBenchmarkLabel, et as ANALYST_BENCHMARK_OBSERVATIONS_FILE, f as createPublicBenchmarkRlmRunner, g as createPublicBenchmarkDirectRunner, h as runReplVariableAnalystDefinition, i as primeAnalystProtocolSha256, it as summarizeAgentRxCalibration, j as CODE_TRACE_BENCH_ANALYST_PROMPT, k as effectiveAnalystProtocolSha256, l as publicBenchmarkDistributions, lt as normalizeAgentRxCategory, m as rlmEngineLimits, n as renderAnalystBenchmarkMarkdown, nt as AGENT_RX_UPSTREAM_REVISION, o as runInlineAnalystDefinition, ot as codeTracerPredictionsToFindings, p as publicRlmAnalystDefinition, q as ANALYST_BENCHMARK_IMPLEMENTATION_FILES, r as createPrimeBenchmarkRunner, rt as renderAgentRxCalibrationMarkdown, s as loadPublicBenchmarkRows, st as agentRxBenchmarkCase, t as runAnalystBenchmarkCommand, tt as compareAnalystRunners, u as publicBenchmarkSelectionReport, ut as roundAgentRxStep, v as runChunkedAnalystDefinition, w as expandCodeTraceFailureBlocks, x as analystDefinitionProtocolSha256, y as AnalystExpressivenessError, z as loadCodeTraceVerificationArtifacts } from "../benchmark-command-CByl9dxG.js";
|
|
12
17
|
import { a as extractPrimeJsonObject, c as primeProtocolSha256, d as runPrimeExchange, f as decodeReplyRows, i as emptyPrimeRawUsage, l as primeReplyDefect, n as buildPrimePrompt, o as mergePrimeRawUsage, r as buildPrimeRepairPrompt, s as normalizePrimeUsage, t as analystUsageReceiptFromPrimeUsage, u as projectPrimeTrajectory } from "../prime-protocol-DcAK9pC8.js";
|
|
13
18
|
import { t as nodeHttpPrimeBridgeTransport } from "../prime-bridge-transport-Z3cpKGoV.js";
|
|
19
|
+
import { t as clusterBootstrapMean } from "../cluster-bootstrap-B3JywQQn.js";
|
|
14
20
|
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
21
|
+
import { z } from "zod";
|
|
15
22
|
import { join } from "node:path";
|
|
23
|
+
import { readFile } from "node:fs/promises";
|
|
16
24
|
//#region src/analyst/adapters.ts
|
|
17
25
|
/**
|
|
18
26
|
* Adapter factory — lifts the semantic-concept judge into the Analyst
|
|
@@ -163,6 +171,673 @@ function defineCustomAnalyst(options) {
|
|
|
163
171
|
};
|
|
164
172
|
}
|
|
165
173
|
//#endregion
|
|
174
|
+
//#region src/analyst/domain-taxonomy.ts
|
|
175
|
+
/**
|
|
176
|
+
* Domain taxonomy — one versioned tree that both the layer ladder and the
|
|
177
|
+
* domain hierarchy live in.
|
|
178
|
+
*
|
|
179
|
+
* Domain nodes nest to any depth ("building systems" → "cryptographic" →
|
|
180
|
+
* "zk proving" → "binary fields"). Problems attach to one primary domain path
|
|
181
|
+
* plus free tags, plays attach to a problem, and runs attach to a play. Every
|
|
182
|
+
* node hangs under the implicit `lab` root, so one parent map answers both
|
|
183
|
+
* "which play holds this run" and "which domain holds this problem", and
|
|
184
|
+
* `promote` walks a single ladder: run → play → problem → domain path → lab.
|
|
185
|
+
*
|
|
186
|
+
* Plain data in, plain data out: every function returns a new value or a
|
|
187
|
+
* typed failure, and nothing here reads a file or the network.
|
|
188
|
+
*/
|
|
189
|
+
const DOMAIN_TAXONOMY_SCHEMA = "agent-eval.domain-taxonomy.v1";
|
|
190
|
+
/** Id of the implicit root every top-level domain hangs under. */
|
|
191
|
+
const LAB_NODE_ID = "lab";
|
|
192
|
+
/** Layers a taxonomy node can sit at, lowest first. */
|
|
193
|
+
const TAXONOMY_NODE_LAYERS = [
|
|
194
|
+
"run",
|
|
195
|
+
"play",
|
|
196
|
+
"problem",
|
|
197
|
+
"domain"
|
|
198
|
+
];
|
|
199
|
+
/** The layer a node of each kind must hang under. */
|
|
200
|
+
const PARENT_LAYER = {
|
|
201
|
+
run: "play",
|
|
202
|
+
play: "problem",
|
|
203
|
+
problem: "domain",
|
|
204
|
+
domain: "domain"
|
|
205
|
+
};
|
|
206
|
+
function fail(message) {
|
|
207
|
+
return {
|
|
208
|
+
succeeded: false,
|
|
209
|
+
error: { message }
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* Validate a taxonomy: unique ids, known parents, each node under a parent of
|
|
214
|
+
* the right layer, no cycles. `lab` is reserved for the implicit root.
|
|
215
|
+
*/
|
|
216
|
+
function defineDomainTaxonomy(input) {
|
|
217
|
+
if (typeof input.version !== "string" || !input.version.trim()) return fail("taxonomy version must be a non-empty string");
|
|
218
|
+
const nodes = input.nodes.map((node) => ({
|
|
219
|
+
id: node.id,
|
|
220
|
+
label: node.label,
|
|
221
|
+
layer: node.layer,
|
|
222
|
+
parent: node.parent,
|
|
223
|
+
...node.tags === void 0 ? {} : { tags: [...node.tags] }
|
|
224
|
+
}));
|
|
225
|
+
const byId = /* @__PURE__ */ new Map();
|
|
226
|
+
for (const node of nodes) {
|
|
227
|
+
if (typeof node.id !== "string" || !node.id.trim()) return fail("node ids must be non-empty");
|
|
228
|
+
if (node.id === "lab") return fail(`'lab' is the implicit root id`);
|
|
229
|
+
if (byId.has(node.id)) return fail(`node '${node.id}' appears twice`);
|
|
230
|
+
if (!TAXONOMY_NODE_LAYERS.includes(node.layer)) return fail(`node '${node.id}' has unknown layer '${String(node.layer)}'`);
|
|
231
|
+
if (typeof node.label !== "string" || !node.label.trim()) return fail(`node '${node.id}' needs a label`);
|
|
232
|
+
byId.set(node.id, node);
|
|
233
|
+
}
|
|
234
|
+
for (const node of nodes) {
|
|
235
|
+
if (node.parent === null) {
|
|
236
|
+
if (node.layer !== "domain") return fail(`${node.layer} '${node.id}' needs a parent`);
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
const parent = byId.get(node.parent);
|
|
240
|
+
if (!parent) return fail(`node '${node.id}' names unknown parent '${node.parent}'`);
|
|
241
|
+
if (parent.layer !== PARENT_LAYER[node.layer]) return fail(`${node.layer} '${node.id}' must hang under a ${PARENT_LAYER[node.layer]}, not ${parent.layer} '${parent.id}'`);
|
|
242
|
+
}
|
|
243
|
+
for (const node of nodes) {
|
|
244
|
+
const seen = /* @__PURE__ */ new Set();
|
|
245
|
+
let current = node;
|
|
246
|
+
while (current) {
|
|
247
|
+
if (seen.has(current.id)) return fail(`node '${node.id}' sits on a parent cycle`);
|
|
248
|
+
seen.add(current.id);
|
|
249
|
+
current = current.parent === null ? void 0 : byId.get(current.parent);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
succeeded: true,
|
|
254
|
+
value: {
|
|
255
|
+
schema: DOMAIN_TAXONOMY_SCHEMA,
|
|
256
|
+
version: input.version,
|
|
257
|
+
nodes
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
/** Attach a problem to one primary domain path, a play to a problem, or a run to a play. */
|
|
262
|
+
function attachToTaxonomy(taxonomy, attachment) {
|
|
263
|
+
let parent;
|
|
264
|
+
if (attachment.layer === "problem") {
|
|
265
|
+
if (!attachment.path || attachment.path.length === 0) return fail(`problem '${attachment.id}' needs a primary domain path`);
|
|
266
|
+
const resolved = resolveDomainPath(taxonomy, attachment.path);
|
|
267
|
+
if (!resolved.succeeded) return resolved;
|
|
268
|
+
parent = attachment.path[attachment.path.length - 1];
|
|
269
|
+
} else {
|
|
270
|
+
if (!attachment.parent) return fail(`${attachment.layer} '${attachment.id}' needs a parent`);
|
|
271
|
+
parent = attachment.parent;
|
|
272
|
+
}
|
|
273
|
+
return defineDomainTaxonomy({
|
|
274
|
+
version: taxonomy.version,
|
|
275
|
+
nodes: [...taxonomy.nodes, {
|
|
276
|
+
id: attachment.id,
|
|
277
|
+
label: attachment.label,
|
|
278
|
+
layer: attachment.layer,
|
|
279
|
+
parent,
|
|
280
|
+
...attachment.tags === void 0 ? {} : { tags: [...attachment.tags] }
|
|
281
|
+
}]
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
/** Check that a path is a top-down chain of domain nodes. */
|
|
285
|
+
function resolveDomainPath(taxonomy, path) {
|
|
286
|
+
const byId = nodeIndex(taxonomy);
|
|
287
|
+
const chain = [];
|
|
288
|
+
let expectedParent = null;
|
|
289
|
+
for (const id of path) {
|
|
290
|
+
const node = byId.get(id);
|
|
291
|
+
if (node?.layer !== "domain") return fail(`'${id}' is not a domain node`);
|
|
292
|
+
if (node.parent !== expectedParent) return fail(`domain path breaks at '${id}': its parent is '${node.parent ?? "lab"}'`);
|
|
293
|
+
chain.push(node);
|
|
294
|
+
expectedParent = id;
|
|
295
|
+
}
|
|
296
|
+
return {
|
|
297
|
+
succeeded: true,
|
|
298
|
+
value: chain
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
/** Layer of a node id, with `lab` for the implicit root; undefined when unknown. */
|
|
302
|
+
function taxonomyLayerOf(taxonomy, id) {
|
|
303
|
+
if (id === "lab") return "lab";
|
|
304
|
+
return nodeIndex(taxonomy).get(id)?.layer;
|
|
305
|
+
}
|
|
306
|
+
/** Parent id, with `lab` above every top-level domain and null above `lab`. */
|
|
307
|
+
function taxonomyParentOf(taxonomy, id) {
|
|
308
|
+
if (id === "lab") return null;
|
|
309
|
+
const node = nodeIndex(taxonomy).get(id);
|
|
310
|
+
if (!node) return void 0;
|
|
311
|
+
return node.parent ?? "lab";
|
|
312
|
+
}
|
|
313
|
+
/** The node and its ancestors, nearest first, ending at `lab`. Empty for an unknown id. */
|
|
314
|
+
function taxonomyAncestry(taxonomy, id) {
|
|
315
|
+
const byId = nodeIndex(taxonomy);
|
|
316
|
+
const chain = [];
|
|
317
|
+
let current = id;
|
|
318
|
+
while (current !== null) {
|
|
319
|
+
if (current === "lab") {
|
|
320
|
+
chain.push(current);
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
const node = byId.get(current);
|
|
324
|
+
if (!node) return [];
|
|
325
|
+
chain.push(current);
|
|
326
|
+
current = node.parent ?? "lab";
|
|
327
|
+
}
|
|
328
|
+
return chain;
|
|
329
|
+
}
|
|
330
|
+
/** Domain path of a node, broadest first: the domain ids above it, and itself when a domain. */
|
|
331
|
+
function domainPathOf(taxonomy, id) {
|
|
332
|
+
const byId = nodeIndex(taxonomy);
|
|
333
|
+
return taxonomyAncestry(taxonomy, id).filter((ancestor) => byId.get(ancestor)?.layer === "domain").reverse();
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Deepest node that is an ancestor of (or equal to) every id, or undefined
|
|
337
|
+
* when an id is unknown. `lab` is the answer when they share nothing else.
|
|
338
|
+
*/
|
|
339
|
+
function lowestCommonAncestor(taxonomy, ids) {
|
|
340
|
+
if (ids.length === 0) return void 0;
|
|
341
|
+
const chains = ids.map((id) => taxonomyAncestry(taxonomy, id));
|
|
342
|
+
if (chains.some((chain) => chain.length === 0)) return void 0;
|
|
343
|
+
const others = chains.slice(1).map((chain) => new Set(chain));
|
|
344
|
+
return chains[0].find((id) => others.every((chain) => chain.has(id)));
|
|
345
|
+
}
|
|
346
|
+
function nodeIndex(taxonomy) {
|
|
347
|
+
return new Map(taxonomy.nodes.map((node) => [node.id, node]));
|
|
348
|
+
}
|
|
349
|
+
//#endregion
|
|
350
|
+
//#region src/analyst/failure-modes.ts
|
|
351
|
+
/**
|
|
352
|
+
* failure-modes — a judged analyst with fixed-choice labels, gated on its own
|
|
353
|
+
* calibration.
|
|
354
|
+
*
|
|
355
|
+
* Labels are adapted from the MAST multi-agent failure taxonomy (Cemri et al.,
|
|
356
|
+
* "Why Do Multi-Agent LLM Systems Fail?", 2025): its fourteen modes in three
|
|
357
|
+
* groups, plus four single-agent modes so the public AgentRx set maps onto the
|
|
358
|
+
* same labels, plus `none`. Two more fixed questions cover delegation-brief
|
|
359
|
+
* completeness and steer handling.
|
|
360
|
+
*
|
|
361
|
+
* The judge runs through the System One decision path (`systemOneAnalyst`).
|
|
362
|
+
* Construction requires a calibration — gold labels compared with the judge's
|
|
363
|
+
* labels on the same items — and refuses to build an analyst whose Cohen's κ
|
|
364
|
+
* is below `FAILURE_MODES_MIN_KAPPA`. A rubric that does not agree with its
|
|
365
|
+
* gold set has nothing to report, so it reports nothing.
|
|
366
|
+
*/
|
|
367
|
+
const FAILURE_MODES_ANALYST_ID = "failure-modes";
|
|
368
|
+
const FAILURE_MODES_VERSION = "1";
|
|
369
|
+
/** κ below this refuses the rubric. */
|
|
370
|
+
const FAILURE_MODES_MIN_KAPPA = .6;
|
|
371
|
+
const FAILURE_MODE_LABELS = {
|
|
372
|
+
"disobey-task-specification": "Ignored or violated the stated task requirements or constraints.",
|
|
373
|
+
"disobey-role-specification": "Acted outside its assigned role or took over another role.",
|
|
374
|
+
"step-repetition": "Repeated a step already completed without new reason.",
|
|
375
|
+
"loss-of-conversation-history": "Lost earlier context and acted as if it was never given.",
|
|
376
|
+
"unaware-of-termination-conditions": "Did not recognize when the task was done or must stop.",
|
|
377
|
+
"conversation-reset": "Restarted the exchange and discarded progress.",
|
|
378
|
+
"fail-to-ask-for-clarification": "Proceeded on an ambiguous instruction without asking.",
|
|
379
|
+
"task-derailment": "Drifted from the task toward a different goal.",
|
|
380
|
+
"information-withholding": "Kept back information another agent needed.",
|
|
381
|
+
"ignored-other-agent-input": "Disregarded another agent's input or recommendation.",
|
|
382
|
+
"reasoning-action-mismatch": "Acted against its own stated reasoning or plan.",
|
|
383
|
+
"premature-termination": "Stopped before the goal of the task was met.",
|
|
384
|
+
"no-or-incomplete-verification": "Delivered without verifying, or verified only part.",
|
|
385
|
+
"incorrect-verification": "Verified, but the check was wrong or passed bad output.",
|
|
386
|
+
"invented-information": "Stated facts, results or tool output that do not exist.",
|
|
387
|
+
"invalid-tool-invocation": "Called a tool with an invalid name, argument or format.",
|
|
388
|
+
"misread-tool-output": "Misinterpreted what a tool or handoff returned.",
|
|
389
|
+
"system-failure": "Failed because of infrastructure, not the agent.",
|
|
390
|
+
none: "No failure mode applies."
|
|
391
|
+
};
|
|
392
|
+
/**
|
|
393
|
+
* AgentRx root-cause category → failure-mode label. `null` categories have no
|
|
394
|
+
* counterpart here and are left out of the gold set, counted as unmapped.
|
|
395
|
+
*/
|
|
396
|
+
const AGENT_RX_FAILURE_MODE_MAP = {
|
|
397
|
+
"instruction-plan-adherence-failure": "disobey-task-specification",
|
|
398
|
+
"invention-of-new-information": "invented-information",
|
|
399
|
+
"invalid-invocation": "invalid-tool-invocation",
|
|
400
|
+
"misinterpretation-of-tool-output-handoff-failure": "misread-tool-output",
|
|
401
|
+
"intent-plan-misalignment": "task-derailment",
|
|
402
|
+
"underspecified-user-intent": "fail-to-ask-for-clarification",
|
|
403
|
+
"intent-not-supported": null,
|
|
404
|
+
"guardrails-triggered": null,
|
|
405
|
+
"system-failure": "system-failure",
|
|
406
|
+
inconclusive: null
|
|
407
|
+
};
|
|
408
|
+
const FAILURE_MODES_QUESTIONS = {
|
|
409
|
+
failure_mode: {
|
|
410
|
+
type: "choice",
|
|
411
|
+
instructions: "Which single failure mode best explains why this agent or team fell short? Choose none when it did not fail.",
|
|
412
|
+
criteria: FAILURE_MODE_LABELS
|
|
413
|
+
},
|
|
414
|
+
delegation_brief: {
|
|
415
|
+
type: "choice",
|
|
416
|
+
instructions: "When this agent delegated work, did the brief state the goal, the constraints, the expected output, and how the result would be checked?",
|
|
417
|
+
criteria: {
|
|
418
|
+
complete: "Every delegated brief stated goal, constraints, output and check.",
|
|
419
|
+
partial: "Some brief left out one or more of goal, constraints, output or check.",
|
|
420
|
+
missing: "Work was delegated with no usable brief.",
|
|
421
|
+
"no-delegation": "Nothing was delegated."
|
|
422
|
+
}
|
|
423
|
+
},
|
|
424
|
+
steer_handling: {
|
|
425
|
+
type: "choice",
|
|
426
|
+
instructions: "When a steer or correction arrived mid-task, what did the agent do with it?",
|
|
427
|
+
criteria: {
|
|
428
|
+
applied: "Changed course as the steer asked.",
|
|
429
|
+
ignored: "Continued as before, or acknowledged without acting.",
|
|
430
|
+
"no-steer": "No steer arrived."
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
};
|
|
434
|
+
function isLabel(value) {
|
|
435
|
+
return typeof value === "string" && Object.hasOwn(FAILURE_MODE_LABELS, value);
|
|
436
|
+
}
|
|
437
|
+
/** Gold labels from AgentRx rows: each trajectory's root-cause category, mapped onto this set. */
|
|
438
|
+
function failureModeGoldFromAgentRx(rows) {
|
|
439
|
+
const gold = [];
|
|
440
|
+
const unmapped = [];
|
|
441
|
+
for (const row of rows) {
|
|
442
|
+
const id = `agentrx:${String(row.trajectory_id)}`;
|
|
443
|
+
const rootId = row.root_cause_failure_id ?? row.root_cause?.failure_id;
|
|
444
|
+
const root = row.failures.find((failure) => String(failure.failure_id) === String(rootId));
|
|
445
|
+
if (!root) throw new TypeError(`AgentRx trajectory '${row.trajectory_id}' has no root-cause failure`);
|
|
446
|
+
const category = normalizeAgentRxCategory(root.failure_category);
|
|
447
|
+
if (!Object.hasOwn(AGENT_RX_FAILURE_MODE_MAP, category)) throw new RangeError(`AgentRx category '${root.failure_category}' is outside the AgentRx taxonomy`);
|
|
448
|
+
const label = AGENT_RX_FAILURE_MODE_MAP[category];
|
|
449
|
+
if (label === null || label === void 0) unmapped.push({
|
|
450
|
+
id,
|
|
451
|
+
category
|
|
452
|
+
});
|
|
453
|
+
else gold.push({
|
|
454
|
+
id,
|
|
455
|
+
label
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
return {
|
|
459
|
+
gold,
|
|
460
|
+
unmapped
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
/** Validate a caller's gold file: a JSON array of `{ id, label }` with unique ids and known labels. */
|
|
464
|
+
function parseFailureModeGold(raw) {
|
|
465
|
+
if (!Array.isArray(raw)) return {
|
|
466
|
+
succeeded: false,
|
|
467
|
+
error: {
|
|
468
|
+
kind: "invalid",
|
|
469
|
+
message: "gold must be a JSON array"
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
const seen = /* @__PURE__ */ new Set();
|
|
473
|
+
const items = [];
|
|
474
|
+
for (const [index, row] of raw.entries()) {
|
|
475
|
+
const id = row?.id;
|
|
476
|
+
const label = row?.label;
|
|
477
|
+
if (typeof id !== "string" || !id.trim() || !isLabel(label)) return {
|
|
478
|
+
succeeded: false,
|
|
479
|
+
error: {
|
|
480
|
+
kind: "invalid",
|
|
481
|
+
message: `gold row ${index} needs a string id and a known label`
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
if (seen.has(id)) return {
|
|
485
|
+
succeeded: false,
|
|
486
|
+
error: {
|
|
487
|
+
kind: "invalid",
|
|
488
|
+
message: `gold id '${id}' repeats`
|
|
489
|
+
}
|
|
490
|
+
};
|
|
491
|
+
seen.add(id);
|
|
492
|
+
items.push({
|
|
493
|
+
id,
|
|
494
|
+
label
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
return {
|
|
498
|
+
succeeded: true,
|
|
499
|
+
value: items
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Load the calibration gold set from the public AgentRx labels and the
|
|
504
|
+
* caller's gold file, through the analyst module's public-dataset loader.
|
|
505
|
+
* Ids must be unique across both sources. No source, or no gold item, is a
|
|
506
|
+
* failure: an empty gold set would calibrate to an undefined κ.
|
|
507
|
+
*/
|
|
508
|
+
async function loadFailureModeGold(sources) {
|
|
509
|
+
const invalid = (message) => ({
|
|
510
|
+
succeeded: false,
|
|
511
|
+
error: {
|
|
512
|
+
kind: "invalid",
|
|
513
|
+
message
|
|
514
|
+
}
|
|
515
|
+
});
|
|
516
|
+
const gold = [];
|
|
517
|
+
let agentRxRows = 0;
|
|
518
|
+
let unmapped = [];
|
|
519
|
+
try {
|
|
520
|
+
if (sources.agentRxLabelsPath !== void 0) {
|
|
521
|
+
const rows = await loadPublicBenchmarkRows(sources.agentRxLabelsPath);
|
|
522
|
+
agentRxRows = rows.length;
|
|
523
|
+
const mapped = failureModeGoldFromAgentRx(rows);
|
|
524
|
+
gold.push(...mapped.gold);
|
|
525
|
+
unmapped = mapped.unmapped;
|
|
526
|
+
}
|
|
527
|
+
if (sources.goldPath !== void 0) {
|
|
528
|
+
const parsed = parseFailureModeGold(JSON.parse(await readFile(sources.goldPath, "utf8")));
|
|
529
|
+
if (!parsed.succeeded) return parsed;
|
|
530
|
+
gold.push(...parsed.value);
|
|
531
|
+
}
|
|
532
|
+
} catch (error) {
|
|
533
|
+
return invalid(error instanceof Error ? error.message : String(error));
|
|
534
|
+
}
|
|
535
|
+
if (gold.length === 0) return invalid("failure-modes calibration needs at least one gold item");
|
|
536
|
+
const ids = /* @__PURE__ */ new Set();
|
|
537
|
+
for (const item of gold) {
|
|
538
|
+
if (ids.has(item.id)) return invalid(`gold id '${item.id}' repeats across sources`);
|
|
539
|
+
ids.add(item.id);
|
|
540
|
+
}
|
|
541
|
+
return {
|
|
542
|
+
succeeded: true,
|
|
543
|
+
value: {
|
|
544
|
+
gold,
|
|
545
|
+
agentRxRows,
|
|
546
|
+
unmapped
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Score the judge's labels against gold labels for the same items, usually
|
|
552
|
+
* the set `loadFailureModeGold` returns.
|
|
553
|
+
*/
|
|
554
|
+
function calibrateFailureModes(gold, predictions) {
|
|
555
|
+
const predicted = /* @__PURE__ */ new Map();
|
|
556
|
+
for (const prediction of predictions) {
|
|
557
|
+
if (predicted.has(prediction.id)) throw new TypeError(`prediction id '${prediction.id}' repeats`);
|
|
558
|
+
predicted.set(prediction.id, prediction.label);
|
|
559
|
+
}
|
|
560
|
+
const ids = /* @__PURE__ */ new Set();
|
|
561
|
+
let missingPredictions = 0;
|
|
562
|
+
const pairs = [];
|
|
563
|
+
for (const item of gold) {
|
|
564
|
+
if (ids.has(item.id)) throw new TypeError(`gold id '${item.id}' repeats`);
|
|
565
|
+
ids.add(item.id);
|
|
566
|
+
const label = predicted.get(item.id);
|
|
567
|
+
if (label === void 0) missingPredictions += 1;
|
|
568
|
+
pairs.push([item.label, label ?? "(no prediction)"]);
|
|
569
|
+
}
|
|
570
|
+
const agreement = nominalAgreement(pairs);
|
|
571
|
+
return {
|
|
572
|
+
gold_n: agreement.n,
|
|
573
|
+
agreement: agreement.agreement,
|
|
574
|
+
kappa: agreement.kappa,
|
|
575
|
+
missingPredictions
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Build the failure-modes analyst, or refuse when its calibration does not
|
|
580
|
+
* clear `FAILURE_MODES_MIN_KAPPA`. An undefined κ (no gold, or one label
|
|
581
|
+
* everywhere) is refused the same way.
|
|
582
|
+
*/
|
|
583
|
+
function createFailureModesAnalyst(options) {
|
|
584
|
+
const { calibration, layer, model, renderState, scope, subject, ...transport } = options;
|
|
585
|
+
if (!(calibration.gold_n > 0) || !(calibration.kappa >= .6)) return {
|
|
586
|
+
succeeded: false,
|
|
587
|
+
error: {
|
|
588
|
+
kind: "uncalibrated",
|
|
589
|
+
message: `failure-modes rubric has kappa ${Number.isNaN(calibration.kappa) ? "undefined" : calibration.kappa.toFixed(3)} over ${calibration.gold_n} gold items; reporting requires kappa >= ${FAILURE_MODES_MIN_KAPPA}`
|
|
590
|
+
}
|
|
591
|
+
};
|
|
592
|
+
const method = {
|
|
593
|
+
kind: "judged",
|
|
594
|
+
name: FAILURE_MODES_ANALYST_ID,
|
|
595
|
+
version: "1",
|
|
596
|
+
calibration: {
|
|
597
|
+
gold_n: calibration.gold_n,
|
|
598
|
+
agreement: calibration.agreement,
|
|
599
|
+
kappa: calibration.kappa
|
|
600
|
+
}
|
|
601
|
+
};
|
|
602
|
+
return {
|
|
603
|
+
succeeded: true,
|
|
604
|
+
value: systemOneAnalyst({
|
|
605
|
+
...transport,
|
|
606
|
+
model,
|
|
607
|
+
renderState,
|
|
608
|
+
id: FAILURE_MODES_ANALYST_ID,
|
|
609
|
+
description: "Labels agent, team and run failures with a calibrated fixed-choice rubric.",
|
|
610
|
+
inputKind: "custom",
|
|
611
|
+
version: "1",
|
|
612
|
+
questions: FAILURE_MODES_QUESTIONS,
|
|
613
|
+
findings: (value, input, context) => {
|
|
614
|
+
const judgedSubject = subject?.(input, context);
|
|
615
|
+
const base = {
|
|
616
|
+
analyst_id: FAILURE_MODES_ANALYST_ID,
|
|
617
|
+
derived_from_judge: true,
|
|
618
|
+
layer,
|
|
619
|
+
scope: scope(input, context),
|
|
620
|
+
method,
|
|
621
|
+
evidence_refs: [{
|
|
622
|
+
kind: "artifact",
|
|
623
|
+
uri: `run://${context.runId}`
|
|
624
|
+
}],
|
|
625
|
+
...judgedSubject === void 0 ? {} : { subject: judgedSubject }
|
|
626
|
+
};
|
|
627
|
+
const out = [];
|
|
628
|
+
const mode = value.answers.failure_mode;
|
|
629
|
+
if (mode.choice !== "none") out.push(makeFinding({
|
|
630
|
+
...base,
|
|
631
|
+
area: "failure-mode",
|
|
632
|
+
severity: "medium",
|
|
633
|
+
confidence: mode.confidence,
|
|
634
|
+
claim: `Failure mode: ${mode.choice}`,
|
|
635
|
+
claim_id: `failure-mode:${mode.choice}`,
|
|
636
|
+
rationale: FAILURE_MODE_LABELS[mode.choice]
|
|
637
|
+
}));
|
|
638
|
+
const brief = value.answers.delegation_brief;
|
|
639
|
+
if (brief.choice === "partial" || brief.choice === "missing") out.push(makeFinding({
|
|
640
|
+
...base,
|
|
641
|
+
area: "delegation-brief",
|
|
642
|
+
severity: brief.choice === "missing" ? "high" : "medium",
|
|
643
|
+
confidence: brief.confidence,
|
|
644
|
+
claim: `Delegation brief ${brief.choice}`,
|
|
645
|
+
claim_id: `delegation-brief:${brief.choice}`
|
|
646
|
+
}));
|
|
647
|
+
const steer = value.answers.steer_handling;
|
|
648
|
+
if (steer.choice === "ignored") out.push(makeFinding({
|
|
649
|
+
...base,
|
|
650
|
+
area: "steer-handling",
|
|
651
|
+
severity: "high",
|
|
652
|
+
confidence: steer.confidence,
|
|
653
|
+
claim: "Steer ignored",
|
|
654
|
+
claim_id: "steer-handling:ignored"
|
|
655
|
+
}));
|
|
656
|
+
return out;
|
|
657
|
+
}
|
|
658
|
+
})
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
//#endregion
|
|
662
|
+
//#region src/analyst/findings-ledger.ts
|
|
663
|
+
/**
|
|
664
|
+
* Findings ledger — analyst findings in the shared ledger-core journal.
|
|
665
|
+
*
|
|
666
|
+
* Each appended finding is one hash-chained, canonical JSON entry, fsynced and
|
|
667
|
+
* pinned to a trusted head, so a later reader can prove nothing was rewritten
|
|
668
|
+
* or cut from the end. A row is validated by `AnalystFindingSchema` before it
|
|
669
|
+
* is written and again when it is read, so a 1.1.0 finding whose declared
|
|
670
|
+
* population disagrees with its evidence never enters the record.
|
|
671
|
+
*
|
|
672
|
+
* Appends are idempotent per (run, finding): replaying a run's findings is a
|
|
673
|
+
* no-op, and the same finding id with different content is a conflict.
|
|
674
|
+
* `FindingsStore` remains the unchained JSONL store for existing consumers.
|
|
675
|
+
*/
|
|
676
|
+
const schema = "agent-eval.findings-ledger.v1";
|
|
677
|
+
const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/);
|
|
678
|
+
const identity = z.string().min(1).refine((value) => value.trim() === value, { message: "must not carry surrounding space" });
|
|
679
|
+
const eventSchema = z.object({
|
|
680
|
+
kind: z.literal("finding"),
|
|
681
|
+
eventId: identity,
|
|
682
|
+
run_id: identity,
|
|
683
|
+
finding: AnalystFindingSchema
|
|
684
|
+
}).strict();
|
|
685
|
+
const entrySchema = z.object({
|
|
686
|
+
schema: z.literal(schema),
|
|
687
|
+
sequence: z.number().int().nonnegative(),
|
|
688
|
+
previousHash: digest.nullable(),
|
|
689
|
+
event: eventSchema,
|
|
690
|
+
entryHash: digest
|
|
691
|
+
}).strict();
|
|
692
|
+
var FindingsLedgerConflictError = class extends Error {
|
|
693
|
+
constructor(message) {
|
|
694
|
+
super(message);
|
|
695
|
+
this.name = "FindingsLedgerConflictError";
|
|
696
|
+
}
|
|
697
|
+
};
|
|
698
|
+
var FindingsLedgerInvalidError = class extends Error {
|
|
699
|
+
constructor(message) {
|
|
700
|
+
super(message);
|
|
701
|
+
this.name = "FindingsLedgerInvalidError";
|
|
702
|
+
}
|
|
703
|
+
};
|
|
704
|
+
function codec() {
|
|
705
|
+
return {
|
|
706
|
+
subject: "findings ledger",
|
|
707
|
+
header: { schema },
|
|
708
|
+
integrityError: (message, options) => new Error(message, options),
|
|
709
|
+
conflictError: (message) => new FindingsLedgerConflictError(message),
|
|
710
|
+
parseEntry: (raw, context) => {
|
|
711
|
+
const decoded = entrySchema.safeParse(raw);
|
|
712
|
+
if (!decoded.success) {
|
|
713
|
+
const issue = decoded.error.issues[0];
|
|
714
|
+
throw new FindingsLedgerInvalidError(`findings ledger ${context.path}:${context.line} is invalid at ${(issue?.path ?? []).join(".")}: ${issue?.message}`);
|
|
715
|
+
}
|
|
716
|
+
return {
|
|
717
|
+
...decoded.data,
|
|
718
|
+
event: decoded.data.event,
|
|
719
|
+
previousHash: decoded.data.previousHash,
|
|
720
|
+
entryHash: decoded.data.entryHash
|
|
721
|
+
};
|
|
722
|
+
},
|
|
723
|
+
checkEntryHeader: (entry) => {
|
|
724
|
+
if (entry.schema !== schema) throw new FindingsLedgerInvalidError("unsupported ledger schema");
|
|
725
|
+
},
|
|
726
|
+
createProjector: () => {
|
|
727
|
+
const rows = [];
|
|
728
|
+
return {
|
|
729
|
+
apply: (entry) => {
|
|
730
|
+
const { event } = entry;
|
|
731
|
+
if (event.eventId !== eventIdFor(event.run_id, event.finding.finding_id)) throw new FindingsLedgerInvalidError(`event id ${event.eventId} does not name its finding`);
|
|
732
|
+
rows.push({
|
|
733
|
+
...event.finding,
|
|
734
|
+
run_id: event.run_id,
|
|
735
|
+
sequence: entry.sequence,
|
|
736
|
+
entryHash: entry.entryHash
|
|
737
|
+
});
|
|
738
|
+
},
|
|
739
|
+
snapshot: () => [...rows]
|
|
740
|
+
};
|
|
741
|
+
}
|
|
742
|
+
};
|
|
743
|
+
}
|
|
744
|
+
function eventIdFor(runId, findingId) {
|
|
745
|
+
return `finding:${runId}:${findingId}`;
|
|
746
|
+
}
|
|
747
|
+
async function outcome(operation) {
|
|
748
|
+
try {
|
|
749
|
+
return {
|
|
750
|
+
succeeded: true,
|
|
751
|
+
value: await operation()
|
|
752
|
+
};
|
|
753
|
+
} catch (error) {
|
|
754
|
+
return {
|
|
755
|
+
succeeded: false,
|
|
756
|
+
error: {
|
|
757
|
+
kind: error instanceof FindingsLedgerConflictError ? "conflict" : error instanceof FindingsLedgerInvalidError || error instanceof z.ZodError ? "invalid" : "unavailable",
|
|
758
|
+
message: error instanceof Error ? error.message : String(error)
|
|
759
|
+
}
|
|
760
|
+
};
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
/** Open (or create on first append) the findings journal at `path`; every reopen requires its pin. */
|
|
764
|
+
function openFindingsLedger(options) {
|
|
765
|
+
if (!options.path.trim()) throw new TypeError("findings ledger: path is empty");
|
|
766
|
+
const journal = new FileLedgerJournal(options.path, codec(), { requireTrustedHead: true });
|
|
767
|
+
return {
|
|
768
|
+
path: journal.path,
|
|
769
|
+
append: (runId, findings) => outcome(async () => {
|
|
770
|
+
identity.parse(runId);
|
|
771
|
+
let appended = 0;
|
|
772
|
+
let replayed = 0;
|
|
773
|
+
for (const finding of findings) {
|
|
774
|
+
const parsed = AnalystFindingSchema.safeParse(finding);
|
|
775
|
+
if (!parsed.success) {
|
|
776
|
+
const issue = parsed.error.issues[0];
|
|
777
|
+
throw new FindingsLedgerInvalidError(`finding ${finding.finding_id} is invalid at ${(issue?.path ?? []).join(".")}: ${issue?.message}`);
|
|
778
|
+
}
|
|
779
|
+
if ((await journal.append({
|
|
780
|
+
kind: "finding",
|
|
781
|
+
eventId: eventIdFor(runId, finding.finding_id),
|
|
782
|
+
run_id: runId,
|
|
783
|
+
finding: parsed.data
|
|
784
|
+
}, { pinHead: true })).appended) appended += 1;
|
|
785
|
+
else replayed += 1;
|
|
786
|
+
}
|
|
787
|
+
return {
|
|
788
|
+
appended,
|
|
789
|
+
replayed
|
|
790
|
+
};
|
|
791
|
+
}),
|
|
792
|
+
read: () => outcome(async () => {
|
|
793
|
+
const findings = await journal.replay();
|
|
794
|
+
return {
|
|
795
|
+
findings,
|
|
796
|
+
merkleRoot: merkleRoot(findings.map((row) => row.entryHash))
|
|
797
|
+
};
|
|
798
|
+
})
|
|
799
|
+
};
|
|
800
|
+
}
|
|
801
|
+
/** Filter ledger rows. Every set criterion must hold; 1.0.0 rows have no layer and never match one. */
|
|
802
|
+
function queryFindings(rows, query) {
|
|
803
|
+
const layers = query.layer === void 0 ? null : new Set(typeof query.layer === "string" ? [query.layer] : query.layer);
|
|
804
|
+
return rows.filter((row) => {
|
|
805
|
+
if (layers && (row.layer === void 0 || !layers.has(row.layer))) return false;
|
|
806
|
+
if (query.subject !== void 0 && row.subject !== query.subject) return false;
|
|
807
|
+
if (query.analystId !== void 0 && row.analyst_id !== query.analystId) return false;
|
|
808
|
+
if (query.runId !== void 0 && row.run_id !== query.runId) return false;
|
|
809
|
+
if (query.claimId !== void 0 && row.claim_id !== query.claimId) return false;
|
|
810
|
+
if (query.domainPath !== void 0) {
|
|
811
|
+
const path = row.scope?.domain_path;
|
|
812
|
+
if (!path || path.length < query.domainPath.length) return false;
|
|
813
|
+
if (query.domainPath.some((id, index) => path[index] !== id)) return false;
|
|
814
|
+
}
|
|
815
|
+
return true;
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
/**
|
|
819
|
+
* Diff each run against the run before it, in the order runs first entered
|
|
820
|
+
* the ledger. Uses `diffFindings`, so `appeared`, `disappeared`, `persisted`
|
|
821
|
+
* and `changed` mean exactly what they mean for two runs.
|
|
822
|
+
*/
|
|
823
|
+
function diffFindingsOverTime(rows, policy = {}) {
|
|
824
|
+
const runs = [];
|
|
825
|
+
const byRun = /* @__PURE__ */ new Map();
|
|
826
|
+
for (const row of [...rows].sort((a, b) => a.sequence - b.sequence)) {
|
|
827
|
+
const list = byRun.get(row.run_id);
|
|
828
|
+
if (list) list.push(row);
|
|
829
|
+
else {
|
|
830
|
+
runs.push(row.run_id);
|
|
831
|
+
byRun.set(row.run_id, [row]);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
return runs.slice(1).map((runId, index) => ({
|
|
835
|
+
previous_run_id: runs[index],
|
|
836
|
+
run_id: runId,
|
|
837
|
+
diff: diffFindings(byRun.get(runs[index]), byRun.get(runId), policy)
|
|
838
|
+
}));
|
|
839
|
+
}
|
|
840
|
+
//#endregion
|
|
166
841
|
//#region src/analyst/kinds/skill-usage.ts
|
|
167
842
|
/**
|
|
168
843
|
* Skill-usage analyst — a DETERMINISTIC `Analyst` over a Claude/Codex skill
|
|
@@ -377,6 +1052,866 @@ var SkillUsageAnalyst = class {
|
|
|
377
1052
|
};
|
|
378
1053
|
const SKILL_USAGE_ANALYST = new SkillUsageAnalyst();
|
|
379
1054
|
//#endregion
|
|
1055
|
+
//#region src/analyst/promote.ts
|
|
1056
|
+
/**
|
|
1057
|
+
* Promotion — the only way a claim moves above the layer it was measured at.
|
|
1058
|
+
*
|
|
1059
|
+
* A claim is restated at a parent node only when at least `k` independent
|
|
1060
|
+
* children carry the same `claim_id` with intervals that agree. Otherwise it
|
|
1061
|
+
* stays where it is, reported as untested above. The rule is the same on every
|
|
1062
|
+
* rung of the ladder: run → play → problem → each domain node up the path →
|
|
1063
|
+
* lab.
|
|
1064
|
+
*
|
|
1065
|
+
* The rule, exactly:
|
|
1066
|
+
* - A finding sits at the node that is the lowest common ancestor of its
|
|
1067
|
+
* `scope.population` in the taxonomy, and that node's layer must equal the
|
|
1068
|
+
* finding's `layer`. Four runs of one play place a claim at that play; the
|
|
1069
|
+
* same claim stored at `domain` or `lab` is refused as exceeding its
|
|
1070
|
+
* evidence.
|
|
1071
|
+
* - Only children with `measure.interval` vote. Voters must name the same
|
|
1072
|
+
* measure and unit, and their intervals must share a common point.
|
|
1073
|
+
* - Children are counted by lineage, not by node: two children whose evidence
|
|
1074
|
+
* units share a lineage root (run 2 built on run 1's code) count once, and
|
|
1075
|
+
* a child with no recorded lineage shares one root with every other such
|
|
1076
|
+
* child.
|
|
1077
|
+
* - Promoted findings cite every voting child by `finding://<finding_id>`.
|
|
1078
|
+
*
|
|
1079
|
+
* Event, turn, agent, team and meta findings sit outside the ladder and never
|
|
1080
|
+
* promote.
|
|
1081
|
+
*/
|
|
1082
|
+
/** Bump when the promotion rule changes, so promoted findings name the rule that lifted them. */
|
|
1083
|
+
const PROMOTE_VERSION = "1";
|
|
1084
|
+
/** The analyst id a promoted finding carries when its children came from different analysts. */
|
|
1085
|
+
const PROMOTE_ANALYST_ID = "promote";
|
|
1086
|
+
const LADDER = [
|
|
1087
|
+
"run",
|
|
1088
|
+
"play",
|
|
1089
|
+
"problem",
|
|
1090
|
+
"domain",
|
|
1091
|
+
"lab"
|
|
1092
|
+
];
|
|
1093
|
+
function promote(findings, taxonomy, options = {}) {
|
|
1094
|
+
const k = options.k ?? 2;
|
|
1095
|
+
if (!Number.isInteger(k) || k < 2) throw new RangeError(`promote: k must be an integer of at least 2, got ${k}`);
|
|
1096
|
+
const unplaced = [];
|
|
1097
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
1098
|
+
const addPlacement = (placement) => {
|
|
1099
|
+
const depth = taxonomyAncestry(taxonomy, placement.node).length;
|
|
1100
|
+
const bucket = buckets.get(depth) ?? [];
|
|
1101
|
+
bucket.push(placement);
|
|
1102
|
+
buckets.set(depth, bucket);
|
|
1103
|
+
};
|
|
1104
|
+
for (const finding of findings) {
|
|
1105
|
+
const placed = place(finding, taxonomy);
|
|
1106
|
+
if ("reason" in placed) unplaced.push(placed);
|
|
1107
|
+
else addPlacement(placed);
|
|
1108
|
+
}
|
|
1109
|
+
const roots = lineageRoots(mergeEvidenceUnits(findings.flatMap((finding) => finding.evidence_units ?? [])));
|
|
1110
|
+
const promoted = [];
|
|
1111
|
+
const untestedAbove = [];
|
|
1112
|
+
const hold = (placement, reason) => {
|
|
1113
|
+
untestedAbove.push({
|
|
1114
|
+
finding_id: placement.finding.finding_id,
|
|
1115
|
+
claim_id: placement.claimId,
|
|
1116
|
+
node: placement.node,
|
|
1117
|
+
reason
|
|
1118
|
+
});
|
|
1119
|
+
};
|
|
1120
|
+
for (let depth = Math.max(0, ...buckets.keys()); depth > 1; depth -= 1) {
|
|
1121
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1122
|
+
for (const placement of buckets.get(depth) ?? []) {
|
|
1123
|
+
const parent = taxonomyParentOf(taxonomy, placement.node);
|
|
1124
|
+
if (parent === null || parent === void 0) continue;
|
|
1125
|
+
const key = JSON.stringify([placement.claimId, parent]);
|
|
1126
|
+
const group = groups.get(key) ?? {
|
|
1127
|
+
parent,
|
|
1128
|
+
members: []
|
|
1129
|
+
};
|
|
1130
|
+
group.members.push(placement);
|
|
1131
|
+
groups.set(key, group);
|
|
1132
|
+
}
|
|
1133
|
+
for (const { parent, members } of groups.values()) {
|
|
1134
|
+
const verdict = judgeGroup(members, k, roots);
|
|
1135
|
+
if (!verdict.lift) {
|
|
1136
|
+
for (const member of members) hold(member, verdict.reason);
|
|
1137
|
+
continue;
|
|
1138
|
+
}
|
|
1139
|
+
const lifted = liftedFinding(verdict.voters, parent, taxonomy, options.produced_at);
|
|
1140
|
+
promoted.push(lifted);
|
|
1141
|
+
addPlacement({
|
|
1142
|
+
finding: lifted,
|
|
1143
|
+
claimId: members[0].claimId,
|
|
1144
|
+
node: parent
|
|
1145
|
+
});
|
|
1146
|
+
for (const member of members) if (!verdict.voters.includes(member)) hold(member, "no interval, so it could not vote; the claim was lifted by its siblings");
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
return {
|
|
1150
|
+
findings: [...findings, ...promoted],
|
|
1151
|
+
promoted,
|
|
1152
|
+
untestedAbove,
|
|
1153
|
+
unplaced
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
function place(finding, taxonomy) {
|
|
1157
|
+
const unplaced = (reason, node = null) => ({
|
|
1158
|
+
finding_id: finding.finding_id,
|
|
1159
|
+
claim_id: finding.claim_id ?? null,
|
|
1160
|
+
node,
|
|
1161
|
+
reason
|
|
1162
|
+
});
|
|
1163
|
+
if (finding.schema_version !== "1.1.0" || !finding.layer || !finding.scope) return unplaced("no layer or scope (schema 1.0.0)");
|
|
1164
|
+
if (finding.layer === "meta") return unplaced("meta findings describe the instrument and never promote");
|
|
1165
|
+
if (!LADDER.includes(finding.layer)) return unplaced(`${finding.layer} findings sit below the run → lab ladder`);
|
|
1166
|
+
if (!finding.claim_id) return unplaced("no claim_id to match siblings by");
|
|
1167
|
+
const population = finding.scope.population;
|
|
1168
|
+
if (population.length === 0) return unplaced("population names no units");
|
|
1169
|
+
const node = lowestCommonAncestor(taxonomy, population);
|
|
1170
|
+
if (node === void 0) return unplaced("population names ids outside the taxonomy");
|
|
1171
|
+
const nodeLayer = taxonomyLayerOf(taxonomy, node);
|
|
1172
|
+
if (nodeLayer !== finding.layer) return unplaced(LADDER.indexOf(finding.layer) > LADDER.indexOf(nodeLayer) ? `stated at ${finding.layer} but its population sits within ${nodeLayer} '${node}'` : `stated at ${finding.layer} but its population spans ${nodeLayer} '${node}'`, node);
|
|
1173
|
+
const path = finding.scope.domain_path;
|
|
1174
|
+
if (path) {
|
|
1175
|
+
const expected = domainPathOf(taxonomy, node);
|
|
1176
|
+
if (path.length !== expected.length || path.some((id, index) => id !== expected[index])) return unplaced(`domain_path ${JSON.stringify(path)} disagrees with the taxonomy`, node);
|
|
1177
|
+
}
|
|
1178
|
+
return {
|
|
1179
|
+
finding,
|
|
1180
|
+
claimId: finding.claim_id,
|
|
1181
|
+
node
|
|
1182
|
+
};
|
|
1183
|
+
}
|
|
1184
|
+
function judgeGroup(members, k, roots) {
|
|
1185
|
+
const voters = members.filter((member) => member.finding.measure?.interval !== void 0);
|
|
1186
|
+
if (voters.length === 0) return {
|
|
1187
|
+
lift: false,
|
|
1188
|
+
reason: "no child carries an interval"
|
|
1189
|
+
};
|
|
1190
|
+
const first = voters[0].finding.measure;
|
|
1191
|
+
if (voters.some(({ finding }) => finding.measure.name !== first.name || finding.measure.unit !== first.unit)) return {
|
|
1192
|
+
lift: false,
|
|
1193
|
+
reason: "children measure different quantities"
|
|
1194
|
+
};
|
|
1195
|
+
if (Math.max(...voters.map(({ finding }) => finding.measure.interval[0])) > Math.min(...voters.map(({ finding }) => finding.measure.interval[1]))) return {
|
|
1196
|
+
lift: false,
|
|
1197
|
+
reason: "child intervals do not agree"
|
|
1198
|
+
};
|
|
1199
|
+
const independent = independentChildren(voters, roots);
|
|
1200
|
+
if (independent < k) {
|
|
1201
|
+
const children = new Set(voters.map((voter) => voter.node)).size;
|
|
1202
|
+
return {
|
|
1203
|
+
lift: false,
|
|
1204
|
+
reason: children > independent ? `${children} children agree but share lineage, leaving ${independent} independent of ${k} required` : `${independent} independent agreeing child${independent === 1 ? "" : "ren"} of ${k} required`
|
|
1205
|
+
};
|
|
1206
|
+
}
|
|
1207
|
+
return {
|
|
1208
|
+
lift: true,
|
|
1209
|
+
voters
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Children joined when they share a lineage root, counted as components. A
|
|
1214
|
+
* child's roots are those of all its voters' evidence units; a voter with no
|
|
1215
|
+
* units has unknown lineage.
|
|
1216
|
+
*/
|
|
1217
|
+
function independentChildren(voters, roots) {
|
|
1218
|
+
const rootsByNode = /* @__PURE__ */ new Map();
|
|
1219
|
+
for (const voter of voters) {
|
|
1220
|
+
const set = rootsByNode.get(voter.node) ?? /* @__PURE__ */ new Set();
|
|
1221
|
+
const units = voter.finding.evidence_units;
|
|
1222
|
+
if (!units) set.add(UNKNOWN_LINEAGE);
|
|
1223
|
+
else for (const unit of units) set.add(roots.get(unit.id) ?? "?unknown-lineage");
|
|
1224
|
+
rootsByNode.set(voter.node, set);
|
|
1225
|
+
}
|
|
1226
|
+
const parent = /* @__PURE__ */ new Map();
|
|
1227
|
+
const find = (id) => {
|
|
1228
|
+
let current = id;
|
|
1229
|
+
while (parent.get(current) !== current) current = parent.get(current);
|
|
1230
|
+
return current;
|
|
1231
|
+
};
|
|
1232
|
+
const owner = /* @__PURE__ */ new Map();
|
|
1233
|
+
for (const [node, nodeRoots] of rootsByNode) {
|
|
1234
|
+
parent.set(node, node);
|
|
1235
|
+
for (const root of nodeRoots) {
|
|
1236
|
+
const other = owner.get(root);
|
|
1237
|
+
if (other === void 0) owner.set(root, node);
|
|
1238
|
+
else parent.set(find(node), find(other));
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
return new Set([...rootsByNode.keys()].map(find)).size;
|
|
1242
|
+
}
|
|
1243
|
+
function liftedFinding(voters, parent, taxonomy, producedAt) {
|
|
1244
|
+
const findings = voters.map((voter) => voter.finding);
|
|
1245
|
+
const first = findings[0];
|
|
1246
|
+
const measures = findings.map((finding) => finding.measure);
|
|
1247
|
+
const level = new Set(measures.map((measure) => measure.level)).size === 1 ? measures[0].level : void 0;
|
|
1248
|
+
const population = [...new Set(findings.flatMap((finding) => finding.scope.population))].sort();
|
|
1249
|
+
const allUnits = findings.every((finding) => finding.evidence_units !== void 0) ? mergeEvidenceUnits(findings.flatMap((finding) => finding.evidence_units)) : void 0;
|
|
1250
|
+
const confounders = [...new Set(findings.flatMap((finding) => finding.confounders ?? []))].sort();
|
|
1251
|
+
const analysts = new Set(findings.map((finding) => finding.analyst_id));
|
|
1252
|
+
const domainPath = domainPathOf(taxonomy, parent);
|
|
1253
|
+
const parentLayer = taxonomyLayerOf(taxonomy, parent);
|
|
1254
|
+
return makeFinding({
|
|
1255
|
+
analyst_id: analysts.size === 1 ? first.analyst_id : PROMOTE_ANALYST_ID,
|
|
1256
|
+
area: first.area,
|
|
1257
|
+
claim: first.claim,
|
|
1258
|
+
claim_id: first.claim_id,
|
|
1259
|
+
id_basis: `${first.claim_id}@${parent}`,
|
|
1260
|
+
subject: parent,
|
|
1261
|
+
severity: mostSevere(findings),
|
|
1262
|
+
confidence: Math.min(...findings.map((finding) => finding.confidence)),
|
|
1263
|
+
evidence_refs: findings.map((finding) => ({
|
|
1264
|
+
kind: "finding",
|
|
1265
|
+
uri: `finding://${finding.finding_id}`
|
|
1266
|
+
})),
|
|
1267
|
+
layer: parentLayer,
|
|
1268
|
+
scope: {
|
|
1269
|
+
population,
|
|
1270
|
+
n: population.length,
|
|
1271
|
+
window: spanningWindow(findings),
|
|
1272
|
+
...domainPath.length > 0 && parent !== "lab" ? { domain_path: domainPath } : {}
|
|
1273
|
+
},
|
|
1274
|
+
method: {
|
|
1275
|
+
kind: "computed",
|
|
1276
|
+
name: "promote",
|
|
1277
|
+
version: "1"
|
|
1278
|
+
},
|
|
1279
|
+
measure: {
|
|
1280
|
+
name: measures[0].name,
|
|
1281
|
+
unit: measures[0].unit,
|
|
1282
|
+
value: measures.reduce((sum, measure) => sum + measure.value, 0) / measures.length,
|
|
1283
|
+
interval: [Math.max(...measures.map((measure) => measure.interval[0])), Math.min(...measures.map((measure) => measure.interval[1]))],
|
|
1284
|
+
...level === void 0 ? {} : { level }
|
|
1285
|
+
},
|
|
1286
|
+
...confounders.length > 0 ? { confounders } : {},
|
|
1287
|
+
...allUnits ? { evidence_units: allUnits } : {},
|
|
1288
|
+
rationale: `Lifted to ${parentLayer} '${parent}': ${new Set(voters.map((voter) => voter.node)).size} children carry claim '${first.claim_id}' with intervals that share [${Math.max(...measures.map((m) => m.interval[0]))}, ${Math.min(...measures.map((m) => m.interval[1]))}].`,
|
|
1289
|
+
metadata: {
|
|
1290
|
+
taxonomy_version: taxonomy.version,
|
|
1291
|
+
promoted_from: [...new Set(voters.map((voter) => voter.node))].sort()
|
|
1292
|
+
},
|
|
1293
|
+
...producedAt === void 0 ? {} : { produced_at: producedAt }
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
//#endregion
|
|
1297
|
+
//#region src/analyst/rollup.ts
|
|
1298
|
+
/**
|
|
1299
|
+
* rollup — run findings aggregated up the ladder, with bootstrap intervals and
|
|
1300
|
+
* denominators, lifted only where `promote` allows.
|
|
1301
|
+
*
|
|
1302
|
+
* Run-layer findings that share a `claim_id` and a measure are pooled per play
|
|
1303
|
+
* into one play-layer finding. Its interval is a cluster bootstrap where each
|
|
1304
|
+
* cluster is a lineage root, so runs that built on each other move together
|
|
1305
|
+
* and four runs of one lineage yield one cluster and no interval. Those play
|
|
1306
|
+
* findings then go through `promote`; each finding it lifts to a problem,
|
|
1307
|
+
* domain node or the lab gets the pooled bootstrap over every run underneath
|
|
1308
|
+
* it, with its observation and cluster counts.
|
|
1309
|
+
*/
|
|
1310
|
+
const ROLLUP_ANALYST_ID = "rollup";
|
|
1311
|
+
const ROLLUP_VERSION = "1";
|
|
1312
|
+
function rollupFindings(findings, taxonomy, options = {}) {
|
|
1313
|
+
const confidence = options.confidence ?? .95;
|
|
1314
|
+
const skipped = [];
|
|
1315
|
+
const skip = (finding, reason) => {
|
|
1316
|
+
skipped.push({
|
|
1317
|
+
finding_id: finding.finding_id,
|
|
1318
|
+
claim_id: finding.claim_id ?? null,
|
|
1319
|
+
node: null,
|
|
1320
|
+
reason
|
|
1321
|
+
});
|
|
1322
|
+
};
|
|
1323
|
+
const roots = lineageRoots(findings.flatMap((finding) => finding.evidence_units ?? []));
|
|
1324
|
+
const observations = [];
|
|
1325
|
+
for (const finding of findings) {
|
|
1326
|
+
if (finding.layer !== "run" || !finding.scope) continue;
|
|
1327
|
+
if (!finding.claim_id) {
|
|
1328
|
+
skip(finding, "no claim_id");
|
|
1329
|
+
continue;
|
|
1330
|
+
}
|
|
1331
|
+
if (!finding.measure) {
|
|
1332
|
+
skip(finding, "no measure to aggregate");
|
|
1333
|
+
continue;
|
|
1334
|
+
}
|
|
1335
|
+
const run = lowestCommonAncestor(taxonomy, finding.scope.population);
|
|
1336
|
+
if (run === void 0 || taxonomyLayerOf(taxonomy, run) !== "run") {
|
|
1337
|
+
skip(finding, "population is not one run in the taxonomy");
|
|
1338
|
+
continue;
|
|
1339
|
+
}
|
|
1340
|
+
const units = finding.evidence_units;
|
|
1341
|
+
const runRoots = new Set(units ? units.map((unit) => roots.get(unit.id) ?? "?unknown-lineage") : [UNKNOWN_LINEAGE]);
|
|
1342
|
+
observations.push({
|
|
1343
|
+
finding,
|
|
1344
|
+
claimId: finding.claim_id,
|
|
1345
|
+
run,
|
|
1346
|
+
value: finding.measure.value,
|
|
1347
|
+
cluster: [...runRoots].sort()[0]
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1351
|
+
for (const observation of observations) {
|
|
1352
|
+
const play = taxonomyParentOf(taxonomy, observation.run);
|
|
1353
|
+
const key = JSON.stringify([observation.claimId, play]);
|
|
1354
|
+
const group = groups.get(key) ?? {
|
|
1355
|
+
play,
|
|
1356
|
+
members: []
|
|
1357
|
+
};
|
|
1358
|
+
group.members.push(observation);
|
|
1359
|
+
groups.set(key, group);
|
|
1360
|
+
}
|
|
1361
|
+
const aggregates = [];
|
|
1362
|
+
for (const { play, members } of groups.values()) {
|
|
1363
|
+
const measure = members[0].finding.measure;
|
|
1364
|
+
if (members.some(({ finding }) => finding.measure.name !== measure.name || finding.measure.unit !== measure.unit)) {
|
|
1365
|
+
for (const member of members) skip(member.finding, `runs of play '${play}' measure different quantities`);
|
|
1366
|
+
continue;
|
|
1367
|
+
}
|
|
1368
|
+
const pooled = pool(members, options, confidence);
|
|
1369
|
+
const findingsInGroup = members.map((member) => member.finding);
|
|
1370
|
+
const population = [...new Set(members.map((member) => member.run))].sort();
|
|
1371
|
+
const units = findingsInGroup.every((finding) => finding.evidence_units !== void 0) ? mergeEvidenceUnits(findingsInGroup.flatMap((finding) => finding.evidence_units)) : void 0;
|
|
1372
|
+
const domainPath = domainPathOf(taxonomy, play);
|
|
1373
|
+
const first = findingsInGroup[0];
|
|
1374
|
+
aggregates.push(makeFinding({
|
|
1375
|
+
analyst_id: ROLLUP_ANALYST_ID,
|
|
1376
|
+
area: first.area,
|
|
1377
|
+
claim: first.claim,
|
|
1378
|
+
claim_id: first.claim_id,
|
|
1379
|
+
id_basis: `${first.claim_id}@${play}`,
|
|
1380
|
+
subject: play,
|
|
1381
|
+
severity: mostSevere(findingsInGroup),
|
|
1382
|
+
confidence: Math.min(...findingsInGroup.map((finding) => finding.confidence)),
|
|
1383
|
+
evidence_refs: findingsInGroup.map((finding) => ({
|
|
1384
|
+
kind: "finding",
|
|
1385
|
+
uri: `finding://${finding.finding_id}`
|
|
1386
|
+
})),
|
|
1387
|
+
layer: "play",
|
|
1388
|
+
scope: {
|
|
1389
|
+
population,
|
|
1390
|
+
n: population.length,
|
|
1391
|
+
window: spanningWindow(findingsInGroup),
|
|
1392
|
+
...domainPath.length > 0 ? { domain_path: domainPath } : {}
|
|
1393
|
+
},
|
|
1394
|
+
method: {
|
|
1395
|
+
kind: "computed",
|
|
1396
|
+
name: ROLLUP_ANALYST_ID,
|
|
1397
|
+
version: "1"
|
|
1398
|
+
},
|
|
1399
|
+
measure: {
|
|
1400
|
+
name: measure.name,
|
|
1401
|
+
unit: measure.unit,
|
|
1402
|
+
...pooled.measure
|
|
1403
|
+
},
|
|
1404
|
+
...units ? { evidence_units: units } : {},
|
|
1405
|
+
metadata: { denominators: pooled.denominators },
|
|
1406
|
+
...options.produced_at === void 0 ? {} : { produced_at: options.produced_at }
|
|
1407
|
+
}));
|
|
1408
|
+
}
|
|
1409
|
+
const promotion = promote(aggregates, taxonomy, {
|
|
1410
|
+
...options.k === void 0 ? {} : { k: options.k },
|
|
1411
|
+
...options.produced_at === void 0 ? {} : { produced_at: options.produced_at }
|
|
1412
|
+
});
|
|
1413
|
+
const promoted = promotion.promoted.map((finding) => {
|
|
1414
|
+
const covered = new Set(finding.scope.population);
|
|
1415
|
+
const pooled = pool(observations.filter((observation) => observation.claimId === finding.claim_id && covered.has(observation.run)), options, confidence);
|
|
1416
|
+
return {
|
|
1417
|
+
...finding,
|
|
1418
|
+
measure: {
|
|
1419
|
+
name: finding.measure.name,
|
|
1420
|
+
unit: finding.measure.unit,
|
|
1421
|
+
...pooled.measure
|
|
1422
|
+
},
|
|
1423
|
+
metadata: {
|
|
1424
|
+
...finding.metadata,
|
|
1425
|
+
denominators: pooled.denominators
|
|
1426
|
+
}
|
|
1427
|
+
};
|
|
1428
|
+
});
|
|
1429
|
+
return {
|
|
1430
|
+
findings: [...aggregates, ...promoted],
|
|
1431
|
+
untestedAbove: promotion.untestedAbove,
|
|
1432
|
+
skipped: [...skipped, ...promotion.unplaced]
|
|
1433
|
+
};
|
|
1434
|
+
}
|
|
1435
|
+
/**
|
|
1436
|
+
* Cluster bootstrap of the run values. With one cluster there is no spread
|
|
1437
|
+
* between independent sources, so the measure carries no interval and cannot
|
|
1438
|
+
* vote in `promote`.
|
|
1439
|
+
*/
|
|
1440
|
+
function pool(members, options, confidence) {
|
|
1441
|
+
const result = clusterBootstrapMean(members.map((member) => ({
|
|
1442
|
+
cluster: member.cluster,
|
|
1443
|
+
value: member.value
|
|
1444
|
+
})), {
|
|
1445
|
+
confidence,
|
|
1446
|
+
...options.seed === void 0 ? {} : { seed: options.seed },
|
|
1447
|
+
...options.resamples === void 0 ? {} : { resamples: options.resamples }
|
|
1448
|
+
});
|
|
1449
|
+
return {
|
|
1450
|
+
measure: result.clusters >= 2 ? {
|
|
1451
|
+
value: result.mean,
|
|
1452
|
+
interval: [result.low, result.high],
|
|
1453
|
+
level: confidence
|
|
1454
|
+
} : { value: result.mean },
|
|
1455
|
+
denominators: {
|
|
1456
|
+
observations: result.n,
|
|
1457
|
+
independent_clusters: result.clusters
|
|
1458
|
+
}
|
|
1459
|
+
};
|
|
1460
|
+
}
|
|
1461
|
+
/** The rollup analyst: emits the play aggregates and every finding promote lifted from them. */
|
|
1462
|
+
function rollupAnalyst() {
|
|
1463
|
+
return {
|
|
1464
|
+
id: ROLLUP_ANALYST_ID,
|
|
1465
|
+
description: "Aggregates run findings up run → play → problem → domain → lab with bootstrap intervals.",
|
|
1466
|
+
inputKind: "custom",
|
|
1467
|
+
cost: { kind: "deterministic" },
|
|
1468
|
+
version: "1",
|
|
1469
|
+
async analyze(input) {
|
|
1470
|
+
return rollupFindings(input.findings, input.taxonomy, input.options).findings;
|
|
1471
|
+
}
|
|
1472
|
+
};
|
|
1473
|
+
}
|
|
1474
|
+
//#endregion
|
|
1475
|
+
//#region src/analyst/team-shape.ts
|
|
1476
|
+
const TEAM_SHAPE_ANALYST_ID = "team-shape";
|
|
1477
|
+
const TEAM_SHAPE_THRESHOLDS = {
|
|
1478
|
+
/** Share of the root's children that delegate at or above which depth ≥ 2 is a hive. */
|
|
1479
|
+
hiveDelegatingShare: .5,
|
|
1480
|
+
/** Lateral share of C edges at or above which a one-dispatcher team is a fleet. */
|
|
1481
|
+
fleetLateralCShare: .2,
|
|
1482
|
+
/** Root share of integrating commits at or above which a flat star reads `root integrates`. */
|
|
1483
|
+
rootIntegrationShare: .5
|
|
1484
|
+
};
|
|
1485
|
+
/** Version 1, provisional, with the thresholds it was decided at. */
|
|
1486
|
+
const TEAM_SHAPE_VERSION = `1-provisional;hive-delegating=${TEAM_SHAPE_THRESHOLDS.hiveDelegatingShare};fleet-lateral-c=${TEAM_SHAPE_THRESHOLDS.fleetLateralCShare};root-integration=${TEAM_SHAPE_THRESHOLDS.rootIntegrationShare}`;
|
|
1487
|
+
const TEAM_SHAPE_CLASSES = [
|
|
1488
|
+
"single",
|
|
1489
|
+
"governed-swarm",
|
|
1490
|
+
"fleet",
|
|
1491
|
+
"partition",
|
|
1492
|
+
"hive",
|
|
1493
|
+
"star-with-sub-leads",
|
|
1494
|
+
"flat-star"
|
|
1495
|
+
];
|
|
1496
|
+
const NOT_OBSERVABLE = "not observable from this record";
|
|
1497
|
+
/** Team-shape input for a run read through a supervisor-run reader. */
|
|
1498
|
+
function teamShapeInputFromSources(sources, integrations) {
|
|
1499
|
+
if (sources.journal === null) return {
|
|
1500
|
+
succeeded: false,
|
|
1501
|
+
error: { message: sources.journalMissingReason ?? `no spawn journal for ${sources.runRef}` }
|
|
1502
|
+
};
|
|
1503
|
+
return {
|
|
1504
|
+
succeeded: true,
|
|
1505
|
+
value: {
|
|
1506
|
+
runRef: sources.runRef,
|
|
1507
|
+
spawnJournal: sources.journal,
|
|
1508
|
+
coordinationLog: sources.coordinationLog ?? null,
|
|
1509
|
+
...integrations === void 0 ? {} : { integrations }
|
|
1510
|
+
}
|
|
1511
|
+
};
|
|
1512
|
+
}
|
|
1513
|
+
function classifyTeamShape(input) {
|
|
1514
|
+
const spawnLog = parseSpawnRows(input.spawnJournal);
|
|
1515
|
+
const coordination = typeof input.coordinationLog === "string" ? parseCoordinationRows(input.coordinationLog) : null;
|
|
1516
|
+
const spawnedAt = /* @__PURE__ */ new Map();
|
|
1517
|
+
for (const row of spawnLog.rows) {
|
|
1518
|
+
if (row.kind !== "spawned") continue;
|
|
1519
|
+
spawnedAt.set(row.id, Math.min(row.at, spawnedAt.get(row.id) ?? Number.POSITIVE_INFINITY));
|
|
1520
|
+
}
|
|
1521
|
+
const rootIds = [...new Set(spawnLog.rows.filter((row) => row.kind === "spawned" && row.journal === row.id).map((r) => r.id))];
|
|
1522
|
+
if (rootIds.length !== 1) return {
|
|
1523
|
+
succeeded: false,
|
|
1524
|
+
error: { message: `spawn journal for ${input.runRef} has ${rootIds.length} root markers; team-shape needs exactly one` }
|
|
1525
|
+
};
|
|
1526
|
+
const rootId = rootIds[0];
|
|
1527
|
+
const agents = new Set(spawnedAt.keys());
|
|
1528
|
+
const parentOf = /* @__PURE__ */ new Map();
|
|
1529
|
+
let unplacedAgents = 0;
|
|
1530
|
+
for (const id of agents) {
|
|
1531
|
+
if (id === rootId) continue;
|
|
1532
|
+
const parent = pathParent(id);
|
|
1533
|
+
if (parent !== null && agents.has(parent) && id.startsWith(`${rootId}:`)) parentOf.set(id, parent);
|
|
1534
|
+
else unplacedAgents += 1;
|
|
1535
|
+
}
|
|
1536
|
+
const children = /* @__PURE__ */ new Map();
|
|
1537
|
+
const command = /* @__PURE__ */ new Map();
|
|
1538
|
+
for (const [child, parent] of parentOf) {
|
|
1539
|
+
children.set(parent, [...children.get(parent) ?? [], child]);
|
|
1540
|
+
addEdge(command, parent, child);
|
|
1541
|
+
}
|
|
1542
|
+
const depthOf = (id) => {
|
|
1543
|
+
let hops = 0;
|
|
1544
|
+
for (let at = parentOf.get(id); at !== void 0; at = parentOf.get(at)) hops += 1;
|
|
1545
|
+
return hops;
|
|
1546
|
+
};
|
|
1547
|
+
const depth = Math.max(0, ...[...parentOf.keys()].map(depthOf));
|
|
1548
|
+
const rootChildren = children.get(rootId) ?? [];
|
|
1549
|
+
const leads = rootChildren.filter((child) => (children.get(child)?.length ?? 0) > 0);
|
|
1550
|
+
const delegatingShare = rootChildren.length === 0 ? null : leads.length / rootChildren.length;
|
|
1551
|
+
const subtreeOf = (id) => {
|
|
1552
|
+
let at = id;
|
|
1553
|
+
while (at !== void 0 && parentOf.get(at) !== rootId) at = parentOf.get(at);
|
|
1554
|
+
return at ?? null;
|
|
1555
|
+
};
|
|
1556
|
+
let peerSpawns = 0;
|
|
1557
|
+
for (const row of spawnLog.rows) {
|
|
1558
|
+
if (row.kind !== "spawned" || row.id === rootId) continue;
|
|
1559
|
+
if (row.owner !== row.id && row.owner !== parentOf.get(row.id)) peerSpawns += 1;
|
|
1560
|
+
}
|
|
1561
|
+
const governance = /* @__PURE__ */ new Map();
|
|
1562
|
+
const settlementsByReceiver = {};
|
|
1563
|
+
const governorsByParent = /* @__PURE__ */ new Map();
|
|
1564
|
+
const governorsOf = (child) => {
|
|
1565
|
+
const parent = parentOf.get(child) ?? `unplaced:${child}`;
|
|
1566
|
+
const entry = governorsByParent.get(parent) ?? {
|
|
1567
|
+
agents: /* @__PURE__ */ new Set(),
|
|
1568
|
+
digests: /* @__PURE__ */ new Set()
|
|
1569
|
+
};
|
|
1570
|
+
governorsByParent.set(parent, entry);
|
|
1571
|
+
return entry;
|
|
1572
|
+
};
|
|
1573
|
+
let gResolved = 0;
|
|
1574
|
+
let gUnresolved = 0;
|
|
1575
|
+
let gNonParent = 0;
|
|
1576
|
+
let firstSettlement = Number.POSITIVE_INFINITY;
|
|
1577
|
+
for (const row of spawnLog.rows) {
|
|
1578
|
+
if (row.kind !== "settled" || row.owner === row.id) continue;
|
|
1579
|
+
firstSettlement = Math.min(firstSettlement, row.at);
|
|
1580
|
+
settlementsByReceiver[row.owner] = (settlementsByReceiver[row.owner] ?? 0) + 1;
|
|
1581
|
+
addEdge(governance, row.owner, row.id);
|
|
1582
|
+
governorsOf(row.id).agents.add(row.owner);
|
|
1583
|
+
gResolved += 1;
|
|
1584
|
+
if (parentOf.get(row.id) !== row.owner) gNonParent += 1;
|
|
1585
|
+
}
|
|
1586
|
+
const reHires = [...spawnedAt.entries()].filter(([id, at]) => id !== rootId && at > firstSettlement).length;
|
|
1587
|
+
const communication = /* @__PURE__ */ new Map();
|
|
1588
|
+
let messages = 0;
|
|
1589
|
+
let cResolved = 0;
|
|
1590
|
+
let cUnresolved = 0;
|
|
1591
|
+
let cLateral = 0;
|
|
1592
|
+
let cCrossSubtree = 0;
|
|
1593
|
+
let gCrossSubtree = 0;
|
|
1594
|
+
const steers = {
|
|
1595
|
+
sent: 0,
|
|
1596
|
+
delivered: 0
|
|
1597
|
+
};
|
|
1598
|
+
const isAgent = (value) => value !== void 0 && agents.has(value);
|
|
1599
|
+
for (const row of coordination?.rows ?? []) {
|
|
1600
|
+
if (row.type === "instruction" || row.type === "delivery-attempt") continue;
|
|
1601
|
+
if (row.toWorker === void 0 && row.from === void 0 && row.by === void 0) continue;
|
|
1602
|
+
messages += 1;
|
|
1603
|
+
if (row.type === "steer") {
|
|
1604
|
+
steers.sent += 1;
|
|
1605
|
+
if (row.delivered === true) steers.delivered += 1;
|
|
1606
|
+
}
|
|
1607
|
+
if (!isAgent(row.toWorker) || row.delivered === false) continue;
|
|
1608
|
+
const receiver = row.toWorker;
|
|
1609
|
+
const agentSender = [row.from, row.by].find(isAgent);
|
|
1610
|
+
const sender = agentSender ?? (row.owner === null ? null : `owner:${row.owner}`);
|
|
1611
|
+
if (sender === null) continue;
|
|
1612
|
+
addEdge(communication, sender, receiver);
|
|
1613
|
+
if (agentSender === void 0) cUnresolved += 1;
|
|
1614
|
+
else {
|
|
1615
|
+
cResolved += 1;
|
|
1616
|
+
const parent = parentOf.get(receiver);
|
|
1617
|
+
if (parent !== void 0 && parent === parentOf.get(agentSender)) cLateral += 1;
|
|
1618
|
+
if (crosses(subtreeOf(agentSender), subtreeOf(receiver))) cCrossSubtree += 1;
|
|
1619
|
+
}
|
|
1620
|
+
if (row.type === "steer") {
|
|
1621
|
+
addEdge(governance, sender, receiver);
|
|
1622
|
+
if (agentSender === void 0) {
|
|
1623
|
+
gUnresolved += 1;
|
|
1624
|
+
governorsOf(receiver).digests.add(sender);
|
|
1625
|
+
} else {
|
|
1626
|
+
gResolved += 1;
|
|
1627
|
+
governorsOf(receiver).agents.add(agentSender);
|
|
1628
|
+
if (parentOf.get(receiver) !== agentSender) gNonParent += 1;
|
|
1629
|
+
if (crosses(subtreeOf(agentSender), subtreeOf(receiver))) gCrossSubtree += 1;
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
}
|
|
1633
|
+
for (const [child, receiver] of spawnLog.rows.filter((row) => row.kind === "settled" && row.owner !== row.id).map((row) => [row.id, row.owner])) if (crosses(subtreeOf(receiver), subtreeOf(child))) gCrossSubtree += 1;
|
|
1634
|
+
const cTotal = cResolved + cUnresolved;
|
|
1635
|
+
const gTotal = gResolved + gUnresolved;
|
|
1636
|
+
const lateralCShare = coordination === null || cTotal === 0 || cUnresolved > 0 ? NOT_OBSERVABLE : cLateral / cTotal;
|
|
1637
|
+
const nonParentGShare = gTotal === 0 || gUnresolved > 0 ? NOT_OBSERVABLE : gNonParent / gTotal;
|
|
1638
|
+
const integrations = input.integrations ?? [];
|
|
1639
|
+
const rootIntegrationShare = integrations.length === 0 ? NOT_OBSERVABLE : integrations.filter((row) => row.by === rootId).length / integrations.length;
|
|
1640
|
+
const features = {
|
|
1641
|
+
agents: agents.size,
|
|
1642
|
+
tasks: agents.size - 1,
|
|
1643
|
+
unplacedAgents,
|
|
1644
|
+
depth,
|
|
1645
|
+
fanOut: {
|
|
1646
|
+
root: rootChildren.length,
|
|
1647
|
+
max: Math.max(0, ...[...children.values()].map((c) => c.length))
|
|
1648
|
+
},
|
|
1649
|
+
delegatingShare,
|
|
1650
|
+
reHires,
|
|
1651
|
+
settlementsByReceiver,
|
|
1652
|
+
steers,
|
|
1653
|
+
messagesPerTask: coordination === null || agents.size < 2 ? null : messages / (agents.size - 1),
|
|
1654
|
+
lateralCShare,
|
|
1655
|
+
nonParentGShare,
|
|
1656
|
+
rootIntegrationShare
|
|
1657
|
+
};
|
|
1658
|
+
const checks = {};
|
|
1659
|
+
checks.single = verdict(agents.size === 1, `${agents.size} agents`);
|
|
1660
|
+
const rotating = [...governorsByParent.entries()].find(([, g]) => g.agents.size > 1);
|
|
1661
|
+
const digestTeams = [...governorsByParent.values()].filter((g) => g.digests.size > 1).length;
|
|
1662
|
+
checks["governed-swarm"] = peerSpawns > 0 ? {
|
|
1663
|
+
result: "yes",
|
|
1664
|
+
reason: `${peerSpawns} spawns recorded by a peer, not the parent`
|
|
1665
|
+
} : rotating !== void 0 ? {
|
|
1666
|
+
result: "yes",
|
|
1667
|
+
reason: `children of ${rotating[0]} are governed by ${[...rotating[1].agents].join(", ")}`
|
|
1668
|
+
} : digestTeams > 0 ? {
|
|
1669
|
+
result: NOT_OBSERVABLE,
|
|
1670
|
+
reason: `${digestTeams} teams were steered by more than one owner digest; a digest is not an agent id, so a rotating governor cannot be told from the operator`
|
|
1671
|
+
} : {
|
|
1672
|
+
result: "no",
|
|
1673
|
+
reason: "every E edge runs parent to child and no team changed governor"
|
|
1674
|
+
};
|
|
1675
|
+
const dispatchers = [...children.keys()];
|
|
1676
|
+
if (dispatchers.length !== 1 || unplacedAgents > 0) checks.fleet = {
|
|
1677
|
+
result: unplacedAgents > 0 && dispatchers.length <= 1 ? NOT_OBSERVABLE : "no",
|
|
1678
|
+
reason: unplacedAgents > 0 && dispatchers.length <= 1 ? `${unplacedAgents} unplaced agents hide who dispatched their work` : `${dispatchers.length} agents dispatch work`
|
|
1679
|
+
};
|
|
1680
|
+
else if (gNonParent > 0) checks.fleet = {
|
|
1681
|
+
result: "yes",
|
|
1682
|
+
reason: `${gNonParent} G edges come from a non-parent agent`
|
|
1683
|
+
};
|
|
1684
|
+
else if (cLateral / Math.max(cTotal, 1) >= TEAM_SHAPE_THRESHOLDS.fleetLateralCShare) checks.fleet = {
|
|
1685
|
+
result: "yes",
|
|
1686
|
+
reason: `lateral C ${cLateral} of ${cTotal}`
|
|
1687
|
+
};
|
|
1688
|
+
else if (gUnresolved > 0 || lateralCShare === "not observable from this record") checks.fleet = {
|
|
1689
|
+
result: NOT_OBSERVABLE,
|
|
1690
|
+
reason: `one dispatcher; ${gUnresolved} steer G and ${cUnresolved} C edges are sent from owner digests that no record joins to an agent${coordination === null ? ", and the run has no coordination log" : ""}`
|
|
1691
|
+
};
|
|
1692
|
+
else checks.fleet = verdict(false, `one dispatcher, no non-parent G, lateral C ${cLateral} of ${cTotal}`);
|
|
1693
|
+
const rootSettledByLeads = spawnLog.rows.filter((row) => row.kind === "settled" && row.owner === rootId && leads.includes(row.id)).length;
|
|
1694
|
+
if (leads.length < 2 || rootSettledByLeads > 0 || gCrossSubtree > 0 || cCrossSubtree > 0) checks.partition = verdict(false, `${leads.length} lead subtrees, ${rootSettledByLeads} lead settlements to the root, ${gCrossSubtree + cCrossSubtree} edges between subtrees`);
|
|
1695
|
+
else if (gUnresolved > 0 || cUnresolved > 0 || coordination === null) checks.partition = {
|
|
1696
|
+
result: NOT_OBSERVABLE,
|
|
1697
|
+
reason: `${leads.length} lead subtrees and no lead settles to the root, but edges sent from owner digests cannot be placed in a subtree`
|
|
1698
|
+
};
|
|
1699
|
+
else checks.partition = verdict(true, `${leads.length} lead subtrees with no C or G between them`);
|
|
1700
|
+
const delegating = `${leads.length} of ${rootChildren.length} root children delegate`;
|
|
1701
|
+
const hive = depth >= 2 && (delegatingShare ?? 0) >= TEAM_SHAPE_THRESHOLDS.hiveDelegatingShare;
|
|
1702
|
+
checks.hive = verdict(hive, `depth ${depth}; ${delegating}`);
|
|
1703
|
+
checks["star-with-sub-leads"] = verdict(depth >= 2 && !hive, `depth ${depth}; ${delegating}`);
|
|
1704
|
+
checks["flat-star"] = verdict(depth === 1, `depth ${depth}`);
|
|
1705
|
+
const shapeClass = TEAM_SHAPE_CLASSES.find((name) => checks[name].result === "yes");
|
|
1706
|
+
if (shapeClass === void 0) return {
|
|
1707
|
+
succeeded: false,
|
|
1708
|
+
error: { message: `no team-shape class matches ${input.runRef} (depth ${depth})` }
|
|
1709
|
+
};
|
|
1710
|
+
const qualifier = shapeClass !== "flat-star" ? null : rootIntegrationShare === "not observable from this record" ? "integration unknown" : rootIntegrationShare >= TEAM_SHAPE_THRESHOLDS.rootIntegrationShare ? "root integrates" : "root does not integrate";
|
|
1711
|
+
const label = shapeClass === "hive" ? `hive depth ${depth}` : qualifier === null ? shapeClass : `${shapeClass} (${qualifier})`;
|
|
1712
|
+
const times = [...spawnLog.rows.map((row) => row.at), ...(coordination?.rows ?? []).flatMap((row) => row.at === null ? [] : [row.at])];
|
|
1713
|
+
return {
|
|
1714
|
+
succeeded: true,
|
|
1715
|
+
value: {
|
|
1716
|
+
class: shapeClass,
|
|
1717
|
+
depth,
|
|
1718
|
+
qualifier,
|
|
1719
|
+
label,
|
|
1720
|
+
rootId,
|
|
1721
|
+
features,
|
|
1722
|
+
checks,
|
|
1723
|
+
graph: {
|
|
1724
|
+
command: [...command.values()],
|
|
1725
|
+
governance: [...governance.values()],
|
|
1726
|
+
communication: [...communication.values()]
|
|
1727
|
+
},
|
|
1728
|
+
invalidRows: {
|
|
1729
|
+
spawn: spawnLog.invalid,
|
|
1730
|
+
coordination: coordination?.invalid ?? 0
|
|
1731
|
+
},
|
|
1732
|
+
window: times.length === 0 ? null : {
|
|
1733
|
+
from: new Date(Math.min(...times)).toISOString(),
|
|
1734
|
+
to: new Date(Math.max(...times)).toISOString()
|
|
1735
|
+
}
|
|
1736
|
+
}
|
|
1737
|
+
};
|
|
1738
|
+
}
|
|
1739
|
+
function verdict(holds, reason) {
|
|
1740
|
+
return {
|
|
1741
|
+
result: holds ? "yes" : "no",
|
|
1742
|
+
reason
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
function crosses(a, b) {
|
|
1746
|
+
return a !== null && b !== null && a !== b;
|
|
1747
|
+
}
|
|
1748
|
+
function pathParent(id) {
|
|
1749
|
+
const match = /^(.+):s\d+$/.exec(id);
|
|
1750
|
+
return match ? match[1] : null;
|
|
1751
|
+
}
|
|
1752
|
+
function addEdge(edges, from, to) {
|
|
1753
|
+
const key = JSON.stringify([from, to]);
|
|
1754
|
+
const edge = edges.get(key);
|
|
1755
|
+
if (edge) edge.count += 1;
|
|
1756
|
+
else edges.set(key, {
|
|
1757
|
+
from,
|
|
1758
|
+
to,
|
|
1759
|
+
count: 1
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
function jsonLines(text) {
|
|
1763
|
+
return text.split("\n").filter((line) => line.trim()).map((line) => {
|
|
1764
|
+
try {
|
|
1765
|
+
const value = JSON.parse(line);
|
|
1766
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
1767
|
+
} catch {
|
|
1768
|
+
return null;
|
|
1769
|
+
}
|
|
1770
|
+
});
|
|
1771
|
+
}
|
|
1772
|
+
function asRecord(value) {
|
|
1773
|
+
return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
|
|
1774
|
+
}
|
|
1775
|
+
function epochMs(value) {
|
|
1776
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
1777
|
+
if (typeof value === "string") {
|
|
1778
|
+
const ms = Date.parse(value);
|
|
1779
|
+
return Number.isNaN(ms) ? null : ms;
|
|
1780
|
+
}
|
|
1781
|
+
return null;
|
|
1782
|
+
}
|
|
1783
|
+
/**
|
|
1784
|
+
* Spawned and settled rows from a Runtime envelope (`{ root, event }`) or a
|
|
1785
|
+
* reduced row (`{ kind, journal, id, at }`). Other Runtime event kinds are
|
|
1786
|
+
* read and skipped; a spawned or settled row without an id, journal or time
|
|
1787
|
+
* is invalid.
|
|
1788
|
+
*/
|
|
1789
|
+
function parseSpawnRows(text) {
|
|
1790
|
+
const rows = [];
|
|
1791
|
+
let invalid = 0;
|
|
1792
|
+
for (const line of jsonLines(text)) {
|
|
1793
|
+
if (line === null) {
|
|
1794
|
+
invalid += 1;
|
|
1795
|
+
continue;
|
|
1796
|
+
}
|
|
1797
|
+
const event = asRecord(line.event) ?? line;
|
|
1798
|
+
const journal = event === line ? line.journal : line.root;
|
|
1799
|
+
const { kind, id } = event;
|
|
1800
|
+
if (kind !== "spawned" && kind !== "settled") continue;
|
|
1801
|
+
const at = epochMs(event.at);
|
|
1802
|
+
if (typeof id !== "string" || typeof journal !== "string" || at === null) {
|
|
1803
|
+
invalid += 1;
|
|
1804
|
+
continue;
|
|
1805
|
+
}
|
|
1806
|
+
rows.push({
|
|
1807
|
+
kind,
|
|
1808
|
+
journal,
|
|
1809
|
+
owner: journal.slice(journal.lastIndexOf("/") + 1),
|
|
1810
|
+
id,
|
|
1811
|
+
at
|
|
1812
|
+
});
|
|
1813
|
+
}
|
|
1814
|
+
return {
|
|
1815
|
+
rows,
|
|
1816
|
+
invalid
|
|
1817
|
+
};
|
|
1818
|
+
}
|
|
1819
|
+
/**
|
|
1820
|
+
* Coordination rows from a Runtime record (`{ at, ownerId, event: { type,
|
|
1821
|
+
* <body> } }`, endpoints inside the event's one object-valued field) or a
|
|
1822
|
+
* reduced row (`{ at, type, owner, ... }`).
|
|
1823
|
+
*/
|
|
1824
|
+
function parseCoordinationRows(text) {
|
|
1825
|
+
const rows = [];
|
|
1826
|
+
let invalid = 0;
|
|
1827
|
+
for (const line of jsonLines(text)) {
|
|
1828
|
+
const event = line === null ? null : asRecord(line.event) ?? line;
|
|
1829
|
+
if (line === null || event === null || typeof event.type !== "string") {
|
|
1830
|
+
invalid += 1;
|
|
1831
|
+
continue;
|
|
1832
|
+
}
|
|
1833
|
+
const body = event === line ? line : Object.entries(event).filter(([key]) => key !== "type").map(([, value]) => asRecord(value)).find((value) => value !== null) ?? {};
|
|
1834
|
+
const owner = typeof line.owner === "string" ? line.owner : line.ownerId;
|
|
1835
|
+
const row = {
|
|
1836
|
+
at: epochMs(line.at),
|
|
1837
|
+
type: event.type,
|
|
1838
|
+
owner: typeof owner === "string" && owner ? owner : null
|
|
1839
|
+
};
|
|
1840
|
+
for (const key of [
|
|
1841
|
+
"toWorker",
|
|
1842
|
+
"from",
|
|
1843
|
+
"by"
|
|
1844
|
+
]) {
|
|
1845
|
+
const value = body[key];
|
|
1846
|
+
if (typeof value === "string") row[key] = value;
|
|
1847
|
+
}
|
|
1848
|
+
const delivered = body.delivered ?? line.delivered;
|
|
1849
|
+
if (typeof delivered === "boolean") row.delivered = delivered;
|
|
1850
|
+
rows.push(row);
|
|
1851
|
+
}
|
|
1852
|
+
return {
|
|
1853
|
+
rows,
|
|
1854
|
+
invalid
|
|
1855
|
+
};
|
|
1856
|
+
}
|
|
1857
|
+
/**
|
|
1858
|
+
* The team-shape analyst: one `team`-layer finding per run naming its class,
|
|
1859
|
+
* with the features and every class check in metadata. A record with no
|
|
1860
|
+
* timestamps fails the analyst instead of emitting a shape it did not measure.
|
|
1861
|
+
*/
|
|
1862
|
+
function teamShapeAnalyst() {
|
|
1863
|
+
return {
|
|
1864
|
+
id: TEAM_SHAPE_ANALYST_ID,
|
|
1865
|
+
description: "Classifies a supervision tree as single, governed swarm, fleet, partition, hive, star with sub-leads or flat star.",
|
|
1866
|
+
inputKind: "custom",
|
|
1867
|
+
cost: { kind: "deterministic" },
|
|
1868
|
+
version: TEAM_SHAPE_VERSION,
|
|
1869
|
+
async analyze(input) {
|
|
1870
|
+
const outcome = classifyTeamShape(input);
|
|
1871
|
+
if (!outcome.succeeded) throw new Error(`team-shape: ${outcome.error.message}`);
|
|
1872
|
+
const shape = outcome.value;
|
|
1873
|
+
if (shape.window === null) throw new Error(`team-shape: record for ${input.runRef} carries no event timestamps`);
|
|
1874
|
+
return [makeFinding({
|
|
1875
|
+
analyst_id: TEAM_SHAPE_ANALYST_ID,
|
|
1876
|
+
area: "team-shape",
|
|
1877
|
+
severity: "info",
|
|
1878
|
+
confidence: 1,
|
|
1879
|
+
subject: shape.rootId,
|
|
1880
|
+
claim: `Team shape is ${shape.label}`,
|
|
1881
|
+
claim_id: `team-shape:${shape.class}`,
|
|
1882
|
+
evidence_refs: [{
|
|
1883
|
+
kind: "artifact",
|
|
1884
|
+
uri: input.runRef
|
|
1885
|
+
}],
|
|
1886
|
+
layer: "team",
|
|
1887
|
+
scope: {
|
|
1888
|
+
population: [input.runRef],
|
|
1889
|
+
n: 1,
|
|
1890
|
+
window: shape.window
|
|
1891
|
+
},
|
|
1892
|
+
method: {
|
|
1893
|
+
kind: "computed",
|
|
1894
|
+
name: TEAM_SHAPE_ANALYST_ID,
|
|
1895
|
+
version: TEAM_SHAPE_VERSION
|
|
1896
|
+
},
|
|
1897
|
+
metadata: {
|
|
1898
|
+
class: shape.class,
|
|
1899
|
+
depth: shape.depth,
|
|
1900
|
+
qualifier: shape.qualifier,
|
|
1901
|
+
features: shape.features,
|
|
1902
|
+
checks: shape.checks,
|
|
1903
|
+
edges: {
|
|
1904
|
+
command: shape.graph.command.length,
|
|
1905
|
+
governance: shape.graph.governance.length,
|
|
1906
|
+
communication: shape.graph.communication.length
|
|
1907
|
+
},
|
|
1908
|
+
invalid_rows: shape.invalidRows
|
|
1909
|
+
}
|
|
1910
|
+
})];
|
|
1911
|
+
}
|
|
1912
|
+
};
|
|
1913
|
+
}
|
|
1914
|
+
//#endregion
|
|
380
1915
|
//#region src/analyst/trace-questions.ts
|
|
381
1916
|
const QUESTION_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u;
|
|
382
1917
|
const DEFINITION_VERSION = "1.0.0";
|
|
@@ -582,6 +2117,6 @@ function errorText(error) {
|
|
|
582
2117
|
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
583
2118
|
}
|
|
584
2119
|
//#endregion
|
|
585
|
-
export { AGENT_RX_UPSTREAM_REVISION, ANALYST_BENCHMARK_COST_LEDGER_FILE, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, ANALYST_BENCHMARK_IMPLEMENTATION_FILES, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, ANALYST_BENCHMARK_MANIFEST_FILE, ANALYST_BENCHMARK_OBSERVATIONS_FILE, AnalystExpressivenessError, AnalystRegistry, CODE_TRACE_BENCH_ANALYST_PROMPT, CONTROL_INTEGRITY_ANALYST, ControlIntegrityAnalyst, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, ExactAnalystRunExecutionError, FAILURE_MODE_KIND_SPEC, FINDING_SUBJECT_KINDS, FINDING_SUBJECT_SYNTAX, FindingsStore, IMPROVEMENT_KIND_SPEC, KIND_EXPECTED_SUBJECTS, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, MAX_INCORRECT_BLOCKS, MAX_INCORRECT_BLOCK_STEPS, RAW_FINDING_SCHEMA_PROMPT, RawAnalystFindingSchema, SKILL_USAGE_ANALYST, adaptPublicBenchmarkFindings, agentRxBenchmarkCase, agentRxPredictionsToFindings, analystBenchmarkDependencyLockDigest, analystBenchmarkImplementationDigest, analystDefinitionAsymmetries, analystDefinitionProtocolSha256, analystInstructionsOverrideFromText, analystUsageReceiptFromPrimeUsage, appendVerificationArtifactsToOtlp, askTraceQuestions, assertProposalFindings, behavioralAnalyst, bindAnalyst, buildDefaultAnalystRegistry, buildPrimePrompt, buildPrimeRepairPrompt, buildSkillUsageReport, buildTraceToolsForGroup, citedSpansOnly, codeTraceBenchCase, codeTracerPredictionsToFindings, coerceJson, compareAnalystRunners, computeFindingId, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createPrimeBenchmarkRunner, createPublicBenchmarkDirectRunner, createPublicBenchmarkRlmRunner, createSemanticConceptJudgeAdapter, createTraceAnalyst, decodeReplyRows, defaultIsMaterial, defineCustomAnalyst, defineTraceAnalyst, deriveEfficiencyFindings, diffFindings, effectiveAnalystProtocolSha256, emitControlIntegrityFindings, emitSkillUsageFindings, emptyPrimeRawUsage, emptyPublicBenchmarkRunner, evidenceRefsFromRawFinding, expandCodeTraceFailureBlocks, extractPrimeJsonObject, findingSubjectGrammarPromptFor, isProposalFinding, loadCodeTraceVerificationArtifacts, loadPublicBenchmarkRows, makeFinding, makeProposalFinding, mergePrimeRawUsage, nodeHttpPrimeBridgeTransport, normalizeAgentRxCategory, normalizeBenchmarkLabel, normalizePrimeUsage, parseFindingSubject, parseRawFinding, parseVerificationOutcome, preparePublicAnalystBenchmark, primeAnalystProtocolSha256, primeCodeTraceAnalystDefinition, primeProtocolSha256, primeReplyDefect, projectPrimeTrajectory, publicBenchmarkDistributions, publicBenchmarkProtocolSha256, publicBenchmarkRlmInstructions, publicBenchmarkSelectionReport, publicBenchmarkSystemPrompt, publicDirectAnalystDefinition, publicRlmAnalystDefinition, readAnalystBenchmarkArtifact, readAnalystInstructionsOverride, registryBenchmarkRunner, renderAgentRxCalibrationMarkdown, renderAnalystBenchmarkMarkdown, renderCodeTraceCalibrationMarkdown, renderFindingSubject, renderPriorFindings, resolveTraceAnalystLimits, rlmEngineLimits, roundAgentRxStep, runAnalystBenchmark, runAnalystBenchmarkCommand, runPrimeExchange, runTraceAnalyst, scoreAnalystFindings, selectPublicBenchmarkRows, stripCodeFences, summarizeAgentRxCalibration, summarizeAnalystBenchmarkRunner, summarizeCodeTraceCalibration, traceStoreEvidenceResolver };
|
|
2120
|
+
export { AGENT_RX_FAILURE_MODE_MAP, AGENT_RX_UPSTREAM_REVISION, ANALYST_BENCHMARK_COST_LEDGER_FILE, ANALYST_BENCHMARK_DEPENDENCY_LOCK_DIGEST_ALGORITHM, ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES, ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256, ANALYST_BENCHMARK_EVIDENCE_DEPENDENCY_LOCK_SHA256, ANALYST_BENCHMARK_EVIDENCE_IMPLEMENTATION_SHA256, ANALYST_BENCHMARK_IMPLEMENTATION_DIGEST_ALGORITHM, ANALYST_BENCHMARK_IMPLEMENTATION_FILES, ANALYST_BENCHMARK_IMPLEMENTATION_SHA256, ANALYST_BENCHMARK_LOCAL_RECEIPT_FILE, ANALYST_BENCHMARK_MANIFEST_FILE, ANALYST_BENCHMARK_OBSERVATIONS_FILE, ANALYST_FINDING_SCHEMA_VERSIONS, AnalystExpressivenessError, AnalystFindingSchema, AnalystRegistry, CODE_TRACE_BENCH_ANALYST_PROMPT, CONTROL_INTEGRITY_ANALYST, ControlIntegrityAnalyst, DEFAULT_MAX_VERIFICATION_ARTIFACT_BYTES, DEFAULT_TRACE_ANALYST_KINDS, DEFAULT_TRACE_ANALYST_OUTPUT_TOKENS, DOMAIN_TAXONOMY_SCHEMA, ExactAnalystRunExecutionError, FAILURE_MODES_ANALYST_ID, FAILURE_MODES_MIN_KAPPA, FAILURE_MODES_QUESTIONS, FAILURE_MODES_VERSION, FAILURE_MODE_KIND_SPEC, FAILURE_MODE_LABELS, FINDING_LAYERS, FINDING_SCOPE_FIELDS, FINDING_SUBJECT_KINDS, FINDING_SUBJECT_SYNTAX, FindingsStore, IMPROVEMENT_KIND_SPEC, KIND_EXPECTED_SUBJECTS, KNOWLEDGE_GAP_KIND_SPEC, KNOWLEDGE_POISONING_KIND_SPEC, LAB_NODE_ID, MAX_INCORRECT_BLOCKS, MAX_INCORRECT_BLOCK_STEPS, NOT_OBSERVABLE, PROMOTE_ANALYST_ID, PROMOTE_VERSION, ProposalFindingSchema, RAW_FINDING_SCHEMA_PROMPT, ROLLUP_ANALYST_ID, ROLLUP_VERSION, RawAnalystFindingSchema, SKILL_USAGE_ANALYST, TAXONOMY_NODE_LAYERS, TEAM_SHAPE_ANALYST_ID, TEAM_SHAPE_CLASSES, TEAM_SHAPE_THRESHOLDS, TEAM_SHAPE_VERSION, UNKNOWN_LINEAGE, adaptPublicBenchmarkFindings, agentRxBenchmarkCase, agentRxPredictionsToFindings, analystBenchmarkDependencyLockDigest, analystBenchmarkImplementationDigest, analystDefinitionAsymmetries, analystDefinitionProtocolSha256, analystInstructionsOverrideFromText, analystUsageReceiptFromPrimeUsage, appendVerificationArtifactsToOtlp, askTraceQuestions, assertProposalFindings, attachToTaxonomy, behavioralAnalyst, bindAnalyst, buildDefaultAnalystRegistry, buildPrimePrompt, buildPrimeRepairPrompt, buildSkillUsageReport, buildTraceToolsForGroup, calibrateFailureModes, citedSpansOnly, classifyTeamShape, codeTraceBenchCase, codeTracerPredictionsToFindings, coerceJson, compareAnalystRunners, computeFindingId, createChatClient, createChatTraceEngine, createDspyRlmTraceEngine, createFailureModesAnalyst, createPrimeBenchmarkRunner, createPublicBenchmarkDirectRunner, createPublicBenchmarkRlmRunner, createSemanticConceptJudgeAdapter, createTraceAnalyst, decodeReplyRows, defaultIsMaterial, defineCustomAnalyst, defineDomainTaxonomy, defineTraceAnalyst, deriveEfficiencyFindings, diffFindings, diffFindingsOverTime, domainPathOf, effectiveAnalystProtocolSha256, emitControlIntegrityFindings, emitSkillUsageFindings, emptyPrimeRawUsage, emptyPublicBenchmarkRunner, evidenceRefsFromRawFinding, expandCodeTraceFailureBlocks, extractPrimeJsonObject, findingLayerRank, findingSubjectGrammarPromptFor, independentUnitCount, isProposalFinding, lineageRoots, loadCodeTraceVerificationArtifacts, loadFailureModeGold, loadPublicBenchmarkRows, lowestCommonAncestor, makeFinding, makeProposalFinding, mergeEvidenceUnits, mergePrimeRawUsage, nodeHttpPrimeBridgeTransport, normalizeAgentRxCategory, normalizeBenchmarkLabel, normalizePrimeUsage, openFindingsLedger, parseAnalystFinding, parseFailureModeGold, parseFindingSubject, parseRawFinding, parseVerificationOutcome, preparePublicAnalystBenchmark, primeAnalystProtocolSha256, primeCodeTraceAnalystDefinition, primeProtocolSha256, primeReplyDefect, projectPrimeTrajectory, promote, publicBenchmarkDistributions, publicBenchmarkProtocolSha256, publicBenchmarkRlmInstructions, publicBenchmarkSelectionReport, publicBenchmarkSystemPrompt, publicDirectAnalystDefinition, publicRlmAnalystDefinition, queryFindings, readAnalystBenchmarkArtifact, readAnalystInstructionsOverride, readFindingScope, registryBenchmarkRunner, renderAgentRxCalibrationMarkdown, renderAnalystBenchmarkMarkdown, renderCodeTraceCalibrationMarkdown, renderFindingSubject, renderPriorFindings, resolveDomainPath, resolveTraceAnalystLimits, rlmEngineLimits, rollupAnalyst, rollupFindings, roundAgentRxStep, runAnalystBenchmark, runAnalystBenchmarkCommand, runPrimeExchange, runTraceAnalyst, scoreAnalystFindings, selectPublicBenchmarkRows, stripCodeFences, summarizeAgentRxCalibration, summarizeAnalystBenchmarkRunner, summarizeCodeTraceCalibration, taxonomyAncestry, taxonomyLayerOf, taxonomyParentOf, teamShapeAnalyst, teamShapeInputFromSources, traceStoreEvidenceResolver };
|
|
586
2121
|
|
|
587
2122
|
//# sourceMappingURL=index.js.map
|