@tangle-network/agent-eval 0.183.0 → 0.185.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/{agent-profile-cell-s__adRnK.d.ts → agent-profile-cell-BHcB-mZX.d.ts} +8 -9
- package/dist/{agent-profile-cell-s__adRnK.d.ts.map → agent-profile-cell-BHcB-mZX.d.ts.map} +1 -1
- package/dist/{agent-profile-cell-Cv6UA-W_.js → agent-profile-cell-DcFW4vOc.js} +16 -15
- package/dist/agent-profile-cell-DcFW4vOc.js.map +1 -0
- package/dist/{agent-profile-CivaSsSy.d.ts → agent-profile-uAIQK3Nr.d.ts} +3 -3
- package/dist/{agent-profile-CivaSsSy.d.ts.map → agent-profile-uAIQK3Nr.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +11 -336
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +10 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyze-runs-CrYhqXop.d.ts +75 -0
- package/dist/analyze-runs-CrYhqXop.d.ts.map +1 -0
- package/dist/{define-agent-eval-CneiWyuA.js → analyze-runs-sTygaDyD.js} +6 -813
- package/dist/analyze-runs-sTygaDyD.js.map +1 -0
- package/dist/{benchmark-DQKzykkO.js → benchmark-BeQJyrRC.js} +3 -3
- package/dist/{benchmark-DQKzykkO.js.map → benchmark-BeQJyrRC.js.map} +1 -1
- package/dist/{benchmark-command-NBrqNwgy.js → benchmark-command-kvq7yFY4.js} +17 -54
- package/dist/benchmark-command-kvq7yFY4.js.map +1 -0
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +6 -5
- package/dist/campaign/index.js +10 -9
- package/dist/{campaign-BwcEBC91.js → campaign-eJgtRs-W.js} +16 -15
- package/dist/{campaign-BwcEBC91.js.map → campaign-eJgtRs-W.js.map} +1 -1
- package/dist/{campaign-evidence-PYPUftRy.js → campaign-evidence-BsrCmBSE.js} +7 -258
- package/dist/campaign-evidence-BsrCmBSE.js.map +1 -0
- package/dist/{canonical-DPyQ_rpt.js → canonical-BsZVgBhW.js} +2 -4
- package/dist/canonical-BsZVgBhW.js.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/{client-kh2jOjTK.d.ts → client-Bc8LANCG.d.ts} +4 -4
- package/dist/{client-kh2jOjTK.d.ts.map → client-Bc8LANCG.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +15 -284
- package/dist/contract/index.js.map +1 -1
- package/dist/{default-registry-BwDSWVzg.d.ts → default-registry-CXv9kHMt.d.ts} +6 -6
- package/dist/{default-registry-BwDSWVzg.d.ts.map → default-registry-CXv9kHMt.d.ts.map} +1 -1
- package/dist/{default-registry-C0-N4PkG.js → default-registry-xiHrldW8.js} +6 -4
- package/dist/{default-registry-C0-N4PkG.js.map → default-registry-xiHrldW8.js.map} +1 -1
- package/dist/define-agent-eval-D4uumjl3.js +813 -0
- package/dist/define-agent-eval-D4uumjl3.js.map +1 -0
- package/dist/{define-agent-eval-CwOWWQt_.d.ts → define-agent-eval-DE67ECH1.d.ts} +7 -7
- package/dist/{define-agent-eval-CwOWWQt_.d.ts.map → define-agent-eval-DE67ECH1.d.ts.map} +1 -1
- package/dist/diagnosis/index.d.ts +319 -0
- package/dist/diagnosis/index.d.ts.map +1 -0
- package/dist/diagnosis/index.js +1328 -0
- package/dist/diagnosis/index.js.map +1 -0
- package/dist/{dspy-rlm-engine-8pgBSiy7.js → dspy-rlm-engine-BE9lCzzL.js} +3 -3
- package/dist/{dspy-rlm-engine-8pgBSiy7.js.map → dspy-rlm-engine-BE9lCzzL.js.map} +1 -1
- package/dist/{engine-DS1cysJy.d.ts → engine-cXUK-WHS.d.ts} +5 -77
- package/dist/engine-cXUK-WHS.d.ts.map +1 -0
- package/dist/equal-terms-Blx9Zk5I.js +25 -0
- package/dist/equal-terms-Blx9Zk5I.js.map +1 -0
- package/dist/{eval-campaign-CCEkwzoH.js → eval-campaign-XFHEpN86.js} +4 -4
- package/dist/{eval-campaign-CCEkwzoH.js.map → eval-campaign-XFHEpN86.js.map} +1 -1
- package/dist/evaluation.d.ts +75 -0
- package/dist/evaluation.d.ts.map +1 -0
- package/dist/evaluation.js +137 -0
- package/dist/evaluation.js.map +1 -0
- package/dist/{exact-types-BZDe0W2D.d.ts → exact-types-BEHN7tnq.d.ts} +2 -2
- package/dist/{exact-types-BZDe0W2D.d.ts.map → exact-types-BEHN7tnq.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +6 -5
- package/dist/experiment/index.js.map +1 -1
- package/dist/{experiment-tracker-C7PfnF4b.d.ts → experiment-tracker-URRGP9ek.d.ts} +2 -2
- package/dist/{experiment-tracker-C7PfnF4b.d.ts.map → experiment-tracker-URRGP9ek.d.ts.map} +1 -1
- package/dist/{external-optimizer-process-DIB4YWEe.js → external-optimizer-process-BbhvXTFx.js} +4 -4
- package/dist/{external-optimizer-process-DIB4YWEe.js.map → external-optimizer-process-BbhvXTFx.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-CjguXN2e.js → external-optimizer-subprocess-DMgmqTB6.js} +3 -3
- package/dist/{external-optimizer-subprocess-CjguXN2e.js.map → external-optimizer-subprocess-DMgmqTB6.js.map} +1 -1
- package/dist/{feedback-trajectory-CXmtITBo.d.ts → feedback-trajectory-CytTR7u4.d.ts} +3 -3
- package/dist/{feedback-trajectory-CXmtITBo.d.ts.map → feedback-trajectory-CytTR7u4.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/{index-DoykkxW0.d.ts → index-BQWfAFeH.d.ts} +11 -11
- package/dist/{index-DoykkxW0.d.ts.map → index-BQWfAFeH.d.ts.map} +1 -1
- package/dist/{index-DbG8QbH9.d.ts → index-C8_iU8KL.d.ts} +6 -6
- package/dist/{index-DbG8QbH9.d.ts.map → index-C8_iU8KL.d.ts.map} +1 -1
- package/dist/{index--_8_Nlpn.d.ts → index-CKZa_rnt.d.ts} +4 -121
- package/dist/index-CKZa_rnt.d.ts.map +1 -0
- package/dist/{index-Bp_6sj3x.d.ts → index-ww8736Am.d.ts} +20 -315
- package/dist/index-ww8736Am.d.ts.map +1 -0
- package/dist/index.d.ts +24 -22
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -24
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-D1qa0HWs.d.ts → insight-report-C3Bfl762.d.ts} +3 -3
- package/dist/{insight-report-D1qa0HWs.d.ts.map → insight-report-C3Bfl762.d.ts.map} +1 -1
- package/dist/jev-protocol.d.ts +74 -0
- package/dist/jev-protocol.d.ts.map +1 -0
- package/dist/jev-protocol.js +91 -0
- package/dist/jev-protocol.js.map +1 -0
- package/dist/jev.d.ts +131 -0
- package/dist/jev.d.ts.map +1 -0
- package/dist/jev.js +312 -0
- package/dist/jev.js.map +1 -0
- package/dist/{journal-Cs9f7385.js → journal-BuRLm6ho.js} +2 -2
- package/dist/{journal-Cs9f7385.js.map → journal-BuRLm6ho.js.map} +1 -1
- package/dist/{kind-factory-DxTzg32u.js → kind-factory-V79Hw0AU.js} +7 -84
- package/dist/kind-factory-V79Hw0AU.js.map +1 -0
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-FfaRgz9X.js → llm-judge-FWWhEb31.js} +11 -9
- package/dist/{llm-judge-FfaRgz9X.js.map → llm-judge-FWWhEb31.js.map} +1 -1
- package/dist/{matrix-CyhW-vgJ.d.ts → matrix-CPKsRY8Y.d.ts} +2 -2
- package/dist/{matrix-CyhW-vgJ.d.ts.map → matrix-CPKsRY8Y.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +54 -2
- package/dist/meta-eval/index.d.ts.map +1 -1
- package/dist/meta-eval/index.js +52 -3
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{mint-ySIIkKlV.js → mint-B4zgSAJW.js} +63 -14
- package/dist/mint-B4zgSAJW.js.map +1 -0
- package/dist/mint-BI14xehJ.d.ts +285 -0
- package/dist/mint-BI14xehJ.d.ts.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/otel-spans-CQY1pNiN.js +275 -0
- package/dist/otel-spans-CQY1pNiN.js.map +1 -0
- package/dist/otlp-attributes-BpC_blLF.js +83 -0
- package/dist/otlp-attributes-BpC_blLF.js.map +1 -0
- package/dist/{run-record-DPa2PeHx.js → paired-promotion-decision-ClJh_9nK.js} +2 -215
- package/dist/paired-promotion-decision-ClJh_9nK.js.map +1 -0
- package/dist/pipelines/index.js +2 -2
- package/dist/prime-bridge-transport-Z3cpKGoV.js +43 -0
- package/dist/prime-bridge-transport-Z3cpKGoV.js.map +1 -0
- package/dist/prime-protocol-CDBcL9NM.d.ts +331 -0
- package/dist/prime-protocol-CDBcL9NM.d.ts.map +1 -0
- package/dist/{prime-protocol-6tZTVsWm.js → prime-protocol-DcAK9pC8.js} +2 -23
- package/dist/prime-protocol-DcAK9pC8.js.map +1 -0
- package/dist/{produced-state-CGoVInrl.js → produced-state-CmB02tr2.js} +23 -19
- package/dist/{produced-state-CGoVInrl.js.map → produced-state-CmB02tr2.js.map} +1 -1
- package/dist/profile-cell.d.ts +1 -1
- package/dist/profile-cell.js +1 -1
- package/dist/{promotion-policy-BKFrl7CC.js → promotion-policy-DdrYWHDh.js} +3 -3
- package/dist/{promotion-policy-BKFrl7CC.js.map → promotion-policy-DdrYWHDh.js.map} +1 -1
- package/dist/{query-D1nLIKt7.js → query-DlV0pdx4.js} +2 -2
- package/dist/{query-D1nLIKt7.js.map → query-DlV0pdx4.js.map} +1 -1
- package/dist/{registry-BRbB6Y0v.d.ts → registry-D5Ly3GRV.d.ts} +3 -3
- package/dist/{registry-BRbB6Y0v.d.ts.map → registry-D5Ly3GRV.d.ts.map} +1 -1
- package/dist/{release-confidence-BcqeQTHW.d.ts → release-confidence-DsETiO4f.d.ts} +3 -3
- package/dist/{release-confidence-BcqeQTHW.d.ts.map → release-confidence-DsETiO4f.d.ts.map} +1 -1
- package/dist/{release-confidence-DMg8n18l.js → release-confidence-RanRUXRW.js} +2 -2
- package/dist/{release-confidence-DMg8n18l.js.map → release-confidence-RanRUXRW.js.map} +1 -1
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +2 -2
- package/dist/{researcher-64T49THL.d.ts → researcher-CJkH-6Ty.d.ts} +4 -4
- package/dist/{researcher-64T49THL.d.ts.map → researcher-CJkH-6Ty.d.ts.map} +1 -1
- package/dist/{reward-hacking-uzO_ihep.d.ts → reward-hacking-BIQ2BWLG.d.ts} +2 -2
- package/dist/{reward-hacking-uzO_ihep.d.ts.map → reward-hacking-BIQ2BWLG.d.ts.map} +1 -1
- package/dist/rl.d.ts +10 -6
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +22 -7
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +2 -2
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-D_54FZVu.js → rollout-C0Ifk3vK.js} +2 -2
- package/dist/{rollout-D_54FZVu.js.map → rollout-C0Ifk3vK.js.map} +1 -1
- package/dist/{rubric-predictive-validity-Bmj2_cll.d.ts → rubric-predictive-validity-CTU8y8Cv.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-Bmj2_cll.d.ts.map → rubric-predictive-validity-CTU8y8Cv.d.ts.map} +1 -1
- package/dist/run-record-CNNzWGpB.js +216 -0
- package/dist/run-record-CNNzWGpB.js.map +1 -0
- package/dist/{run-record-BiTWauyO.d.ts → run-record-D9wS35fK.d.ts} +2 -2
- package/dist/{run-record-BiTWauyO.d.ts.map → run-record-D9wS35fK.d.ts.map} +1 -1
- package/dist/{run-record-DualPTn2.js → run-record-tg_5H2IK.js} +2 -2
- package/dist/{run-record-DualPTn2.js.map → run-record-tg_5H2IK.js.map} +1 -1
- package/dist/search-ledger-types-D7KDVg-o.d.ts +302 -0
- package/dist/search-ledger-types-D7KDVg-o.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-_3Fp6Q_3.js → semantic-concept-judge-Cnt0TWp0.js} +4 -4
- package/dist/{semantic-concept-judge-_3Fp6Q_3.js.map → semantic-concept-judge-Cnt0TWp0.js.map} +1 -1
- package/dist/{sequential-D_xHrBZe.js → sequential-PgLEqcTJ.js} +3 -3
- package/dist/{sequential-D_xHrBZe.js.map → sequential-PgLEqcTJ.js.map} +1 -1
- package/dist/{server-BZQR1f0C.js → server-DmpcFiVK.js} +2 -2
- package/dist/{server-BZQR1f0C.js.map → server-DmpcFiVK.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DY1aQ-aL.js → skillopt-optimization-method-FSk8b5FQ.js} +8 -7
- package/dist/{skillopt-optimization-method-DY1aQ-aL.js.map → skillopt-optimization-method-FSk8b5FQ.js.map} +1 -1
- package/dist/{statistical-heldout-CpVd6FmY.d.ts → statistical-heldout-Bu4oNOJh.d.ts} +3 -3
- package/dist/{statistical-heldout-CpVd6FmY.d.ts.map → statistical-heldout-Bu4oNOJh.d.ts.map} +1 -1
- package/dist/statistical-heldout-CeWT_W7y.js +256 -0
- package/dist/statistical-heldout-CeWT_W7y.js.map +1 -0
- package/dist/{store-otlp-DAaEbMLf.js → store-otlp-2gYCy-v0.js} +2 -2
- package/dist/{store-otlp-DAaEbMLf.js.map → store-otlp-2gYCy-v0.js.map} +1 -1
- package/dist/{store-tool-spans-BZ8MPKFe.js → store-tool-spans-B1e6Auc2.js} +69 -6
- package/dist/store-tool-spans-B1e6Auc2.js.map +1 -0
- package/dist/{store-tool-spans-Dt-YdAuE.d.ts → store-tool-spans-BrdfPaVh.d.ts} +54 -6
- package/dist/store-tool-spans-BrdfPaVh.d.ts.map +1 -0
- package/dist/{summary-report-IGAOtgPd.js → summary-report-BMvou2oZ.js} +2 -2
- package/dist/{summary-report-IGAOtgPd.js.map → summary-report-BMvou2oZ.js.map} +1 -1
- package/dist/{summary-report-D1h4dlrK.d.ts → summary-report-BzFS5UhZ.d.ts} +2 -2
- package/dist/{summary-report-D1h4dlrK.d.ts.map → summary-report-BzFS5UhZ.d.ts.map} +1 -1
- package/dist/{task-failure-attributes-B_JjrOVo.js → task-failure-attributes-B34TiMBe.js} +2 -2
- package/dist/{task-failure-attributes-B_JjrOVo.js.map → task-failure-attributes-B34TiMBe.js.map} +1 -1
- package/dist/{tool-groups-B2bSNaJB.d.ts → tool-groups-eOOgYGaM.d.ts} +3 -3
- package/dist/tool-groups-eOOgYGaM.d.ts.map +1 -0
- package/dist/{tool-waste-C7MU9u1e.js → tool-waste-nbHPAxei.js} +2 -2
- package/dist/{tool-waste-C7MU9u1e.js.map → tool-waste-nbHPAxei.js.map} +1 -1
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/trace-repair/index.js +5 -4
- package/dist/trace-repair/index.js.map +1 -1
- package/dist/traces.d.ts +5 -5
- package/dist/traces.js +8 -7
- package/dist/traces.js.map +1 -1
- package/dist/trajectory-replay/index.js +1 -1
- package/dist/{types-CS0qk_Yp.d.ts → types-BTRcIqFr.d.ts} +3 -3
- package/dist/{types-CS0qk_Yp.d.ts.map → types-BTRcIqFr.d.ts.map} +1 -1
- package/dist/{types-D7gEdPoQ.d.ts → types-BjB2wZM9.d.ts} +2 -2
- package/dist/{types-D7gEdPoQ.d.ts.map → types-BjB2wZM9.d.ts.map} +1 -1
- package/dist/types-C8VBbiw6.js +91 -0
- package/dist/types-C8VBbiw6.js.map +1 -0
- package/dist/{types-BvZoPTGa.d.ts → types-c3ZRTbw9.d.ts} +3 -3
- package/dist/{types-BvZoPTGa.d.ts.map → types-c3ZRTbw9.d.ts.map} +1 -1
- package/dist/{types-DQ0e2E7y.js → usage-receipt-Cx3xNeyB.js} +2 -89
- package/dist/usage-receipt-Cx3xNeyB.js.map +1 -0
- package/dist/{verdict-BQ3pCFf8.js → verdict-DhPoXApk.js} +2 -2
- package/dist/{verdict-BQ3pCFf8.js.map → verdict-DhPoXApk.js.map} +1 -1
- package/dist/{verdict-cache-B3eCVQtY.js → verdict-cache-BQEsVy9n.js} +2 -2
- package/dist/{verdict-cache-B3eCVQtY.js.map → verdict-cache-BQEsVy9n.js.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/changes/evaluation-definition-identity.md +27 -0
- package/docs/changes/jev-evidence-review.md +18 -0
- package/docs/changes/jev-product-adoption.md +18 -0
- package/docs/changes/probability-policy-audit.md +81 -0
- package/docs/diagnosis.md +42 -0
- package/docs/jev-behavior-review.md +174 -0
- package/docs/jev-integrations.md +50 -0
- package/docs/jev.md +140 -0
- package/docs/produced-state.md +13 -0
- package/docs/profile-json-identity.md +33 -0
- package/package.json +22 -1
- package/dist/agent-profile-cell-Cv6UA-W_.js.map +0 -1
- package/dist/benchmark-command-NBrqNwgy.js.map +0 -1
- package/dist/campaign-evidence-PYPUftRy.js.map +0 -1
- package/dist/canonical-DPyQ_rpt.js.map +0 -1
- package/dist/define-agent-eval-CneiWyuA.js.map +0 -1
- package/dist/engine-DS1cysJy.d.ts.map +0 -1
- package/dist/exporters-D98GPi1n.d.ts +0 -161
- package/dist/exporters-D98GPi1n.d.ts.map +0 -1
- package/dist/index--_8_Nlpn.d.ts.map +0 -1
- package/dist/index-Bp_6sj3x.d.ts.map +0 -1
- package/dist/kind-factory-DxTzg32u.js.map +0 -1
- package/dist/mint-ySIIkKlV.js.map +0 -1
- package/dist/prime-protocol-6tZTVsWm.js.map +0 -1
- package/dist/run-record-DPa2PeHx.js.map +0 -1
- package/dist/store-tool-spans-BZ8MPKFe.js.map +0 -1
- package/dist/store-tool-spans-Dt-YdAuE.d.ts.map +0 -1
- package/dist/tool-groups-B2bSNaJB.d.ts.map +0 -1
- package/dist/types-DQ0e2E7y.js.map +0 -1
|
@@ -0,0 +1,1328 @@
|
|
|
1
|
+
import { n as summarizeExecution } from "../analyze-runs-sTygaDyD.js";
|
|
2
|
+
import { t as packageVersion } from "../package-version-D7lQHt_-.js";
|
|
3
|
+
import { c as primeProtocolSha256, d as runPrimeExchange, n as buildPrimePrompt } from "../prime-protocol-DcAK9pC8.js";
|
|
4
|
+
import { t as nodeHttpPrimeBridgeTransport } from "../prime-bridge-transport-Z3cpKGoV.js";
|
|
5
|
+
import { t as fromOtelSpans } from "../otel-spans-CQY1pNiN.js";
|
|
6
|
+
import { createHash, createHmac, randomBytes } from "node:crypto";
|
|
7
|
+
import { MODEL_ATTR_KEYS, SPAN_KINDS, TOOL_NAME_ATTR_KEYS, resolveSpanKind, validateTraceSpans } from "@tangle-network/agent-trace-contract";
|
|
8
|
+
//#region src/diagnosis/secret-filter.ts
|
|
9
|
+
const marker = (id) => `[REDACTED:${id}]`;
|
|
10
|
+
const whole = (id) => () => marker(id);
|
|
11
|
+
/**
|
|
12
|
+
* The trace-archive `SECRET_ASSIGNMENT` pattern (tangle-tools
|
|
13
|
+
* trace-archive/archive.py), with the prefix captured so the key name survives
|
|
14
|
+
* and only the value is replaced.
|
|
15
|
+
*/
|
|
16
|
+
const SECRET_ASSIGNMENT_PATTERN = /^([ \t]*(?:export[ \t]+)?["']?(?:[A-Za-z_][A-Za-z0-9_.-]*)?(?:key|token|secret|password|passwd|passphrase|credentials?)[A-Za-z0-9_]*["']?[ \t]*[=:][ \t]*["']?)(?!encrypted:)(?![$`])(?=[^\s"'#,]*[0-9])(?=[^\s"'#,]*[A-Za-z])[^\s"'#,]{16,}(["']?[ \t]*,?[ \t]*(?:#.*)?\r?)$/gim;
|
|
17
|
+
const DIAGNOSIS_SECRET_RULES = [
|
|
18
|
+
{
|
|
19
|
+
id: "pem-block",
|
|
20
|
+
pattern: /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z0-9 ]*PRIVATE KEY-----|$)/g,
|
|
21
|
+
replace: whole("pem-block")
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
id: "secret-assignment",
|
|
25
|
+
pattern: SECRET_ASSIGNMENT_PATTERN,
|
|
26
|
+
replace: (_match, prefix, suffix) => `${prefix}${marker("secret-assignment")}${suffix}`
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
id: "json-secret",
|
|
30
|
+
pattern: /("[A-Za-z0-9_.-]*(?:key|token|secret|password|passwd|passphrase|credentials?)[A-Za-z0-9_]*"[ \t]*:[ \t]*")(?!encrypted:)(?![$`])(?=[^"\s]*[0-9])(?=[^"\s]*[A-Za-z])[^"\s]{16,}(")/gi,
|
|
31
|
+
replace: (_match, prefix, suffix) => `${prefix}${marker("json-secret")}${suffix}`
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
id: "inline-assignment",
|
|
35
|
+
pattern: /\b([A-Za-z0-9_.-]*(?:key|token|secret|password|passwd|passphrase|credentials?)[A-Za-z0-9_]*["']?[ \t]*[=:][ \t]*["']?)(?!encrypted:)(?![$`[])(?=[^\s"'#,&;]*[0-9])(?=[^\s"'#,&;]*[A-Za-z])[^\s"'#,&;]{16,}/gi,
|
|
36
|
+
replace: (_match, prefix) => `${prefix}${marker("inline-assignment")}`
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
id: "anthropic-key",
|
|
40
|
+
pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}/g,
|
|
41
|
+
replace: whole("anthropic-key")
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
id: "openai-key",
|
|
45
|
+
pattern: /\bsk-[A-Za-z0-9_-]{20,}/g,
|
|
46
|
+
replace: whole("openai-key")
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
id: "github-token",
|
|
50
|
+
pattern: /\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,})/g,
|
|
51
|
+
replace: whole("github-token")
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
id: "slack-token",
|
|
55
|
+
pattern: /\bxox[abeoprs]-[A-Za-z0-9-]{10,}/g,
|
|
56
|
+
replace: whole("slack-token")
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: "aws-access-key",
|
|
60
|
+
pattern: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g,
|
|
61
|
+
replace: whole("aws-access-key")
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
id: "google-api-key",
|
|
65
|
+
pattern: /\bAIza[0-9A-Za-z_-]{35}/g,
|
|
66
|
+
replace: whole("google-api-key")
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
id: "jwt",
|
|
70
|
+
pattern: /\beyJ[A-Za-z0-9_-]{8,}\.eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g,
|
|
71
|
+
replace: whole("jwt")
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: "bearer",
|
|
75
|
+
pattern: /\b(Bearer)[ \t]+[A-Za-z0-9._~+/=-]{16,}/gi,
|
|
76
|
+
replace: (_match, word) => `${word} ${marker("bearer")}`
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
id: "url-credentials",
|
|
80
|
+
pattern: /\b([a-z][a-z0-9+.-]*:\/\/[^\s:/@]+:)[^\s@/]+@/gi,
|
|
81
|
+
replace: (_match, prefix) => `${prefix}${marker("url-credentials")}@`
|
|
82
|
+
}
|
|
83
|
+
];
|
|
84
|
+
/** Redact one string, adding the per-rule counts to `report`. */
|
|
85
|
+
function redactSecrets(input, report = emptySecretFilterReport(), rules = DIAGNOSIS_SECRET_RULES) {
|
|
86
|
+
let output = input;
|
|
87
|
+
for (const rule of rules) {
|
|
88
|
+
let hits = 0;
|
|
89
|
+
rule.pattern.lastIndex = 0;
|
|
90
|
+
output = output.replace(rule.pattern, (match, ...rest) => {
|
|
91
|
+
hits += 1;
|
|
92
|
+
const captured = rest.slice(0, countGroups(rule.pattern)).map((value) => typeof value === "string" ? value : "");
|
|
93
|
+
return rule.replace(match, ...captured);
|
|
94
|
+
});
|
|
95
|
+
if (hits > 0) {
|
|
96
|
+
report.redactionCount += hits;
|
|
97
|
+
report.byRule[rule.id] = (report.byRule[rule.id] ?? 0) + hits;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return output;
|
|
101
|
+
}
|
|
102
|
+
/** Whether any rule matches: the miner's equivalent of trace-archive `holds_secret`. */
|
|
103
|
+
function holdsSecret(input, rules = DIAGNOSIS_SECRET_RULES) {
|
|
104
|
+
return rules.some((rule) => {
|
|
105
|
+
rule.pattern.lastIndex = 0;
|
|
106
|
+
const found = rule.pattern.test(input);
|
|
107
|
+
rule.pattern.lastIndex = 0;
|
|
108
|
+
return found;
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
/** Redact every string leaf of a JSON-like value. Keys are kept; values are filtered. */
|
|
112
|
+
function redactSecretsDeep(value, report) {
|
|
113
|
+
if (typeof value === "string") return redactSecrets(value, report);
|
|
114
|
+
if (Array.isArray(value)) return value.map((item) => redactSecretsDeep(item, report));
|
|
115
|
+
if (value !== null && typeof value === "object") {
|
|
116
|
+
const next = {};
|
|
117
|
+
for (const [key, item] of Object.entries(value)) next[key] = redactSecretsDeep(item, report);
|
|
118
|
+
return next;
|
|
119
|
+
}
|
|
120
|
+
return value;
|
|
121
|
+
}
|
|
122
|
+
function emptySecretFilterReport() {
|
|
123
|
+
return {
|
|
124
|
+
redactionCount: 0,
|
|
125
|
+
byRule: {}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const groupCounts = /* @__PURE__ */ new WeakMap();
|
|
129
|
+
function countGroups(pattern) {
|
|
130
|
+
const cached = groupCounts.get(pattern);
|
|
131
|
+
if (cached !== void 0) return cached;
|
|
132
|
+
const count = (new RegExp(`${pattern.source}|`).exec("")?.length ?? 1) - 1;
|
|
133
|
+
groupCounts.set(pattern, count);
|
|
134
|
+
return count;
|
|
135
|
+
}
|
|
136
|
+
//#endregion
|
|
137
|
+
//#region src/diagnosis/spans.ts
|
|
138
|
+
/**
|
|
139
|
+
* Ingest for the diagnosis engine: flat spans in, filtered and normalized spans
|
|
140
|
+
* out. Nothing downstream of this file sees an unfiltered value.
|
|
141
|
+
*
|
|
142
|
+
* Input is the flat-span shape trace-adapters and the traces CLI emit, one
|
|
143
|
+
* object per span: `{trace_id, span_id, parent_span_id, name, kind,
|
|
144
|
+
* start_time, end_time, status | status_code, status_message, attributes,
|
|
145
|
+
* resource?}`. Times may be OTLP nanosecond strings, epoch milliseconds, or
|
|
146
|
+
* ISO-8601; the generic OTLP line reader treats every digit string as
|
|
147
|
+
* milliseconds, so time parsing is owned here.
|
|
148
|
+
*/
|
|
149
|
+
/**
|
|
150
|
+
* Attributes that carry prompt, response, or tool payload text. When content is
|
|
151
|
+
* not included they are removed before anything reads the span, rather than
|
|
152
|
+
* trusting pattern redaction over free-form prose.
|
|
153
|
+
*/
|
|
154
|
+
const CONTENT_ATTRIBUTE = /^(?:input\.value|output\.value|input|output|result|prompt|completion|text|thinking|message|messages|tool\.(?:input|output|arguments|result)|tool_input|tool_output|arguments|gen_ai\.(?:prompt|completion|input\.messages|output\.messages|system_instructions|tool\.call\.(?:arguments|result))(?:\..*)?|llm\.(?:input|output)_messages(?:\..*)?|llm\.prompts?(?:\..*)?|content|body|request|response|command)$|\.content$/;
|
|
155
|
+
const INPUT_ATTRIBUTE_KEYS = [
|
|
156
|
+
"input.value",
|
|
157
|
+
"input",
|
|
158
|
+
"tool.input",
|
|
159
|
+
"tool.arguments",
|
|
160
|
+
"tool_input",
|
|
161
|
+
"arguments",
|
|
162
|
+
"gen_ai.tool.call.arguments"
|
|
163
|
+
];
|
|
164
|
+
function isContentAttribute(key) {
|
|
165
|
+
return CONTENT_ATTRIBUTE.test(key);
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Filter secrets from every string, drop content attributes unless content is
|
|
169
|
+
* included, and normalize each span. Order matters: the input fingerprint is
|
|
170
|
+
* keyed and taken before redaction and the content drop, so repeated-call
|
|
171
|
+
* detection works on metadata-only runs without the payload itself surviving.
|
|
172
|
+
*/
|
|
173
|
+
function ingestSpans(raw, options) {
|
|
174
|
+
const secrets = {
|
|
175
|
+
redactionCount: 0,
|
|
176
|
+
byRule: {}
|
|
177
|
+
};
|
|
178
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
179
|
+
const seen = /* @__PURE__ */ new Set();
|
|
180
|
+
const spans = [];
|
|
181
|
+
let unreadable = 0;
|
|
182
|
+
let duplicateSpans = 0;
|
|
183
|
+
const traceOfSpan = /* @__PURE__ */ new Map();
|
|
184
|
+
const ambiguous = /* @__PURE__ */ new Set();
|
|
185
|
+
const digestKey = randomBytes(32);
|
|
186
|
+
for (const line of raw) {
|
|
187
|
+
if (line === null || typeof line !== "object" || Array.isArray(line)) {
|
|
188
|
+
unreadable += 1;
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const record = line;
|
|
192
|
+
const traceId = idField(record, "trace_id", "traceId");
|
|
193
|
+
const spanId = idField(record, "span_id", "spanId");
|
|
194
|
+
if (!traceId || !spanId) {
|
|
195
|
+
unreadable += 1;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const key = `${traceId}\u0000${spanId}`;
|
|
199
|
+
if (seen.has(key)) {
|
|
200
|
+
duplicateSpans += 1;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
seen.add(key);
|
|
204
|
+
const firstTrace = traceOfSpan.get(spanId);
|
|
205
|
+
if (firstTrace === void 0) traceOfSpan.set(spanId, traceId);
|
|
206
|
+
else if (firstTrace !== traceId) ambiguous.add(spanId);
|
|
207
|
+
const merged = mergedAttributes(record);
|
|
208
|
+
const inputDigest = digestInput(merged, digestKey);
|
|
209
|
+
const attributes = redactSecretsDeep(merged, secrets);
|
|
210
|
+
if (!options.contentIncluded) {
|
|
211
|
+
for (const key of Object.keys(attributes)) if (isContentAttribute(key)) {
|
|
212
|
+
delete attributes[key];
|
|
213
|
+
dropped.add(key);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const status = readStatus(record);
|
|
217
|
+
const statusMessage = status.message === null ? null : redactSecrets(status.message, secrets);
|
|
218
|
+
const toolName = firstString(attributes, TOOL_NAME_ATTR_KEYS);
|
|
219
|
+
const model = firstString(attributes, MODEL_ATTR_KEYS);
|
|
220
|
+
const name = redactSecrets(stringOr(record.name, "unknown"), secrets);
|
|
221
|
+
spans.push({
|
|
222
|
+
traceId,
|
|
223
|
+
spanId,
|
|
224
|
+
parentSpanId: idField(record, "parent_span_id", "parentSpanId") ?? null,
|
|
225
|
+
name,
|
|
226
|
+
kind: resolveKind(record.kind, name, attributes),
|
|
227
|
+
startMs: epochMillis(record.start_time ?? record.startTime ?? record.startTimeUnixNano),
|
|
228
|
+
endMs: epochMillis(record.end_time ?? record.endTime ?? record.endTimeUnixNano),
|
|
229
|
+
status: status.code,
|
|
230
|
+
statusMessage: options.contentIncluded ? statusMessage : truncate(statusMessage, 160),
|
|
231
|
+
toolName,
|
|
232
|
+
model,
|
|
233
|
+
inputDigest,
|
|
234
|
+
attributes
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
spans,
|
|
239
|
+
report: {
|
|
240
|
+
received: raw.length,
|
|
241
|
+
accepted: spans.length,
|
|
242
|
+
unreadable,
|
|
243
|
+
duplicateSpans,
|
|
244
|
+
ambiguousSpanIds: [...ambiguous].sort(),
|
|
245
|
+
secrets,
|
|
246
|
+
droppedAttributes: [...dropped].sort()
|
|
247
|
+
}
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
/** The contract shape `validateTraceSpans` reads. */
|
|
251
|
+
function toContractSpan(span) {
|
|
252
|
+
return {
|
|
253
|
+
trace_id: span.traceId,
|
|
254
|
+
span_id: span.spanId,
|
|
255
|
+
parent_span_id: span.parentSpanId,
|
|
256
|
+
name: span.name,
|
|
257
|
+
...span.kind === "UNKNOWN" ? {} : { kind: span.kind },
|
|
258
|
+
start_time: isoOrEmpty(span.startMs),
|
|
259
|
+
end_time: isoOrEmpty(span.endMs),
|
|
260
|
+
status: {
|
|
261
|
+
code: span.status === "ERROR" ? "STATUS_CODE_ERROR" : span.status === "OK" ? "STATUS_CODE_OK" : "STATUS_CODE_UNSET",
|
|
262
|
+
...span.statusMessage ? { message: span.statusMessage } : {}
|
|
263
|
+
},
|
|
264
|
+
attributes: span.attributes
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function durationMs(span) {
|
|
268
|
+
return span.startMs !== null && span.endMs !== null ? Math.max(0, span.endMs - span.startMs) : null;
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* Epoch milliseconds from OTLP nanoseconds, epoch milliseconds, epoch seconds,
|
|
272
|
+
* or ISO-8601. Digit strings are classified by length: 17 or more digits are
|
|
273
|
+
* nanoseconds, 15-16 microseconds, 12-14 milliseconds, and fewer are seconds.
|
|
274
|
+
*/
|
|
275
|
+
function epochMillis(value) {
|
|
276
|
+
if (typeof value === "number") return Number.isFinite(value) ? scaleEpoch(value, String(Math.trunc(value)).length) : null;
|
|
277
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
278
|
+
if (/^\d+$/.test(value)) {
|
|
279
|
+
if (value.length >= 17) return Number(BigInt(value) / 1000000n);
|
|
280
|
+
return scaleEpoch(Number(value), value.length);
|
|
281
|
+
}
|
|
282
|
+
const parsed = Date.parse(value);
|
|
283
|
+
return Number.isNaN(parsed) ? null : parsed;
|
|
284
|
+
}
|
|
285
|
+
function scaleEpoch(value, digits) {
|
|
286
|
+
if (digits >= 17) return Math.trunc(value / 1e6);
|
|
287
|
+
if (digits >= 15) return Math.trunc(value / 1e3);
|
|
288
|
+
if (digits >= 12) return value;
|
|
289
|
+
return value * 1e3;
|
|
290
|
+
}
|
|
291
|
+
function isoOrEmpty(ms) {
|
|
292
|
+
return ms === null ? "" : new Date(ms).toISOString();
|
|
293
|
+
}
|
|
294
|
+
function mergedAttributes(record) {
|
|
295
|
+
const out = {};
|
|
296
|
+
const resource = record.resource;
|
|
297
|
+
if (resource && typeof resource === "object" && !Array.isArray(resource)) {
|
|
298
|
+
const attrs = resource.attributes;
|
|
299
|
+
if (attrs && typeof attrs === "object" && !Array.isArray(attrs)) Object.assign(out, attrs);
|
|
300
|
+
}
|
|
301
|
+
const attrs = record.attributes;
|
|
302
|
+
if (attrs && typeof attrs === "object" && !Array.isArray(attrs)) Object.assign(out, attrs);
|
|
303
|
+
if (typeof record.scope_name === "string" && out["otel.scope.name"] === void 0) out["otel.scope.name"] = record.scope_name;
|
|
304
|
+
return out;
|
|
305
|
+
}
|
|
306
|
+
function readStatus(record) {
|
|
307
|
+
let code = record.status_code;
|
|
308
|
+
let message = record.status_message;
|
|
309
|
+
const status = record.status;
|
|
310
|
+
if (status && typeof status === "object" && !Array.isArray(status)) {
|
|
311
|
+
const object = status;
|
|
312
|
+
code = object.code ?? code;
|
|
313
|
+
message = object.message ?? message;
|
|
314
|
+
}
|
|
315
|
+
const text = typeof code === "string" ? code.toUpperCase() : code;
|
|
316
|
+
return {
|
|
317
|
+
code: text === 2 || text === "STATUS_CODE_ERROR" || text === "ERROR" || text === "2" ? "ERROR" : text === 1 || text === "STATUS_CODE_OK" || text === "OK" || text === "1" ? "OK" : "UNSET",
|
|
318
|
+
message: typeof message === "string" && message.length > 0 ? message : null
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* The contract's classifier decides the kind, so the capability table and the
|
|
323
|
+
* findings count the same tool and LLM spans. A top-level `kind` counts as a
|
|
324
|
+
* declaration only when it is a contract kind; OTLP words such as
|
|
325
|
+
* SPAN_KIND_INTERNAL fall through to the attribute and name signals.
|
|
326
|
+
*/
|
|
327
|
+
function resolveKind(kind, name, attributes) {
|
|
328
|
+
const declared = typeof kind === "string" && SPAN_KINDS.includes(kind.toUpperCase()) ? kind.toUpperCase() : void 0;
|
|
329
|
+
return resolveSpanKind({
|
|
330
|
+
...declared ? { kind: declared } : {},
|
|
331
|
+
name,
|
|
332
|
+
attributes
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
function digestInput(attributes, key) {
|
|
336
|
+
for (const name of INPUT_ATTRIBUTE_KEYS) {
|
|
337
|
+
const value = attributes[name];
|
|
338
|
+
if (value === void 0 || value === null || value === "") continue;
|
|
339
|
+
const text = typeof value === "string" ? value : JSON.stringify(value);
|
|
340
|
+
return createHmac("sha256", key).update(text).digest("hex").slice(0, 16);
|
|
341
|
+
}
|
|
342
|
+
return null;
|
|
343
|
+
}
|
|
344
|
+
function idField(record, snake, camel) {
|
|
345
|
+
const value = record[snake] ?? record[camel];
|
|
346
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
347
|
+
}
|
|
348
|
+
function firstString(attributes, keys) {
|
|
349
|
+
for (const key of keys) {
|
|
350
|
+
const value = attributes[key];
|
|
351
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
352
|
+
}
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
function stringOr(value, fallback) {
|
|
356
|
+
return typeof value === "string" && value.length > 0 ? value : fallback;
|
|
357
|
+
}
|
|
358
|
+
function truncate(value, max) {
|
|
359
|
+
if (value === null || value.length <= max) return value;
|
|
360
|
+
return `${value.slice(0, max)}…`;
|
|
361
|
+
}
|
|
362
|
+
/** The analysis each capability gates, as the report names it. */
|
|
363
|
+
const CAPABILITY_ANALYSES = {
|
|
364
|
+
"token-accounting": "token accounting",
|
|
365
|
+
"cost-attribution": "cost attribution",
|
|
366
|
+
"tool-usage": "tool failure rates and repeated-call detection",
|
|
367
|
+
"loop-convergence": "loop convergence (did round N+1 improve on round N)",
|
|
368
|
+
"tree-comparison": "arm-vs-arm comparison across branches of one run",
|
|
369
|
+
"steering-chain": "steering chain (which verdict caused which retry)",
|
|
370
|
+
"latency-analysis": "latency distribution"
|
|
371
|
+
};
|
|
372
|
+
/** Capabilities the engine reports but has no analysis for yet. */
|
|
373
|
+
const NOT_BUILT = /* @__PURE__ */ new Map([
|
|
374
|
+
["loop-convergence", "the engine has no round-over-round convergence analysis yet"],
|
|
375
|
+
["tree-comparison", "the engine has no arm-vs-arm comparison yet"],
|
|
376
|
+
["steering-chain", "the engine has no steering-chain analysis yet"]
|
|
377
|
+
]);
|
|
378
|
+
function runDeterministicPass(spans) {
|
|
379
|
+
const validation = validateTraceSpans(spans.map(toContractSpan));
|
|
380
|
+
const capabilities = {};
|
|
381
|
+
const skipped = [];
|
|
382
|
+
for (const capability of validation.capabilities) {
|
|
383
|
+
capabilities[capability.name] = capability.available ? { available: true } : {
|
|
384
|
+
available: false,
|
|
385
|
+
reason: capability.reason ?? "the trace contract validator gave no reason"
|
|
386
|
+
};
|
|
387
|
+
const analysis = CAPABILITY_ANALYSES[capability.name] ?? capability.name;
|
|
388
|
+
if (!capability.available) skipped.push({
|
|
389
|
+
analysis,
|
|
390
|
+
reason: capability.reason ?? "capability unavailable"
|
|
391
|
+
});
|
|
392
|
+
else if (NOT_BUILT.has(capability.name)) skipped.push({
|
|
393
|
+
analysis,
|
|
394
|
+
reason: NOT_BUILT.get(capability.name)
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
const traces = groupByTrace(spans);
|
|
398
|
+
const findings = [];
|
|
399
|
+
const tools = spans.filter((span) => span.kind === "TOOL");
|
|
400
|
+
if (tools.length > 0) {
|
|
401
|
+
findings.push(...toolFailureFindings(tools));
|
|
402
|
+
if (tools.filter((span) => span.inputDigest !== null).length > 0) findings.push(...repeatedCallFindings(traces));
|
|
403
|
+
else skipped.push({
|
|
404
|
+
analysis: "repeated identical tool calls",
|
|
405
|
+
reason: `none of ${tools.length} tool spans carries its input, so identical calls cannot be told apart`
|
|
406
|
+
});
|
|
407
|
+
} else skipped.push({
|
|
408
|
+
analysis: "tool failure rates",
|
|
409
|
+
reason: "no span resolves to kind TOOL"
|
|
410
|
+
});
|
|
411
|
+
findings.push(...failedRunFindings(traces));
|
|
412
|
+
return {
|
|
413
|
+
validation,
|
|
414
|
+
capabilities,
|
|
415
|
+
skipped,
|
|
416
|
+
facts: executionFacts(spans, traces.size),
|
|
417
|
+
findings,
|
|
418
|
+
window: timeWindow(spans)
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
function groupByTrace(spans) {
|
|
422
|
+
const traces = /* @__PURE__ */ new Map();
|
|
423
|
+
for (const span of spans) {
|
|
424
|
+
const list = traces.get(span.traceId);
|
|
425
|
+
if (list) list.push(span);
|
|
426
|
+
else traces.set(span.traceId, [span]);
|
|
427
|
+
}
|
|
428
|
+
for (const list of traces.values()) list.sort(byStart);
|
|
429
|
+
return traces;
|
|
430
|
+
}
|
|
431
|
+
function toolFailureFindings(tools) {
|
|
432
|
+
const byTool = /* @__PURE__ */ new Map();
|
|
433
|
+
for (const span of tools) {
|
|
434
|
+
const name = span.toolName ?? span.name;
|
|
435
|
+
const list = byTool.get(name);
|
|
436
|
+
if (list) list.push(span);
|
|
437
|
+
else byTool.set(name, [span]);
|
|
438
|
+
}
|
|
439
|
+
const findings = [];
|
|
440
|
+
for (const [tool, calls] of byTool) {
|
|
441
|
+
const failed = calls.filter((span) => span.status === "ERROR");
|
|
442
|
+
if (calls.length < 3 || failed.length < 2) continue;
|
|
443
|
+
const rate = failed.length / calls.length;
|
|
444
|
+
const severity = rate >= .5 && failed.length >= 5 ? "high" : rate >= .2 ? "medium" : "low";
|
|
445
|
+
findings.push({
|
|
446
|
+
id: stableId("tool-failures", tool),
|
|
447
|
+
severity,
|
|
448
|
+
claim: `${failed.length} of ${calls.length} calls to the ${tool} tool failed.`,
|
|
449
|
+
consequence: `Each failed call spends a model turn noticing the error and retrying, so ${pct(rate)} of this tool's calls bought no progress.`,
|
|
450
|
+
measure: {
|
|
451
|
+
name: `${tool} tool failure rate`,
|
|
452
|
+
value: round(rate * 100),
|
|
453
|
+
unit: "%",
|
|
454
|
+
denominator: `${calls.length} calls to ${tool}`
|
|
455
|
+
},
|
|
456
|
+
evidence: failed.slice(0, 8).map((span) => span.spanId),
|
|
457
|
+
confidence: "observed",
|
|
458
|
+
recommendation: `Read the failed ${tool} calls' status messages and fix the most common cause, or give the agent the precondition it keeps missing.`
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
return findings.sort((a, b) => (b.measure?.value ?? 0) - (a.measure?.value ?? 0)).slice(0, 10);
|
|
462
|
+
}
|
|
463
|
+
function repeatedCallFindings(traces) {
|
|
464
|
+
const runs = [];
|
|
465
|
+
let digestCalls = 0;
|
|
466
|
+
for (const spans of traces.values()) {
|
|
467
|
+
let current = [];
|
|
468
|
+
for (const span of spans) {
|
|
469
|
+
if (span.kind !== "TOOL" || span.inputDigest === null) continue;
|
|
470
|
+
digestCalls += 1;
|
|
471
|
+
const last = current[current.length - 1];
|
|
472
|
+
if (last && last.inputDigest === span.inputDigest && (last.toolName ?? last.name) === (span.toolName ?? span.name)) current.push(span);
|
|
473
|
+
else {
|
|
474
|
+
if (current.length >= 3) runs.push(current);
|
|
475
|
+
current = [span];
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if (current.length >= 3) runs.push(current);
|
|
479
|
+
}
|
|
480
|
+
if (runs.length === 0) return [];
|
|
481
|
+
const wasted = runs.reduce((sum, run) => sum + run.length - 1, 0);
|
|
482
|
+
const tracesAffected = new Set(runs.map((run) => run[0].traceId)).size;
|
|
483
|
+
runs.sort((a, b) => b.length - a.length);
|
|
484
|
+
const evidence = runs.flatMap((run) => run.slice(0, 2).map((span) => span.spanId)).slice(0, 8);
|
|
485
|
+
const rate = wasted / digestCalls;
|
|
486
|
+
return [{
|
|
487
|
+
id: stableId("repeated-calls", String(runs.length)),
|
|
488
|
+
severity: rate >= .1 ? "high" : rate >= .03 ? "medium" : "low",
|
|
489
|
+
claim: `The agent made the same tool call with the same input 3 or more times with no other tool call in between, ${runs.length} times across ${tracesAffected} runs; the longest streak was ${runs[0].length} calls.`,
|
|
490
|
+
consequence: `${wasted} calls repeated the previous tool call with the same input, which is work the agent paid for twice without new information.`,
|
|
491
|
+
measure: {
|
|
492
|
+
name: "repeated identical tool calls",
|
|
493
|
+
value: wasted,
|
|
494
|
+
unit: "calls",
|
|
495
|
+
denominator: `${digestCalls} tool calls with recorded input`
|
|
496
|
+
},
|
|
497
|
+
evidence,
|
|
498
|
+
confidence: "observed",
|
|
499
|
+
recommendation: "Detect a repeated identical call in the harness and stop or change strategy after the second attempt."
|
|
500
|
+
}];
|
|
501
|
+
}
|
|
502
|
+
function failedRunFindings(traces) {
|
|
503
|
+
const failedRoots = [];
|
|
504
|
+
for (const spans of traces.values()) {
|
|
505
|
+
const ids = new Set(spans.map((span) => span.spanId));
|
|
506
|
+
const root = spans.find((span) => span.parentSpanId === null || !ids.has(span.parentSpanId));
|
|
507
|
+
if (root?.status === "ERROR") failedRoots.push(root);
|
|
508
|
+
}
|
|
509
|
+
if (failedRoots.length === 0) return [];
|
|
510
|
+
const rate = failedRoots.length / traces.size;
|
|
511
|
+
return [{
|
|
512
|
+
id: stableId("failed-runs", String(traces.size)),
|
|
513
|
+
severity: rate >= .25 ? "critical" : rate >= .1 ? "high" : "medium",
|
|
514
|
+
claim: `${failedRoots.length} of ${traces.size} runs ended with an error on their root span.`,
|
|
515
|
+
consequence: "A run that ends in an error delivered nothing its caller can use, and any tokens it spent are lost.",
|
|
516
|
+
measure: {
|
|
517
|
+
name: "failed runs",
|
|
518
|
+
value: round(rate * 100),
|
|
519
|
+
unit: "%",
|
|
520
|
+
denominator: `${traces.size} runs`
|
|
521
|
+
},
|
|
522
|
+
evidence: failedRoots.slice(0, 8).map((span) => span.spanId),
|
|
523
|
+
confidence: "observed",
|
|
524
|
+
recommendation: "Group the failed runs by their root status message and fix the largest group first."
|
|
525
|
+
}];
|
|
526
|
+
}
|
|
527
|
+
function executionFacts(spans, runs) {
|
|
528
|
+
const spansByKind = {};
|
|
529
|
+
const models = /* @__PURE__ */ new Set();
|
|
530
|
+
let errorSpans = 0;
|
|
531
|
+
let toolCalls = 0;
|
|
532
|
+
for (const span of spans) {
|
|
533
|
+
spansByKind[span.kind] = (spansByKind[span.kind] ?? 0) + 1;
|
|
534
|
+
if (span.status === "ERROR") errorSpans += 1;
|
|
535
|
+
if (span.kind === "TOOL") toolCalls += 1;
|
|
536
|
+
if (span.model) models.add(span.model);
|
|
537
|
+
}
|
|
538
|
+
const facts = {
|
|
539
|
+
runs,
|
|
540
|
+
spans: spans.length,
|
|
541
|
+
spansByKind,
|
|
542
|
+
errorSpans,
|
|
543
|
+
toolCalls,
|
|
544
|
+
models: [...models].sort(),
|
|
545
|
+
execution: null
|
|
546
|
+
};
|
|
547
|
+
try {
|
|
548
|
+
facts.execution = summarizeExecution({ runs: fromOtelSpans({ spans: spans.map(toTraceSpanEvent) }) });
|
|
549
|
+
} catch (error) {
|
|
550
|
+
facts.executionError = error instanceof Error ? error.message : String(error);
|
|
551
|
+
}
|
|
552
|
+
return facts;
|
|
553
|
+
}
|
|
554
|
+
function toTraceSpanEvent(span) {
|
|
555
|
+
const attributes = {};
|
|
556
|
+
for (const [key, value] of Object.entries(span.attributes)) if (typeof value === "string" || typeof value === "boolean") attributes[key] = value;
|
|
557
|
+
else if (typeof value === "number" && Number.isFinite(value)) attributes[key] = value;
|
|
558
|
+
if (span.kind !== "UNKNOWN" && attributes["openinference.span.kind"] === void 0) attributes["openinference.span.kind"] = span.kind;
|
|
559
|
+
const start = span.startMs ?? 0;
|
|
560
|
+
const end = span.endMs ?? start;
|
|
561
|
+
return {
|
|
562
|
+
traceId: span.traceId,
|
|
563
|
+
spanId: span.spanId,
|
|
564
|
+
...span.parentSpanId ? { parentSpanId: span.parentSpanId } : {},
|
|
565
|
+
name: span.name,
|
|
566
|
+
startTimeUnixNano: `${BigInt(Math.trunc(start)) * 1000000n}`,
|
|
567
|
+
endTimeUnixNano: `${BigInt(Math.trunc(end)) * 1000000n}`,
|
|
568
|
+
attributes,
|
|
569
|
+
status: {
|
|
570
|
+
code: span.status,
|
|
571
|
+
...span.statusMessage ? { message: span.statusMessage } : {}
|
|
572
|
+
}
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
function timeWindow(spans) {
|
|
576
|
+
let first = Number.POSITIVE_INFINITY;
|
|
577
|
+
let last = Number.NEGATIVE_INFINITY;
|
|
578
|
+
for (const span of spans) {
|
|
579
|
+
if (span.startMs !== null) first = Math.min(first, span.startMs);
|
|
580
|
+
const end = span.endMs ?? span.startMs;
|
|
581
|
+
if (end !== null) last = Math.max(last, end);
|
|
582
|
+
}
|
|
583
|
+
if (!Number.isFinite(first) || !Number.isFinite(last)) return "unknown";
|
|
584
|
+
return `${new Date(first).toISOString()}/${new Date(last).toISOString()}`;
|
|
585
|
+
}
|
|
586
|
+
function byStart(a, b) {
|
|
587
|
+
return (a.startMs ?? 0) - (b.startMs ?? 0) || (durationMs(b) ?? 0) - (durationMs(a) ?? 0);
|
|
588
|
+
}
|
|
589
|
+
function stableId(prefix, ...parts) {
|
|
590
|
+
return `${prefix}-${createHash("sha256").update(parts.join("\0")).digest("hex").slice(0, 12)}`;
|
|
591
|
+
}
|
|
592
|
+
function pct(rate) {
|
|
593
|
+
return `${round(rate * 100)}%`;
|
|
594
|
+
}
|
|
595
|
+
function round(value) {
|
|
596
|
+
return Math.round(value * 10) / 10;
|
|
597
|
+
}
|
|
598
|
+
//#endregion
|
|
599
|
+
//#region src/diagnosis/findings.ts
|
|
600
|
+
/**
|
|
601
|
+
* The diagnosis findings document: what the engine returns and what the
|
|
602
|
+
* diagnosis report renders. It mirrors the kit's JSON Schema
|
|
603
|
+
* (`https://tangle.tools/schemas/diagnosis-findings-v1.json`, owned by the
|
|
604
|
+
* traces repository's diagnosis kit) field for field, including its closed
|
|
605
|
+
* objects, so a document that passes {@link validateDiagnosisFindings} passes
|
|
606
|
+
* that schema.
|
|
607
|
+
*/
|
|
608
|
+
const DIAGNOSIS_FINDINGS_SCHEMA_ID = "https://tangle.tools/schemas/diagnosis-findings-v1.json";
|
|
609
|
+
const SEVERITIES$1 = /* @__PURE__ */ new Set([
|
|
610
|
+
"critical",
|
|
611
|
+
"high",
|
|
612
|
+
"medium",
|
|
613
|
+
"low"
|
|
614
|
+
]);
|
|
615
|
+
const SEVERITY_RANK = {
|
|
616
|
+
critical: 0,
|
|
617
|
+
high: 1,
|
|
618
|
+
medium: 2,
|
|
619
|
+
low: 3
|
|
620
|
+
};
|
|
621
|
+
/**
|
|
622
|
+
* Structural check against the v1 schema plus the two rules the schema states
|
|
623
|
+
* in prose: an unavailable capability carries a reason, and a measure carries a
|
|
624
|
+
* denominator. When `spanIds` is given, every evidence id must resolve to it.
|
|
625
|
+
* Returns the list of defects; empty means valid.
|
|
626
|
+
*/
|
|
627
|
+
function validateDiagnosisFindings(document, options = {}) {
|
|
628
|
+
const errors = [];
|
|
629
|
+
if (!isObject(document)) return ["document is not an object"];
|
|
630
|
+
closed(document, [
|
|
631
|
+
"schemaVersion",
|
|
632
|
+
"subject",
|
|
633
|
+
"coverage",
|
|
634
|
+
"findings"
|
|
635
|
+
], "", errors);
|
|
636
|
+
if (document.schemaVersion !== 1) errors.push("schemaVersion must be 1");
|
|
637
|
+
const subject = document.subject;
|
|
638
|
+
if (!isObject(subject)) errors.push("subject must be an object");
|
|
639
|
+
else {
|
|
640
|
+
closed(subject, [
|
|
641
|
+
"label",
|
|
642
|
+
"runCount",
|
|
643
|
+
"window",
|
|
644
|
+
"contentIncluded"
|
|
645
|
+
], "subject", errors);
|
|
646
|
+
if (typeof subject.label !== "string") errors.push("subject.label must be a string");
|
|
647
|
+
if (!Number.isInteger(subject.runCount) || subject.runCount < 0) errors.push("subject.runCount must be a non-negative integer");
|
|
648
|
+
if (typeof subject.window !== "string") errors.push("subject.window must be a string");
|
|
649
|
+
if (subject.contentIncluded !== void 0 && typeof subject.contentIncluded !== "boolean") errors.push("subject.contentIncluded must be a boolean");
|
|
650
|
+
}
|
|
651
|
+
const coverage = document.coverage;
|
|
652
|
+
if (!isObject(coverage)) errors.push("coverage must be an object");
|
|
653
|
+
else {
|
|
654
|
+
closed(coverage, [
|
|
655
|
+
"capabilities",
|
|
656
|
+
"skipped",
|
|
657
|
+
"redaction"
|
|
658
|
+
], "coverage", errors);
|
|
659
|
+
if (!isObject(coverage.capabilities)) errors.push("coverage.capabilities must be an object");
|
|
660
|
+
else for (const [name, capability] of Object.entries(coverage.capabilities)) if (!isObject(capability) || typeof capability.available !== "boolean") errors.push(`coverage.capabilities.${name}.available must be a boolean`);
|
|
661
|
+
else if (!capability.available && !(typeof capability.reason === "string" && capability.reason)) errors.push(`coverage.capabilities.${name} is unavailable without a reason`);
|
|
662
|
+
if (!Array.isArray(coverage.skipped)) errors.push("coverage.skipped must be an array");
|
|
663
|
+
else coverage.skipped.forEach((entry, index) => {
|
|
664
|
+
if (!isObject(entry) || typeof entry.analysis !== "string" || typeof entry.reason !== "string") errors.push(`coverage.skipped[${index}] needs analysis and reason strings`);
|
|
665
|
+
});
|
|
666
|
+
if (coverage.redaction !== void 0) {
|
|
667
|
+
const redaction = coverage.redaction;
|
|
668
|
+
if (!isObject(redaction)) errors.push("coverage.redaction must be an object");
|
|
669
|
+
else {
|
|
670
|
+
if (!Number.isInteger(redaction.redactionCount) || redaction.redactionCount < 0) errors.push("coverage.redaction.redactionCount must be a non-negative integer");
|
|
671
|
+
if (redaction.byRule !== void 0 && !isObject(redaction.byRule)) errors.push("coverage.redaction.byRule must be an object");
|
|
672
|
+
if (redaction.droppedAttributes !== void 0 && !Array.isArray(redaction.droppedAttributes)) errors.push("coverage.redaction.droppedAttributes must be an array");
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
if (!Array.isArray(document.findings)) errors.push("findings must be an array");
|
|
677
|
+
else document.findings.forEach((finding, index) => {
|
|
678
|
+
const at = `findings[${index}]`;
|
|
679
|
+
if (!isObject(finding)) {
|
|
680
|
+
errors.push(`${at} must be an object`);
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
closed(finding, [
|
|
684
|
+
"id",
|
|
685
|
+
"severity",
|
|
686
|
+
"claim",
|
|
687
|
+
"consequence",
|
|
688
|
+
"measure",
|
|
689
|
+
"evidence",
|
|
690
|
+
"reproduce",
|
|
691
|
+
"confidence",
|
|
692
|
+
"recommendation"
|
|
693
|
+
], at, errors);
|
|
694
|
+
for (const key of [
|
|
695
|
+
"id",
|
|
696
|
+
"claim",
|
|
697
|
+
"consequence"
|
|
698
|
+
]) if (typeof finding[key] !== "string" || !finding[key]) errors.push(`${at}.${key} must be a non-empty string`);
|
|
699
|
+
if (!SEVERITIES$1.has(finding.severity)) errors.push(`${at}.severity is invalid`);
|
|
700
|
+
if (finding.confidence !== "observed" && finding.confidence !== "inferred") errors.push(`${at}.confidence must be observed or inferred`);
|
|
701
|
+
if (!Array.isArray(finding.evidence) || finding.evidence.length === 0) errors.push(`${at}.evidence must be a non-empty array`);
|
|
702
|
+
else for (const id of finding.evidence) if (typeof id !== "string") errors.push(`${at}.evidence holds a non-string`);
|
|
703
|
+
else if (options.spanIds && !options.spanIds.has(id)) errors.push(`${at}.evidence ${id} does not resolve`);
|
|
704
|
+
if (finding.measure !== void 0) {
|
|
705
|
+
const measure = finding.measure;
|
|
706
|
+
if (!isObject(measure) || typeof measure.name !== "string" || typeof measure.value !== "number" || !Number.isFinite(measure.value) || typeof measure.unit !== "string") errors.push(`${at}.measure needs name, finite value and unit`);
|
|
707
|
+
else if (!(typeof measure.denominator === "string" && measure.denominator)) errors.push(`${at}.measure has no denominator`);
|
|
708
|
+
}
|
|
709
|
+
for (const key of ["reproduce", "recommendation"]) if (finding[key] !== void 0 && typeof finding[key] !== "string") errors.push(`${at}.${key} must be a string`);
|
|
710
|
+
});
|
|
711
|
+
return errors;
|
|
712
|
+
}
|
|
713
|
+
function closed(value, allowed, at, errors) {
|
|
714
|
+
for (const key of Object.keys(value)) if (!allowed.includes(key)) errors.push(`${at ? `${at}.` : ""}${key} is not allowed`);
|
|
715
|
+
}
|
|
716
|
+
function isObject(value) {
|
|
717
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
718
|
+
}
|
|
719
|
+
//#endregion
|
|
720
|
+
//#region src/diagnosis/model-pass.ts
|
|
721
|
+
const MAX_ROWS = 12;
|
|
722
|
+
const MAX_STRING = 400;
|
|
723
|
+
const SEVERITIES = /* @__PURE__ */ new Set([
|
|
724
|
+
"critical",
|
|
725
|
+
"high",
|
|
726
|
+
"medium",
|
|
727
|
+
"low"
|
|
728
|
+
]);
|
|
729
|
+
function modelQuestion(input) {
|
|
730
|
+
const focus = input.focus ? ` The owner's question: ${input.focus}` : "";
|
|
731
|
+
const projects = input.projects && input.projects.length > 0 ? ` The runs belong to these projects: ${input.projects.join(", ")}; set "project" on a row to one of these names when the run shows which one it concerns.` : "";
|
|
732
|
+
const withheld = input.contentIncluded ? "" : " Prompt, response and tool payload content was withheld at the owner's choice; do not report its absence as a defect, and reason from span names, kinds, order, timing, status and the remaining attributes.";
|
|
733
|
+
return input.subject === "internal" ? `Diagnose this coding-agent session from our own fleet: find concrete agent failures, wasted work, unsupported claims of success, and where the human operator's instructions caused trouble.${withheld}${projects}${focus}` : `Diagnose this agent run: find concrete failures, wasted work, and unsupported claims of success that cost the owner money, time, or correct results.${withheld}${projects}${focus}`;
|
|
734
|
+
}
|
|
735
|
+
function modelContractLines(subject) {
|
|
736
|
+
return [
|
|
737
|
+
"Reply with ONE fenced ```json block and nothing after it. The object has exactly two fields:",
|
|
738
|
+
" \"answer\": one paragraph (at most 400 characters) summarising what went wrong in this run, or that nothing did.",
|
|
739
|
+
` "rows": an array of at most ${MAX_ROWS} objects, each with "kind": ${subject === "internal" ? "\"finding\" | \"question\" | \"operator\"" : "\"finding\" | \"question\""}.`,
|
|
740
|
+
"A \"finding\" row: {\"kind\":\"finding\",\"severity\":\"critical|high|medium|low\",\"claim\":\"one sentence stating the defect\",\"consequence\":\"what it cost in money, time, failed work or escalations\",\"recommendation\":\"the fix\",\"evidence\":[\"<span_id>\", ...]}.",
|
|
741
|
+
"A \"question\" row: {\"kind\":\"question\",\"question\":\"a concrete question the owner should answer\",\"evidence\":[]}.",
|
|
742
|
+
...subject === "internal" ? ["An \"operator\" row: {\"kind\":\"operator\",\"claim\":\"one sentence critiquing how the human operator instructed or steered the agent\",\"evidence\":[\"<span_id>\", ...]}."] : [],
|
|
743
|
+
"Add \"project\": \"<repo or project name>\" to any row when the run shows which project it belongs to.",
|
|
744
|
+
"Every \"finding\" and \"operator\" row cites 1-8 span ids copied VERBATIM from the \"id\" fields of the trajectory. Rows citing ids that are not in the trajectory are discarded.",
|
|
745
|
+
"Keep every string under 400 characters. Report only what the trajectory shows; an empty rows array is a valid answer."
|
|
746
|
+
];
|
|
747
|
+
}
|
|
748
|
+
async function runModelPass(traces, ambiguousSpanIds, input, options, signal) {
|
|
749
|
+
const question = modelQuestion(input);
|
|
750
|
+
const contractLines = modelContractLines(input.subject);
|
|
751
|
+
const result = {
|
|
752
|
+
findings: [],
|
|
753
|
+
questions: [],
|
|
754
|
+
notes: [],
|
|
755
|
+
summaries: [],
|
|
756
|
+
rejected: [],
|
|
757
|
+
uncovered: [],
|
|
758
|
+
runs: [],
|
|
759
|
+
promptSha256: primeProtocolSha256({
|
|
760
|
+
question,
|
|
761
|
+
contractLines,
|
|
762
|
+
repairContractLines: contractLines,
|
|
763
|
+
limits: {
|
|
764
|
+
maxRows: MAX_ROWS,
|
|
765
|
+
maxString: MAX_STRING
|
|
766
|
+
}
|
|
767
|
+
})
|
|
768
|
+
};
|
|
769
|
+
const maxInlineChars = options.maxInlineChars ?? 4e5;
|
|
770
|
+
const attributeChars = options.attributeChars ?? 600;
|
|
771
|
+
const maxSegments = options.maxSegmentsPerRun ?? 3;
|
|
772
|
+
for (const { traceId, spans } of traces) {
|
|
773
|
+
const segments = segmentRun(spans, attributeChars, input.contentIncluded, maxInlineChars);
|
|
774
|
+
const chosen = segments.map((segment, index) => ({
|
|
775
|
+
...segment,
|
|
776
|
+
index
|
|
777
|
+
})).sort((a, b) => b.errors - a.errors || a.index - b.index).slice(0, maxSegments).sort((a, b) => a.index - b.index);
|
|
778
|
+
if (segments.length > chosen.length) result.uncovered.push({
|
|
779
|
+
traceId,
|
|
780
|
+
reason: `the run renders to ${segments.length} segments of at most ${maxInlineChars} characters; the model read the ${chosen.length} with the most errors (${chosen.map((segment) => segment.index + 1).join(", ")})`
|
|
781
|
+
});
|
|
782
|
+
for (const segment of chosen) {
|
|
783
|
+
const label = segments.length > 1 ? `${traceId} segment ${segment.index + 1}/${segments.length}` : traceId;
|
|
784
|
+
const ids = new Set(segment.spans.map((span) => span.spanId));
|
|
785
|
+
const outcome = await runPrimeExchange({
|
|
786
|
+
contract: {
|
|
787
|
+
rowsField: "rows",
|
|
788
|
+
contractLines,
|
|
789
|
+
repairContractLines: contractLines,
|
|
790
|
+
maxRows: MAX_ROWS,
|
|
791
|
+
decodeRow: (row) => decodeRow(row, input.subject)
|
|
792
|
+
},
|
|
793
|
+
prompt: buildPrimePrompt({
|
|
794
|
+
question,
|
|
795
|
+
contractLines,
|
|
796
|
+
trajectoryHeader: `TRAJECTORY (run ${label}; ${segment.spans.length} of the run's ${spans.length} spans, in start order; secrets redacted as [REDACTED:<kind>]):`,
|
|
797
|
+
renderedTrajectory: segment.rendered
|
|
798
|
+
}),
|
|
799
|
+
transport: options.transport,
|
|
800
|
+
url: options.url,
|
|
801
|
+
model: options.model,
|
|
802
|
+
timeoutMs: options.timeoutMs ?? 6e5,
|
|
803
|
+
repair: true,
|
|
804
|
+
...signal ? { signal } : {}
|
|
805
|
+
});
|
|
806
|
+
const delivery = {
|
|
807
|
+
segment: segment.index + 1,
|
|
808
|
+
segments: segments.length,
|
|
809
|
+
renderedChars: segment.rendered.length
|
|
810
|
+
};
|
|
811
|
+
if (!outcome.ok) {
|
|
812
|
+
result.runs.push({
|
|
813
|
+
traceId,
|
|
814
|
+
ok: false,
|
|
815
|
+
failure: `${label}: ${outcome.failure.kind}: ${outcome.failure.message}`,
|
|
816
|
+
delivery,
|
|
817
|
+
usage: outcome.usage,
|
|
818
|
+
rows: 0
|
|
819
|
+
});
|
|
820
|
+
continue;
|
|
821
|
+
}
|
|
822
|
+
for (const rejection of outcome.rejected) result.rejected.push({
|
|
823
|
+
traceId,
|
|
824
|
+
reason: `row ${rejection.index}: ${rejection.reason}`
|
|
825
|
+
});
|
|
826
|
+
if (outcome.overflow > 0) result.rejected.push({
|
|
827
|
+
traceId,
|
|
828
|
+
reason: `${outcome.overflow} valid rows beyond the ${MAX_ROWS}-row cap were dropped`
|
|
829
|
+
});
|
|
830
|
+
if (outcome.answer) result.summaries.push({
|
|
831
|
+
traceId,
|
|
832
|
+
answer: outcome.answer.slice(0, MAX_STRING)
|
|
833
|
+
});
|
|
834
|
+
acceptRows(result, outcome.rows, traceId, ids, ambiguousSpanIds);
|
|
835
|
+
result.runs.push({
|
|
836
|
+
traceId,
|
|
837
|
+
ok: true,
|
|
838
|
+
delivery,
|
|
839
|
+
usage: outcome.usage,
|
|
840
|
+
rows: outcome.rows.length
|
|
841
|
+
});
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
return result;
|
|
845
|
+
}
|
|
846
|
+
/** The topology reading: one exchange over the caller's run-graph summary, internal runs only. */
|
|
847
|
+
async function runTopologyPass(topology, runIds, options, signal) {
|
|
848
|
+
const contractLines = [
|
|
849
|
+
"Reply with ONE fenced ```json block and nothing after it. The object has exactly two fields:",
|
|
850
|
+
" \"answer\": one paragraph (at most 400 characters) on how the agent fleet was organised today.",
|
|
851
|
+
` "rows": at most ${MAX_ROWS} objects, each either {"kind":"topology","claim":"one sentence on a spawn, messaging, hand-off or resume pattern that helped or hurt","evidence":["<run id>", ...],"project":"<optional>"} or {"kind":"question","question":"...","project":"<optional>","evidence":[]}.`,
|
|
852
|
+
"Every \"topology\" row cites 1-8 run ids copied VERBATIM from the graph. Keep every string under 400 characters."
|
|
853
|
+
];
|
|
854
|
+
const rendered = JSON.stringify(topology);
|
|
855
|
+
const outcome = await runPrimeExchange({
|
|
856
|
+
contract: {
|
|
857
|
+
rowsField: "rows",
|
|
858
|
+
contractLines,
|
|
859
|
+
repairContractLines: contractLines,
|
|
860
|
+
maxRows: MAX_ROWS,
|
|
861
|
+
decodeRow: (row) => decodeRow(row, "topology")
|
|
862
|
+
},
|
|
863
|
+
prompt: buildPrimePrompt({
|
|
864
|
+
question: "Analyse the topology of this agent fleet's runs: who spawned whom, who messaged whom, hand-offs and resumes. Name patterns that wasted work or lost context, and what to change.",
|
|
865
|
+
contractLines,
|
|
866
|
+
trajectoryHeader: `RUN GRAPH (${runIds.size} runs):`,
|
|
867
|
+
renderedTrajectory: rendered.length > (options.maxInlineChars ?? 4e5) ? rendered.slice(0, options.maxInlineChars ?? 4e5) : rendered
|
|
868
|
+
}),
|
|
869
|
+
transport: options.transport,
|
|
870
|
+
url: options.url,
|
|
871
|
+
model: options.model,
|
|
872
|
+
timeoutMs: options.timeoutMs ?? 6e5,
|
|
873
|
+
repair: true,
|
|
874
|
+
...signal ? { signal } : {}
|
|
875
|
+
});
|
|
876
|
+
const notes = [];
|
|
877
|
+
const questions = [];
|
|
878
|
+
const rejected = [];
|
|
879
|
+
if (outcome.ok && outcome.overflow > 0) rejected.push({ reason: `${outcome.overflow} valid topology rows beyond the ${MAX_ROWS}-row cap were dropped` });
|
|
880
|
+
if (!outcome.ok) return {
|
|
881
|
+
notes,
|
|
882
|
+
questions,
|
|
883
|
+
rejected,
|
|
884
|
+
run: {
|
|
885
|
+
traceId: "topology",
|
|
886
|
+
ok: false,
|
|
887
|
+
failure: `${outcome.failure.kind}: ${outcome.failure.message}`,
|
|
888
|
+
usage: outcome.usage,
|
|
889
|
+
rows: 0
|
|
890
|
+
}
|
|
891
|
+
};
|
|
892
|
+
for (const row of outcome.rows) {
|
|
893
|
+
if (row.kind === "question") {
|
|
894
|
+
questions.push({
|
|
895
|
+
question: row.question,
|
|
896
|
+
...row.project ? { project: row.project } : {}
|
|
897
|
+
});
|
|
898
|
+
continue;
|
|
899
|
+
}
|
|
900
|
+
const cited = row.evidence.filter((id) => runIds.has(id));
|
|
901
|
+
if (cited.length === 0) {
|
|
902
|
+
rejected.push({ reason: `topology row "${(row.claim ?? "").slice(0, 80)}" cites no run id from the graph` });
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
notes.push({
|
|
906
|
+
kind: "topology",
|
|
907
|
+
text: row.claim,
|
|
908
|
+
evidence: cited,
|
|
909
|
+
...row.project ? { project: row.project } : {}
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
if (outcome.answer) notes.unshift({
|
|
913
|
+
kind: "topology",
|
|
914
|
+
text: outcome.answer.slice(0, MAX_STRING),
|
|
915
|
+
evidence: []
|
|
916
|
+
});
|
|
917
|
+
return {
|
|
918
|
+
notes,
|
|
919
|
+
questions,
|
|
920
|
+
rejected,
|
|
921
|
+
run: {
|
|
922
|
+
traceId: "topology",
|
|
923
|
+
ok: true,
|
|
924
|
+
usage: outcome.usage,
|
|
925
|
+
rows: outcome.rows.length
|
|
926
|
+
}
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
function acceptRows(result, rows, traceId, ids, ambiguousSpanIds) {
|
|
930
|
+
for (const row of rows) {
|
|
931
|
+
const unresolved = row.evidence.filter((id) => !ids.has(id));
|
|
932
|
+
const ambiguous = row.evidence.filter((id) => ambiguousSpanIds.has(id));
|
|
933
|
+
const cited = row.evidence.filter((id) => ids.has(id) && !ambiguousSpanIds.has(id));
|
|
934
|
+
if (unresolved.length > 0) result.rejected.push({
|
|
935
|
+
traceId,
|
|
936
|
+
reason: `${row.kind} row cites ids not in the segment it read: ${unresolved.slice(0, 5).join(", ")}`
|
|
937
|
+
});
|
|
938
|
+
if (ambiguous.length > 0) result.rejected.push({
|
|
939
|
+
traceId,
|
|
940
|
+
reason: `${row.kind} row cites ids shared by several runs: ${ambiguous.slice(0, 5).join(", ")}`
|
|
941
|
+
});
|
|
942
|
+
if (row.kind === "question") {
|
|
943
|
+
result.questions.push({
|
|
944
|
+
question: row.question,
|
|
945
|
+
traceId,
|
|
946
|
+
...row.project ? { project: row.project } : {}
|
|
947
|
+
});
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
if (unresolved.length > 0 || ambiguous.length > 0) {
|
|
951
|
+
result.rejected.push({
|
|
952
|
+
traceId,
|
|
953
|
+
reason: `${row.kind} row "${(row.claim ?? "").slice(0, 80)}" discarded: it cites invalid evidence`
|
|
954
|
+
});
|
|
955
|
+
continue;
|
|
956
|
+
}
|
|
957
|
+
if (cited.length === 0) {
|
|
958
|
+
result.rejected.push({
|
|
959
|
+
traceId,
|
|
960
|
+
reason: `${row.kind} row "${(row.claim ?? "").slice(0, 80)}" has no resolvable evidence`
|
|
961
|
+
});
|
|
962
|
+
continue;
|
|
963
|
+
}
|
|
964
|
+
if (row.kind === "operator") {
|
|
965
|
+
result.notes.push({
|
|
966
|
+
kind: "operator",
|
|
967
|
+
text: row.claim,
|
|
968
|
+
evidence: cited,
|
|
969
|
+
traceId,
|
|
970
|
+
...row.project ? { project: row.project } : {}
|
|
971
|
+
});
|
|
972
|
+
continue;
|
|
973
|
+
}
|
|
974
|
+
result.findings.push({
|
|
975
|
+
id: stableId("inferred", traceId, row.claim),
|
|
976
|
+
severity: row.severity,
|
|
977
|
+
claim: row.claim,
|
|
978
|
+
consequence: row.consequence,
|
|
979
|
+
evidence: cited,
|
|
980
|
+
confidence: "inferred",
|
|
981
|
+
...row.recommendation ? { recommendation: row.recommendation } : {}
|
|
982
|
+
});
|
|
983
|
+
}
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Split a run, in start order, into contiguous segments that each render
|
|
987
|
+
* within the inline budget. Inline is prime's only delivery, so a long run is
|
|
988
|
+
* read as whole segments rather than silently truncated; which segments were
|
|
989
|
+
* read and which were not is recorded by the caller.
|
|
990
|
+
*/
|
|
991
|
+
function segmentRun(spans, attributeChars, contentIncluded, maxInlineChars) {
|
|
992
|
+
const segments = [];
|
|
993
|
+
let current = [];
|
|
994
|
+
let parts = [];
|
|
995
|
+
let size = 2;
|
|
996
|
+
let errors = 0;
|
|
997
|
+
const flush = () => {
|
|
998
|
+
if (current.length === 0) return;
|
|
999
|
+
segments.push({
|
|
1000
|
+
spans: current,
|
|
1001
|
+
rendered: `[${parts.join(",")}]`,
|
|
1002
|
+
errors
|
|
1003
|
+
});
|
|
1004
|
+
current = [];
|
|
1005
|
+
parts = [];
|
|
1006
|
+
size = 2;
|
|
1007
|
+
errors = 0;
|
|
1008
|
+
};
|
|
1009
|
+
for (const span of spans) {
|
|
1010
|
+
let part = JSON.stringify(renderSpan(span, attributeChars, contentIncluded));
|
|
1011
|
+
if (part.length + 2 > maxInlineChars) part = JSON.stringify(renderSpan(span, 120, contentIncluded));
|
|
1012
|
+
if (size + part.length + 1 > maxInlineChars) flush();
|
|
1013
|
+
current.push(span);
|
|
1014
|
+
parts.push(part);
|
|
1015
|
+
size += part.length + 1;
|
|
1016
|
+
if (span.status === "ERROR") errors += 1;
|
|
1017
|
+
}
|
|
1018
|
+
flush();
|
|
1019
|
+
return segments;
|
|
1020
|
+
}
|
|
1021
|
+
function decodeRow(row, mode) {
|
|
1022
|
+
if (row === null || typeof row !== "object" || Array.isArray(row)) return {
|
|
1023
|
+
ok: false,
|
|
1024
|
+
reason: "row is not an object"
|
|
1025
|
+
};
|
|
1026
|
+
const value = row;
|
|
1027
|
+
const kind = value.kind;
|
|
1028
|
+
if (typeof kind !== "string" || !(mode === "internal" ? [
|
|
1029
|
+
"finding",
|
|
1030
|
+
"question",
|
|
1031
|
+
"operator"
|
|
1032
|
+
] : mode === "customer" ? ["finding", "question"] : ["topology", "question"]).includes(kind)) return {
|
|
1033
|
+
ok: false,
|
|
1034
|
+
reason: `kind ${String(kind)} is not allowed here`
|
|
1035
|
+
};
|
|
1036
|
+
const evidence = Array.isArray(value.evidence) ? value.evidence.filter((id) => typeof id === "string" && id.length > 0).slice(0, 8) : [];
|
|
1037
|
+
const project = shortString(value.project);
|
|
1038
|
+
if (kind === "question") {
|
|
1039
|
+
const question = shortString(value.question);
|
|
1040
|
+
if (!question) return {
|
|
1041
|
+
ok: false,
|
|
1042
|
+
reason: "question row has no question"
|
|
1043
|
+
};
|
|
1044
|
+
return {
|
|
1045
|
+
ok: true,
|
|
1046
|
+
row: {
|
|
1047
|
+
kind,
|
|
1048
|
+
question,
|
|
1049
|
+
evidence,
|
|
1050
|
+
...project ? { project } : {}
|
|
1051
|
+
}
|
|
1052
|
+
};
|
|
1053
|
+
}
|
|
1054
|
+
const claim = shortString(value.claim);
|
|
1055
|
+
if (!claim) return {
|
|
1056
|
+
ok: false,
|
|
1057
|
+
reason: `${kind} row has no claim`
|
|
1058
|
+
};
|
|
1059
|
+
if (kind !== "finding") return {
|
|
1060
|
+
ok: true,
|
|
1061
|
+
row: {
|
|
1062
|
+
kind,
|
|
1063
|
+
claim,
|
|
1064
|
+
evidence,
|
|
1065
|
+
...project ? { project } : {}
|
|
1066
|
+
}
|
|
1067
|
+
};
|
|
1068
|
+
const severity = value.severity;
|
|
1069
|
+
if (typeof severity !== "string" || !SEVERITIES.has(severity)) return {
|
|
1070
|
+
ok: false,
|
|
1071
|
+
reason: `severity ${String(severity)} is invalid`
|
|
1072
|
+
};
|
|
1073
|
+
const consequence = shortString(value.consequence);
|
|
1074
|
+
if (!consequence) return {
|
|
1075
|
+
ok: false,
|
|
1076
|
+
reason: "finding row has no consequence"
|
|
1077
|
+
};
|
|
1078
|
+
const recommendation = shortString(value.recommendation);
|
|
1079
|
+
return {
|
|
1080
|
+
ok: true,
|
|
1081
|
+
row: {
|
|
1082
|
+
kind: "finding",
|
|
1083
|
+
severity,
|
|
1084
|
+
claim,
|
|
1085
|
+
consequence,
|
|
1086
|
+
evidence,
|
|
1087
|
+
...recommendation ? { recommendation } : {},
|
|
1088
|
+
...project ? { project } : {}
|
|
1089
|
+
}
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
/** One span as the model reads it: the pivots, then the attributes that remain after filtering. */
|
|
1093
|
+
function renderSpan(span, attributeChars, contentIncluded) {
|
|
1094
|
+
const out = {
|
|
1095
|
+
id: span.spanId,
|
|
1096
|
+
parent: span.parentSpanId,
|
|
1097
|
+
kind: span.kind,
|
|
1098
|
+
name: span.name
|
|
1099
|
+
};
|
|
1100
|
+
if (span.toolName) out.tool = span.toolName;
|
|
1101
|
+
if (span.model) out.model = span.model;
|
|
1102
|
+
if (span.status !== "UNSET") out.status = span.status;
|
|
1103
|
+
if (span.statusMessage) out.status_message = clip(span.statusMessage, attributeChars);
|
|
1104
|
+
const duration = durationMs(span);
|
|
1105
|
+
if (duration !== null) out.ms = duration;
|
|
1106
|
+
const attrs = {};
|
|
1107
|
+
for (const [key, value] of Object.entries(span.attributes)) {
|
|
1108
|
+
if (SKIP_ATTRIBUTES.has(key)) continue;
|
|
1109
|
+
if (typeof value === "string") attrs[key] = clip(value, attributeChars);
|
|
1110
|
+
else if (typeof value === "number" || typeof value === "boolean") attrs[key] = value;
|
|
1111
|
+
else if (value !== null && value !== void 0) attrs[key] = clip(JSON.stringify(value), attributeChars);
|
|
1112
|
+
}
|
|
1113
|
+
if (Object.keys(attrs).length > 0) out.attrs = attrs;
|
|
1114
|
+
if (!contentIncluded) out.content = "withheld";
|
|
1115
|
+
return out;
|
|
1116
|
+
}
|
|
1117
|
+
/** Pivots already rendered above, and resource noise that repeats on every span. */
|
|
1118
|
+
const SKIP_ATTRIBUTES = /* @__PURE__ */ new Set([
|
|
1119
|
+
"openinference.span.kind",
|
|
1120
|
+
"tool.name",
|
|
1121
|
+
"gen_ai.tool.name",
|
|
1122
|
+
"llm.model_name",
|
|
1123
|
+
"gen_ai.request.model",
|
|
1124
|
+
"service.name",
|
|
1125
|
+
"agent.name",
|
|
1126
|
+
"source_file",
|
|
1127
|
+
"otel.scope.name",
|
|
1128
|
+
"telemetry.sdk.name",
|
|
1129
|
+
"telemetry.sdk.language",
|
|
1130
|
+
"telemetry.sdk.version"
|
|
1131
|
+
]);
|
|
1132
|
+
function clip(value, max) {
|
|
1133
|
+
return value.length <= max ? value : `${value.slice(0, max)}…[+${value.length - max} chars]`;
|
|
1134
|
+
}
|
|
1135
|
+
function shortString(value) {
|
|
1136
|
+
if (typeof value !== "string") return void 0;
|
|
1137
|
+
const trimmed = value.trim();
|
|
1138
|
+
if (!trimmed) return void 0;
|
|
1139
|
+
return trimmed.length <= MAX_STRING ? trimmed : `${trimmed.slice(0, MAX_STRING)}…`;
|
|
1140
|
+
}
|
|
1141
|
+
//#endregion
|
|
1142
|
+
//#region src/diagnosis/engine.ts
|
|
1143
|
+
/**
|
|
1144
|
+
* The agent-failure diagnosis engine: flat spans plus context in, a findings
|
|
1145
|
+
* document out.
|
|
1146
|
+
*
|
|
1147
|
+
* It is a plain function. Its only side effect is the model call made through
|
|
1148
|
+
* the caller's transport, and in deterministic mode it makes none. It keeps no
|
|
1149
|
+
* storage, files no issues, and retains nothing: where the spans came from and
|
|
1150
|
+
* where the findings go belong to the caller. That is what lets one engine
|
|
1151
|
+
* serve both the internal daily mining run and a customer diagnosis while their
|
|
1152
|
+
* data paths stay separate.
|
|
1153
|
+
*
|
|
1154
|
+
* Order of operations, each step reading only the previous step's output:
|
|
1155
|
+
* 1. secret filter and content drop over every input (spans, focus, topology);
|
|
1156
|
+
* 2. the deterministic pass: capability table, execution facts, observed findings;
|
|
1157
|
+
* 3. in model mode, the prime-protocol pass over the highest-signal runs;
|
|
1158
|
+
* 4. evidence resolution: every cited id must be a span id of the input that
|
|
1159
|
+
* no other run shares; findings left without evidence are rejected.
|
|
1160
|
+
*/
|
|
1161
|
+
const DEFAULT_MAX_MODEL_TRACES = 6;
|
|
1162
|
+
async function diagnoseSpans(spans, context, options) {
|
|
1163
|
+
if (context.subject !== "internal" && context.subject !== "customer") throw new TypeError(`diagnoseSpans: subject must be internal or customer, got ${String(context.subject)}`);
|
|
1164
|
+
if (context.subject === "customer" && context.topology !== void 0) throw new TypeError("diagnoseSpans: topology analysis is internal only; a customer context must not carry one");
|
|
1165
|
+
const contentIncluded = context.contentIncluded === true;
|
|
1166
|
+
const { spans: filtered, report: ingest } = ingestSpans(spans, { contentIncluded });
|
|
1167
|
+
const focus = context.focus === void 0 ? void 0 : redactSecrets(context.focus, ingest.secrets);
|
|
1168
|
+
const deterministic = runDeterministicPass(filtered);
|
|
1169
|
+
const skipped = [...deterministic.skipped];
|
|
1170
|
+
const ambiguous = new Set(ingest.ambiguousSpanIds);
|
|
1171
|
+
const rejected = [];
|
|
1172
|
+
const questions = [];
|
|
1173
|
+
const notes = [];
|
|
1174
|
+
const summaries = [];
|
|
1175
|
+
const runs = [];
|
|
1176
|
+
let promptSha256 = null;
|
|
1177
|
+
let modelFindings = [];
|
|
1178
|
+
const traces = groupByTrace(filtered);
|
|
1179
|
+
if (options.mode === "deterministic") skipped.push({
|
|
1180
|
+
analysis: "model reading of the runs",
|
|
1181
|
+
reason: options.reason ?? "deterministic mode: no model call was made"
|
|
1182
|
+
});
|
|
1183
|
+
else {
|
|
1184
|
+
const maxTraces = options.model.maxTraces ?? 6;
|
|
1185
|
+
const ranked = rankTraces(traces);
|
|
1186
|
+
const selected = ranked.slice(0, maxTraces);
|
|
1187
|
+
const short = traces.size - ranked.length;
|
|
1188
|
+
if (ranked.length === 0) skipped.push({
|
|
1189
|
+
analysis: "model reading of the runs",
|
|
1190
|
+
reason: `no run has the 3 spans the model needs to read it; ${traces.size} runs were too short`
|
|
1191
|
+
});
|
|
1192
|
+
else if (short > 0) skipped.push({
|
|
1193
|
+
analysis: `model reading of ${short} of ${traces.size} runs`,
|
|
1194
|
+
reason: `those runs have fewer than 3 spans, too few for the model to read`
|
|
1195
|
+
});
|
|
1196
|
+
if (ranked.length > selected.length) skipped.push({
|
|
1197
|
+
analysis: `model reading of ${ranked.length - selected.length} of ${ranked.length} runs`,
|
|
1198
|
+
reason: `the model reads at most ${maxTraces} runs per diagnosis, chosen by error count then size`
|
|
1199
|
+
});
|
|
1200
|
+
const pass = await runModelPass(selected, ambiguous, {
|
|
1201
|
+
subject: context.subject,
|
|
1202
|
+
contentIncluded,
|
|
1203
|
+
...focus ? { focus } : {},
|
|
1204
|
+
...context.projects ? { projects: context.projects } : {}
|
|
1205
|
+
}, options.model, options.signal);
|
|
1206
|
+
promptSha256 = pass.promptSha256;
|
|
1207
|
+
modelFindings = pass.findings;
|
|
1208
|
+
questions.push(...pass.questions);
|
|
1209
|
+
notes.push(...context.subject === "internal" ? pass.notes : []);
|
|
1210
|
+
summaries.push(...pass.summaries);
|
|
1211
|
+
rejected.push(...pass.rejected);
|
|
1212
|
+
runs.push(...pass.runs);
|
|
1213
|
+
for (const gap of pass.uncovered) skipped.push({
|
|
1214
|
+
analysis: `model reading of part of run ${gap.traceId}`,
|
|
1215
|
+
reason: gap.reason
|
|
1216
|
+
});
|
|
1217
|
+
for (const run of pass.runs) if (!run.ok) skipped.push({
|
|
1218
|
+
analysis: `model reading of run ${run.traceId}`,
|
|
1219
|
+
reason: run.failure ?? "failed"
|
|
1220
|
+
});
|
|
1221
|
+
if (context.subject === "internal" && context.topology) {
|
|
1222
|
+
const topology = redactSecretsDeep(context.topology, ingest.secrets);
|
|
1223
|
+
const reading = await runTopologyPass(topology, new Set((topology.runs ?? []).map((run) => run.id).filter((id) => typeof id === "string")), options.model, options.signal);
|
|
1224
|
+
notes.push(...reading.notes);
|
|
1225
|
+
questions.push(...reading.questions);
|
|
1226
|
+
rejected.push(...reading.rejected);
|
|
1227
|
+
runs.push(reading.run);
|
|
1228
|
+
if (!reading.run.ok) skipped.push({
|
|
1229
|
+
analysis: "topology reading",
|
|
1230
|
+
reason: reading.run.failure ?? "failed"
|
|
1231
|
+
});
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
const spanIds = new Set(filtered.map((span) => span.spanId));
|
|
1235
|
+
const findings = resolveEvidence([...deterministic.findings, ...modelFindings], spanIds, ambiguous, rejected).sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || (a.confidence === b.confidence ? 0 : a.confidence === "observed" ? -1 : 1));
|
|
1236
|
+
const document = {
|
|
1237
|
+
schemaVersion: 1,
|
|
1238
|
+
subject: {
|
|
1239
|
+
label: redactSecrets(context.label, ingest.secrets),
|
|
1240
|
+
runCount: traces.size,
|
|
1241
|
+
window: deterministic.window,
|
|
1242
|
+
contentIncluded
|
|
1243
|
+
},
|
|
1244
|
+
coverage: {
|
|
1245
|
+
capabilities: deterministic.capabilities,
|
|
1246
|
+
skipped,
|
|
1247
|
+
redaction: {
|
|
1248
|
+
redactionCount: ingest.secrets.redactionCount,
|
|
1249
|
+
byRule: { ...ingest.secrets.byRule },
|
|
1250
|
+
droppedAttributes: ingest.droppedAttributes
|
|
1251
|
+
}
|
|
1252
|
+
},
|
|
1253
|
+
findings
|
|
1254
|
+
};
|
|
1255
|
+
const defects = validateDiagnosisFindings(document, { spanIds });
|
|
1256
|
+
if (defects.length > 0) throw new Error(`diagnoseSpans produced an invalid findings document: ${defects.slice(0, 5).join("; ")}`);
|
|
1257
|
+
const pricing = options.mode === "model" ? options.model.pricing : void 0;
|
|
1258
|
+
return {
|
|
1259
|
+
document,
|
|
1260
|
+
questions,
|
|
1261
|
+
notes,
|
|
1262
|
+
summaries,
|
|
1263
|
+
facts: deterministic.facts,
|
|
1264
|
+
conformance: deterministic.validation.findings,
|
|
1265
|
+
ingest,
|
|
1266
|
+
rejected,
|
|
1267
|
+
model: {
|
|
1268
|
+
used: options.mode === "model",
|
|
1269
|
+
model: options.mode === "model" ? options.model.model : null,
|
|
1270
|
+
runs,
|
|
1271
|
+
promptSha256,
|
|
1272
|
+
usage: totalUsage(runs, pricing)
|
|
1273
|
+
},
|
|
1274
|
+
engine: {
|
|
1275
|
+
name: "agent-eval/diagnosis",
|
|
1276
|
+
version: packageVersion()
|
|
1277
|
+
}
|
|
1278
|
+
};
|
|
1279
|
+
}
|
|
1280
|
+
/** Runs with more errors first, then larger runs; runs under MIN_MODEL_SPANS are left out. */
|
|
1281
|
+
function rankTraces(traces) {
|
|
1282
|
+
return [...traces.entries()].filter(([, spans]) => spans.length >= 3).map(([traceId, spans]) => ({
|
|
1283
|
+
traceId,
|
|
1284
|
+
spans,
|
|
1285
|
+
errors: spans.filter((span) => span.status === "ERROR").length
|
|
1286
|
+
})).sort((a, b) => b.errors - a.errors || b.spans.length - a.spans.length || a.traceId.localeCompare(b.traceId));
|
|
1287
|
+
}
|
|
1288
|
+
function resolveEvidence(findings, spanIds, ambiguous, rejected) {
|
|
1289
|
+
const kept = [];
|
|
1290
|
+
for (const finding of findings) {
|
|
1291
|
+
const evidence = finding.evidence.filter((id) => spanIds.has(id) && !ambiguous.has(id));
|
|
1292
|
+
if (evidence.length === 0) {
|
|
1293
|
+
rejected.push({ reason: `finding ${finding.id} lost all evidence: its span ids are missing or shared by several runs` });
|
|
1294
|
+
continue;
|
|
1295
|
+
}
|
|
1296
|
+
kept.push({
|
|
1297
|
+
...finding,
|
|
1298
|
+
evidence
|
|
1299
|
+
});
|
|
1300
|
+
}
|
|
1301
|
+
return kept;
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Token and cost totals over the model exchanges. No exchange attempted is a
|
|
1305
|
+
* measured zero. An exchange that failed without reporting usage may still
|
|
1306
|
+
* have been billed, so it makes the totals unknown (null) rather than zero.
|
|
1307
|
+
*/
|
|
1308
|
+
function totalUsage(runs, pricing) {
|
|
1309
|
+
let inputTokens = 0;
|
|
1310
|
+
let outputTokens = 0;
|
|
1311
|
+
let exchanges = 0;
|
|
1312
|
+
for (const run of runs) {
|
|
1313
|
+
exchanges += 1;
|
|
1314
|
+
inputTokens = inputTokens === null || run.usage.inputTokens === null ? null : inputTokens + run.usage.inputTokens;
|
|
1315
|
+
outputTokens = outputTokens === null || run.usage.outputTokens === null ? null : outputTokens + run.usage.outputTokens;
|
|
1316
|
+
}
|
|
1317
|
+
const usd = exchanges === 0 ? 0 : pricing && inputTokens !== null && outputTokens !== null ? (inputTokens * pricing.inputUsdPerMillion + outputTokens * pricing.outputUsdPerMillion) / 1e6 : null;
|
|
1318
|
+
return {
|
|
1319
|
+
exchanges,
|
|
1320
|
+
inputTokens,
|
|
1321
|
+
outputTokens,
|
|
1322
|
+
usd
|
|
1323
|
+
};
|
|
1324
|
+
}
|
|
1325
|
+
//#endregion
|
|
1326
|
+
export { DEFAULT_MAX_MODEL_TRACES, DIAGNOSIS_FINDINGS_SCHEMA_ID, DIAGNOSIS_SECRET_RULES, SECRET_ASSIGNMENT_PATTERN, diagnoseSpans, epochMillis, holdsSecret, ingestSpans, isContentAttribute, nodeHttpPrimeBridgeTransport, redactSecrets, redactSecretsDeep, validateDiagnosisFindings };
|
|
1327
|
+
|
|
1328
|
+
//# sourceMappingURL=index.js.map
|