@tangle-network/agent-eval 0.188.0 → 0.190.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +119 -0
- package/README.md +3 -2
- package/dist/adapters/http.d.ts +11 -2
- package/dist/adapters/http.d.ts.map +1 -1
- package/dist/adapters/http.js +75 -7
- package/dist/adapters/http.js.map +1 -1
- package/dist/{agent-profile-CfbuZk8b.d.ts → agent-profile-DNV_U6QR.d.ts} +3 -3
- package/dist/{agent-profile-CfbuZk8b.d.ts.map → agent-profile-DNV_U6QR.d.ts.map} +1 -1
- package/dist/analyst/index.d.ts +75 -12
- package/dist/analyst/index.d.ts.map +1 -1
- package/dist/analyst/index.js +213 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/{analyze-runs-CLz5O01x.d.ts → analyze-runs-BvK4-wSs.d.ts} +5 -5
- package/dist/{analyze-runs-CLz5O01x.d.ts.map → analyze-runs-BvK4-wSs.d.ts.map} +1 -1
- package/dist/{analyze-runs-DCY7LMt4.js → analyze-runs-SCSWYpy0.js} +17 -9
- package/dist/analyze-runs-SCSWYpy0.js.map +1 -0
- package/dist/{benchmark-command-CdUGku3W.js → benchmark-command-TZlsuAh9.js} +12 -12
- package/dist/{benchmark-command-CdUGku3W.js.map → benchmark-command-TZlsuAh9.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +4 -4
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +7 -7
- package/dist/campaign/index.js +8 -8
- package/dist/{campaign-DZuysBbS.js → campaign-_29Qpc9Z.js} +258 -13
- package/dist/campaign-_29Qpc9Z.js.map +1 -0
- package/dist/{campaign-evidence-IF1p0oDa.js → campaign-evidence-CWF8pEIk.js} +9 -8
- package/dist/campaign-evidence-CWF8pEIk.js.map +1 -0
- package/dist/{chat-client-C4dg7Ng_.js → chat-client-DbH06Ek6.js} +2 -2
- package/dist/{chat-client-C4dg7Ng_.js.map → chat-client-DbH06Ek6.js.map} +1 -1
- package/dist/{chat-json-call-JVGbfx95.js → chat-json-call-B0xTEY0I.js} +2 -2
- package/dist/{chat-json-call-JVGbfx95.js.map → chat-json-call-B0xTEY0I.js.map} +1 -1
- package/dist/claim-BGOUqoMq.d.ts +38 -0
- package/dist/claim-BGOUqoMq.d.ts.map +1 -0
- package/dist/cli.js +3 -3
- package/dist/{client-CFj9dfcn.d.ts → client-BhEyAdfe.d.ts} +4 -4
- package/dist/{client-CFj9dfcn.d.ts.map → client-BhEyAdfe.d.ts.map} +1 -1
- package/dist/{client-CXE-U1SA.js → client-QwJpsNYU.js} +5 -1
- package/dist/client-QwJpsNYU.js.map +1 -0
- package/dist/contract/index.d.ts +10 -10
- package/dist/contract/index.js +22 -13
- package/dist/contract/index.js.map +1 -1
- package/dist/deep-freeze-CurcOQg3.js +16 -0
- package/dist/deep-freeze-CurcOQg3.js.map +1 -0
- package/dist/{default-registry-BtZ_t-9b.js → default-registry-16I-Bxlf.js} +4 -3
- package/dist/default-registry-16I-Bxlf.js.map +1 -0
- package/dist/{default-registry-BLsGfnoO.d.ts → default-registry-B5HrwrPM.d.ts} +6 -6
- package/dist/{default-registry-BLsGfnoO.d.ts.map → default-registry-B5HrwrPM.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CsicmDxt.d.ts → define-agent-eval-BWeyIpny.d.ts} +8 -7
- package/dist/define-agent-eval-BWeyIpny.d.ts.map +1 -0
- package/dist/{define-agent-eval-ChtddasD.js → define-agent-eval-DrGfrCTZ.js} +8 -8
- package/dist/{define-agent-eval-ChtddasD.js.map → define-agent-eval-DrGfrCTZ.js.map} +1 -1
- package/dist/diagnosis/index.d.ts +3 -3
- package/dist/diagnosis/index.d.ts.map +1 -1
- package/dist/diagnosis/index.js +4 -4
- package/dist/diagnosis/index.js.map +1 -1
- package/dist/{dspy-rlm-engine-sPX7B5ac.js → dspy-rlm-engine-CTIGomgr.js} +3 -3
- package/dist/{dspy-rlm-engine-sPX7B5ac.js.map → dspy-rlm-engine-CTIGomgr.js.map} +1 -1
- package/dist/{engine-Gw2n1irt.d.ts → engine-BkLCnZQX.d.ts} +4 -4
- package/dist/{engine-Gw2n1irt.d.ts.map → engine-BkLCnZQX.d.ts.map} +1 -1
- package/dist/{eval-campaign-CeotQ1ZR.js → eval-campaign-CMKRjWl0.js} +3 -3
- package/dist/{eval-campaign-CeotQ1ZR.js.map → eval-campaign-CMKRjWl0.js.map} +1 -1
- package/dist/evaluation.d.ts +2 -2
- package/dist/{exact-types-CrhPoDp8.d.ts → exact-types-CfpP2A4Z.d.ts} +2 -2
- package/dist/{exact-types-CrhPoDp8.d.ts.map → exact-types-CfpP2A4Z.d.ts.map} +1 -1
- package/dist/experiment/index.d.ts +2 -1
- package/dist/experiment/index.d.ts.map +1 -1
- package/dist/experiment/index.js +1 -1
- package/dist/exporters-XMCYJlqT.d.ts +161 -0
- package/dist/exporters-XMCYJlqT.d.ts.map +1 -0
- package/dist/{external-optimizer-process-D3BLpZyR.js → external-optimizer-process-Mc83UoWl.js} +3 -3
- package/dist/{external-optimizer-process-D3BLpZyR.js.map → external-optimizer-process-Mc83UoWl.js.map} +1 -1
- package/dist/{external-optimizer-subprocess-BrfsgL92.js → external-optimizer-subprocess-C0dHXH8e.js} +3 -3
- package/dist/{external-optimizer-subprocess-BrfsgL92.js.map → external-optimizer-subprocess-C0dHXH8e.js.map} +1 -1
- package/dist/{feedback-trajectory-BJi4t3Vf.d.ts → feedback-trajectory-BYjh7NEM.d.ts} +4 -4
- package/dist/{feedback-trajectory-BJi4t3Vf.d.ts.map → feedback-trajectory-BYjh7NEM.d.ts.map} +1 -1
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.d.ts.map +1 -1
- package/dist/hosted/index.js +1 -1
- package/dist/{index-Ca0_IjFu.d.ts → index-2e_MTznj.d.ts} +7 -7
- package/dist/{index-Ca0_IjFu.d.ts.map → index-2e_MTznj.d.ts.map} +1 -1
- package/dist/{index-BD1oSbHr.d.ts → index-BR_R1eBE.d.ts} +11 -11
- package/dist/{index-BD1oSbHr.d.ts.map → index-BR_R1eBE.d.ts.map} +1 -1
- package/dist/{index-u0d1Jp4F.d.ts → index-BqDJ6fUK.d.ts} +45 -24
- package/dist/index-BqDJ6fUK.d.ts.map +1 -0
- package/dist/{index-eTTnPe3r.d.ts → index-Ci0-EYzc.d.ts} +678 -249
- package/dist/index-Ci0-EYzc.d.ts.map +1 -0
- package/dist/{index-B-s8qDRJ.d.ts → index-D4pC7Tvd.d.ts} +4 -3
- package/dist/index-D4pC7Tvd.d.ts.map +1 -0
- package/dist/index.d.ts +81 -32
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +385 -41
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-DeE1solA.d.ts → insight-report-CvPkEIcC.d.ts} +13 -6
- package/dist/insight-report-CvPkEIcC.d.ts.map +1 -0
- package/dist/jev.d.ts +2 -2
- package/dist/jev.js +2 -1
- package/dist/jev.js.map +1 -1
- package/dist/{journal-BuRLm6ho.js → journal-CDP0Vhx_.js} +262 -115
- package/dist/journal-CDP0Vhx_.js.map +1 -0
- package/dist/{kind-factory-g1ROw67A.js → kind-factory-ByZeXDqF.js} +4 -3
- package/dist/{kind-factory-g1ROw67A.js.map → kind-factory-ByZeXDqF.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +1 -1
- package/dist/{llm-client-DF9bPNYA.js → llm-client-BqmRnibA.js} +2 -2
- package/dist/{llm-client-DF9bPNYA.js.map → llm-client-BqmRnibA.js.map} +1 -1
- package/dist/{llm-judge-DF18J3CR.js → llm-judge-bwbgDMP0.js} +1895 -1379
- package/dist/llm-judge-bwbgDMP0.js.map +1 -0
- package/dist/{matrix-BvA1WUpj.d.ts → matrix-CNypzP-v.d.ts} +2 -2
- package/dist/{matrix-BvA1WUpj.d.ts.map → matrix-CNypzP-v.d.ts.map} +1 -1
- package/dist/meta-eval/index.d.ts +1 -1
- package/dist/mint-C9Lojt1V.d.ts +147 -0
- package/dist/mint-C9Lojt1V.d.ts.map +1 -0
- package/dist/{mint-B4zgSAJW.js → mint-iCD-W43s.js} +30 -12
- package/dist/mint-iCD-W43s.js.map +1 -0
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +1 -1
- package/dist/{prime-protocol-CF8Z43Fj.d.ts → prime-protocol-Fwvctmhm.d.ts} +2 -2
- package/dist/{prime-protocol-CF8Z43Fj.d.ts.map → prime-protocol-Fwvctmhm.d.ts.map} +1 -1
- package/dist/{produced-state-xsMMjYVc.js → produced-state-zSaGDC_Q.js} +4 -4
- package/dist/{produced-state-xsMMjYVc.js.map → produced-state-zSaGDC_Q.js.map} +1 -1
- package/dist/{raw-provider-sink-C96CcqeG.js → raw-provider-sink-DIRCZjP_.js} +2 -2
- package/dist/{raw-provider-sink-C96CcqeG.js.map → raw-provider-sink-DIRCZjP_.js.map} +1 -1
- package/dist/{redact-DxwlUAvh.js → redact-8fX8-uKr.js} +16 -7
- package/dist/redact-8fX8-uKr.js.map +1 -0
- package/dist/{redact-DPoH1-OW.d.ts → redact-Du3WKUOP.d.ts} +7 -5
- package/dist/redact-Du3WKUOP.d.ts.map +1 -0
- package/dist/{registry-BM2Kdk8T.d.ts → registry-mAQVtuLx.d.ts} +3 -3
- package/dist/{registry-BM2Kdk8T.d.ts.map → registry-mAQVtuLx.d.ts.map} +1 -1
- package/dist/{release-confidence-CG4Z9QLB.d.ts → release-confidence-CkoXFx9I.d.ts} +3 -3
- package/dist/{release-confidence-CG4Z9QLB.d.ts.map → release-confidence-CkoXFx9I.d.ts.map} +1 -1
- package/dist/{release-confidence-RanRUXRW.js → release-confidence-XQm_xjhR.js} +3 -3
- package/dist/release-confidence-XQm_xjhR.js.map +1 -0
- package/dist/reporting.d.ts +3 -3
- package/dist/reporting.js +1 -1
- package/dist/{researcher-Bjcd_8T_.d.ts → researcher-DojEMb7E.d.ts} +3 -3
- package/dist/{researcher-Bjcd_8T_.d.ts.map → researcher-DojEMb7E.d.ts.map} +1 -1
- package/dist/{reward-hacking-DrKXJtgH.d.ts → reward-hacking-Cyk-QHk9.d.ts} +2 -2
- package/dist/{reward-hacking-DrKXJtgH.d.ts.map → reward-hacking-Cyk-QHk9.d.ts.map} +1 -1
- package/dist/rl.d.ts +10 -8
- package/dist/rl.d.ts.map +1 -1
- package/dist/rl.js +7 -6
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +5 -4
- package/dist/rollout/index.js +2 -2
- package/dist/{rollout-C0Ifk3vK.js → rollout-B-XBReTC.js} +2 -2
- package/dist/{rollout-C0Ifk3vK.js.map → rollout-B-XBReTC.js.map} +1 -1
- package/dist/{rubric-predictive-validity-B14yBAmM.d.ts → rubric-predictive-validity-DIUwJJ9Y.d.ts} +2 -2
- package/dist/{rubric-predictive-validity-B14yBAmM.d.ts.map → rubric-predictive-validity-DIUwJJ9Y.d.ts.map} +1 -1
- package/dist/{run-record-tg_5H2IK.js → run-record-BRlM99rX.js} +41 -3
- package/dist/run-record-BRlM99rX.js.map +1 -0
- package/dist/{run-record-xEULpgML.d.ts → run-record-DF-fuu31.d.ts} +58 -6
- package/dist/run-record-DF-fuu31.d.ts.map +1 -0
- package/dist/{run-record-CNNzWGpB.js → run-record-DnMFyg_g.js} +26 -13
- package/dist/run-record-DnMFyg_g.js.map +1 -0
- package/dist/{schema-BzWDXhOR.d.ts → schema-B5LtYvn9.d.ts} +7 -5
- package/dist/{schema-BzWDXhOR.d.ts.map → schema-B5LtYvn9.d.ts.map} +1 -1
- package/dist/schema-C1aaAxTf.js.map +1 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts +489 -0
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +1 -0
- package/dist/{semantic-concept-judge-70LxrCSL.js → semantic-concept-judge-BY0me-Jq.js} +6 -6
- package/dist/{semantic-concept-judge-70LxrCSL.js.map → semantic-concept-judge-BY0me-Jq.js.map} +1 -1
- package/dist/{server-C8XvqCg8.js → server-DnJCDLf8.js} +2 -2
- package/dist/{server-C8XvqCg8.js.map → server-DnJCDLf8.js.map} +1 -1
- package/dist/{skillopt-optimization-method-C60h0Got.js → skillopt-optimization-method-CfnQEjgT.js} +364 -19
- package/dist/skillopt-optimization-method-CfnQEjgT.js.map +1 -0
- package/dist/{spans-W0o8pbEg.d.ts → spans-D1RftP_w.d.ts} +2 -2
- package/dist/{spans-W0o8pbEg.d.ts.map → spans-D1RftP_w.d.ts.map} +1 -1
- package/dist/{statistical-heldout-CPv-y6sK.d.ts → statistical-heldout-CLpjYffJ.d.ts} +4 -38
- package/dist/statistical-heldout-CLpjYffJ.d.ts.map +1 -0
- package/dist/{store-otlp-DPgzSQKS.js → store-otlp-84DH0OEH.js} +2 -2
- package/dist/{store-otlp-DPgzSQKS.js.map → store-otlp-84DH0OEH.js.map} +1 -1
- package/dist/{store-tool-spans-C2lLwDzg.d.ts → store-tool-spans-BBXs8hDP.d.ts} +6 -6
- package/dist/{store-tool-spans-C2lLwDzg.d.ts.map → store-tool-spans-BBXs8hDP.d.ts.map} +1 -1
- package/dist/{store-tool-spans-ClPxcvh2.js → store-tool-spans-DSbYpl7f.js} +5 -5
- package/dist/{store-tool-spans-ClPxcvh2.js.map → store-tool-spans-DSbYpl7f.js.map} +1 -1
- package/dist/{summary-report-SNvJoAvD.d.ts → summary-report-DvvesgR_.d.ts} +5 -4
- package/dist/summary-report-DvvesgR_.d.ts.map +1 -0
- package/dist/supervisor-run/index.d.ts +2 -2
- package/dist/{tool-groups-CCQqAa16.d.ts → tool-groups-W7MJobX_.d.ts} +3 -3
- package/dist/tool-groups-W7MJobX_.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +2 -2
- package/dist/traces.d.ts +6 -66
- package/dist/traces.d.ts.map +1 -1
- package/dist/traces.js +7 -196
- package/dist/traces.js.map +1 -1
- package/dist/{types-DDp2Mb8Y.d.ts → types-Bay1dh2o.d.ts} +5 -5
- package/dist/{types-DDp2Mb8Y.d.ts.map → types-Bay1dh2o.d.ts.map} +1 -1
- package/dist/{types-B1vDZ694.d.ts → types-BvmWNjjs.d.ts} +3 -3
- package/dist/{types-B1vDZ694.d.ts.map → types-BvmWNjjs.d.ts.map} +1 -1
- package/dist/{types-C8VBbiw6.js → types-CW5b90JH.js} +2 -15
- package/dist/types-CW5b90JH.js.map +1 -0
- package/dist/{types-5ak2PqnJ.d.ts → types-Cvqy7asY.d.ts} +3 -3
- package/dist/{types-5ak2PqnJ.d.ts.map → types-Cvqy7asY.d.ts.map} +1 -1
- package/dist/{types-Tp5WhYL9.d.ts → types-DRNlLbBR.d.ts} +2 -2
- package/dist/{types-Tp5WhYL9.d.ts.map → types-DRNlLbBR.d.ts.map} +1 -1
- package/dist/wire/index.d.ts +1 -1
- package/dist/wire/index.js +1 -1
- package/docs/adapters-observability.md +31 -27
- package/docs/charter.md +1 -1
- package/docs/code-agent-intake.md +1 -1
- package/docs/design/mlbenchmarks-book-review.md +1 -1
- package/docs/distributed-driver.md +1 -5
- package/docs/experiment.md +15 -0
- package/docs/insight-report.md +3 -2
- package/docs/public-api.md +174 -68
- package/docs/redaction.md +7 -1
- package/docs/rollout.md +5 -5
- package/docs/search-ledger.md +174 -0
- package/docs/trace-contracts.md +48 -3
- package/package.json +1 -1
- package/dist/analyze-runs-DCY7LMt4.js.map +0 -1
- package/dist/campaign-DZuysBbS.js.map +0 -1
- package/dist/campaign-evidence-IF1p0oDa.js.map +0 -1
- package/dist/client-CXE-U1SA.js.map +0 -1
- package/dist/default-registry-BtZ_t-9b.js.map +0 -1
- package/dist/define-agent-eval-CsicmDxt.d.ts.map +0 -1
- package/dist/index-B-s8qDRJ.d.ts.map +0 -1
- package/dist/index-eTTnPe3r.d.ts.map +0 -1
- package/dist/index-u0d1Jp4F.d.ts.map +0 -1
- package/dist/insight-report-DeE1solA.d.ts.map +0 -1
- package/dist/journal-BuRLm6ho.js.map +0 -1
- package/dist/llm-judge-DF18J3CR.js.map +0 -1
- package/dist/mint-B4zgSAJW.js.map +0 -1
- package/dist/mint-CJFSYBIU.d.ts +0 -285
- package/dist/mint-CJFSYBIU.d.ts.map +0 -1
- package/dist/redact-DPoH1-OW.d.ts.map +0 -1
- package/dist/redact-DxwlUAvh.js.map +0 -1
- package/dist/release-confidence-RanRUXRW.js.map +0 -1
- package/dist/run-record-CNNzWGpB.js.map +0 -1
- package/dist/run-record-tg_5H2IK.js.map +0 -1
- package/dist/run-record-xEULpgML.d.ts.map +0 -1
- package/dist/search-ledger-types-D7KDVg-o.d.ts +0 -302
- package/dist/search-ledger-types-D7KDVg-o.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-C60h0Got.js.map +0 -1
- package/dist/statistical-heldout-CPv-y6sK.d.ts.map +0 -1
- package/dist/summary-report-SNvJoAvD.d.ts.map +0 -1
- package/dist/tool-groups-CCQqAa16.d.ts.map +0 -1
- package/dist/types-C8VBbiw6.js.map +0 -1
- package/docs/search-history-receipts.md +0 -232
package/docs/redaction.md
CHANGED
|
@@ -24,9 +24,11 @@ The report lists the JSON Pointer, category and detector of each change, never t
|
|
|
24
24
|
| `share` | Also phone, IP and postal-address fields; pseudonymizes user, account, tenant, session and request ids | 64 KiB |
|
|
25
25
|
| `strict` | Also prompt, completion, message, tool payload and other raw-content fields, and `data:` media | 4 KiB |
|
|
26
26
|
|
|
27
|
-
A string that holds a credential is replaced whole with `[REDACTED:<detector>]`.
|
|
27
|
+
A string that holds a credential shape is replaced whole with `[REDACTED:<detector>]`.
|
|
28
28
|
Personal data inside prose is replaced in place.
|
|
29
29
|
Pass `knownSecrets` to remove exact values in any form: as written, base64, base64url or URL-encoded.
|
|
30
|
+
Each occurrence is cut out where it stands, so an error message that contains the key keeps the rest of its text.
|
|
31
|
+
A whole-string base64 payload that holds a known secret at any byte offset is replaced whole.
|
|
30
32
|
|
|
31
33
|
## Field names
|
|
32
34
|
|
|
@@ -34,6 +36,10 @@ Pass `knownSecrets` to remove exact values in any form: as written, base64, base
|
|
|
34
36
|
`apiKey`, `x-api-key`, `client.secret`, `refreshToken` and `OPENAI_API_KEY` are credentials.
|
|
35
37
|
Token counts and model limits are not: `inputTokens`, `outputTokens`, `max_tokens`, `token_count`, `gen_ai.usage.input_tokens` and `next_page_token` survive.
|
|
36
38
|
`author`, `auth_type`, `secret_name` and `content_type` also survive, because they name or describe a value instead of holding it.
|
|
39
|
+
|
|
40
|
+
agent-interface's `looksLikeCredential` stays the candidate schemas' refusal check, and the core does not call it.
|
|
41
|
+
On 629,707 strings from real sessions and VerticalBench runs, it flagged 280 strings the core passes, and all 280 were `Bearer ${token}`-style code or placeholders.
|
|
42
|
+
The core flagged 39 credentials that `looksLikeCredential` passes.
|
|
37
43
|
A number under a credential name survives, because it is a limit or a count.
|
|
38
44
|
|
|
39
45
|
## Share-safety verdict
|
package/docs/rollout.md
CHANGED
|
@@ -72,13 +72,13 @@ Rule applied: where fields conflicted, RunRecord-derived semantics won; the ledg
|
|
|
72
72
|
| field | from | decision |
|
|
73
73
|
| --- | --- | --- |
|
|
74
74
|
| `schema: "tangle.rollout.v1"` | ledger | wire key is `schema` (PR #410's `format` key retired) |
|
|
75
|
-
| `rollout_id` / `parent_rollout_id` | ledger | minted lines use `runId` as `rollout_id` (deterministic); multi-agent producers mint UUIDs and point workers at their supervisor |
|
|
75
|
+
| `rollout_id` / `parent_rollout_id` | ledger | minted lines use `runId` as `rollout_id` (deterministic; a search run's id is `cellId:attempt`); multi-agent producers mint UUIDs and point workers at their supervisor; a nested search run points at the run of the cell that contains the search |
|
|
76
76
|
| `run_id` | both | `RunRecord.runId` |
|
|
77
77
|
| `experiment_id`, `candidate_id` | PR #410 | required keys; `null` means the producer did not record the value |
|
|
78
|
-
| `generation`, `candidate_index` | ledger |
|
|
78
|
+
| `generation`, `candidate_index` | ledger | search coordinates: node depth (`0` = the search root) and node registration order; `null` = not a search run. Mint fills both from `searchLineage` for a record with `search` coordinates and refuses such a record without it |
|
|
79
79
|
| `role` | ledger | enum extended with `agent` for solo eval runs (mint default) |
|
|
80
80
|
| `task.split` | conflict | **RunRecord semantics win**: `search` is the trainable pool; `dev` and `holdout` follow `RunSplitTag`; `canary` is retained for release checks |
|
|
81
|
-
| `task.seed`, `task.rep` | ledger | seed from `RunRecord.seed`; rep 0 for minted
|
|
81
|
+
| `task.seed`, `task.rep` | ledger | seed from `RunRecord.seed`; rep from the search cell for a search run, 0 for other minted runs |
|
|
82
82
|
| `policy.*` | ledger | + `prompt_hash`, `config_hash`, `agent_profile_cell_id` from PR #410's RunRecord provenance |
|
|
83
83
|
| `messages` | ledger | canonical OpenAI chat-with-tools incl. `reasoning_content`; minted lines inline the final llm span's conversation |
|
|
84
84
|
| `steps` | PR #410 | optional trace-span projections (llm/tool), absent on harness-store-derived lines |
|
|
@@ -86,8 +86,8 @@ Rule applied: where fields conflicted, RunRecord-derived semantics won; the ledg
|
|
|
86
86
|
| `outcome.realness_gated` | PR #410 | required boolean; the anti-Goodhart gate travels into the data, the validator refuses it beside a positive reward, and SFT export refuses gated lines outright. `outcome.realness_screened` rides beside it (optional tri-state) so a screened-clean reward is distinguishable from a never-screened one |
|
|
87
87
|
| `outcome.reward_source` / `verdict` / `metrics` | ledger | unchanged; mint fills `metrics` from `RunRecord.outcome.raw` |
|
|
88
88
|
| `outcome.is_completed` / `is_truncated` / `error` | conflict | mint derives terminal fields from the required `RunRecord.terminalOutcome`; producers use the explicit `unknown` value when terminal evidence is unavailable |
|
|
89
|
-
| `cost.*` | ledger | superset of PR #410's costUsd/totalTokens; `cost.usd` is `null`
|
|
90
|
-
| `artifacts.*`, `provenance.*` | ledger | `provenance.capture` gains `mint` alongside `settle-time` / `backfill` |
|
|
89
|
+
| `cost.*` | ledger | superset of PR #410's costUsd/totalTokens; `cost.usd` is `null` unless `costProvenance.kind` is `observed` or `estimated`: an `uncaptured` cost is unknown and a `lower-bound` cost is only a floor (never a fake 0, never a floor read as a total) |
|
|
90
|
+
| `artifacts.*`, `provenance.*` | ledger | `provenance.capture` gains `mint` alongside `settle-time` / `backfill`; minted lines set `artifacts.transcript_ref` to `trace:<traceId>` from `RunRecord.traceRef` |
|
|
91
91
|
| gap discipline | ledger | records without trace spans become labeled gap lines (`messages: []`, `provenance.gap`) AND are listed in `missingTraces`; PR #410's silent skip retired |
|
|
92
92
|
|
|
93
93
|
## Export filters (fail-closed)
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# Search ledger
|
|
2
|
+
|
|
3
|
+
A selected prompt, profile, or patch is not a record of the search that produced it.
|
|
4
|
+
The search ledger is that record: a hash-chained event log of one search's nodes, edges, and cells, where every decision carries the evidence that justified it.
|
|
5
|
+
It is the search's only lineage record and its only resume checkpoint.
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
search-ledger.jsonl canonical JSONL, SHA-256 hash chain, trusted-head pin
|
|
9
|
+
├── blobs/<sha256>.json surfaces, profiles, rationales, diffs, RunRecords
|
|
10
|
+
├── SearchState the one projection: invariants and the read model
|
|
11
|
+
└── SearchHistoryReceipt bounded proof envelope for compareOptimizationMethods
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Terms
|
|
15
|
+
|
|
16
|
+
- **Node:** a content-addressed artifact, for example an AgentProfile or a prompt.
|
|
17
|
+
`nodeId` is `searchNodeId(searchId, artifactDigest)`, so identical content in one search is one node.
|
|
18
|
+
- **Edge:** the proposal that derived a node from its parents.
|
|
19
|
+
It records the operator, the proposer, the redacted rationale, and one parent-to-child diff per parent.
|
|
20
|
+
`edgeId` is `searchEdgeId(searchId, childNodeId, proposalKey)`, where the proposal key names the proposal within the search, so a resumed search recognizes its own edges.
|
|
21
|
+
- **Cell:** one node on one task in one split at one repeat.
|
|
22
|
+
`cellId` is `searchCellId(searchId, nodeId, taskId, split, rep)`.
|
|
23
|
+
Each attempt at a cell has the run id `cellId:attempt`, the id its RunRecord carries.
|
|
24
|
+
- **Unit:** the claim's independent unit, usually the task's source.
|
|
25
|
+
Repeats and sibling tasks of one unit average inside it before any statistic.
|
|
26
|
+
- **Splits:** `train` is the proposer's feedback, `selection` is private to the policy and allocator, and `test` is sealed for the claim.
|
|
27
|
+
|
|
28
|
+
## Events
|
|
29
|
+
|
|
30
|
+
| Event | Records |
|
|
31
|
+
|---|---|
|
|
32
|
+
| `search-opened` | Subject, process, artifact kind, objective with judge and claim, the three splits with their tasks and units, policy, budget, containment, derivation, identity. First and once. |
|
|
33
|
+
| `operation-started`, `operation-recorded` | A non-cell spend, such as one proposer call, with its reservation and then its outcome and accounting. |
|
|
34
|
+
| `node-registered` | A node, its artifact digest and declared surfaces. |
|
|
35
|
+
| `edge-recorded` | Parents (primary first), operator (`seed`, `draft`, `improve`, `debug`, `merge`, `derive`), attribution, proposer, selection rule, rationale, diffs, label. |
|
|
36
|
+
| `cell-allocated` | A planned cell with its stage (`root`, `train`, `screen`, `rung`, `claim`, `external`), lane, and reservation. |
|
|
37
|
+
| `cell-settled` | One attempt: outcome, accounting, time, placement, identity, surface evidence, trace reference, and optionally its RunRecord blob. |
|
|
38
|
+
| `cell-cancelled` | A cell that will not run: `pruned`, `budget`, `deadline`, or `aborted`. |
|
|
39
|
+
| `node-decided` | `advanced {rung}`, `pruned`, `invalid`, `finalist`, `selected`, or `rejected`, with its rule, reason, and the estimate it used. May repeat; the latest wins. |
|
|
40
|
+
| `search-closed` | The stop reason and the claim: power, finalists, the selected node, and `ship`, `hold`, or `test-cannot-resolve`. Last and once. |
|
|
41
|
+
|
|
42
|
+
Edge attribution says how the parents are known and is never inferred from timing or order:
|
|
43
|
+
|
|
44
|
+
- `explicit`: the proposer that created the child emitted the edge.
|
|
45
|
+
- `correlated`: an importer joined an optimizer's own parent record by content digest, for example GEPA's `parentIndices`.
|
|
46
|
+
- `unknown`: no parent record exists, and the edge names no parent.
|
|
47
|
+
|
|
48
|
+
## Invariants
|
|
49
|
+
|
|
50
|
+
`SearchState` applies every entry and refuses one that breaks an invariant.
|
|
51
|
+
The same code runs in the producer's journal, the kernel, and the Intelligence verifier.
|
|
52
|
+
|
|
53
|
+
- **Order:** `search-opened` comes first; nothing follows `search-closed`. More work on a closed search is a derived search.
|
|
54
|
+
- **Graph:** a node exists before an edge or cell names it, and a cell's node already has an edge. A parent registered before its child and already in the tree, so the graph is acyclic. A parent in another search appears only on a `derive` edge that matches `derivedFrom`.
|
|
55
|
+
- **Re-proposal:** identical content is a second edge into the existing node, counted as a re-proposal.
|
|
56
|
+
- **Splits:** a task belongs to one split and one unit. With `heldOutUnits`, no test unit appears in train or selection. Stages match splits: `claim` cells run on test, `screen` and `rung` on selection.
|
|
57
|
+
- **Seal:** only the root (the first registered node) and nodes decided `finalist` run test cells.
|
|
58
|
+
- **Attempts:** attempts count from 1 without gaps. A `passed` or `failed` outcome is final; only a retryable `errored` outcome admits another attempt.
|
|
59
|
+
- **Budget:** at every `cell-allocated` and `operation-started`, committed spend plus open reservations plus the unspent claim reserve plus the new reservation stays within `maxUsd`. Spend above a reservation is recorded as overspend, never refused. An unknown cost counts as its proven floor.
|
|
60
|
+
- **Completion:** `search-closed` needs every allocated cell settled or cancelled, every started operation recorded, and every node with an edge and a terminal decision. A `ship` claim needs a promoted finalist and held-out test units; any other claim keeps the root.
|
|
61
|
+
|
|
62
|
+
A ledger written under another schema tag, such as the retired `tangle.search-ledger.v1` candidate-slot format, is refused with the tag named.
|
|
63
|
+
It is never translated.
|
|
64
|
+
|
|
65
|
+
## Reading the state
|
|
66
|
+
|
|
67
|
+
`ledger.state()` and every append return a `SearchStateView`.
|
|
68
|
+
Its `header`, `head`, `audit`, `completion`, and `closed` are plain values.
|
|
69
|
+
Its node, edge, cell, unit-score, and lineage reads go to the live indexes, so taking a view costs the same at any search size.
|
|
70
|
+
A view is valid until the ledger applies its next entry; a later read throws instead of mixing two ledger positions.
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
const state = await ledger.state()
|
|
74
|
+
state.audit.cells // { allocated, settled, cancelled, open }
|
|
75
|
+
state.scoredCells(nodeId, 'selection') // [{ cellId, unitId, attempt, score }]
|
|
76
|
+
state.unitScores(nodeId, 'selection') // [{ unitId, sum, count, mean }]
|
|
77
|
+
state.lineage(record.search) // { depth, ordinal, rep, containingRunId } for mintRolloutRows
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
A unit's mean sums its cells in cellId order, so it depends on the set of cells and not on the order they settled in.
|
|
81
|
+
|
|
82
|
+
## Estimates
|
|
83
|
+
|
|
84
|
+
`estimateNode(state, nodeId, { against, split })` is the one statistic of a node.
|
|
85
|
+
It averages each node's scored cells inside their units, pairs the units both nodes scored, and runs `pairedDeltaTest` on the per-unit deltas.
|
|
86
|
+
Unscored cells (errored, cancelled or in flight) are absent, never zero.
|
|
87
|
+
|
|
88
|
+
| Pairs | `method` | Reported |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| 0 or 1 | `none` | nothing; the view shows "unknown (1 unit)" |
|
|
91
|
+
| 2 to 5 | `insufficient` | `delta` only |
|
|
92
|
+
| 6 to 19 | `descriptive` | `delta`, the bootstrap `interval` as spread, and `exactSignP` |
|
|
93
|
+
| 20 or more | `bootstrap` | `delta` and the decision-grade `interval` |
|
|
94
|
+
|
|
95
|
+
The thresholds are the library's own: `minimumPairsForPairedDeltaTest(0.95)` and `BOOTSTRAP_GATE_MIN_N`.
|
|
96
|
+
`units` counts the units the node scored; `pairs` counts the units both nodes scored.
|
|
97
|
+
`delta` is node minus `against` in the metric's units, so an improvement on a `minimize` objective is negative.
|
|
98
|
+
`exactSignP` is one-sided toward improvement in the objective's direction.
|
|
99
|
+
When every paired delta is equal, the estimate is `indeterminate`: its interval would have zero width, so it carries neither an interval nor a p-value.
|
|
100
|
+
|
|
101
|
+
`cellSetDigest` digests the contrast and exactly the cells read: id, unit, attempt and score, in cellId order.
|
|
102
|
+
Its first 32 bits seed the bootstrap, and `estimator` is `SEARCH_ESTIMATOR`, whose revision digests every parameter of the computation.
|
|
103
|
+
Equal cells therefore give equal bits in any process and in any row order.
|
|
104
|
+
A verifier that reads cells from its own store calls `estimateNodeFromCells` with them; `searchCellSetDigest` tells it whether a node's cells changed.
|
|
105
|
+
|
|
106
|
+
```ts
|
|
107
|
+
const estimate = estimateNode(state, childId, { against: parentId, split: 'selection' })
|
|
108
|
+
await recorder.decideNode({ nodeId: childId, decision, basis: estimate, rule, reason })
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`searchPosterior(state, { split })` gives every node a normal posterior on its improvement over the root, for sampling parents.
|
|
112
|
+
Its mean is the node's mean per-unit improvement over the root, oriented so that larger is better in either direction.
|
|
113
|
+
Its variance is the search's pooled between-unit variance of those improvements divided by the node's shared units, so a node measured on one unit has a wide posterior, not none.
|
|
114
|
+
The root sits at exactly 0, and the pooled variance stays null until some node shares 2 units with the root.
|
|
115
|
+
Posterior numbers steer spend and claim nothing; a claim comes from the sealed test split.
|
|
116
|
+
|
|
117
|
+
## Record a search
|
|
118
|
+
|
|
119
|
+
`SearchRecorder` writes each fact the moment it exists.
|
|
120
|
+
Ids are deterministic and every write checks the ledger's state first, so a resumed search continues its ledger instead of conflicting with its own history.
|
|
121
|
+
Rationales and labels are redacted with the `share` profile before they are hashed; surfaces are stored as they ran.
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
import { openSearchLedger, SearchRecorder, surfaceNode } from '@tangle-network/agent-eval/campaign'
|
|
125
|
+
|
|
126
|
+
const ledger = openSearchLedger({ path: `${runDir}/search-ledger.jsonl`, searchId })
|
|
127
|
+
const recorder = await SearchRecorder.open({ ledger }, opening)
|
|
128
|
+
const root = await recorder.registerNode(surfaceNode(recorder, baseline))
|
|
129
|
+
await recorder.recordEdge({
|
|
130
|
+
childNodeId: root.nodeId, parents: [], operator: 'seed', attribution: 'explicit',
|
|
131
|
+
proposer: null, proposalKey: 'baseline', rationale: { unknown: 'the starting surface' }, diffs: [],
|
|
132
|
+
})
|
|
133
|
+
const cellId = await recorder.allocateCell({ nodeId: root.nodeId, taskId, split: 'selection', rep: 0, stage: 'root' })
|
|
134
|
+
await recorder.settleCell({ cellId, outcome, accounting, identity, runRecord })
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`runOptimization({ searchLedger: { ledger, identity } })` and `selfImprove` record through the recorder and return the receipt on `searchHistory`.
|
|
138
|
+
The loop's scenarios are its proposer's feedback, so they are the train split; its promotions are budget decisions and it makes no claim.
|
|
139
|
+
|
|
140
|
+
`gepaOptimizationMethod({ searchLedger: { identity } })` records GEPA's search when it finishes.
|
|
141
|
+
`importGepaPopulation` turns the population into nodes and `correlated` edges and reports collapsed duplicates.
|
|
142
|
+
`importExternalEvaluations` turns every callback evaluation into an `external` cell; a candidate GEPA evaluated but kept out of its population gets an `unknown` edge and is decided `pruned`.
|
|
143
|
+
|
|
144
|
+
## Receipts and `require-complete`
|
|
145
|
+
|
|
146
|
+
`createSearchHistoryReceipt({ producerId, runId, ledger })` reads the ledger's bytes, replays them, and binds the byte digest, the audit digest, and a bounded summary.
|
|
147
|
+
A receipt is complete exactly when the ledger holds `search-closed`, which the completion invariant admits only when nothing is outstanding.
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
const comparison = await compareOptimizationMethods({
|
|
151
|
+
// ...methods, partitions, dispatch, judges, runDir
|
|
152
|
+
searchHistoryPolicy: 'require-complete',
|
|
153
|
+
searchHistoryVerification: 'ledger',
|
|
154
|
+
storage,
|
|
155
|
+
})
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Under `require-complete`, a missing, malformed, producer-mismatched, or open search refuses the final comparison before its first test dispatch.
|
|
159
|
+
`searchHistoryVerification: 'ledger'` also resolves the receipt's URI through `storage.read`, checks the bytes' SHA-256 and length, and replays them.
|
|
160
|
+
`verifySearchHistoryReceipt` checks the envelope alone; `assertSearchHistoryMatchesState` checks it against a replayed state.
|
|
161
|
+
|
|
162
|
+
A receipt does not prove that the optimizer searched well, that a node is correct or safe, or that a winner generalizes.
|
|
163
|
+
Those claims need the sealed test split, the claim's power check, and held-out evaluation.
|
|
164
|
+
|
|
165
|
+
## Files
|
|
166
|
+
|
|
167
|
+
- `src/campaign/search-ledger-types.ts`: the event and audit types.
|
|
168
|
+
- `src/campaign/search-ledger.ts`: schemas, canonical ordering, the codec, and `FileSearchLedger`.
|
|
169
|
+
- `src/campaign/search-state.ts`: `SearchState`, the invariants and read model, and the id functions.
|
|
170
|
+
- `src/campaign/estimate-node.ts`: `estimateNode`, `estimateNodeFromCells` and `searchPosterior`.
|
|
171
|
+
- `src/campaign/search-ledger-recording.ts`: `SearchRecorder` and the surface helpers.
|
|
172
|
+
- `src/campaign/gepa-search-import.ts`: the GEPA population and evaluation importers.
|
|
173
|
+
- `src/campaign/search-history-receipt.ts`: receipts and admission.
|
|
174
|
+
- `src/ledger-core/`: hashing, locking, durable appends, chain verification, and the trusted-head pin.
|
package/docs/trace-contracts.md
CHANGED
|
@@ -4,7 +4,7 @@ A trace contract states which path an agent run must take, and checks the record
|
|
|
4
4
|
Two runs can give the same answer while one of them calls a forbidden tool; an output check passes both, and a contract fails the wrong one.
|
|
5
5
|
|
|
6
6
|
The checker is deterministic and calls no model.
|
|
7
|
-
It reads any span array that has `name`, `kind` or span-kind attributes, timestamps, and attributes: agent-eval `TraceStore` spans
|
|
7
|
+
It reads any span array that has `name`, `kind` or span-kind attributes, timestamps, and attributes: agent-eval `TraceStore` spans or OTLP rows that `traces check` maps.
|
|
8
8
|
|
|
9
9
|
## Write a contract
|
|
10
10
|
|
|
@@ -27,7 +27,7 @@ Write the contract as JSON and compile it with `compileTraceContractSpec`:
|
|
|
27
27
|
|
|
28
28
|
| Key | Checks |
|
|
29
29
|
|---|---|
|
|
30
|
-
| `run.requireCompleted` | The run
|
|
30
|
+
| `run.requireCompleted` | The run status is `completed` and its end time is recorded. A failed or aborted run fails. |
|
|
31
31
|
| `run.allowedStatuses` | The run status is one of `running`, `completed`, `failed`, `aborted`. |
|
|
32
32
|
| `run.maxDurationMs` | End minus start. An unknown duration fails. |
|
|
33
33
|
| `tools.required` | Each tool is called at least once. |
|
|
@@ -36,7 +36,9 @@ Write the contract as JSON and compile it with `compileTraceContractSpec`:
|
|
|
36
36
|
| `tools.maxCalls`, `tools.maxCallsPerTool` | Ceilings on tool calls, overall and per tool. |
|
|
37
37
|
| `tools.requiredOrder`, `tools.orderMode` | Each tool is called, and each comes before the next. |
|
|
38
38
|
| `tools.arguments` | An RFC 6901 JSON Pointer check (`exists`, `equals`, `oneOf`, `type`) on tool-call arguments. A call without captured arguments fails. |
|
|
39
|
-
| `llm.maxCalls`, `llm.maxTotalTokens` | Ceilings on model calls and on input plus output tokens. An unrecorded token count fails. |
|
|
39
|
+
| `llm.maxCalls`, `llm.maxTotalTokens` | Ceilings on model calls and on input plus output tokens. Input includes cache-read and cache-write tokens when the span records them, per the OTel GenAI definition — a cached call's prompt-token count alone omits most of what it spent. An unrecorded token count fails. |
|
|
40
|
+
| `tools.enforced` | The trace records the tools the harness offered the model (`gen_ai.tool.definitions`), every call is one of them, and every offered tool is in `tools.allowed`. A trace that records no offered tools fails. |
|
|
41
|
+
| `retries.reads`, `retries.writes` | A tool called again with the same arguments repeats its side effect. A read may repeat. A write may repeat only when every call carries the same idempotency key at `idempotencyKey`. Any other tool may not repeat. |
|
|
40
42
|
| `llm.allowedModels` | Every model call names a listed model. A call without a model fails. |
|
|
41
43
|
| `scope.root` | Check only the one span that matches this predicate and its descendants: one sub-agent or one search-tree node. Zero or several matches make every rule an error. |
|
|
42
44
|
| `alternatives.anyOf` | Named paths, each with its own `run`, `tools`, `llm` and `rules`. At least one must pass completely, as well as the base checks. Use it for a legitimate shortcut such as a cache hit. |
|
|
@@ -50,6 +52,45 @@ An unknown key or an unknown status is an error that names the closest known wor
|
|
|
50
52
|
`lintTraceContractSpec` reports contradictions (a tool that is both required and forbidden) and likely mistakes (an order that uses the default `start-order` mode, which passes overlapping calls).
|
|
51
53
|
`explainTraceContract` states each compiled rule in one line.
|
|
52
54
|
|
|
55
|
+
## Check what the harness enforced
|
|
56
|
+
|
|
57
|
+
A tool list in a prompt or a CLI flag is a declaration, not proof of enforcement.
|
|
58
|
+
`tools.enforced` compares three sets: the tools the contract allows, the tools the harness offered the model, and the tools the run called.
|
|
59
|
+
`traces` records the offered set from the `system` `init` record of `claude -p --output-format stream-json --verbose` output.
|
|
60
|
+
Any OTel GenAI producer that sets `gen_ai.tool.definitions` works the same way.
|
|
61
|
+
|
|
62
|
+
In a recorded run, `claude -p --tools Read,Grep` also offered eight claude.ai connector tools, so the contract below failed.
|
|
63
|
+
The same run with `--strict-mcp-config` offered only `Read` and `Grep`, and passed.
|
|
64
|
+
|
|
65
|
+
```json
|
|
66
|
+
{
|
|
67
|
+
"name": "refund-desk-enforced",
|
|
68
|
+
"tools": { "required": ["Read"], "allowed": ["Read", "Grep"], "enforced": true }
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Check that a retry cannot apply twice
|
|
73
|
+
|
|
74
|
+
A write that times out may still have applied.
|
|
75
|
+
Calling it again with the same arguments applies it twice unless the target deduplicates on an idempotency key.
|
|
76
|
+
|
|
77
|
+
```json
|
|
78
|
+
{
|
|
79
|
+
"name": "payments",
|
|
80
|
+
"retries": {
|
|
81
|
+
"reads": ["lookup_order"],
|
|
82
|
+
"writes": [{ "tool": "charge_card", "idempotencyKey": "/idempotency_key" }]
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The check groups calls by tool and arguments, without the idempotency key.
|
|
88
|
+
A group of more than one call passes for a read, and for a write only when every call carries the same key.
|
|
89
|
+
A repeated call of an unlisted tool fails, because its side effect is unknown.
|
|
90
|
+
A call without captured arguments fails when its tool is called more than once, because the checker cannot tell whether it repeats.
|
|
91
|
+
Grouping matches arguments exactly: a retry that re-encodes a value (`42.5` vs `"42.50"`) is not recognized as the same call.
|
|
92
|
+
Independent of grouping, a declared write whose call itself errored (including a timeout) and carries no idempotency key always fails — its outcome at the target is unknown, so nothing proves a retry, however it was encoded, would be safe.
|
|
93
|
+
|
|
53
94
|
## Rule semantics
|
|
54
95
|
|
|
55
96
|
Every rule reports `pass`, `fail` or `error` in `ContractVerdict.ruleExecutions`.
|
|
@@ -68,6 +109,8 @@ A required successor that never ran fails `precedes`, a span without the timesta
|
|
|
68
109
|
| `atMost(p, max)`, `tokensAtMost(p, max)` | Call and token ceilings. |
|
|
69
110
|
| `run({ ... })` | The run status and duration. |
|
|
70
111
|
| `argument(p, { pointer, check, occurrence })` | A JSON Pointer check on tool arguments. |
|
|
112
|
+
| `toolsOffered(declared)` | The offered tools are recorded, every call is one of them, and every offered tool is declared. |
|
|
113
|
+
| `retrySafe({ reads, writes })` | No call repeats a side effect that is not proven safe to repeat. |
|
|
71
114
|
|
|
72
115
|
Ordering has three modes.
|
|
73
116
|
`start-order` (the default) needs an `a` that started before each `b`.
|
|
@@ -87,6 +130,8 @@ const result = checkTraceContracts(await store.spans({ runId }), [contract])
|
|
|
87
130
|
|
|
88
131
|
## Gate CI on a contract
|
|
89
132
|
|
|
133
|
+
**Unreleased.** `traces check` lives on `feat/traces-check` in `/traces` (PR #128), proven against this package but not yet mergeable: it needs the traces/agent-runtime redaction-core migration first, so a `/traces` build on this package's contract API does not crash at load. Until then, call `evaluateTraceContract` / `checkTraceContracts` directly, as shown above.
|
|
134
|
+
|
|
90
135
|
`traces check` runs a contract over a recorded trace and exits with a code a CI step can read:
|
|
91
136
|
|
|
92
137
|
```sh
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/agent-eval",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.190.0",
|
|
4
4
|
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
|
|
5
5
|
"homepage": "https://github.com/tangle-network/agent-eval#readme",
|
|
6
6
|
"repository": {
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"analyze-runs-DCY7LMt4.js","names":[],"sources":["../src/contamination-guard.ts","../src/contract/analyze-runs.ts"],"sourcesContent":["/**\n * ContaminationGuard — ensures held-out scenarios don't leak into\n * training/prompt paths, and flags model memorization.\n *\n * Three probes:\n * 1. `checkCanaries(output, scenario)` — if the scenario carries a\n * canary token, it MUST NOT appear in the agent's output.\n * Canaries are strings that are statistically impossible to\n * reconstruct from the scenario description alone — so if they\n * echo back, the model memorized them.\n * 2. `canaryLeakView(store)` — cross-corpus view of every run whose\n * output contained a canary, with the offending scenario + run.\n * 3. `HoldoutAuditor` — wraps a Dataset and emits a structured error\n * on any code path that reads holdout scenarios but doesn't flag\n * `purpose: 'evaluation'`. Keeps engineers honest.\n */\n\nimport type { DatasetScenario } from './dataset'\nimport { llmSpans } from './trace/query'\nimport type { TraceStore } from './trace/store'\n\nexport interface CanaryLeak {\n scenarioId: string\n canary: string\n runId?: string\n evidence: string\n}\n\nexport function checkCanaries(output: string, scenarios: DatasetScenario[]): CanaryLeak[] {\n const leaks: CanaryLeak[] = []\n for (const s of scenarios) {\n if (!s.canary) continue\n if (output.includes(s.canary)) {\n leaks.push({ scenarioId: s.id, canary: s.canary, evidence: excerpt(output, s.canary) })\n }\n }\n return leaks\n}\n\n/**\n * Behavioral canary: tests the AGENT, not the eval grep.\n *\n * - `forbiddenPattern` PRESENT in output ⇒ AGENT EMITTED BAD CONTENT ⇒ LEAK.\n * - `forbiddenPattern` ABSENT from output ⇒ AGENT HELD THE LINE ⇒ PASS.\n *\n * Use when running known-bad-prompt scenarios against the agent under\n * test and you want to know if the agent misbehaved. The classical\n * {@link checkCanaries} / {@link import('./canary').runCanaries | runCanaries}\n * test whether the eval check fires when the bad output is forced\n * into the eval flow — different question, different answer.\n *\n * Pattern resolution order (first match wins):\n * 1. `scenario.forbiddenPattern` — if it parses as `/body/flags`,\n * treated as a regex; otherwise a literal substring.\n * 2. `scenario.canary` — literal substring fallback so the helper\n * works on existing scenario fixtures.\n *\n * Returns `null` when nothing forbidden was found OR the scenario\n * declared no pattern.\n */\nexport function checkBehavioralCanary(\n output: string,\n scenario: DatasetScenario,\n): CanaryLeak | null {\n const pattern = scenario.forbiddenPattern ?? scenario.canary\n if (!pattern) return null\n const hit = matchForbidden(output, pattern)\n if (!hit) return null\n return {\n scenarioId: scenario.id,\n canary: pattern,\n evidence: excerpt(output, hit),\n }\n}\n\n/**\n * Behavioral canary over many (scenario, output) pairs. Sibling to\n * {@link import('./canary').runCanaries | runCanaries} — same idea\n * (run-many → report) but the question being answered is \"did the\n * AGENT misbehave?\" rather than \"did the EVAL grep fire?\".\n *\n * Returns one `CanaryLeak` per pair where the agent's output\n * contained its scenario's `forbiddenPattern` (or `canary` fallback).\n */\nexport function runBehavioralCanaries(\n cases: Array<{ scenario: DatasetScenario; output: string; runId?: string }>,\n): CanaryLeak[] {\n const leaks: CanaryLeak[] = []\n for (const c of cases) {\n const leak = checkBehavioralCanary(c.output, c.scenario)\n if (leak) leaks.push({ ...leak, runId: c.runId ?? leak.runId })\n }\n return leaks\n}\n\n/**\n * Resolve a forbidden-pattern string to the matched substring inside\n * `output`. `/body/flags` notation is interpreted as a regex; anything\n * else is a literal substring.\n */\nfunction matchForbidden(output: string, pattern: string): string | null {\n const re = tryParseRegex(pattern)\n if (re) {\n const m = output.match(re)\n return m && m[0].length > 0 ? m[0] : null\n }\n return output.includes(pattern) ? pattern : null\n}\n\nfunction tryParseRegex(pattern: string): RegExp | null {\n if (pattern.length < 2 || pattern[0] !== '/') return null\n const last = pattern.lastIndexOf('/')\n if (last <= 0) return null\n const body = pattern.slice(1, last)\n const flags = pattern.slice(last + 1)\n if (!/^[gimsuy]*$/.test(flags)) return null\n try {\n return new RegExp(body, flags)\n } catch {\n return null\n }\n}\n\n/**\n * Scan the LLM-output history in a corpus; returns every case where a\n * canary from a known scenario appeared in agent output. Pass the full\n * set of scenarios whose canaries you care about (typically the whole\n * held-out slice).\n */\nexport async function canaryLeakView(\n store: TraceStore,\n scenarios: DatasetScenario[],\n): Promise<CanaryLeak[]> {\n const targets = scenarios.filter((s) => !!s.canary)\n if (targets.length === 0) return []\n const spans = await llmSpans(store)\n const leaks: CanaryLeak[] = []\n for (const span of spans) {\n const output = span.output ?? ''\n for (const s of targets) {\n if (s.canary && output.includes(s.canary)) {\n leaks.push({\n scenarioId: s.id,\n canary: s.canary,\n runId: span.runId,\n evidence: excerpt(output, s.canary),\n })\n }\n }\n }\n return leaks\n}\n\nexport class HoldoutAuditor {\n private scenarios: DatasetScenario[]\n private accessLog: Array<{ scenarioId: string; purpose: string; at: number }> = []\n\n constructor(scenarios: DatasetScenario[]) {\n this.scenarios = scenarios\n }\n\n /** Retrieve a holdout scenario for a declared purpose. Non-'evaluation' throws. */\n get(scenarioId: string, purpose: 'evaluation' | 'debugging'): DatasetScenario {\n if (purpose !== 'evaluation' && purpose !== 'debugging') {\n throw new Error(\n `HoldoutAuditor.get: purpose must be 'evaluation' or 'debugging', got ${purpose}`,\n )\n }\n const s = this.scenarios.find((x) => x.id === scenarioId)\n if (!s) throw new Error(`holdout scenario \"${scenarioId}\" not found`)\n this.accessLog.push({ scenarioId, purpose, at: Date.now() })\n return s\n }\n\n getAccessLog(): ReadonlyArray<{ scenarioId: string; purpose: string; at: number }> {\n return this.accessLog\n }\n}\n\nfunction excerpt(source: string, needle: string): string {\n const at = source.indexOf(needle)\n if (at < 0) return ''\n const start = Math.max(0, at - 30)\n const end = Math.min(source.length, at + needle.length + 30)\n return (start > 0 ? '…' : '') + source.slice(start, end) + (end < source.length ? '…' : '')\n}\n","/**\n * # `analyzeRuns()` — turn a set of agent runs into an actionable decision packet.\n *\n * Wires the substrate's statistical, calibration, clustering, Pareto, and\n * release-confidence primitives into one `InsightReport`. Two top-level\n * entry points use this function:\n *\n * - `selfImprove()` calls it on the campaign output to attach a packet\n * to every run.\n * - Consumers with observed `RunRecord[]` (production traces, gold\n * corpora, approve/reject tables) call it directly via `analyzeRuns()`\n * for analysis without a closed loop.\n *\n * Every section is opt-in based on what the input data supports — the\n * function never invents signal. If runs carry no judge scores, `judges`\n * is empty. If there's no baseline/candidate split, `lift` is undefined.\n * If no `analyst` is wired, `failureClusters` is undefined.\n *\n * The `recommendations` array is the human-readable layer; everything\n * else is the evidence backing each recommendation.\n */\n\nimport type { AnalystRegistry } from '../analyst/registry'\nimport type { AnalystFinding } from '../analyst/types'\nimport { welchsTTest } from '../baseline'\nimport { aggregatePairedHoldout } from '../campaign/gates/statistical-heldout'\nimport { checkCanaries } from '../contamination-guard'\nimport type { DatasetScenario } from '../dataset'\nimport { continuousAgreement } from '../judge-calibration'\nimport { pairRunRecords } from '../paired-arms'\nimport { observedSplitScore } from '../rollout/reward'\nimport {\n type RunRecord,\n type RunTerminalOutcome,\n type RunTokenUsage,\n validateRunRecord,\n} from '../run-record'\nimport {\n BOOTSTRAP_GATE_MIN_N,\n pairedBootstrap,\n pairedCohensDz,\n pairedMde,\n pairedTTest,\n pearsonR,\n requiredPairedSampleSize,\n spearmanR,\n} from '../statistics'\nimport { type ParetoFigureSpec, paretoChart } from '../summary-report'\nimport type { FailureClass } from '../trace/schema'\n\nimport type {\n CostProvenanceSummary,\n ExecutionInsight,\n FailureClassTally,\n FailureClusterInsight,\n InsightReport,\n InterRaterInsight,\n JudgeInsight,\n LiftInsight,\n MetricDelta,\n OutcomeCorrelationInsight,\n PriorPeriodComparison,\n Recommendation,\n ScalarDistribution,\n TokenUsageInsight,\n} from './insight-report'\n\n// ── Public API ───────────────────────────────────────────────────────\n\nexport interface AnalyzeRunsOptions {\n /** The runs to analyze. */\n runs: RunRecord[]\n /** Which split to score against when reading composite from RunOutcome.\n * Default: holdout when ANY run has a `holdoutScore`, else search. */\n split?: 'search' | 'holdout' | 'auto'\n /** Pairwise analysis configuration. When both `baselineCandidateId` and\n * `candidateCandidateId` are present, lift is computed on paired\n * (experimentId, scenarioId, seed) identities shared between the two sides.\n * Unmatched rows remain visible in the lift result. */\n baselineCandidateId?: string\n candidateCandidateId?: string\n /** Average matched run scores within these units before estimating lift.\n * Raw run distributions and unmatched-run counts remain unchanged. */\n independentUnitByScenarioId?: ReadonlyMap<string, string>\n /** Canary scenarios — checked against every run's raw output for\n * holdout contamination. */\n canaryScenarios?: DatasetScenario[]\n /** Analyst registry for failure clustering. When omitted, the\n * `failureClusters` section is left undefined. */\n analyst?: AnalystRegistry\n /** Downstream outcome metric per run (e.g. engagement rate, approval\n * rate, downstream pass rate). When present, the report includes\n * `outcomeCorrelation` + a simple linear reward model fit. */\n outcomeSignal?: {\n metric: string\n valueByRunId: Record<string, number>\n }\n /** Multi-rater feedback for inter-rater agreement. Each entry is one\n * rater's score for one run. Two or more raters → kappa + disagreement\n * triage list. */\n raterScores?: Array<{ runId: string; rater: string; score: number }>\n /** Number of histogram bins for distributional summaries. Default 12. */\n histogramBins?: number\n /** Decision threshold — the smallest composite lift the caller cares\n * about. Used by the recommendations engine to call ship vs hold.\n * Default 0.02. */\n decisionThreshold?: number\n /** Optional prior-period runs. When set, the report includes\n * `priorPeriodComparison` with per-metric Welch-CI deltas and\n * recommendations fire on statistically significant regressions.\n * The two windows do NOT have to share scenarios — the comparison\n * is two-sample unpaired (the substrate's `lift` field uses paired\n * bootstrap on shared (experimentId, scenarioId, seed) identities; this is the\n * shape for \"this week vs last week\" rather than \"candidate vs\n * baseline within a campaign\"). */\n baselineRuns?: RunRecord[]\n /** Human-readable label for the baseline window, e.g. \"vs prior 7\n * days\", \"vs v3.1 release\". Surfaces in recommendations + UI. */\n baselineLabel?: string\n}\n\nexport interface SummarizeExecutionOptions {\n runs: RunRecord[]\n histogramBins?: number\n}\n\nexport interface ExecutionReport {\n execution: ExecutionInsight\n costProvenance: CostProvenanceSummary\n}\n\n/** Summarize runtime facts without interpreting task quality or promotion readiness. */\nexport function summarizeExecution(opts: SummarizeExecutionOptions): ExecutionReport {\n const runs = opts.runs.map(validateRunRecord)\n const bins = opts.histogramBins ?? 12\n return {\n execution: computeExecutionInsight(runs, bins),\n costProvenance: summarizeCostProvenance(runs),\n }\n}\n\n/** A bootstrap interval with no spread: every resample landed on the same\n * value, so the interval carries no information about how far the point\n * estimate could be wrong and cannot support a directional claim. */\nfunction zeroWidth(ci: readonly [number, number]): boolean {\n return !Number.isFinite(ci[0]) || !Number.isFinite(ci[1]) || ci[0] === ci[1]\n}\n\nexport async function analyzeRuns(opts: AnalyzeRunsOptions): Promise<InsightReport> {\n const runs = opts.runs.map(validateRunRecord)\n const bins = opts.histogramBins ?? 12\n const threshold = opts.decisionThreshold ?? 0.02\n if (!Number.isFinite(threshold)) {\n throw new Error(`analyzeRuns: decisionThreshold must be finite, got ${threshold}`)\n }\n const split = resolveSplit(runs, opts.split ?? 'auto')\n\n const compositeWithIds = runs\n .map((r) => ({ runId: r.runId, score: compositeOf(r, split) }))\n .filter((p) => Number.isFinite(p.score))\n const composite = distributionOf(\n compositeWithIds.map((p) => p.score),\n bins,\n compositeWithIds,\n )\n\n const perDimension = computePerDimension(runs, bins)\n const { execution, costProvenance: provenance } = summarizeExecution({\n runs,\n histogramBins: bins,\n })\n const knownCostRuns = runs.filter((run) => run.costProvenance.kind !== 'uncaptured')\n const costs = knownCostRuns.map((r) => r.costUsd).filter(isFiniteNumber)\n const costDist = distributionOf(costs, bins)\n const pareto = paretoChart(knownCostRuns, { split })\n const degraded: { cost?: string; pareto?: string } = {}\n if (provenance.uncaptured.n > 0) {\n degraded.cost = diagnoseCostCoverage(runs, provenance)\n } else if (costs.length === 0 || costs.every((c) => c === 0)) {\n degraded.cost = `all ${runs.length} explicitly observed or estimated USD values are $0`\n }\n if (pareto.points.length < 2) {\n degraded.pareto =\n pareto.points.length === 0\n ? 'no candidates — Pareto unavailable'\n : 'single candidate — Pareto is a single point, not a frontier'\n }\n const costQuality = {\n cost: costDist,\n pareto,\n provenance,\n ...(degraded.cost || degraded.pareto ? { degraded } : {}),\n }\n\n const judges = computeJudgeInsights(runs)\n\n const interRater = opts.raterScores ? computeInterRater(opts.raterScores) : undefined\n\n const lift = computeLift(\n runs,\n opts.baselineCandidateId,\n opts.candidateCandidateId,\n split,\n opts.independentUnitByScenarioId,\n )\n\n const failureClusters = opts.analyst\n ? await computeFailureClusters(runs, opts.analyst, split)\n : undefined\n\n const failureClasses = computeFailureClasses(runs, split)\n\n const contamination = opts.canaryScenarios\n ? computeContamination(runs, opts.canaryScenarios)\n : undefined\n\n const outcomeCorrelation = opts.outcomeSignal\n ? computeOutcomeCorrelation(runs, opts.outcomeSignal, split)\n : undefined\n\n const release = buildReleaseScorecard(composite, lift, contamination)\n\n const priorPeriodComparison = opts.baselineRuns\n ? computePriorPeriodComparison(runs, opts.baselineRuns, split, opts.baselineLabel)\n : undefined\n\n const recommendations = buildRecommendations({\n composite,\n judges,\n interRater,\n lift,\n failureClusters,\n failureClasses,\n contamination,\n outcomeCorrelation,\n priorPeriodComparison,\n threshold,\n })\n\n return {\n n: runs.length,\n execution,\n composite,\n perDimension,\n costQuality,\n judges,\n interRater,\n lift,\n failureClusters,\n contamination,\n outcomeCorrelation,\n release,\n ...(failureClasses ? { failureClasses } : {}),\n ...(priorPeriodComparison ? { priorPeriodComparison } : {}),\n recommendations,\n }\n}\n\nfunction computeExecutionInsight(runs: RunRecord[], bins: number): ExecutionInsight {\n const aggregateRows = runs.flatMap((run) => {\n const usage = aggregateTokenUsage(run)\n return usage ? [{ usage, costUsd: finiteRaw(run, 'aggregate_cost_usd') }] : []\n })\n const aggregateCosts = aggregateRows.flatMap((row) =>\n row.costUsd !== undefined ? [row.costUsd] : [],\n )\n const modelCounts = new Map<string, number>()\n let executionErrorRuns = 0\n let executionErrorEvents = 0\n let errorReportingRuns = 0\n let errorSpanEvents = 0\n let errorSpanReportingRuns = 0\n const terminalOutcomes: Record<RunTerminalOutcome, number> = {\n succeeded: 0,\n failed: 0,\n cancelled: 0,\n incomplete: 0,\n unknown: 0,\n }\n const errorsByTerminalOutcome: ExecutionInsight['executionErrors']['byTerminalOutcome'] = {\n succeeded: { withErrors: 0, withoutErrors: 0, unreported: 0 },\n failed: { withErrors: 0, withoutErrors: 0, unreported: 0 },\n cancelled: { withErrors: 0, withoutErrors: 0, unreported: 0 },\n incomplete: { withErrors: 0, withoutErrors: 0, unreported: 0 },\n unknown: { withErrors: 0, withoutErrors: 0, unreported: 0 },\n }\n let modelCallRuns = 0\n let modelCallEvents = 0\n let modelCallReportingRuns = 0\n\n for (const run of runs) {\n modelCounts.set(run.model, (modelCounts.get(run.model) ?? 0) + 1)\n const terminalOutcome = run.terminalOutcome\n terminalOutcomes[terminalOutcome] += 1\n const modelCalls = nonNegativeCountRaw(run, 'llm_span_count')\n if (modelCalls !== undefined) {\n modelCallEvents += modelCalls\n modelCallReportingRuns += 1\n }\n const usage = run.tokenUsage\n if (\n (modelCalls ?? 0) > 0 ||\n usage.input > 0 ||\n usage.output > 0 ||\n (usage.cached ?? 0) > 0 ||\n (usage.cacheWrite ?? 0) > 0\n ) {\n modelCallRuns += 1\n }\n const errorEvents = reportedExecutionErrorEvents(run)\n if (errorEvents !== undefined) {\n executionErrorEvents += errorEvents\n errorReportingRuns += 1\n if (errorEvents > 0) {\n executionErrorRuns += 1\n errorsByTerminalOutcome[terminalOutcome].withErrors += 1\n } else errorsByTerminalOutcome[terminalOutcome].withoutErrors += 1\n } else errorsByTerminalOutcome[terminalOutcome].unreported += 1\n const reportedErrorSpans = nonNegativeCountRaw(run, 'error_span_count')\n if (reportedErrorSpans !== undefined) {\n errorSpanEvents += reportedErrorSpans\n errorSpanReportingRuns += 1\n }\n }\n\n return {\n durationMs: distributionOf(\n runs.map((run) => run.wallMs),\n bins,\n ),\n queueMs: distributionOf(\n runs.filter((run) => run.queueMs !== undefined).map((run) => run.queueMs!),\n bins,\n ),\n tokenUsage: summarizeTokenUsage(\n runs.map((run) => run.tokenUsage),\n bins,\n ),\n aggregateUsage: {\n runs: aggregateRows.length,\n tokenUsage: summarizeTokenUsage(\n aggregateRows.map((row) => row.usage),\n bins,\n ),\n costUsd: distributionOf(aggregateCosts, bins),\n totalCostUsd: aggregateCosts.reduce((total, value) => total + value, 0),\n },\n models: [...modelCounts.entries()]\n .map(([model, count]) => ({ model, runs: count }))\n .sort((left, right) => right.runs - left.runs || left.model.localeCompare(right.model)),\n modelCalls: {\n runs: modelCallRuns,\n events: modelCallEvents,\n reportingRuns: modelCallReportingRuns,\n },\n executionErrors: {\n runs: executionErrorRuns,\n fraction: errorReportingRuns > 0 ? executionErrorRuns / errorReportingRuns : null,\n events: executionErrorEvents,\n reportingRuns: errorReportingRuns,\n errorSpanEvents,\n errorSpanReportingRuns,\n byTerminalOutcome: errorsByTerminalOutcome,\n },\n terminalOutcomes,\n }\n}\n\nfunction reportedExecutionErrorEvents(run: RunRecord): number | undefined {\n return nonNegativeCountRaw(run, 'execution_error_count')\n}\n\nfunction nonNegativeCountRaw(run: RunRecord, key: string): number | undefined {\n const value = finiteRaw(run, key)\n return value !== undefined && Number.isInteger(value) && value >= 0 ? value : undefined\n}\n\nfunction summarizeTokenUsage(usages: RunTokenUsage[], bins: number): TokenUsageInsight {\n const reasoning = usages.flatMap((usage) =>\n usage.reasoning !== undefined ? [usage.reasoning] : [],\n )\n const cached = usages.flatMap((usage) => (usage.cached !== undefined ? [usage.cached] : []))\n const cacheWrite = usages.flatMap((usage) =>\n usage.cacheWrite !== undefined ? [usage.cacheWrite] : [],\n )\n return {\n input: distributionOf(\n usages.map((usage) => usage.input),\n bins,\n ),\n output: distributionOf(\n usages.map((usage) => usage.output),\n bins,\n ),\n reasoning: distributionOf(reasoning, bins),\n cached: distributionOf(cached, bins),\n cacheWrite: distributionOf(cacheWrite, bins),\n totals: {\n input: usages.reduce((total, usage) => total + usage.input, 0),\n output: usages.reduce((total, usage) => total + usage.output, 0),\n reasoning: reasoning.reduce((total, value) => total + value, 0),\n cached: cached.reduce((total, value) => total + value, 0),\n cacheWrite: cacheWrite.reduce((total, value) => total + value, 0),\n },\n }\n}\n\nfunction aggregateTokenUsage(run: RunRecord): RunTokenUsage | undefined {\n const input = finiteRaw(run, 'aggregate_prompt_tokens')\n const output = finiteRaw(run, 'aggregate_completion_tokens')\n const reasoning = finiteRaw(run, 'aggregate_reasoning_tokens')\n const cached = finiteRaw(run, 'aggregate_cached_tokens')\n const cacheWrite = finiteRaw(run, 'aggregate_cache_write_tokens')\n if (\n input === undefined &&\n output === undefined &&\n reasoning === undefined &&\n cached === undefined &&\n cacheWrite === undefined\n )\n return undefined\n return {\n input: input ?? 0,\n output: output ?? 0,\n ...(reasoning !== undefined ? { reasoning } : {}),\n ...(cached !== undefined ? { cached } : {}),\n ...(cacheWrite !== undefined ? { cacheWrite } : {}),\n }\n}\n\nfunction finiteRaw(run: RunRecord, key: string): number | undefined {\n const value = run.outcome.raw[key]\n return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined\n}\n\nfunction summarizeCostProvenance(runs: RunRecord[]): CostProvenanceSummary {\n const summary: CostProvenanceSummary = {\n observed: { n: 0, totalUsd: 0 },\n estimated: { n: 0, totalUsd: 0 },\n uncaptured: { n: 0 },\n knownFraction: 0,\n }\n for (const run of runs) {\n const cost = run.costProvenance\n if (cost.kind === 'uncaptured') {\n summary.uncaptured.n += 1\n } else {\n summary[cost.kind].n += 1\n summary[cost.kind].totalUsd += cost.usd\n }\n }\n const known = summary.observed.n + summary.estimated.n\n summary.knownFraction = runs.length > 0 ? known / runs.length : 0\n return summary\n}\n\nfunction diagnoseCostCoverage(runs: RunRecord[], provenance: CostProvenanceSummary): string {\n const uncaptured = provenance.uncaptured.n\n const known = provenance.observed.n + provenance.estimated.n\n if (uncaptured === runs.length) {\n return `USD cost uncaptured for all ${runs.length} runs — no observed or estimated USD values; token and wall-time metrics remain available.`\n }\n return `USD cost uncaptured for ${uncaptured}/${runs.length} runs; excluded those rows from cost statistics (${known}/${runs.length} retained: ${provenance.observed.n} observed, ${provenance.estimated.n} estimated).`\n}\n\n/**\n * Model-free task-failure tally.\n *\n * Explicit non-success classes are task-failure evidence.\n * A low task score without a class is counted as `unknown`.\n */\nfunction computeFailureClasses(\n runs: RunRecord[],\n split: 'search' | 'holdout',\n): FailureClassTally[] | undefined {\n const counts = new Map<FailureClass, number>()\n for (const r of runs) {\n if (!isTaskFailure(r, split)) continue\n const key =\n r.failureClass !== undefined && r.failureClass !== 'success' ? r.failureClass : 'unknown'\n counts.set(key, (counts.get(key) ?? 0) + 1)\n }\n if (counts.size === 0) return undefined\n const n = runs.length\n return [...counts.entries()]\n .map(([failureClass, count]) => ({\n failureClass,\n count,\n share: n > 0 ? count / n : 0,\n }))\n .sort((a, b) => b.count - a.count || a.failureClass.localeCompare(b.failureClass))\n}\n\n// ── Prior-period comparison ─────────────────────────────────────────\n\n/** Direction of the metric — does \"higher current\" mean better or worse?\n * Composite + judge dimensions: higher is better. Cost + duration: lower\n * is better. The recommendations engine flips the sign before judging\n * regressed vs improved. */\ntype MetricDirection = 'higher-is-better' | 'lower-is-better'\n\nfunction computePriorPeriodComparison(\n current: RunRecord[],\n baseline: RunRecord[],\n split: 'search' | 'holdout',\n windowLabel: string | undefined,\n): PriorPeriodComparison | undefined {\n if (current.length === 0 || baseline.length === 0) return undefined\n\n const metrics: Record<string, MetricDelta> = {}\n const directions: Record<string, MetricDirection> = {}\n\n const compositeCurrent = current\n .map((r) => compositeOf(r, split))\n .filter(Number.isFinite) as number[]\n const compositeBaseline = baseline\n .map((r) => compositeOf(r, split))\n .filter(Number.isFinite) as number[]\n if (compositeCurrent.length > 0 && compositeBaseline.length > 0) {\n metrics.composite = welchCompare(compositeBaseline, compositeCurrent)\n directions.composite = 'higher-is-better'\n }\n\n const costCurrent = knownCostValues(current)\n const costBaseline = knownCostValues(baseline)\n if (costCurrent.length > 0 && costBaseline.length > 0) {\n metrics.cost = welchCompare(costBaseline, costCurrent)\n directions.cost = 'lower-is-better'\n }\n\n const durCurrent = current.map((r) => r.wallMs).filter(Number.isFinite)\n const durBaseline = baseline.map((r) => r.wallMs).filter(Number.isFinite)\n if (durCurrent.length > 0 && durBaseline.length > 0) {\n metrics.duration = welchCompare(durBaseline, durCurrent)\n directions.duration = 'lower-is-better'\n }\n\n const tokCurrent = current\n .map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0))\n .filter(Number.isFinite)\n const tokBaseline = baseline\n .map((r) => (r.tokenUsage.input ?? 0) + (r.tokenUsage.output ?? 0))\n .filter(Number.isFinite)\n if (tokCurrent.length > 0 && tokBaseline.length > 0) {\n metrics.tokenUsage = welchCompare(tokBaseline, tokCurrent)\n directions.tokenUsage = 'lower-is-better'\n }\n\n // Per-dimension judge comparisons — only for dimensions present in BOTH\n // windows. We use perDimMean since per-judge nesting is finicky for\n // two-sample comparisons across different judge configurations.\n const dimsCurrent = collectPerDimension(current)\n const dimsBaseline = collectPerDimension(baseline)\n for (const dim of Object.keys(dimsCurrent)) {\n const b = dimsBaseline[dim]\n const c = dimsCurrent[dim]\n if (!b || b.length === 0 || !c || c.length === 0) continue\n metrics[`dim.${dim}`] = welchCompare(b, c)\n directions[`dim.${dim}`] = 'higher-is-better'\n }\n\n const regressedMetrics: string[] = []\n const improvedMetrics: string[] = []\n const inconclusiveMetrics: string[] = []\n for (const [name, delta] of Object.entries(metrics)) {\n if (delta.status !== 'ok') {\n inconclusiveMetrics.push(name)\n continue\n }\n if (!delta.significant) continue\n const dir = directions[name] ?? 'higher-is-better'\n const better = dir === 'higher-is-better' ? delta.delta > 0 : delta.delta < 0\n if (better) improvedMetrics.push(name)\n else regressedMetrics.push(name)\n }\n\n return {\n baselineN: baseline.length,\n currentN: current.length,\n ...(windowLabel ? { windowLabel } : {}),\n metrics,\n regressedMetrics,\n improvedMetrics,\n inconclusiveMetrics,\n }\n}\n\nfunction knownCostValues(runs: RunRecord[]): number[] {\n return runs\n .filter((run) => run.costProvenance.kind !== 'uncaptured')\n .map((run) => run.costUsd)\n .filter(isFiniteNumber)\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value)\n}\n\n/** Collect per-dimension values across runs (from outcome.judgeScores.perDimMean). */\nfunction collectPerDimension(runs: RunRecord[]): Record<string, number[]> {\n const out: Record<string, number[]> = {}\n for (const r of runs) {\n const perDim = r.outcome.judgeScores?.perDimMean\n if (!perDim) continue\n for (const [dim, value] of Object.entries(perDim)) {\n if (!Number.isFinite(value)) continue\n if (!out[dim]) out[dim] = []\n out[dim].push(value as number)\n }\n }\n return out\n}\n\n/** Adapt the shared two-sample Welch result to the report contract. */\nfunction welchCompare(baseline: number[], current: number[]): MetricDelta {\n const result = welchsTTest(baseline, current)\n const base = {\n current: result.meanB,\n baseline: result.meanA,\n delta: result.delta,\n baselineN: baseline.length,\n currentN: current.length,\n }\n if (result.status !== 'ok') {\n return {\n ...base,\n status: result.status,\n ci95: null,\n pValue: null,\n cohensD: null,\n significant: false,\n }\n }\n return {\n ...base,\n status: 'ok',\n ci95: result.ci95,\n pValue: result.p,\n cohensD: result.cohensD,\n significant: result.p < 0.05 && Math.abs(result.cohensD) >= 0.2,\n }\n}\n\n// ── Composite + split selection ─────────────────────────────────────\n\nfunction resolveSplit(\n runs: RunRecord[],\n pref: 'search' | 'holdout' | 'auto',\n): 'search' | 'holdout' {\n if (pref !== 'auto') return pref\n const hasHoldout = runs.some((r) => Number.isFinite(observedSplitScore(r, 'holdout')))\n return hasHoldout ? 'holdout' : 'search'\n}\n\n/**\n * RAW (`observedSplitScore`): `analyzeRuns` describes what a set of runs\n * reported, and every downstream reader of this composite — distributions,\n * per-candidate summaries, the reward-hacking correlation — needs the ungated\n * number to see an inflated run at all.\n */\nfunction compositeOf(run: RunRecord, split: 'search' | 'holdout'): number {\n // Split-exact, no cross-split fallthrough: answering \"what did this run\n // score on the split I am summarising\" with the other split's number\n // silently mixes populations.\n const score = observedSplitScore(run, split)\n return Number.isFinite(score) ? (score as number) : Number.NaN\n}\n\n// ── Distribution helpers ────────────────────────────────────────────\n\nfunction distributionOf(\n values: number[],\n bins: number,\n withIds?: Array<{ runId: string; score: number }>,\n): ScalarDistribution {\n if (values.length === 0) {\n return {\n n: 0,\n mean: null,\n p50: null,\n p95: null,\n stddev: null,\n min: null,\n max: null,\n histogram: [],\n }\n }\n const sorted = [...values].sort((a, b) => a - b)\n const n = sorted.length\n const mean = sorted.reduce((s, v) => s + v, 0) / n\n const variance = sorted.reduce((s, v) => s + (v - mean) ** 2, 0) / n\n const stddev = Math.sqrt(variance)\n const tailRuns = withIds\n ? [...withIds].sort((a, b) => a.score - b.score).slice(0, Math.min(5, withIds.length))\n : undefined\n return {\n n,\n mean,\n p50: percentile(sorted, 0.5),\n p95: percentile(sorted, 0.95),\n stddev,\n min: sorted[0]!,\n max: sorted[n - 1]!,\n histogram: histogram(sorted, bins),\n ...(tailRuns ? { tailRuns } : {}),\n }\n}\n\nfunction percentile(sorted: number[], q: number): number {\n if (sorted.length === 0) return 0\n if (sorted.length === 1) return sorted[0]!\n const idx = (sorted.length - 1) * q\n const lo = Math.floor(idx)\n const hi = Math.ceil(idx)\n if (lo === hi) return sorted[lo]!\n const w = idx - lo\n return sorted[lo]! * (1 - w) + sorted[hi]! * w\n}\n\n/** Even-width histogram over the value range. Returns inclusive-lo /\n * exclusive-hi bins (closed on right for the last bin) compatible with\n * the substrate's `GainDistributionBin` shape. */\nfunction histogram(sorted: number[], bins: number): ScalarDistribution['histogram'] {\n if (sorted.length === 0 || bins < 1) return []\n const min = sorted[0]!\n const max = sorted[sorted.length - 1]!\n if (min === max) return [{ lo: min, hi: max, count: sorted.length }]\n const width = (max - min) / bins\n const out: ScalarDistribution['histogram'] = []\n for (let i = 0; i < bins; i++) {\n const lo = min + i * width\n const hi = i === bins - 1 ? max : lo + width\n out.push({ lo, hi, count: 0 })\n }\n for (const v of sorted) {\n const idx = Math.min(bins - 1, Math.floor((v - min) / width))\n out[idx]!.count++\n }\n return out\n}\n\nfunction computePerDimension(runs: RunRecord[], bins: number): Record<string, ScalarDistribution> {\n // JudgeScoresRecord pre-aggregates `perDimMean` (mean across judges per\n // dimension). We collect those means across runs to produce a per-dim\n // distribution at the corpus level. Consumers who want per-judge\n // dimension values reach into `perJudge[judgeId][dim]` themselves.\n const byDim = new Map<string, number[]>()\n for (const run of runs) {\n const scores = run.outcome.judgeScores\n if (!scores) continue\n for (const [dim, value] of Object.entries(scores.perDimMean ?? {})) {\n if (!Number.isFinite(value)) continue\n const arr = byDim.get(dim) ?? []\n arr.push(value)\n byDim.set(dim, arr)\n }\n }\n const out: Record<string, ScalarDistribution> = {}\n for (const [dim, values] of byDim) out[dim] = distributionOf(values, bins)\n return out\n}\n\n// ── Judge insights ──────────────────────────────────────────────────\n\nfunction computeJudgeInsights(runs: RunRecord[]): Record<string, JudgeInsight> {\n // Each judge's per-run mean is the average of its per-dimension scores\n // for that run. We aggregate those means across all runs each judge\n // scored — giving consumers a \"this judge's typical verdict\" reading.\n const out: Record<string, JudgeInsight> = {}\n const byJudge = new Map<string, number[]>()\n for (const run of runs) {\n const scores = run.outcome.judgeScores\n if (!scores?.perJudge) continue\n for (const [judgeId, dims] of Object.entries(scores.perJudge)) {\n const dimValues = Object.values(dims).filter(Number.isFinite) as number[]\n if (dimValues.length === 0) continue\n const judgeMean = dimValues.reduce((s, v) => s + v, 0) / dimValues.length\n const arr = byJudge.get(judgeId) ?? []\n arr.push(judgeMean)\n byJudge.set(judgeId, arr)\n }\n }\n for (const [judgeId, values] of byJudge) {\n out[judgeId] = {\n n: values.length,\n meanScore: values.reduce((s, v) => s + v, 0) / values.length,\n }\n }\n return out\n}\n\n// ── Inter-rater agreement ───────────────────────────────────────────\n\nfunction computeInterRater(\n ratings: Array<{ runId: string; rater: string; score: number }>,\n): InterRaterInsight | undefined {\n const byRun = new Map<string, Array<{ rater: string; score: number }>>()\n for (const r of ratings) {\n if (!Number.isFinite(r.score)) continue\n const list = byRun.get(r.runId) ?? []\n list.push({ rater: r.rater, score: r.score })\n byRun.set(r.runId, list)\n }\n const raters = new Set(ratings.map((r) => r.rater))\n const jointlyRated: string[] = []\n for (const [runId, ratersForRun] of byRun) {\n const seen = new Set(ratersForRun.map((r) => r.rater))\n let all = true\n for (const r of raters) if (!seen.has(r)) all = false\n if (all) jointlyRated.push(runId)\n }\n if (raters.size < 2 || jointlyRated.length === 0) return undefined\n\n const raterList = [...raters].sort()\n const perPair: Record<string, number> = {}\n for (let i = 0; i < raterList.length; i++) {\n for (let j = i + 1; j < raterList.length; j++) {\n const a = raterList[i]!\n const b = raterList[j]!\n const aScores: number[] = []\n const bScores: number[] = []\n for (const runId of jointlyRated) {\n const ratersForRun = byRun.get(runId)!\n const sa = ratersForRun.find((r) => r.rater === a)?.score\n const sb = ratersForRun.find((r) => r.rater === b)?.score\n if (sa !== undefined && sb !== undefined) {\n aScores.push(sa)\n bScores.push(sb)\n }\n }\n const agreement = continuousAgreement(\n aScores.map((score, index) => [score, bScores[index]!]),\n { bootstrap: 0 },\n )\n perPair[`${a}::${b}`] = agreement.weightedKappa\n }\n }\n const matrix = jointlyRated.map((runId) => {\n const ratingsByRater = new Map(byRun.get(runId)!.map((rating) => [rating.rater, rating.score]))\n return raterList.map((rater) => ratingsByRater.get(rater)!)\n })\n const agreement = continuousAgreement(matrix, { bootstrap: 0 })\n\n const disagreementCases = jointlyRated\n .map((runId) => {\n const ratersForRun = byRun.get(runId)!\n const scores = ratersForRun.map((r) => r.score)\n const range = Math.max(...scores) - Math.min(...scores)\n return { runId, ratings: ratersForRun, range }\n })\n .sort((a, b) => b.range - a.range)\n .slice(0, 20)\n\n return {\n raters: raters.size,\n jointlyRated: jointlyRated.length,\n kappa: Number.isFinite(agreement.weightedKappa) ? agreement.weightedKappa : 0,\n icc: agreement.icc,\n pearson: agreement.pearson,\n spearman: agreement.spearman,\n perPair,\n disagreementCases,\n }\n}\n\n// ── Lift ────────────────────────────────────────────────────────────\n\nfunction computeLift(\n runs: RunRecord[],\n baselineId: string | undefined,\n candidateId: string | undefined,\n split: 'search' | 'holdout',\n independentUnitByScenarioId?: ReadonlyMap<string, string>,\n): LiftInsight | undefined {\n let bId = baselineId\n let cId = candidateId\n if (!bId || !cId) {\n // Auto-detect: when exactly two distinct candidateIds appear, treat the\n // lower-mean side as baseline.\n const ids = [...new Set(runs.map((r) => r.candidateId))]\n if (ids.length !== 2) return undefined\n const [idA, idB] = ids as [string, string]\n const scoresA = finiteCompositeScores(\n runs.filter((run) => run.candidateId === idA),\n split,\n )\n const scoresB = finiteCompositeScores(\n runs.filter((run) => run.candidateId === idB),\n split,\n )\n if (scoresA.length === 0 || scoresB.length === 0) return undefined\n const meanA = mean(scoresA)\n const meanB = mean(scoresB)\n bId = meanA <= meanB ? idA : idB\n cId = meanA <= meanB ? idB : idA\n }\n\n const baseline = runs.filter((r) => r.candidateId === bId)\n const candidate = runs.filter((r) => r.candidateId === cId)\n if (baseline.length === 0 || candidate.length === 0) return undefined\n\n const scoredBaseline = baseline.filter((run) => Number.isFinite(compositeOf(run, split)))\n const scoredCandidate = candidate.filter((run) => Number.isFinite(compositeOf(run, split)))\n const pairing = pairRunRecords(scoredBaseline, scoredCandidate)\n let pairedBaseline = pairing.pairs.map((pair) => compositeOf(pair.baseline, split))\n let pairedCandidate = pairing.pairs.map((pair) => compositeOf(pair.treatment, split))\n if (pairedBaseline.length === 0) return undefined\n const grouped = independentUnitByScenarioId\n ? aggregatePairedHoldout(\n {\n before: pairedBaseline,\n after: pairedCandidate,\n // Pairing is already fixed by experiment, scenario, and seed.\n // The ordinal labels each matched observation for the shared reducer.\n cellIds: pairing.pairs.map((pair, index) => `${pair.pairKey}:${index}`),\n },\n new Map(\n pairing.pairs.map((pair) => [\n pair.pairKey,\n independentUnitByScenarioId.get(pair.baseline.scenarioId)!,\n ]),\n ),\n )\n : undefined\n if (grouped) {\n pairedBaseline = grouped.before\n pairedCandidate = grouped.after\n }\n const reverseInferredArms =\n grouped !== undefined &&\n (!baselineId || !candidateId) &&\n mean(pairedBaseline) > mean(pairedCandidate)\n if (reverseInferredArms) [pairedBaseline, pairedCandidate] = [pairedCandidate, pairedBaseline]\n\n const baselineMean = mean(pairedBaseline)\n const candidateMean = mean(pairedCandidate)\n const delta = candidateMean - baselineMean\n\n const bootstrap = pairedBootstrap(pairedBaseline, pairedCandidate, {\n confidence: 0.95,\n resamples: 2000,\n statistic: 'mean',\n })\n const tTest = pairedTTest(pairedBaseline, pairedCandidate)\n const d = pairedCohensDz(pairedBaseline, pairedCandidate)\n const mde = pairedMde({ nPaired: pairedBaseline.length, power: 0.8, alpha: 0.05 })\n const requiredN =\n d === null || d === 0\n ? null\n : requiredPairedSampleSize({\n effect: Math.abs(d),\n power: 0.8,\n alpha: 0.05,\n })\n\n return {\n baselineMean,\n candidateMean,\n delta,\n ci95: [bootstrap.low, bootstrap.high],\n pValue: tTest.p,\n n: pairedBaseline.length,\n ...(grouped ? { pairedRunN: pairing.pairs.length, independentUnitIds: grouped.unitIds } : {}),\n minimumRequired: BOOTSTRAP_GATE_MIN_N,\n decisionEligible: bootstrap.gateEligible,\n unpairedBaseline: (reverseInferredArms ? pairing.unpairedTreatment : pairing.unpairedBaseline)\n .length,\n unpairedCandidate: (reverseInferredArms ? pairing.unpairedBaseline : pairing.unpairedTreatment)\n .length,\n cohensD: d,\n mde,\n requiredN,\n }\n}\n\nfunction mean(arr: number[]): number {\n return arr.length === 0 ? 0 : arr.reduce((s, v) => s + v, 0) / arr.length\n}\n\n// ── Failure clustering ──────────────────────────────────────────────\n\nasync function computeFailureClusters(\n runs: RunRecord[],\n analyst: AnalystRegistry,\n split: 'search' | 'holdout',\n): Promise<FailureClusterInsight | undefined> {\n const failed = runs.filter((run) => isTaskFailure(run, split))\n if (failed.length === 0) return { clusters: [], totalFailures: 0 }\n\n const clusters = new Map<string, { exemplars: string[]; runs: number }>()\n const recordCluster = (key: string, runId: string) => {\n const cluster = clusters.get(key) ?? { exemplars: [], runs: 0 }\n cluster.runs += 1\n if (cluster.exemplars.length < 5 && !cluster.exemplars.includes(runId)) {\n cluster.exemplars.push(runId)\n }\n clusters.set(key, cluster)\n }\n for (const run of failed) {\n try {\n // AnalystRunInputs routes by field name: run-record analysts read\n // `runRecord`. Any other shape makes every analyst skip with\n // \"missing input\" and the clusters come back silently empty.\n const result = await analyst.run(run.runId, { runRecord: run })\n // One run can produce several findings in one cluster; count it once.\n const keys = new Set(\n result.findings.map(\n (finding: AnalystFinding) => finding.area || finding.analyst_id || 'unclassified',\n ),\n )\n for (const key of keys) recordCluster(key, run.runId)\n } catch {\n recordCluster('analyst-error', run.runId)\n }\n }\n const clusterList = [...clusters.entries()].map(([id, c]) => ({\n id,\n name: id,\n share: c.runs / failed.length,\n exemplars: c.exemplars,\n }))\n clusterList.sort((a, b) => b.share - a.share)\n return { clusters: clusterList, totalFailures: failed.length }\n}\n\nfunction finiteCompositeScores(runs: readonly RunRecord[], split: 'search' | 'holdout'): number[] {\n return runs.map((run) => compositeOf(run, split)).filter(Number.isFinite)\n}\n\nfunction isTaskFailure(run: RunRecord, split: 'search' | 'holdout'): boolean {\n if (run.failureClass !== undefined && run.failureClass !== 'success') return true\n const score = compositeOf(run, split)\n return Number.isFinite(score) && score < 0.5\n}\n\n// ── Contamination ──────────────────────────────────────────────────\n\nfunction computeContamination(\n runs: RunRecord[],\n canaries: DatasetScenario[],\n): InsightReport['contamination'] {\n let leaks = 0\n const details: Array<{ runId: string; canary: string; matched: string }> = []\n for (const run of runs) {\n const output = stringifyOutput(run)\n if (!output) continue\n const leaksHere = checkCanaries(output, canaries)\n for (const leak of leaksHere) {\n leaks++\n details.push({ runId: run.runId, canary: leak.canary, matched: leak.evidence })\n }\n }\n return { leaks, holdoutAuditPassed: leaks === 0, details }\n}\n\nfunction stringifyOutput(run: RunRecord): string | undefined {\n // RunRecord doesn't fix where \"the agent's output\" lives — different\n // consumers stash it differently. We probe the common shapes: the\n // outcome.raw map (numeric only by design — unlikely to contain text),\n // and any string-valued fields tucked under metadata via type casting.\n // Consumers with bespoke shapes pass canaryScenarios only when they\n // know their runs carry a stringifiable surface.\n const metadata = (run as unknown as { metadata?: Record<string, unknown> }).metadata\n if (typeof metadata?.output === 'string') return metadata.output\n if (typeof metadata?.text === 'string') return metadata.text\n return undefined\n}\n\n// ── Outcome correlation + linear reward model ──────────────────────\n\nfunction computeOutcomeCorrelation(\n runs: RunRecord[],\n outcome: { metric: string; valueByRunId: Record<string, number> },\n split: 'search' | 'holdout',\n): OutcomeCorrelationInsight | undefined {\n const xs: number[] = []\n const ys: number[] = []\n for (const run of runs) {\n const y = outcome.valueByRunId[run.runId]\n if (y === undefined || !Number.isFinite(y)) continue\n const x = compositeOf(run, split)\n if (!Number.isFinite(x)) continue\n xs.push(x)\n ys.push(y)\n }\n if (xs.length < 3) return undefined\n\n const p = pearsonR(xs, ys)\n const s = spearmanR(xs, ys)\n const meanX = mean(xs)\n const meanY = mean(ys)\n let num = 0\n let denom = 0\n for (let i = 0; i < xs.length; i++) {\n num += (xs[i]! - meanX) * (ys[i]! - meanY)\n denom += (xs[i]! - meanX) ** 2\n }\n const slope = denom === 0 ? 0 : num / denom\n const intercept = meanY - slope * meanX\n const ssTot = ys.reduce((a, y) => a + (y - meanY) ** 2, 0)\n const ssRes = ys.reduce((a, y, i) => a + (y - (intercept + slope * xs[i]!)) ** 2, 0)\n const r2 = ssTot === 0 ? 0 : 1 - ssRes / ssTot\n\n return {\n metric: outcome.metric,\n n: xs.length,\n pearson: p,\n spearman: s,\n rewardModel: { intercept, slope, r2 },\n }\n}\n\n// ── Release confidence scorecard ───────────────────────────────────\n\nfunction buildReleaseScorecard(\n composite: ScalarDistribution,\n lift: LiftInsight | undefined,\n contamination: InsightReport['contamination'],\n): InsightReport['release'] {\n // Synthesise a minimal scorecard from the rolled-up signal. The\n // substrate's `evaluateReleaseConfidence` primitive consumes a richer\n // input shape that callers can produce by wiring SLO definitions; the\n // shape here is the contract `selfImprove`/`analyzeRuns` consumers\n // receive automatically. They can call `evaluateReleaseConfidence`\n // directly when they want SLO-based axis evaluation.\n const axes: InsightReport['release']['axes'] = []\n const liftPass =\n lift === undefined\n ? ('not_evaluated' as const)\n : !lift.decisionEligible\n ? ('not_evaluated' as const)\n : lift.ci95[0] > 0 && !zeroWidth(lift.ci95)\n ? ('pass' as const)\n : lift.delta > 0\n ? ('warn' as const)\n : ('fail' as const)\n axes.push({\n name: 'quality-lift',\n status: liftPass,\n detail: lift\n ? `delta=${lift.delta.toFixed(3)}, CI95=[${lift.ci95[0].toFixed(3)}, ${lift.ci95[1].toFixed(3)}], n=${lift.n}${lift.decisionEligible ? '' : ` (descriptive only; ${lift.minimumRequired} required)`}`\n : 'no baseline/candidate pair available',\n })\n const contamPass =\n contamination === undefined\n ? ('not_evaluated' as const)\n : contamination.leaks === 0\n ? ('pass' as const)\n : ('fail' as const)\n axes.push({\n name: 'contamination',\n status: contamPass,\n detail: contamination ? `${contamination.leaks} canary leak(s)` : 'no canaries supplied',\n })\n axes.push(\n composite.n === 0\n ? {\n name: 'composite-distribution',\n status: 'not_evaluated',\n detail: 'no task-quality scores available',\n }\n : {\n name: 'composite-distribution',\n status:\n composite.mean !== null && composite.mean >= 0.5\n ? 'pass'\n : composite.mean !== null && composite.mean >= 0.3\n ? 'warn'\n : 'fail',\n detail:\n composite.mean === null || composite.p50 === null || composite.p95 === null\n ? 'task-quality distribution is internally incomplete'\n : `mean=${composite.mean.toFixed(3)}, p50=${composite.p50.toFixed(3)}, p95=${composite.p95.toFixed(3)} over n=${composite.n}`,\n },\n )\n const status = axes.some((a) => a.status === 'fail')\n ? 'fail'\n : axes.some((a) => a.status === 'warn' || a.status === 'not_evaluated')\n ? 'warn'\n : 'pass'\n return {\n status,\n axes,\n issues: [],\n }\n}\n\n// ── Recommendations engine ─────────────────────────────────────────\n\ninterface RecommendationContext {\n composite: ScalarDistribution\n judges: Record<string, JudgeInsight>\n interRater?: InterRaterInsight\n lift?: LiftInsight\n failureClusters?: FailureClusterInsight\n failureClasses?: FailureClassTally[]\n contamination?: InsightReport['contamination']\n outcomeCorrelation?: OutcomeCorrelationInsight\n priorPeriodComparison?: PriorPeriodComparison\n threshold: number\n}\n\nfunction buildRecommendations(ctx: RecommendationContext): Recommendation[] {\n const out: Recommendation[] = []\n\n // Prior-period regressions — highest customer-impact signal when present.\n // \"Did my last change help?\" with a falsifiable answer.\n if (ctx.priorPeriodComparison) {\n const ppc = ctx.priorPeriodComparison\n const label = ppc.windowLabel ?? 'baseline period'\n for (const name of ppc.regressedMetrics) {\n const d = ppc.metrics[name]\n if (d?.status !== 'ok') continue\n out.push({\n priority: 'critical',\n kind: 'investigate',\n title: `${name} regressed from ${d.baseline.toFixed(3)} → ${d.current.toFixed(3)} vs ${label}`,\n detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). The regression is statistically significant at p<0.05 with at-least-small effect size.`,\n evidencePath: `priorPeriodComparison.metrics.${name}`,\n })\n }\n for (const name of ppc.improvedMetrics) {\n const d = ppc.metrics[name]\n if (d?.status !== 'ok') continue\n out.push({\n priority: 'low',\n kind: 'ship',\n title: `${name} improved from ${d.baseline.toFixed(3)} → ${d.current.toFixed(3)} vs ${label}`,\n detail: `Welch CI95 = [${d.ci95[0].toFixed(3)}, ${d.ci95[1].toFixed(3)}], p=${d.pValue.toFixed(4)}, Cohen's d=${d.cohensD.toFixed(2)} (n_current=${d.currentN}, n_baseline=${d.baselineN}). Statistically significant improvement worth flagging.`,\n evidencePath: `priorPeriodComparison.metrics.${name}`,\n })\n }\n for (const name of ppc.inconclusiveMetrics) {\n const d = ppc.metrics[name]\n if (!d || d.status === 'ok' || d.delta === 0) continue\n const reason =\n d.status === 'zero-variance'\n ? 'both periods have zero observed variance'\n : 'one or both periods have fewer than two observations'\n out.push({\n priority: 'high',\n kind: 'investigate',\n title: `${name} changed from ${d.baseline.toFixed(3)} → ${d.current.toFixed(3)} vs ${label}; inference unavailable`,\n detail: `Observed delta ${d.delta.toFixed(3)} across n_current=${d.currentN} and n_baseline=${d.baselineN}, but ${reason}. The report does not fabricate a p-value, confidence interval, or effect size; inspect independence and data capture before acting.`,\n evidencePath: `priorPeriodComparison.metrics.${name}`,\n })\n }\n }\n\n // Composite-distribution branch. Fires when the overall quality signal is\n // poor regardless of lift / contamination / clusters — the customer needs\n // to know they have a problem AND which specific runs to inspect.\n if (\n ctx.composite.n > 0 &&\n ctx.composite.mean !== null &&\n ctx.composite.p50 !== null &&\n ctx.composite.p95 !== null\n ) {\n if (ctx.composite.mean < 0.3) {\n const tail = ctx.composite.tailRuns ?? []\n const names = tail\n .slice(0, 5)\n .map((t) => `${t.runId}=${t.score.toFixed(3)}`)\n .join(', ')\n out.push({\n priority: 'critical',\n kind: 'investigate',\n title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below the 0.3 floor — the agent is broken on this corpus`,\n detail:\n tail.length > 0\n ? `Worst ${tail.length} run${tail.length === 1 ? '' : 's'} to inspect first: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`\n : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,\n evidencePath: 'composite.tailRuns',\n })\n } else if (ctx.composite.mean < 0.5) {\n const tail = ctx.composite.tailRuns ?? []\n const names = tail\n .slice(0, 3)\n .map((t) => `${t.runId}=${t.score.toFixed(3)}`)\n .join(', ')\n out.push({\n priority: 'high',\n kind: 'investigate',\n title: `Composite mean ${ctx.composite.mean.toFixed(3)} is below 0.5 — investigate the lower tail before claiming the agent is healthy`,\n detail:\n tail.length > 0\n ? `Worst ${tail.length} run${tail.length === 1 ? '' : 's'}: ${names}. Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`\n : `Histogram p50=${ctx.composite.p50.toFixed(3)}, p95=${ctx.composite.p95.toFixed(3)}.`,\n evidencePath: 'composite.tailRuns',\n })\n }\n }\n\n // A healthy-looking mean can hide a group of failed tasks sharing one\n // producer-reported cause. This path does not require an analyst.\n if (ctx.failureClasses && ctx.failureClasses.length > 0) {\n const top = ctx.failureClasses[0]!\n if (top.count >= 3 && top.share >= 0.15) {\n out.push({\n priority: top.share >= 0.25 ? 'high' : 'medium',\n kind: 'investigate',\n title: `'${top.failureClass}' is the dominant failure class — ${top.count} runs (${(top.share * 100).toFixed(0)}% of the corpus)`,\n detail: `The mean composite can look acceptable while one failure class dominates the lower tail. ${top.count} of ${ctx.composite.n} runs failed with '${top.failureClass}'${ctx.failureClasses.length > 1 ? ` (next: '${ctx.failureClasses[1]!.failureClass}' ×${ctx.failureClasses[1]!.count})` : ''}. Fix this cause first.`,\n evidencePath: 'failureClasses',\n })\n }\n }\n\n // Missing-judges branch. The report can't surface per-dimension or\n // calibration signal when `outcome.judgeScores` is empty across the\n // corpus. Tell the customer how to enrich.\n if (Object.keys(ctx.judges).length === 0 && ctx.composite.n > 0) {\n out.push({\n priority: 'medium',\n kind: 'expand-corpus',\n title: 'No judge scores recorded — per-dimension + calibration insights unavailable',\n detail:\n 'Records have no `outcome.judgeScores`. To unlock perDimension, judges, and calibration, attach a Judge run during your eval pass and populate `outcome.judgeScores.perJudge[judgeName][dimension] = score`. See `docs/insight-report.md` for the expected shape.',\n evidencePath: 'judges',\n })\n }\n\n if (ctx.lift) {\n if (!ctx.lift.decisionEligible) {\n out.push({\n priority: 'high',\n kind: 'expand-corpus',\n title: `Inconclusive — ${ctx.lift.n} paired runs; ${ctx.lift.minimumRequired} required`,\n detail: `The bootstrap interval is descriptive below ${ctx.lift.minimumRequired} paired observations and cannot support a ship decision.`,\n evidencePath: 'lift',\n })\n } else {\n const pairedEffect =\n ctx.lift.cohensD === null ? 'undefined (zero delta variance)' : ctx.lift.cohensD.toFixed(2)\n const pairedP =\n ctx.lift.pValue === null ? 'undefined (zero delta variance)' : ctx.lift.pValue.toFixed(4)\n const requiredRuns =\n ctx.lift.requiredN === null ? 'not estimable' : `~${ctx.lift.requiredN} paired runs`\n // A ZERO-WIDTH interval never reads as \"ship\": n identical paired deltas\n // make every resample identical, so `[g, g]` clears any threshold below g\n // and `[0, 0]` clears any negative `decisionThreshold`, on no spread at\n // all. It falls through to the inconclusive/hold arms, which is where a\n // sample carrying no information about its own error belongs.\n const decisive = !zeroWidth(ctx.lift.ci95) && ctx.lift.ci95[0] > ctx.threshold\n const inconclusive = ctx.lift.ci95[0] <= ctx.threshold && ctx.lift.ci95[1] > ctx.threshold\n if (decisive) {\n out.push({\n priority: 'critical',\n kind: 'ship',\n title: `Ship — lift ${ctx.lift.delta.toFixed(3)} (95% CI ${ctx.lift.ci95[0].toFixed(3)}..${ctx.lift.ci95[1].toFixed(3)})`,\n detail: `Holdout lift exceeds threshold ${ctx.threshold} with 95% bootstrap confidence (n=${ctx.lift.n}, p=${pairedP}, paired d=${pairedEffect}).`,\n evidencePath: 'lift',\n })\n } else if (inconclusive) {\n out.push({\n priority: 'high',\n kind: 'expand-corpus',\n title: `Inconclusive — required sample is ${requiredRuns} (have ${ctx.lift.n}) at current effect size`,\n detail: `CI straddles threshold. Current MDE at 80% power is ${ctx.lift.mde.toFixed(3)}; observed delta is ${ctx.lift.delta.toFixed(3)}.`,\n evidencePath: 'lift',\n })\n } else {\n out.push({\n priority: 'critical',\n kind: 'hold',\n title: `Hold — lift CI lower bound ${ctx.lift.ci95[0].toFixed(3)} is at or below threshold ${ctx.threshold}`,\n detail: `Bootstrap CI provides no statistical evidence the candidate is better. Consider tightening the mutation or expanding the holdout.`,\n evidencePath: 'lift',\n })\n }\n }\n }\n\n if (ctx.contamination && ctx.contamination.leaks > 0) {\n out.push({\n priority: 'critical',\n kind: 'fix',\n title: `${ctx.contamination.leaks} canary leak${ctx.contamination.leaks === 1 ? '' : 's'} detected`,\n detail: `Holdout integrity is compromised. The lift number is unreliable until you investigate.`,\n evidencePath: 'contamination',\n })\n }\n\n if (ctx.interRater && ctx.interRater.kappa < 0.5) {\n out.push({\n priority: 'high',\n kind: 'recalibrate',\n title: `Inter-rater weighted kappa ${ctx.interRater.kappa.toFixed(2)} is below 0.5`,\n detail:\n 'Raters disagree on what good looks like. Review the largest disagreement cases and refine the rubric before automating these decisions.',\n evidencePath: 'interRater',\n })\n }\n\n if (ctx.failureClusters && ctx.failureClusters.clusters.length > 0) {\n const top = ctx.failureClusters.clusters[0]!\n out.push({\n priority: 'high',\n kind: 'investigate',\n title: `Top failure cluster: ${top.name} (${(top.share * 100).toFixed(0)}% of failures)`,\n detail: `${ctx.failureClusters.totalFailures} runs failed. The largest cluster groups ${top.exemplars.length} exemplars under '${top.name}'.`,\n evidencePath: 'failureClusters.clusters[0]',\n })\n }\n\n if (ctx.outcomeCorrelation && Math.abs(ctx.outcomeCorrelation.spearman) < 0.3) {\n out.push({\n priority: 'medium',\n kind: 'recalibrate',\n title: `Judge scores decoupled from ${ctx.outcomeCorrelation.metric} (Spearman ρ=${ctx.outcomeCorrelation.spearman.toFixed(2)})`,\n detail: `Your judges score what they were trained to score, but it isn't predicting downstream ${ctx.outcomeCorrelation.metric}. Consider retraining the judge against ${ctx.outcomeCorrelation.metric} as the gold signal.`,\n evidencePath: 'outcomeCorrelation',\n })\n }\n\n return out\n}\n\n// ── Re-export pareto figure spec for hosted-side rendering ─────────\n\nexport type { ParetoFigureSpec }\n"],"mappings":";;;;;;;;;;;;;AA4BA,SAAgB,cAAc,QAAgB,WAA4C;CACxF,MAAM,QAAsB,CAAC;CAC7B,KAAK,MAAM,KAAK,WAAW;EACzB,IAAI,CAAC,EAAE,QAAQ;EACf,IAAI,OAAO,SAAS,EAAE,MAAM,GAC1B,MAAM,KAAK;GAAE,YAAY,EAAE;GAAI,QAAQ,EAAE;GAAQ,UAAU,QAAQ,QAAQ,EAAE,MAAM;EAAE,CAAC;CAE1F;CACA,OAAO;AACT;AA8IA,SAAS,QAAQ,QAAgB,QAAwB;CACvD,MAAM,KAAK,OAAO,QAAQ,MAAM;CAChC,IAAI,KAAK,GAAG,OAAO;CACnB,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,EAAE;CACjC,MAAM,MAAM,KAAK,IAAI,OAAO,QAAQ,KAAK,OAAO,SAAS,EAAE;CAC3D,QAAQ,QAAQ,IAAI,MAAM,MAAM,OAAO,MAAM,OAAO,GAAG,KAAK,MAAM,OAAO,SAAS,MAAM;AAC1F;;;;ACrDA,SAAgB,mBAAmB,MAAkD;CACnF,MAAM,OAAO,KAAK,KAAK,IAAI,iBAAiB;CAE5C,OAAO;EACL,WAAW,wBAAwB,MAFxB,KAAK,iBAAiB,EAEY;EAC7C,gBAAgB,wBAAwB,IAAI;CAC9C;AACF;;;;AAKA,SAAS,UAAU,IAAwC;CACzD,OAAO,CAAC,OAAO,SAAS,GAAG,EAAE,KAAK,CAAC,OAAO,SAAS,GAAG,EAAE,KAAK,GAAG,OAAO,GAAG;AAC5E;AAEA,eAAsB,YAAY,MAAkD;CAClF,MAAM,OAAO,KAAK,KAAK,IAAI,iBAAiB;CAC5C,MAAM,OAAO,KAAK,iBAAiB;CACnC,MAAM,YAAY,KAAK,qBAAqB;CAC5C,IAAI,CAAC,OAAO,SAAS,SAAS,GAC5B,MAAM,IAAI,MAAM,sDAAsD,WAAW;CAEnF,MAAM,QAAQ,aAAa,MAAM,KAAK,SAAS,MAAM;CAErD,MAAM,mBAAmB,KACtB,KAAK,OAAO;EAAE,OAAO,EAAE;EAAO,OAAO,YAAY,GAAG,KAAK;CAAE,EAAE,CAAC,CAC9D,QAAQ,MAAM,OAAO,SAAS,EAAE,KAAK,CAAC;CACzC,MAAM,YAAY,eAChB,iBAAiB,KAAK,MAAM,EAAE,KAAK,GACnC,MACA,gBACF;CAEA,MAAM,eAAe,oBAAoB,MAAM,IAAI;CACnD,MAAM,EAAE,WAAW,gBAAgB,eAAe,mBAAmB;EACnE;EACA,eAAe;CACjB,CAAC;CACD,MAAM,gBAAgB,KAAK,QAAQ,QAAQ,IAAI,eAAe,SAAS,YAAY;CACnF,MAAM,QAAQ,cAAc,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC,OAAO,cAAc;CACvE,MAAM,WAAW,eAAe,OAAO,IAAI;CAC3C,MAAM,SAAS,YAAY,eAAe,EAAE,MAAM,CAAC;CACnD,MAAM,WAA+C,CAAC;CACtD,IAAI,WAAW,WAAW,IAAI,GAC5B,SAAS,OAAO,qBAAqB,MAAM,UAAU;MAChD,IAAI,MAAM,WAAW,KAAK,MAAM,OAAO,MAAM,MAAM,CAAC,GACzD,SAAS,OAAO,OAAO,KAAK,OAAO;CAErC,IAAI,OAAO,OAAO,SAAS,GACzB,SAAS,SACP,OAAO,OAAO,WAAW,IACrB,uCACA;CAER,MAAM,cAAc;EAClB,MAAM;EACN;EACA;EACA,GAAI,SAAS,QAAQ,SAAS,SAAS,EAAE,SAAS,IAAI,CAAC;CACzD;CAEA,MAAM,SAAS,qBAAqB,IAAI;CAExC,MAAM,aAAa,KAAK,cAAc,kBAAkB,KAAK,WAAW,IAAI,KAAA;CAE5E,MAAM,OAAO,YACX,MACA,KAAK,qBACL,KAAK,sBACL,OACA,KAAK,2BACP;CAEA,MAAM,kBAAkB,KAAK,UACzB,MAAM,uBAAuB,MAAM,KAAK,SAAS,KAAK,IACtD,KAAA;CAEJ,MAAM,iBAAiB,sBAAsB,MAAM,KAAK;CAExD,MAAM,gBAAgB,KAAK,kBACvB,qBAAqB,MAAM,KAAK,eAAe,IAC/C,KAAA;CAEJ,MAAM,qBAAqB,KAAK,gBAC5B,0BAA0B,MAAM,KAAK,eAAe,KAAK,IACzD,KAAA;CAEJ,MAAM,UAAU,sBAAsB,WAAW,MAAM,aAAa;CAEpE,MAAM,wBAAwB,KAAK,eAC/B,6BAA6B,MAAM,KAAK,cAAc,OAAO,KAAK,aAAa,IAC/E,KAAA;CAEJ,MAAM,kBAAkB,qBAAqB;EAC3C;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC;CAED,OAAO;EACL,GAAG,KAAK;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,GAAI,iBAAiB,EAAE,eAAe,IAAI,CAAC;EAC3C,GAAI,wBAAwB,EAAE,sBAAsB,IAAI,CAAC;EACzD;CACF;AACF;AAEA,SAAS,wBAAwB,MAAmB,MAAgC;CAClF,MAAM,gBAAgB,KAAK,SAAS,QAAQ;EAC1C,MAAM,QAAQ,oBAAoB,GAAG;EACrC,OAAO,QAAQ,CAAC;GAAE;GAAO,SAAS,UAAU,KAAK,oBAAoB;EAAE,CAAC,IAAI,CAAC;CAC/E,CAAC;CACD,MAAM,iBAAiB,cAAc,SAAS,QAC5C,IAAI,YAAY,KAAA,IAAY,CAAC,IAAI,OAAO,IAAI,CAAC,CAC/C;CACA,MAAM,8BAAc,IAAI,IAAoB;CAC5C,IAAI,qBAAqB;CACzB,IAAI,uBAAuB;CAC3B,IAAI,qBAAqB;CACzB,IAAI,kBAAkB;CACtB,IAAI,yBAAyB;CAC7B,MAAM,mBAAuD;EAC3D,WAAW;EACX,QAAQ;EACR,WAAW;EACX,YAAY;EACZ,SAAS;CACX;CACA,MAAM,0BAAoF;EACxF,WAAW;GAAE,YAAY;GAAG,eAAe;GAAG,YAAY;EAAE;EAC5D,QAAQ;GAAE,YAAY;GAAG,eAAe;GAAG,YAAY;EAAE;EACzD,WAAW;GAAE,YAAY;GAAG,eAAe;GAAG,YAAY;EAAE;EAC5D,YAAY;GAAE,YAAY;GAAG,eAAe;GAAG,YAAY;EAAE;EAC7D,SAAS;GAAE,YAAY;GAAG,eAAe;GAAG,YAAY;EAAE;CAC5D;CACA,IAAI,gBAAgB;CACpB,IAAI,kBAAkB;CACtB,IAAI,yBAAyB;CAE7B,KAAK,MAAM,OAAO,MAAM;EACtB,YAAY,IAAI,IAAI,QAAQ,YAAY,IAAI,IAAI,KAAK,KAAK,KAAK,CAAC;EAChE,MAAM,kBAAkB,IAAI;EAC5B,iBAAiB,oBAAoB;EACrC,MAAM,aAAa,oBAAoB,KAAK,gBAAgB;EAC5D,IAAI,eAAe,KAAA,GAAW;GAC5B,mBAAmB;GACnB,0BAA0B;EAC5B;EACA,MAAM,QAAQ,IAAI;EAClB,KACG,cAAc,KAAK,KACpB,MAAM,QAAQ,KACd,MAAM,SAAS,MACd,MAAM,UAAU,KAAK,MACrB,MAAM,cAAc,KAAK,GAE1B,iBAAiB;EAEnB,MAAM,cAAc,6BAA6B,GAAG;EACpD,IAAI,gBAAgB,KAAA,GAAW;GAC7B,wBAAwB;GACxB,sBAAsB;GACtB,IAAI,cAAc,GAAG;IACnB,sBAAsB;IACtB,wBAAwB,gBAAgB,CAAC,cAAc;GACzD,OAAO,wBAAwB,gBAAgB,CAAC,iBAAiB;EACnE,OAAO,wBAAwB,gBAAgB,CAAC,cAAc;EAC9D,MAAM,qBAAqB,oBAAoB,KAAK,kBAAkB;EACtE,IAAI,uBAAuB,KAAA,GAAW;GACpC,mBAAmB;GACnB,0BAA0B;EAC5B;CACF;CAEA,OAAO;EACL,YAAY,eACV,KAAK,KAAK,QAAQ,IAAI,MAAM,GAC5B,IACF;EACA,SAAS,eACP,KAAK,QAAQ,QAAQ,IAAI,YAAY,KAAA,CAAS,CAAC,CAAC,KAAK,QAAQ,IAAI,OAAQ,GACzE,IACF;EACA,YAAY,oBACV,KAAK,KAAK,QAAQ,IAAI,UAAU,GAChC,IACF;EACA,gBAAgB;GACd,MAAM,cAAc;GACpB,YAAY,oBACV,cAAc,KAAK,QAAQ,IAAI,KAAK,GACpC,IACF;GACA,SAAS,eAAe,gBAAgB,IAAI;GAC5C,cAAc,eAAe,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC;EACxE;EACA,QAAQ,CAAC,GAAG,YAAY,QAAQ,CAAC,CAAC,CAC/B,KAAK,CAAC,OAAO,YAAY;GAAE;GAAO,MAAM;EAAM,EAAE,CAAC,CACjD,MAAM,MAAM,UAAU,MAAM,OAAO,KAAK,QAAQ,KAAK,MAAM,cAAc,MAAM,KAAK,CAAC;EACxF,YAAY;GACV,MAAM;GACN,QAAQ;GACR,eAAe;EACjB;EACA,iBAAiB;GACf,MAAM;GACN,UAAU,qBAAqB,IAAI,qBAAqB,qBAAqB;GAC7E,QAAQ;GACR,eAAe;GACf;GACA;GACA,mBAAmB;EACrB;EACA;CACF;AACF;AAEA,SAAS,6BAA6B,KAAoC;CACxE,OAAO,oBAAoB,KAAK,uBAAuB;AACzD;AAEA,SAAS,oBAAoB,KAAgB,KAAiC;CAC5E,MAAM,QAAQ,UAAU,KAAK,GAAG;CAChC,OAAO,UAAU,KAAA,KAAa,OAAO,UAAU,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AAChF;AAEA,SAAS,oBAAoB,QAAyB,MAAiC;CACrF,MAAM,YAAY,OAAO,SAAS,UAChC,MAAM,cAAc,KAAA,IAAY,CAAC,MAAM,SAAS,IAAI,CAAC,CACvD;CACA,MAAM,SAAS,OAAO,SAAS,UAAW,MAAM,WAAW,KAAA,IAAY,CAAC,MAAM,MAAM,IAAI,CAAC,CAAE;CAC3F,MAAM,aAAa,OAAO,SAAS,UACjC,MAAM,eAAe,KAAA,IAAY,CAAC,MAAM,UAAU,IAAI,CAAC,CACzD;CACA,OAAO;EACL,OAAO,eACL,OAAO,KAAK,UAAU,MAAM,KAAK,GACjC,IACF;EACA,QAAQ,eACN,OAAO,KAAK,UAAU,MAAM,MAAM,GAClC,IACF;EACA,WAAW,eAAe,WAAW,IAAI;EACzC,QAAQ,eAAe,QAAQ,IAAI;EACnC,YAAY,eAAe,YAAY,IAAI;EAC3C,QAAQ;GACN,OAAO,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,OAAO,CAAC;GAC7D,QAAQ,OAAO,QAAQ,OAAO,UAAU,QAAQ,MAAM,QAAQ,CAAC;GAC/D,WAAW,UAAU,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC;GAC9D,QAAQ,OAAO,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC;GACxD,YAAY,WAAW,QAAQ,OAAO,UAAU,QAAQ,OAAO,CAAC;EAClE;CACF;AACF;AAEA,SAAS,oBAAoB,KAA2C;CACtE,MAAM,QAAQ,UAAU,KAAK,yBAAyB;CACtD,MAAM,SAAS,UAAU,KAAK,6BAA6B;CAC3D,MAAM,YAAY,UAAU,KAAK,4BAA4B;CAC7D,MAAM,SAAS,UAAU,KAAK,yBAAyB;CACvD,MAAM,aAAa,UAAU,KAAK,8BAA8B;CAChE,IACE,UAAU,KAAA,KACV,WAAW,KAAA,KACX,cAAc,KAAA,KACd,WAAW,KAAA,KACX,eAAe,KAAA,GAEf,OAAO,KAAA;CACT,OAAO;EACL,OAAO,SAAS;EAChB,QAAQ,UAAU;EAClB,GAAI,cAAc,KAAA,IAAY,EAAE,UAAU,IAAI,CAAC;EAC/C,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;EACzC,GAAI,eAAe,KAAA,IAAY,EAAE,WAAW,IAAI,CAAC;CACnD;AACF;AAEA,SAAS,UAAU,KAAgB,KAAiC;CAClE,MAAM,QAAQ,IAAI,QAAQ,IAAI;CAC9B,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAA;AACrF;AAEA,SAAS,wBAAwB,MAA0C;CACzE,MAAM,UAAiC;EACrC,UAAU;GAAE,GAAG;GAAG,UAAU;EAAE;EAC9B,WAAW;GAAE,GAAG;GAAG,UAAU;EAAE;EAC/B,YAAY,EAAE,GAAG,EAAE;EACnB,eAAe;CACjB;CACA,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,IAAI;EACjB,IAAI,KAAK,SAAS,cAChB,QAAQ,WAAW,KAAK;OACnB;GACL,QAAQ,KAAK,KAAK,CAAC,KAAK;GACxB,QAAQ,KAAK,KAAK,CAAC,YAAY,KAAK;EACtC;CACF;CACA,MAAM,QAAQ,QAAQ,SAAS,IAAI,QAAQ,UAAU;CACrD,QAAQ,gBAAgB,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS;CAChE,OAAO;AACT;AAEA,SAAS,qBAAqB,MAAmB,YAA2C;CAC1F,MAAM,aAAa,WAAW,WAAW;CACzC,MAAM,QAAQ,WAAW,SAAS,IAAI,WAAW,UAAU;CAC3D,IAAI,eAAe,KAAK,QACtB,OAAO,+BAA+B,KAAK,OAAO;CAEpD,OAAO,2BAA2B,WAAW,GAAG,KAAK,OAAO,mDAAmD,MAAM,GAAG,KAAK,OAAO,aAAa,WAAW,SAAS,EAAE,aAAa,WAAW,UAAU,EAAE;AAC7M;;;;;;;AAQA,SAAS,sBACP,MACA,OACiC;CACjC,MAAM,yBAAS,IAAI,IAA0B;CAC7C,KAAK,MAAM,KAAK,MAAM;EACpB,IAAI,CAAC,cAAc,GAAG,KAAK,GAAG;EAC9B,MAAM,MACJ,EAAE,iBAAiB,KAAA,KAAa,EAAE,iBAAiB,YAAY,EAAE,eAAe;EAClF,OAAO,IAAI,MAAM,OAAO,IAAI,GAAG,KAAK,KAAK,CAAC;CAC5C;CACA,IAAI,OAAO,SAAS,GAAG,OAAO,KAAA;CAC9B,MAAM,IAAI,KAAK;CACf,OAAO,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CACzB,KAAK,CAAC,cAAc,YAAY;EAC/B;EACA;EACA,OAAO,IAAI,IAAI,QAAQ,IAAI;CAC7B,EAAE,CAAC,CACF,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,cAAc,EAAE,YAAY,CAAC;AACrF;AAUA,SAAS,6BACP,SACA,UACA,OACA,aACmC;CACnC,IAAI,QAAQ,WAAW,KAAK,SAAS,WAAW,GAAG,OAAO,KAAA;CAE1D,MAAM,UAAuC,CAAC;CAC9C,MAAM,aAA8C,CAAC;CAErD,MAAM,mBAAmB,QACtB,KAAK,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CACjC,OAAO,OAAO,QAAQ;CACzB,MAAM,oBAAoB,SACvB,KAAK,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CACjC,OAAO,OAAO,QAAQ;CACzB,IAAI,iBAAiB,SAAS,KAAK,kBAAkB,SAAS,GAAG;EAC/D,QAAQ,YAAY,aAAa,mBAAmB,gBAAgB;EACpE,WAAW,YAAY;CACzB;CAEA,MAAM,cAAc,gBAAgB,OAAO;CAC3C,MAAM,eAAe,gBAAgB,QAAQ;CAC7C,IAAI,YAAY,SAAS,KAAK,aAAa,SAAS,GAAG;EACrD,QAAQ,OAAO,aAAa,cAAc,WAAW;EACrD,WAAW,OAAO;CACpB;CAEA,MAAM,aAAa,QAAQ,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,OAAO,OAAO,QAAQ;CACtE,MAAM,cAAc,SAAS,KAAK,MAAM,EAAE,MAAM,CAAC,CAAC,OAAO,OAAO,QAAQ;CACxE,IAAI,WAAW,SAAS,KAAK,YAAY,SAAS,GAAG;EACnD,QAAQ,WAAW,aAAa,aAAa,UAAU;EACvD,WAAW,WAAW;CACxB;CAEA,MAAM,aAAa,QAChB,KAAK,OAAO,EAAE,WAAW,SAAS,MAAM,EAAE,WAAW,UAAU,EAAE,CAAC,CAClE,OAAO,OAAO,QAAQ;CACzB,MAAM,cAAc,SACjB,KAAK,OAAO,EAAE,WAAW,SAAS,MAAM,EAAE,WAAW,UAAU,EAAE,CAAC,CAClE,OAAO,OAAO,QAAQ;CACzB,IAAI,WAAW,SAAS,KAAK,YAAY,SAAS,GAAG;EACnD,QAAQ,aAAa,aAAa,aAAa,UAAU;EACzD,WAAW,aAAa;CAC1B;CAKA,MAAM,cAAc,oBAAoB,OAAO;CAC/C,MAAM,eAAe,oBAAoB,QAAQ;CACjD,KAAK,MAAM,OAAO,OAAO,KAAK,WAAW,GAAG;EAC1C,MAAM,IAAI,aAAa;EACvB,MAAM,IAAI,YAAY;EACtB,IAAI,CAAC,KAAK,EAAE,WAAW,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG;EAClD,QAAQ,OAAO,SAAS,aAAa,GAAG,CAAC;EACzC,WAAW,OAAO,SAAS;CAC7B;CAEA,MAAM,mBAA6B,CAAC;CACpC,MAAM,kBAA4B,CAAC;CACnC,MAAM,sBAAgC,CAAC;CACvC,KAAK,MAAM,CAAC,MAAM,UAAU,OAAO,QAAQ,OAAO,GAAG;EACnD,IAAI,MAAM,WAAW,MAAM;GACzB,oBAAoB,KAAK,IAAI;GAC7B;EACF;EACA,IAAI,CAAC,MAAM,aAAa;EAGxB,KAFY,WAAW,SAAS,wBACT,qBAAqB,MAAM,QAAQ,IAAI,MAAM,QAAQ,GAChE,gBAAgB,KAAK,IAAI;OAChC,iBAAiB,KAAK,IAAI;CACjC;CAEA,OAAO;EACL,WAAW,SAAS;EACpB,UAAU,QAAQ;EAClB,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;EACrC;EACA;EACA;EACA;CACF;AACF;AAEA,SAAS,gBAAgB,MAA6B;CACpD,OAAO,KACJ,QAAQ,QAAQ,IAAI,eAAe,SAAS,YAAY,CAAC,CACzD,KAAK,QAAQ,IAAI,OAAO,CAAC,CACzB,OAAO,cAAc;AAC1B;AAEA,SAAS,eAAe,OAAiC;CACvD,OAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;;AAGA,SAAS,oBAAoB,MAA6C;CACxE,MAAM,MAAgC,CAAC;CACvC,KAAK,MAAM,KAAK,MAAM;EACpB,MAAM,SAAS,EAAE,QAAQ,aAAa;EACtC,IAAI,CAAC,QAAQ;EACb,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,MAAM,GAAG;GACjD,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG;GAC7B,IAAI,CAAC,IAAI,MAAM,IAAI,OAAO,CAAC;GAC3B,IAAI,IAAI,CAAC,KAAK,KAAe;EAC/B;CACF;CACA,OAAO;AACT;;AAGA,SAAS,aAAa,UAAoB,SAAgC;CACxE,MAAM,SAAS,YAAY,UAAU,OAAO;CAC5C,MAAM,OAAO;EACX,SAAS,OAAO;EAChB,UAAU,OAAO;EACjB,OAAO,OAAO;EACd,WAAW,SAAS;EACpB,UAAU,QAAQ;CACpB;CACA,IAAI,OAAO,WAAW,MACpB,OAAO;EACL,GAAG;EACH,QAAQ,OAAO;EACf,MAAM;EACN,QAAQ;EACR,SAAS;EACT,aAAa;CACf;CAEF,OAAO;EACL,GAAG;EACH,QAAQ;EACR,MAAM,OAAO;EACb,QAAQ,OAAO;EACf,SAAS,OAAO;EAChB,aAAa,OAAO,IAAI,OAAQ,KAAK,IAAI,OAAO,OAAO,KAAK;CAC9D;AACF;AAIA,SAAS,aACP,MACA,MACsB;CACtB,IAAI,SAAS,QAAQ,OAAO;CAE5B,OADmB,KAAK,MAAM,MAAM,OAAO,SAAS,mBAAmB,GAAG,SAAS,CAAC,CACpE,IAAI,YAAY;AAClC;;;;;;;AAQA,SAAS,YAAY,KAAgB,OAAqC;CAIxE,MAAM,QAAQ,mBAAmB,KAAK,KAAK;CAC3C,OAAO,OAAO,SAAS,KAAK,IAAK,QAAmB;AACtD;AAIA,SAAS,eACP,QACA,MACA,SACoB;CACpB,IAAI,OAAO,WAAW,GACpB,OAAO;EACL,GAAG;EACH,MAAM;EACN,KAAK;EACL,KAAK;EACL,QAAQ;EACR,KAAK;EACL,KAAK;EACL,WAAW,CAAC;CACd;CAEF,MAAM,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;CAC/C,MAAM,IAAI,OAAO;CACjB,MAAM,OAAO,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;CACjD,MAAM,WAAW,OAAO,QAAQ,GAAG,MAAM,KAAK,IAAI,SAAS,GAAG,CAAC,IAAI;CACnE,MAAM,SAAS,KAAK,KAAK,QAAQ;CACjC,MAAM,WAAW,UACb,CAAC,GAAG,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,IAAI,GAAG,QAAQ,MAAM,CAAC,IACnF,KAAA;CACJ,OAAO;EACL;EACA;EACA,KAAK,WAAW,QAAQ,EAAG;EAC3B,KAAK,WAAW,QAAQ,GAAI;EAC5B;EACA,KAAK,OAAO;EACZ,KAAK,OAAO,IAAI;EAChB,WAAW,UAAU,QAAQ,IAAI;EACjC,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;CACjC;AACF;AAEA,SAAS,WAAW,QAAkB,GAAmB;CACvD,IAAI,OAAO,WAAW,GAAG,OAAO;CAChC,IAAI,OAAO,WAAW,GAAG,OAAO,OAAO;CACvC,MAAM,OAAO,OAAO,SAAS,KAAK;CAClC,MAAM,KAAK,KAAK,MAAM,GAAG;CACzB,MAAM,KAAK,KAAK,KAAK,GAAG;CACxB,IAAI,OAAO,IAAI,OAAO,OAAO;CAC7B,MAAM,IAAI,MAAM;CAChB,OAAO,OAAO,OAAQ,IAAI,KAAK,OAAO,MAAO;AAC/C;;;;AAKA,SAAS,UAAU,QAAkB,MAA+C;CAClF,IAAI,OAAO,WAAW,KAAK,OAAO,GAAG,OAAO,CAAC;CAC7C,MAAM,MAAM,OAAO;CACnB,MAAM,MAAM,OAAO,OAAO,SAAS;CACnC,IAAI,QAAQ,KAAK,OAAO,CAAC;EAAE,IAAI;EAAK,IAAI;EAAK,OAAO,OAAO;CAAO,CAAC;CACnE,MAAM,SAAS,MAAM,OAAO;CAC5B,MAAM,MAAuC,CAAC;CAC9C,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,KAAK;EAC7B,MAAM,KAAK,MAAM,IAAI;EACrB,MAAM,KAAK,MAAM,OAAO,IAAI,MAAM,KAAK;EACvC,IAAI,KAAK;GAAE;GAAI;GAAI,OAAO;EAAE,CAAC;CAC/B;CACA,KAAK,MAAM,KAAK,QAAQ;EACtB,MAAM,MAAM,KAAK,IAAI,OAAO,GAAG,KAAK,OAAO,IAAI,OAAO,KAAK,CAAC;EAC5D,IAAI,IAAI,CAAE;CACZ;CACA,OAAO;AACT;AAEA,SAAS,oBAAoB,MAAmB,MAAkD;CAKhG,MAAM,wBAAQ,IAAI,IAAsB;CACxC,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,IAAI,QAAQ;EAC3B,IAAI,CAAC,QAAQ;EACb,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,OAAO,cAAc,CAAC,CAAC,GAAG;GAClE,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG;GAC7B,MAAM,MAAM,MAAM,IAAI,GAAG,KAAK,CAAC;GAC/B,IAAI,KAAK,KAAK;GACd,MAAM,IAAI,KAAK,GAAG;EACpB;CACF;CACA,MAAM,MAA0C,CAAC;CACjD,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,IAAI,OAAO,eAAe,QAAQ,IAAI;CACzE,OAAO;AACT;AAIA,SAAS,qBAAqB,MAAiD;CAI7E,MAAM,MAAoC,CAAC;CAC3C,MAAM,0BAAU,IAAI,IAAsB;CAC1C,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,IAAI,QAAQ;EAC3B,IAAI,CAAC,QAAQ,UAAU;EACvB,KAAK,MAAM,CAAC,SAAS,SAAS,OAAO,QAAQ,OAAO,QAAQ,GAAG;GAC7D,MAAM,YAAY,OAAO,OAAO,IAAI,CAAC,CAAC,OAAO,OAAO,QAAQ;GAC5D,IAAI,UAAU,WAAW,GAAG;GAC5B,MAAM,YAAY,UAAU,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,UAAU;GACnE,MAAM,MAAM,QAAQ,IAAI,OAAO,KAAK,CAAC;GACrC,IAAI,KAAK,SAAS;GAClB,QAAQ,IAAI,SAAS,GAAG;EAC1B;CACF;CACA,KAAK,MAAM,CAAC,SAAS,WAAW,SAC9B,IAAI,WAAW;EACb,GAAG,OAAO;EACV,WAAW,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;CACxD;CAEF,OAAO;AACT;AAIA,SAAS,kBACP,SAC+B;CAC/B,MAAM,wBAAQ,IAAI,IAAqD;CACvE,KAAK,MAAM,KAAK,SAAS;EACvB,IAAI,CAAC,OAAO,SAAS,EAAE,KAAK,GAAG;EAC/B,MAAM,OAAO,MAAM,IAAI,EAAE,KAAK,KAAK,CAAC;EACpC,KAAK,KAAK;GAAE,OAAO,EAAE;GAAO,OAAO,EAAE;EAAM,CAAC;EAC5C,MAAM,IAAI,EAAE,OAAO,IAAI;CACzB;CACA,MAAM,SAAS,IAAI,IAAI,QAAQ,KAAK,MAAM,EAAE,KAAK,CAAC;CAClD,MAAM,eAAyB,CAAC;CAChC,KAAK,MAAM,CAAC,OAAO,iBAAiB,OAAO;EACzC,MAAM,OAAO,IAAI,IAAI,aAAa,KAAK,MAAM,EAAE,KAAK,CAAC;EACrD,IAAI,MAAM;EACV,KAAK,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,MAAM;EAChD,IAAI,KAAK,aAAa,KAAK,KAAK;CAClC;CACA,IAAI,OAAO,OAAO,KAAK,aAAa,WAAW,GAAG,OAAO,KAAA;CAEzD,MAAM,YAAY,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK;CACnC,MAAM,UAAkC,CAAC;CACzC,KAAK,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KACpC,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;EAC7C,MAAM,IAAI,UAAU;EACpB,MAAM,IAAI,UAAU;EACpB,MAAM,UAAoB,CAAC;EAC3B,MAAM,UAAoB,CAAC;EAC3B,KAAK,MAAM,SAAS,cAAc;GAChC,MAAM,eAAe,MAAM,IAAI,KAAK;GACpC,MAAM,KAAK,aAAa,MAAM,MAAM,EAAE,UAAU,CAAC,CAAC,EAAE;GACpD,MAAM,KAAK,aAAa,MAAM,MAAM,EAAE,UAAU,CAAC,CAAC,EAAE;GACpD,IAAI,OAAO,KAAA,KAAa,OAAO,KAAA,GAAW;IACxC,QAAQ,KAAK,EAAE;IACf,QAAQ,KAAK,EAAE;GACjB;EACF;EACA,MAAM,YAAY,oBAChB,QAAQ,KAAK,OAAO,UAAU,CAAC,OAAO,QAAQ,MAAO,CAAC,GACtD,EAAE,WAAW,EAAE,CACjB;EACA,QAAQ,GAAG,EAAE,IAAI,OAAO,UAAU;CACpC;CAMF,MAAM,YAAY,oBAJH,aAAa,KAAK,UAAU;EACzC,MAAM,iBAAiB,IAAI,IAAI,MAAM,IAAI,KAAK,CAAC,CAAE,KAAK,WAAW,CAAC,OAAO,OAAO,OAAO,KAAK,CAAC,CAAC;EAC9F,OAAO,UAAU,KAAK,UAAU,eAAe,IAAI,KAAK,CAAE;CAC5D,CAC2C,GAAG,EAAE,WAAW,EAAE,CAAC;CAE9D,MAAM,oBAAoB,aACvB,KAAK,UAAU;EACd,MAAM,eAAe,MAAM,IAAI,KAAK;EACpC,MAAM,SAAS,aAAa,KAAK,MAAM,EAAE,KAAK;EAE9C,OAAO;GAAE;GAAO,SAAS;GAAc,OADzB,KAAK,IAAI,GAAG,MAAM,IAAI,KAAK,IAAI,GAAG,MAAM;EACT;CAC/C,CAAC,CAAC,CACD,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CACjC,MAAM,GAAG,EAAE;CAEd,OAAO;EACL,QAAQ,OAAO;EACf,cAAc,aAAa;EAC3B,OAAO,OAAO,SAAS,UAAU,aAAa,IAAI,UAAU,gBAAgB;EAC5E,KAAK,UAAU;EACf,SAAS,UAAU;EACnB,UAAU,UAAU;EACpB;EACA;CACF;AACF;AAIA,SAAS,YACP,MACA,YACA,aACA,OACA,6BACyB;CACzB,IAAI,MAAM;CACV,IAAI,MAAM;CACV,IAAI,CAAC,OAAO,CAAC,KAAK;EAGhB,MAAM,MAAM,CAAC,GAAG,IAAI,IAAI,KAAK,KAAK,MAAM,EAAE,WAAW,CAAC,CAAC;EACvD,IAAI,IAAI,WAAW,GAAG,OAAO,KAAA;EAC7B,MAAM,CAAC,KAAK,OAAO;EACnB,MAAM,UAAU,sBACd,KAAK,QAAQ,QAAQ,IAAI,gBAAgB,GAAG,GAC5C,KACF;EACA,MAAM,UAAU,sBACd,KAAK,QAAQ,QAAQ,IAAI,gBAAgB,GAAG,GAC5C,KACF;EACA,IAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,GAAG,OAAO,KAAA;EACzD,MAAM,QAAQ,KAAK,OAAO;EAC1B,MAAM,QAAQ,KAAK,OAAO;EAC1B,MAAM,SAAS,QAAQ,MAAM;EAC7B,MAAM,SAAS,QAAQ,MAAM;CAC/B;CAEA,MAAM,WAAW,KAAK,QAAQ,MAAM,EAAE,gBAAgB,GAAG;CACzD,MAAM,YAAY,KAAK,QAAQ,MAAM,EAAE,gBAAgB,GAAG;CAC1D,IAAI,SAAS,WAAW,KAAK,UAAU,WAAW,GAAG,OAAO,KAAA;CAI5D,MAAM,UAAU,eAFO,SAAS,QAAQ,QAAQ,OAAO,SAAS,YAAY,KAAK,KAAK,CAAC,CAE3C,GADpB,UAAU,QAAQ,QAAQ,OAAO,SAAS,YAAY,KAAK,KAAK,CAAC,CAC5B,CAAC;CAC9D,IAAI,iBAAiB,QAAQ,MAAM,KAAK,SAAS,YAAY,KAAK,UAAU,KAAK,CAAC;CAClF,IAAI,kBAAkB,QAAQ,MAAM,KAAK,SAAS,YAAY,KAAK,WAAW,KAAK,CAAC;CACpF,IAAI,eAAe,WAAW,GAAG,OAAO,KAAA;CACxC,MAAM,UAAU,8BACZ,uBACE;EACE,QAAQ;EACR,OAAO;EAGP,SAAS,QAAQ,MAAM,KAAK,MAAM,UAAU,GAAG,KAAK,QAAQ,GAAG,OAAO;CACxE,GACA,IAAI,IACF,QAAQ,MAAM,KAAK,SAAS,CAC1B,KAAK,SACL,4BAA4B,IAAI,KAAK,SAAS,UAAU,CAC1D,CAAC,CACH,CACF,IACA,KAAA;CACJ,IAAI,SAAS;EACX,iBAAiB,QAAQ;EACzB,kBAAkB,QAAQ;CAC5B;CACA,MAAM,sBACJ,YAAY,KAAA,MACX,CAAC,cAAc,CAAC,gBACjB,KAAK,cAAc,IAAI,KAAK,eAAe;CAC7C,IAAI,qBAAqB,CAAC,gBAAgB,mBAAmB,CAAC,iBAAiB,cAAc;CAE7F,MAAM,eAAe,KAAK,cAAc;CACxC,MAAM,gBAAgB,KAAK,eAAe;CAC1C,MAAM,QAAQ,gBAAgB;CAE9B,MAAM,YAAY,gBAAgB,gBAAgB,iBAAiB;EACjE,YAAY;EACZ,WAAW;EACX,WAAW;CACb,CAAC;CACD,MAAM,QAAQ,YAAY,gBAAgB,eAAe;CACzD,MAAM,IAAI,eAAe,gBAAgB,eAAe;CACxD,MAAM,MAAM,UAAU;EAAE,SAAS,eAAe;EAAQ,OAAO;EAAK,OAAO;CAAK,CAAC;CACjF,MAAM,YACJ,MAAM,QAAQ,MAAM,IAChB,OACA,yBAAyB;EACvB,QAAQ,KAAK,IAAI,CAAC;EAClB,OAAO;EACP,OAAO;CACT,CAAC;CAEP,OAAO;EACL;EACA;EACA;EACA,MAAM,CAAC,UAAU,KAAK,UAAU,IAAI;EACpC,QAAQ,MAAM;EACd,GAAG,eAAe;EAClB,GAAI,UAAU;GAAE,YAAY,QAAQ,MAAM;GAAQ,oBAAoB,QAAQ;EAAQ,IAAI,CAAC;EAC3F,iBAAA;EACA,kBAAkB,UAAU;EAC5B,mBAAmB,sBAAsB,QAAQ,oBAAoB,QAAQ,iBAAA,CAC1E;EACH,oBAAoB,sBAAsB,QAAQ,mBAAmB,QAAQ,kBAAA,CAC1E;EACH,SAAS;EACT;EACA;CACF;AACF;AAEA,SAAS,KAAK,KAAuB;CACnC,OAAO,IAAI,WAAW,IAAI,IAAI,IAAI,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI;AACrE;AAIA,eAAe,uBACb,MACA,SACA,OAC4C;CAC5C,MAAM,SAAS,KAAK,QAAQ,QAAQ,cAAc,KAAK,KAAK,CAAC;CAC7D,IAAI,OAAO,WAAW,GAAG,OAAO;EAAE,UAAU,CAAC;EAAG,eAAe;CAAE;CAEjE,MAAM,2BAAW,IAAI,IAAmD;CACxE,MAAM,iBAAiB,KAAa,UAAkB;EACpD,MAAM,UAAU,SAAS,IAAI,GAAG,KAAK;GAAE,WAAW,CAAC;GAAG,MAAM;EAAE;EAC9D,QAAQ,QAAQ;EAChB,IAAI,QAAQ,UAAU,SAAS,KAAK,CAAC,QAAQ,UAAU,SAAS,KAAK,GACnE,QAAQ,UAAU,KAAK,KAAK;EAE9B,SAAS,IAAI,KAAK,OAAO;CAC3B;CACA,KAAK,MAAM,OAAO,QAChB,IAAI;EAIF,MAAM,SAAS,MAAM,QAAQ,IAAI,IAAI,OAAO,EAAE,WAAW,IAAI,CAAC;EAE9D,MAAM,OAAO,IAAI,IACf,OAAO,SAAS,KACb,YAA4B,QAAQ,QAAQ,QAAQ,cAAc,cACrE,CACF;EACA,KAAK,MAAM,OAAO,MAAM,cAAc,KAAK,IAAI,KAAK;CACtD,QAAQ;EACN,cAAc,iBAAiB,IAAI,KAAK;CAC1C;CAEF,MAAM,cAAc,CAAC,GAAG,SAAS,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,QAAQ;EAC5D;EACA,MAAM;EACN,OAAO,EAAE,OAAO,OAAO;EACvB,WAAW,EAAE;CACf,EAAE;CACF,YAAY,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;CAC5C,OAAO;EAAE,UAAU;EAAa,eAAe,OAAO;CAAO;AAC/D;AAEA,SAAS,sBAAsB,MAA4B,OAAuC;CAChG,OAAO,KAAK,KAAK,QAAQ,YAAY,KAAK,KAAK,CAAC,CAAC,CAAC,OAAO,OAAO,QAAQ;AAC1E;AAEA,SAAS,cAAc,KAAgB,OAAsC;CAC3E,IAAI,IAAI,iBAAiB,KAAA,KAAa,IAAI,iBAAiB,WAAW,OAAO;CAC7E,MAAM,QAAQ,YAAY,KAAK,KAAK;CACpC,OAAO,OAAO,SAAS,KAAK,KAAK,QAAQ;AAC3C;AAIA,SAAS,qBACP,MACA,UACgC;CAChC,IAAI,QAAQ;CACZ,MAAM,UAAqE,CAAC;CAC5E,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,SAAS,gBAAgB,GAAG;EAClC,IAAI,CAAC,QAAQ;EACb,MAAM,YAAY,cAAc,QAAQ,QAAQ;EAChD,KAAK,MAAM,QAAQ,WAAW;GAC5B;GACA,QAAQ,KAAK;IAAE,OAAO,IAAI;IAAO,QAAQ,KAAK;IAAQ,SAAS,KAAK;GAAS,CAAC;EAChF;CACF;CACA,OAAO;EAAE;EAAO,oBAAoB,UAAU;EAAG;CAAQ;AAC3D;AAEA,SAAS,gBAAgB,KAAoC;CAO3D,MAAM,WAAY,IAA0D;CAC5E,IAAI,OAAO,UAAU,WAAW,UAAU,OAAO,SAAS;CAC1D,IAAI,OAAO,UAAU,SAAS,UAAU,OAAO,SAAS;AAE1D;AAIA,SAAS,0BACP,MACA,SACA,OACuC;CACvC,MAAM,KAAe,CAAC;CACtB,MAAM,KAAe,CAAC;CACtB,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,IAAI,QAAQ,aAAa,IAAI;EACnC,IAAI,MAAM,KAAA,KAAa,CAAC,OAAO,SAAS,CAAC,GAAG;EAC5C,MAAM,IAAI,YAAY,KAAK,KAAK;EAChC,IAAI,CAAC,OAAO,SAAS,CAAC,GAAG;EACzB,GAAG,KAAK,CAAC;EACT,GAAG,KAAK,CAAC;CACX;CACA,IAAI,GAAG,SAAS,GAAG,OAAO,KAAA;CAE1B,MAAM,IAAI,SAAS,IAAI,EAAE;CACzB,MAAM,IAAI,UAAU,IAAI,EAAE;CAC1B,MAAM,QAAQ,KAAK,EAAE;CACrB,MAAM,QAAQ,KAAK,EAAE;CACrB,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,KAAK,IAAI,IAAI,GAAG,IAAI,GAAG,QAAQ,KAAK;EAClC,QAAQ,GAAG,KAAM,UAAU,GAAG,KAAM;EACpC,UAAU,GAAG,KAAM,UAAU;CAC/B;CACA,MAAM,QAAQ,UAAU,IAAI,IAAI,MAAM;CACtC,MAAM,YAAY,QAAQ,QAAQ;CAClC,MAAM,QAAQ,GAAG,QAAQ,GAAG,MAAM,KAAK,IAAI,UAAU,GAAG,CAAC;CACzD,MAAM,QAAQ,GAAG,QAAQ,GAAG,GAAG,MAAM,KAAK,KAAK,YAAY,QAAQ,GAAG,QAAS,GAAG,CAAC;CACnF,MAAM,KAAK,UAAU,IAAI,IAAI,IAAI,QAAQ;CAEzC,OAAO;EACL,QAAQ,QAAQ;EAChB,GAAG,GAAG;EACN,SAAS;EACT,UAAU;EACV,aAAa;GAAE;GAAW;GAAO;EAAG;CACtC;AACF;AAIA,SAAS,sBACP,WACA,MACA,eAC0B;CAO1B,MAAM,OAAyC,CAAC;CAChD,MAAM,WACJ,SAAS,KAAA,IACJ,kBACD,CAAC,KAAK,mBACH,kBACD,KAAK,KAAK,KAAK,KAAK,CAAC,UAAU,KAAK,IAAI,IACrC,SACD,KAAK,QAAQ,IACV,SACA;CACb,KAAK,KAAK;EACR,MAAM;EACN,QAAQ;EACR,QAAQ,OACJ,SAAS,KAAK,MAAM,QAAQ,CAAC,EAAE,UAAU,KAAK,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO,KAAK,IAAI,KAAK,mBAAmB,KAAK,uBAAuB,KAAK,gBAAgB,gBACtL;CACN,CAAC;CACD,MAAM,aACJ,kBAAkB,KAAA,IACb,kBACD,cAAc,UAAU,IACrB,SACA;CACT,KAAK,KAAK;EACR,MAAM;EACN,QAAQ;EACR,QAAQ,gBAAgB,GAAG,cAAc,MAAM,mBAAmB;CACpE,CAAC;CACD,KAAK,KACH,UAAU,MAAM,IACZ;EACE,MAAM;EACN,QAAQ;EACR,QAAQ;CACV,IACA;EACE,MAAM;EACN,QACE,UAAU,SAAS,QAAQ,UAAU,QAAQ,KACzC,SACA,UAAU,SAAS,QAAQ,UAAU,QAAQ,KAC3C,SACA;EACR,QACE,UAAU,SAAS,QAAQ,UAAU,QAAQ,QAAQ,UAAU,QAAQ,OACnE,uDACA,QAAQ,UAAU,KAAK,QAAQ,CAAC,EAAE,QAAQ,UAAU,IAAI,QAAQ,CAAC,EAAE,QAAQ,UAAU,IAAI,QAAQ,CAAC,EAAE,UAAU,UAAU;CAChI,CACN;CAMA,OAAO;EACL,QANa,KAAK,MAAM,MAAM,EAAE,WAAW,MAAM,IAC/C,SACA,KAAK,MAAM,MAAM,EAAE,WAAW,UAAU,EAAE,WAAW,eAAe,IAClE,SACA;EAGJ;EACA,QAAQ,CAAC;CACX;AACF;AAiBA,SAAS,qBAAqB,KAA8C;CAC1E,MAAM,MAAwB,CAAC;CAI/B,IAAI,IAAI,uBAAuB;EAC7B,MAAM,MAAM,IAAI;EAChB,MAAM,QAAQ,IAAI,eAAe;EACjC,KAAK,MAAM,QAAQ,IAAI,kBAAkB;GACvC,MAAM,IAAI,IAAI,QAAQ;GACtB,IAAI,GAAG,WAAW,MAAM;GACxB,IAAI,KAAK;IACP,UAAU;IACV,MAAM;IACN,OAAO,GAAG,KAAK,kBAAkB,EAAE,SAAS,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,EAAE,MAAM;IACvF,QAAQ,iBAAiB,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,QAAQ,CAAC,EAAE,cAAc,EAAE,QAAQ,QAAQ,CAAC,EAAE,cAAc,EAAE,SAAS,eAAe,EAAE,UAAU;IACzL,cAAc,iCAAiC;GACjD,CAAC;EACH;EACA,KAAK,MAAM,QAAQ,IAAI,iBAAiB;GACtC,MAAM,IAAI,IAAI,QAAQ;GACtB,IAAI,GAAG,WAAW,MAAM;GACxB,IAAI,KAAK;IACP,UAAU;IACV,MAAM;IACN,OAAO,GAAG,KAAK,iBAAiB,EAAE,SAAS,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,EAAE,MAAM;IACtF,QAAQ,iBAAiB,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,OAAO,QAAQ,CAAC,EAAE,cAAc,EAAE,QAAQ,QAAQ,CAAC,EAAE,cAAc,EAAE,SAAS,eAAe,EAAE,UAAU;IACzL,cAAc,iCAAiC;GACjD,CAAC;EACH;EACA,KAAK,MAAM,QAAQ,IAAI,qBAAqB;GAC1C,MAAM,IAAI,IAAI,QAAQ;GACtB,IAAI,CAAC,KAAK,EAAE,WAAW,QAAQ,EAAE,UAAU,GAAG;GAC9C,MAAM,SACJ,EAAE,WAAW,kBACT,6CACA;GACN,IAAI,KAAK;IACP,UAAU;IACV,MAAM;IACN,OAAO,GAAG,KAAK,gBAAgB,EAAE,SAAS,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,QAAQ,CAAC,EAAE,MAAM,MAAM;IAC3F,QAAQ,kBAAkB,EAAE,MAAM,QAAQ,CAAC,EAAE,oBAAoB,EAAE,SAAS,kBAAkB,EAAE,UAAU,QAAQ,OAAO;IACzH,cAAc,iCAAiC;GACjD,CAAC;EACH;CACF;CAKA,IACE,IAAI,UAAU,IAAI,KAClB,IAAI,UAAU,SAAS,QACvB,IAAI,UAAU,QAAQ,QACtB,IAAI,UAAU,QAAQ,MAElB;MAAA,IAAI,UAAU,OAAO,IAAK;GAC5B,MAAM,OAAO,IAAI,UAAU,YAAY,CAAC;GACxC,MAAM,QAAQ,KACX,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,MAAM,GAAG,EAAE,MAAM,GAAG,EAAE,MAAM,QAAQ,CAAC,GAAG,CAAC,CAC9C,KAAK,IAAI;GACZ,IAAI,KAAK;IACP,UAAU;IACV,MAAM;IACN,OAAO,kBAAkB,IAAI,UAAU,KAAK,QAAQ,CAAC,EAAE;IACvD,QACE,KAAK,SAAS,IACV,SAAS,KAAK,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,IAAI,qBAAqB,MAAM,kBAAkB,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,QAAQ,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,KACvK,iBAAiB,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,QAAQ,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE;IACzF,cAAc;GAChB,CAAC;EACH,OAAO,IAAI,IAAI,UAAU,OAAO,IAAK;GACnC,MAAM,OAAO,IAAI,UAAU,YAAY,CAAC;GACxC,MAAM,QAAQ,KACX,MAAM,GAAG,CAAC,CAAC,CACX,KAAK,MAAM,GAAG,EAAE,MAAM,GAAG,EAAE,MAAM,QAAQ,CAAC,GAAG,CAAC,CAC9C,KAAK,IAAI;GACZ,IAAI,KAAK;IACP,UAAU;IACV,MAAM;IACN,OAAO,kBAAkB,IAAI,UAAU,KAAK,QAAQ,CAAC,EAAE;IACvD,QACE,KAAK,SAAS,IACV,SAAS,KAAK,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,IAAI,IAAI,MAAM,kBAAkB,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,QAAQ,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,KACtJ,iBAAiB,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE,QAAQ,IAAI,UAAU,IAAI,QAAQ,CAAC,EAAE;IACzF,cAAc;GAChB,CAAC;EACH;;CAKF,IAAI,IAAI,kBAAkB,IAAI,eAAe,SAAS,GAAG;EACvD,MAAM,MAAM,IAAI,eAAe;EAC/B,IAAI,IAAI,SAAS,KAAK,IAAI,SAAS,KACjC,IAAI,KAAK;GACP,UAAU,IAAI,SAAS,MAAO,SAAS;GACvC,MAAM;GACN,OAAO,IAAI,IAAI,aAAa,oCAAoC,IAAI,MAAM,UAAU,IAAI,QAAQ,IAAA,CAAK,QAAQ,CAAC,EAAE;GAChH,QAAQ,4FAA4F,IAAI,MAAM,MAAM,IAAI,UAAU,EAAE,qBAAqB,IAAI,aAAa,GAAG,IAAI,eAAe,SAAS,IAAI,YAAY,IAAI,eAAe,EAAE,CAAE,aAAa,KAAK,IAAI,eAAe,EAAE,CAAE,MAAM,KAAK,GAAG;GACvS,cAAc;EAChB,CAAC;CAEL;CAKA,IAAI,OAAO,KAAK,IAAI,MAAM,CAAC,CAAC,WAAW,KAAK,IAAI,UAAU,IAAI,GAC5D,IAAI,KAAK;EACP,UAAU;EACV,MAAM;EACN,OAAO;EACP,QACE;EACF,cAAc;CAChB,CAAC;CAGH,IAAI,IAAI,MACN,IAAI,CAAC,IAAI,KAAK,kBACZ,IAAI,KAAK;EACP,UAAU;EACV,MAAM;EACN,OAAO,kBAAkB,IAAI,KAAK,EAAE,gBAAgB,IAAI,KAAK,gBAAgB;EAC7E,QAAQ,+CAA+C,IAAI,KAAK,gBAAgB;EAChF,cAAc;CAChB,CAAC;MACI;EACL,MAAM,eACJ,IAAI,KAAK,YAAY,OAAO,oCAAoC,IAAI,KAAK,QAAQ,QAAQ,CAAC;EAC5F,MAAM,UACJ,IAAI,KAAK,WAAW,OAAO,oCAAoC,IAAI,KAAK,OAAO,QAAQ,CAAC;EAC1F,MAAM,eACJ,IAAI,KAAK,cAAc,OAAO,kBAAkB,IAAI,IAAI,KAAK,UAAU;EAMzE,MAAM,WAAW,CAAC,UAAU,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI;EACrE,MAAM,eAAe,IAAI,KAAK,KAAK,MAAM,IAAI,aAAa,IAAI,KAAK,KAAK,KAAK,IAAI;EACjF,IAAI,UACF,IAAI,KAAK;GACP,UAAU;GACV,MAAM;GACN,OAAO,eAAe,IAAI,KAAK,MAAM,QAAQ,CAAC,EAAE,WAAW,IAAI,KAAK,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE;GACvH,QAAQ,kCAAkC,IAAI,UAAU,oCAAoC,IAAI,KAAK,EAAE,MAAM,QAAQ,aAAa,aAAa;GAC/I,cAAc;EAChB,CAAC;OACI,IAAI,cACT,IAAI,KAAK;GACP,UAAU;GACV,MAAM;GACN,OAAO,qCAAqC,aAAa,SAAS,IAAI,KAAK,EAAE;GAC7E,QAAQ,uDAAuD,IAAI,KAAK,IAAI,QAAQ,CAAC,EAAE,sBAAsB,IAAI,KAAK,MAAM,QAAQ,CAAC,EAAE;GACvI,cAAc;EAChB,CAAC;OAED,IAAI,KAAK;GACP,UAAU;GACV,MAAM;GACN,OAAO,8BAA8B,IAAI,KAAK,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,4BAA4B,IAAI;GACjG,QAAQ;GACR,cAAc;EAChB,CAAC;CAEL;CAGF,IAAI,IAAI,iBAAiB,IAAI,cAAc,QAAQ,GACjD,IAAI,KAAK;EACP,UAAU;EACV,MAAM;EACN,OAAO,GAAG,IAAI,cAAc,MAAM,cAAc,IAAI,cAAc,UAAU,IAAI,KAAK,IAAI;EACzF,QAAQ;EACR,cAAc;CAChB,CAAC;CAGH,IAAI,IAAI,cAAc,IAAI,WAAW,QAAQ,IAC3C,IAAI,KAAK;EACP,UAAU;EACV,MAAM;EACN,OAAO,8BAA8B,IAAI,WAAW,MAAM,QAAQ,CAAC,EAAE;EACrE,QACE;EACF,cAAc;CAChB,CAAC;CAGH,IAAI,IAAI,mBAAmB,IAAI,gBAAgB,SAAS,SAAS,GAAG;EAClE,MAAM,MAAM,IAAI,gBAAgB,SAAS;EACzC,IAAI,KAAK;GACP,UAAU;GACV,MAAM;GACN,OAAO,wBAAwB,IAAI,KAAK,KAAK,IAAI,QAAQ,IAAA,CAAK,QAAQ,CAAC,EAAE;GACzE,QAAQ,GAAG,IAAI,gBAAgB,cAAc,2CAA2C,IAAI,UAAU,OAAO,oBAAoB,IAAI,KAAK;GAC1I,cAAc;EAChB,CAAC;CACH;CAEA,IAAI,IAAI,sBAAsB,KAAK,IAAI,IAAI,mBAAmB,QAAQ,IAAI,IACxE,IAAI,KAAK;EACP,UAAU;EACV,MAAM;EACN,OAAO,+BAA+B,IAAI,mBAAmB,OAAO,eAAe,IAAI,mBAAmB,SAAS,QAAQ,CAAC,EAAE;EAC9H,QAAQ,yFAAyF,IAAI,mBAAmB,OAAO,0CAA0C,IAAI,mBAAmB,OAAO;EACvM,cAAc;CAChB,CAAC;CAGH,OAAO;AACT"}
|