@devflow-tools/benchmark 0.17.8 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/accuracy-command-options.d.ts +3 -0
- package/dist/accuracy-command-options.d.ts.map +1 -0
- package/dist/accuracy-command-options.js +17 -0
- package/dist/accuracy-command-options.js.map +1 -0
- package/dist/context-symbol-quality-command.d.ts +59 -0
- package/dist/context-symbol-quality-command.d.ts.map +1 -0
- package/dist/context-symbol-quality-command.js +482 -0
- package/dist/context-symbol-quality-command.js.map +1 -0
- package/dist/evaluation-bundle-builder.d.ts +48 -0
- package/dist/evaluation-bundle-builder.d.ts.map +1 -0
- package/dist/evaluation-bundle-builder.js +496 -0
- package/dist/evaluation-bundle-builder.js.map +1 -0
- package/dist/evaluation-bundle-command.d.ts +2 -0
- package/dist/evaluation-bundle-command.d.ts.map +1 -0
- package/dist/evaluation-bundle-command.js +15 -0
- package/dist/evaluation-bundle-command.js.map +1 -0
- package/dist/evaluation-bundle.d.ts +55 -0
- package/dist/evaluation-bundle.d.ts.map +1 -0
- package/dist/evaluation-bundle.js +272 -0
- package/dist/evaluation-bundle.js.map +1 -0
- package/dist/evaluation-run-state.d.ts +23 -0
- package/dist/evaluation-run-state.d.ts.map +1 -0
- package/dist/evaluation-run-state.js +115 -0
- package/dist/evaluation-run-state.js.map +1 -0
- package/dist/evidence-quality-metrics.d.ts +15 -0
- package/dist/evidence-quality-metrics.d.ts.map +1 -1
- package/dist/evidence-quality-metrics.js +51 -1
- package/dist/evidence-quality-metrics.js.map +1 -1
- package/dist/graders/deterministic.d.ts.map +1 -1
- package/dist/graders/deterministic.js +5 -3
- package/dist/graders/deterministic.js.map +1 -1
- package/dist/index.d.ts +15 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -1
- package/dist/index.js.map +1 -1
- package/dist/release-quality-bundle-command.d.ts +3 -0
- package/dist/release-quality-bundle-command.d.ts.map +1 -0
- package/dist/release-quality-bundle-command.js +72 -0
- package/dist/release-quality-bundle-command.js.map +1 -0
- package/dist/release-quality-bundle-evaluator.d.ts +61 -0
- package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
- package/dist/release-quality-bundle-evaluator.js +321 -0
- package/dist/release-quality-bundle-evaluator.js.map +1 -0
- package/dist/release-quality-bundle-materializer.d.ts +79 -0
- package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
- package/dist/release-quality-bundle-materializer.js +395 -0
- package/dist/release-quality-bundle-materializer.js.map +1 -0
- package/dist/release-quality-comparison.d.ts +34 -0
- package/dist/release-quality-comparison.d.ts.map +1 -0
- package/dist/release-quality-comparison.js +60 -0
- package/dist/release-quality-comparison.js.map +1 -0
- package/dist/release-quality-evaluation-command.d.ts +42 -0
- package/dist/release-quality-evaluation-command.d.ts.map +1 -0
- package/dist/release-quality-evaluation-command.js +91 -0
- package/dist/release-quality-evaluation-command.js.map +1 -0
- package/dist/release-quality-host-proof.d.ts +30 -0
- package/dist/release-quality-host-proof.d.ts.map +1 -0
- package/dist/release-quality-host-proof.js +109 -0
- package/dist/release-quality-host-proof.js.map +1 -0
- package/dist/release-quality-output-cleanup.d.ts +2 -0
- package/dist/release-quality-output-cleanup.d.ts.map +1 -0
- package/dist/release-quality-output-cleanup.js +18 -0
- package/dist/release-quality-output-cleanup.js.map +1 -0
- package/dist/release-quality-policy.d.ts +87 -0
- package/dist/release-quality-policy.d.ts.map +1 -0
- package/dist/release-quality-policy.js +179 -0
- package/dist/release-quality-policy.js.map +1 -0
- package/dist/release-quality-query-sampling.d.ts +36 -0
- package/dist/release-quality-query-sampling.d.ts.map +1 -0
- package/dist/release-quality-query-sampling.js +87 -0
- package/dist/release-quality-query-sampling.js.map +1 -0
- package/dist/release-quality-source-snapshot.d.ts +18 -0
- package/dist/release-quality-source-snapshot.d.ts.map +1 -0
- package/dist/release-quality-source-snapshot.js +46 -0
- package/dist/release-quality-source-snapshot.js.map +1 -0
- package/dist/retrieval-quality-corpus.d.ts +0 -1
- package/dist/retrieval-quality-corpus.d.ts.map +1 -1
- package/dist/retrieval-quality-corpus.js +1 -1
- package/dist/retrieval-quality-corpus.js.map +1 -1
- package/dist/retrieval-quality.d.ts.map +1 -1
- package/dist/retrieval-quality.js +29 -26
- package/dist/retrieval-quality.js.map +1 -1
- package/dist/run-accuracy-tests.js +82 -2
- package/dist/run-accuracy-tests.js.map +1 -1
- package/dist/suite-runner.js +1 -1
- package/dist/suite-runner.js.map +1 -1
- package/dist/unified-runtime-corpus.d.ts +64 -0
- package/dist/unified-runtime-corpus.d.ts.map +1 -0
- package/dist/unified-runtime-corpus.js +161 -0
- package/dist/unified-runtime-corpus.js.map +1 -0
- package/dist/unified-runtime-evaluation.d.ts +133 -0
- package/dist/unified-runtime-evaluation.d.ts.map +1 -0
- package/dist/unified-runtime-evaluation.js +388 -0
- package/dist/unified-runtime-evaluation.js.map +1 -0
- package/dist/unified-runtime-observation.d.ts +101 -0
- package/dist/unified-runtime-observation.d.ts.map +1 -0
- package/dist/unified-runtime-observation.js +2 -0
- package/dist/unified-runtime-observation.js.map +1 -0
- package/dist/unified-runtime-runner.d.ts +73 -0
- package/dist/unified-runtime-runner.d.ts.map +1 -0
- package/dist/unified-runtime-runner.js +620 -0
- package/dist/unified-runtime-runner.js.map +1 -0
- package/fixtures/evidence-runtime-corpus.json +0 -100
- package/fixtures/release-quality/bundles.json +1 -0
- package/fixtures/release-quality/calibration.json +602 -0
- package/fixtures/release-quality/ecc/corpus.json +257 -0
- package/fixtures/release-quality/ecc/input.json +62 -0
- package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
- package/fixtures/release-quality/llm-wiki/input.json +72 -0
- package/fixtures/release-quality/mem0/corpus.json +288 -0
- package/fixtures/release-quality/mem0/input.json +68 -0
- package/fixtures/release-quality/projects.json +29 -0
- package/fixtures/release-quality/release-quality-policy.json +232 -0
- package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
- package/fixtures/release-quality/rsbuild-react/input.json +62 -0
- package/fixtures/retrieval-quality-corpus.json +7 -7
- package/fixtures/semantic-regex-inventory.json +3 -10
- package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
- package/fixtures/unified-runtime/context.json +206 -0
- package/fixtures/unified-runtime/knowledge.json +224 -0
- package/fixtures/unified-runtime/memory.json +224 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
- package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
- package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
- package/fixtures/unified-runtime/semantic.json +409 -0
- package/package.json +24 -6
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unified-runtime-corpus.js","sourceRoot":"","sources":["../src/unified-runtime-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA0HpC,MAAM,SAAS,GAAG,gDAAgD,CAAC;AACnE,MAAM,iBAAiB,GAAqB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;AAC5H,MAAM,YAAY,GAAuB,CAAC,UAAU,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;AACjG,MAAM,0BAA0B,GAAG,IAAI,GAAG,CAAC;IACzC,UAAU,EAAE,UAAU,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,gBAAgB;IACvF,+BAA+B,EAAE,WAAW,EAAE,qBAAqB;IACnE,qBAAqB,EAAE,uBAAuB,EAAE,kBAAkB;CACnE,CAAC,CAAC;AAEH,MAAM,UAAU,wBAAwB,CAAC,gBAAgB,GAAG,uBAAuB,EAAE;IACnF,MAAM,eAAe,GAAG,QAAQ,CAAkB,OAAO,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC;IAC9F,MAAM,iBAAiB,GAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACpF,QAAQ,CAAmB,OAAO,CAAC,gBAAgB,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5E,qBAAqB,CAAC,eAAe,EAAE,qCAAqC,CAAC,CAAC;IAC9E,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,qBAAqB,CAAC,OAAO,EAAE,sCAAsC,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,WAAW,OAAO,CAAC,OAAO,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACvJ,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACrG,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1J,IAAI,OAAO,CAAC,eAAe,CAAC,cAAc,KAAK,eAAe,CAAC,eAAe,CAAC,cAAc;eACxF,OAAO,CAAC,eAAe,CAAC,MAAM,KAAK,eAAe,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;YAC/E,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAmD,CAAC;IACtK,MAAM,MAAM,GAAyB;QACnC,aAAa,EAAE,iCAAiC,EAAE,UAAU,EAAE;YAC5D,QAAQ,EAAE,eAAe,CAAC,SAAS;YACnC,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAE,CAAC,SAAS;YAC9E,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAE,CAAC,SAAS;YAC5E,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,WAAW,CAAE,CAAC,SAAS;SACnF,EAAE,eAAe,EAAE,eAAe,CAAC,eAAe;QACnD,QAAQ,EAAE,cAAc,CAAC,eAAe,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS;KAChI,CAAC;IACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,OAAwB;IAC9C,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtD,OAAO,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QAC1E,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO;YACL,EAAE,EAAE,YAAY,QAAQ,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM;YAClG,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;YAC9C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,WAAoB,EAAE;YAClH,QAAQ,EAAE;gBACR,SAAS,EAAE,oBAAoB,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;gBAClE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAClC,kBAAkB,EAAE,aAAa,KAAK,uBAAuB;gBAC7D,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE;wBAC1C,GAAG,QAAQ,CAAC,YAAY;wBACxB,mBAAmB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC;qBAClF,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACT,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7E;SACF,CAAC;IACJ,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CACxB,IAA8B,EAC9B,aAAqB;IAErB,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,UAAU,CAAC;IAC5C,IAAI,aAAa,KAAK,uBAAuB;QAAE,OAAO,UAAU,CAAC;IACjE,IAAI,aAAa,KAAK,wBAAwB;QAAE,OAAO,sBAAsB,CAAC;IAC9E,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IAChD,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtD,OAAO,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5E,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;QACnF,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS;QACrF,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE;QACnK,QAAQ,EAAE;YACR,WAAW,EAAE,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS;YACrE,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,KAAK,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnJ,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE;KACF,CAAC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA0B,EAAE,OAAuB;IAC/E,IAAI,OAAO,KAAK,UAAU;QAAE,OAAO,uBAAuB,CAAC;IAC3D,IAAI,OAAO,KAAK,aAAa;QAAE,OAAO,wBAAwB,CAAC;IAC/D,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,aAAa;QAAE,OAAO,0BAA0B,CAAC;IAC5F,OAAO,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,0BAA0B,CAAC;AAC5F,CAAC;AAED,SAAS,oBAAoB,CAAC,SAAmB,EAAE,aAAqB;IACtE,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,YAAY,CAAC,MAAqB,EAAE,OAAuB;IAClE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA2C,EAAE,aAAqB;IAC/F,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,IAAI,OAAO,CAAC,aAAa,KAAK,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACrH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7O,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,QAA0B;IACrE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1C,IAAI,GAAG,CAAC,MAAM,KAAK,iBAAiB,CAAC,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,qDAAqD,CAAC,CAAC;AACjO,CAAC;AAED,SAAS,sBAAsB,CAAC,MAA4B;IAC1D,MAAM,cAAc,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAW,CAAC;IAC1F,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAsC,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,6BAA6B,OAAO,QAAQ,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3F,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACjG,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACzI,MAAM,IAAI,GAAG,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAChG,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,SAAS,KAAK,cAAc,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACrI,IAAI,IAAI,CAAC,SAAS,KAAK,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU;gBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAChJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,IAAI,GAAG,CAAC,IAAa,EAAQ,EAAE;QACnC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO;QAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,0BAA0B,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;YACpH,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,KAAa,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC,CAAC,CAAC;AAClH,SAAS,QAAQ,CAAI,IAAY,IAAO,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import type { AggregatedEvidenceQualityMetric, EvidenceQualityMetric } from './evidence-quality-metrics.js';
|
|
2
|
+
import { type EvaluationBundleManifestV1 } from '@devflow-tools/sdk';
|
|
3
|
+
import type { EvaluationLabelSetV1 } from './evaluation-bundle.js';
|
|
4
|
+
import type { UnifiedChannel, UnifiedRuntimeCorpus } from './unified-runtime-corpus.js';
|
|
5
|
+
import type { UnifiedRuntimeIdentityV2, UnifiedRuntimeObservationV4 } from './unified-runtime-observation.js';
|
|
6
|
+
export interface UnifiedLatencyMetric {
|
|
7
|
+
sampleCount: number;
|
|
8
|
+
meanMs: number | null;
|
|
9
|
+
p95Ms: number | null;
|
|
10
|
+
}
|
|
11
|
+
export interface UnifiedRerankerMetric {
|
|
12
|
+
sampleCount: number;
|
|
13
|
+
meanRelevantAt5Delta: number | null;
|
|
14
|
+
}
|
|
15
|
+
export interface UnifiedRuntimeEvaluationReport {
|
|
16
|
+
schemaVersion: 'devflow.unified-runtime-evaluation.v4';
|
|
17
|
+
createdAt: number;
|
|
18
|
+
status: 'passed' | 'failed';
|
|
19
|
+
evaluationPolicy: {
|
|
20
|
+
labelAuthority: 'stable_ids';
|
|
21
|
+
llmJudge: 'diagnostic_only';
|
|
22
|
+
productionLogsWithoutLabelsProveRecall: false;
|
|
23
|
+
observationsProducedByRuntime: true;
|
|
24
|
+
};
|
|
25
|
+
datasetIds: UnifiedRuntimeCorpus['datasetIds'];
|
|
26
|
+
runtime: UnifiedRuntimeIdentityV2;
|
|
27
|
+
samples: Record<'semantic' | 'context' | 'memory' | 'knowledge', number>;
|
|
28
|
+
suites: {
|
|
29
|
+
safety: {
|
|
30
|
+
status: 'passed' | 'failed';
|
|
31
|
+
metrics: {
|
|
32
|
+
mutationFalseAuthorization: EvidenceQualityMetric;
|
|
33
|
+
contextHashAgreement: EvidenceQualityMetric;
|
|
34
|
+
terminalFalseSuccess: EvidenceQualityMetric;
|
|
35
|
+
boundedCycles: EvidenceQualityMetric;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
semanticCapability: {
|
|
39
|
+
status: 'passed' | 'failed' | 'unavailable';
|
|
40
|
+
metrics: {
|
|
41
|
+
fieldAccuracy: EvidenceQualityMetric;
|
|
42
|
+
abstentionAccuracy: EvidenceQualityMetric;
|
|
43
|
+
domainActionAccuracy: EvidenceQualityMetric;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
planningCoverage: StageSuite<{
|
|
47
|
+
validPlans: EvidenceQualityMetric;
|
|
48
|
+
}>;
|
|
49
|
+
producerExecution: StageSuite<{
|
|
50
|
+
originalLegOutcomes: EvidenceQualityMetric;
|
|
51
|
+
}>;
|
|
52
|
+
candidateQuality: StageSuite<{
|
|
53
|
+
context: RetrievalMetricGroup;
|
|
54
|
+
memory: RetrievalMetricGroup;
|
|
55
|
+
knowledge: RetrievalMetricGroup;
|
|
56
|
+
}>;
|
|
57
|
+
selectionQuality: StageSuite<{
|
|
58
|
+
context: RetrievalMetricGroup;
|
|
59
|
+
memory: RetrievalMetricGroup;
|
|
60
|
+
knowledge: RetrievalMetricGroup;
|
|
61
|
+
}>;
|
|
62
|
+
consumptionProof: StageSuite<{
|
|
63
|
+
consumption: EvidenceQualityMetric;
|
|
64
|
+
knownOutcome: EvidenceQualityMetric;
|
|
65
|
+
}>;
|
|
66
|
+
terminalProof: StageSuite<{
|
|
67
|
+
verified: EvidenceQualityMetric;
|
|
68
|
+
}>;
|
|
69
|
+
externalReplay: {
|
|
70
|
+
status: 'passed' | 'unavailable';
|
|
71
|
+
replayId?: string;
|
|
72
|
+
};
|
|
73
|
+
};
|
|
74
|
+
metrics: {
|
|
75
|
+
semantic: {
|
|
76
|
+
fieldAccuracy: EvidenceQualityMetric;
|
|
77
|
+
abstentionAccuracy: EvidenceQualityMetric;
|
|
78
|
+
mutationFalseAuthorization: EvidenceQualityMetric;
|
|
79
|
+
domainActionAccuracy: EvidenceQualityMetric;
|
|
80
|
+
};
|
|
81
|
+
context: RetrievalMetricGroup;
|
|
82
|
+
memory: RetrievalMetricGroup & {
|
|
83
|
+
temporalCorrectness: EvidenceQualityMetric;
|
|
84
|
+
explicitSaveDurability: EvidenceQualityMetric;
|
|
85
|
+
};
|
|
86
|
+
knowledge: RetrievalMetricGroup & {
|
|
87
|
+
sourceVersionCorrectness: EvidenceQualityMetric;
|
|
88
|
+
abstentionPrecision: EvidenceQualityMetric;
|
|
89
|
+
};
|
|
90
|
+
runtime: {
|
|
91
|
+
contextHashAgreement: EvidenceQualityMetric;
|
|
92
|
+
attributionConsumption: EvidenceQualityMetric;
|
|
93
|
+
attributionKnownOutcome: EvidenceQualityMetric;
|
|
94
|
+
attributionVerification: EvidenceQualityMetric;
|
|
95
|
+
terminalFalseSuccess: EvidenceQualityMetric;
|
|
96
|
+
boundedCycles: EvidenceQualityMetric;
|
|
97
|
+
noGainStopObservable: EvidenceQualityMetric;
|
|
98
|
+
};
|
|
99
|
+
};
|
|
100
|
+
slices: Record<string, UnifiedEvaluationSlice>;
|
|
101
|
+
gates: Record<UnifiedReleaseGate, boolean>;
|
|
102
|
+
diagnostics: string[];
|
|
103
|
+
}
|
|
104
|
+
export interface EvaluationBundleEvidenceV1 {
|
|
105
|
+
manifest: EvaluationBundleManifestV1;
|
|
106
|
+
labels: Record<UnifiedChannel, EvaluationLabelSetV1>;
|
|
107
|
+
}
|
|
108
|
+
interface StageSuite<Metrics> {
|
|
109
|
+
status: 'passed' | 'failed' | 'unavailable';
|
|
110
|
+
metrics?: Metrics;
|
|
111
|
+
}
|
|
112
|
+
interface RetrievalMetricGroup {
|
|
113
|
+
candidateRecall: AggregatedEvidenceQualityMetric;
|
|
114
|
+
eligibilitySurvival: AggregatedEvidenceQualityMetric;
|
|
115
|
+
precisionAt5: AggregatedEvidenceQualityMetric;
|
|
116
|
+
recallAt5: AggregatedEvidenceQualityMetric;
|
|
117
|
+
forbiddenContamination: AggregatedEvidenceQualityMetric;
|
|
118
|
+
rejectionReasonCounts: Record<string, number>;
|
|
119
|
+
latency: UnifiedLatencyMetric;
|
|
120
|
+
reranker: UnifiedRerankerMetric;
|
|
121
|
+
}
|
|
122
|
+
export interface UnifiedEvaluationSlice {
|
|
123
|
+
status: 'passed' | 'failed' | 'unavailable';
|
|
124
|
+
sampleCount: number;
|
|
125
|
+
metrics: {
|
|
126
|
+
precisionAt5: AggregatedEvidenceQualityMetric;
|
|
127
|
+
recallAt5: AggregatedEvidenceQualityMetric;
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
type UnifiedReleaseGate = 'runtimeIdentity' | 'semanticCapabilityAvailable' | 'candidateGenerationAvailable' | 'externalReplayAvailable' | 'semanticFieldAccuracy' | 'semanticAbstention' | 'mutationFalseAuthorization' | 'planningCoverage' | 'producerExecution' | 'consumptionProofAvailable' | 'terminalProofAvailable' | 'domainActionAccuracy' | 'contextRetrieval' | 'memoryRetrieval' | 'memoryContamination' | 'memoryTemporalCorrectness' | 'knowledgeRetrieval' | 'knowledgeVersionCorrectness' | 'knowledgeAbstention' | 'explicitSaveDurability' | 'attributionConsumption' | 'hashAgreement' | 'terminalTruth' | 'boundedCycles';
|
|
131
|
+
export declare function evaluateUnifiedRuntimeCorpus(corpus: UnifiedRuntimeCorpus, observation: UnifiedRuntimeObservationV4, currentRuntime?: UnifiedRuntimeIdentityV2, bundleEvidence?: EvaluationBundleEvidenceV1): UnifiedRuntimeEvaluationReport;
|
|
132
|
+
export {};
|
|
133
|
+
//# sourceMappingURL=unified-runtime-evaluation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unified-runtime-evaluation.d.ts","sourceRoot":"","sources":["../src/unified-runtime-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,+BAA+B,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAW5G,OAAO,EAAgC,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAEV,cAAc,EAEd,oBAAoB,EAErB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAEV,wBAAwB,EACxB,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,oBAAoB;IAAG,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAC3G,MAAM,WAAW,qBAAqB;IAAG,WAAW,EAAE,MAAM,CAAC;IAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAEpG,MAAM,WAAW,8BAA8B;IAC7C,aAAa,EAAE,uCAAuC,CAAC;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,gBAAgB,EAAE;QAAE,cAAc,EAAE,YAAY,CAAC;QAAC,QAAQ,EAAE,iBAAiB,CAAC;QAAC,sCAAsC,EAAE,KAAK,CAAC;QAAC,6BAA6B,EAAE,IAAI,CAAC;KAAE,CAAC;IACrK,UAAU,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAC/C,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,EAAE,MAAM,CAAC,CAAC;IACzE,MAAM,EAAE;QACN,MAAM,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;YAAC,OAAO,EAAE;gBAAE,0BAA0B,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;gBAAC,aAAa,EAAE,qBAAqB,CAAC;aAAE,CAAA;SAAE,CAAC;QACzO,kBAAkB,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;YAAC,OAAO,EAAE;gBAAE,aAAa,EAAE,qBAAqB,CAAC;gBAAC,kBAAkB,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;aAAE,CAAA;SAAE,CAAC;QAChN,gBAAgB,EAAE,UAAU,CAAC;YAAE,UAAU,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QACpE,iBAAiB,EAAE,UAAU,CAAC;YAAE,mBAAmB,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC9E,gBAAgB,EAAE,UAAU,CAAC;YAAE,OAAO,EAAE,oBAAoB,CAAC;YAAC,MAAM,EAAE,oBAAoB,CAAC;YAAC,SAAS,EAAE,oBAAoB,CAAA;SAAE,CAAC,CAAC;QAC/H,gBAAgB,EAAE,UAAU,CAAC;YAAE,OAAO,EAAE,oBAAoB,CAAC;YAAC,MAAM,EAAE,oBAAoB,CAAC;YAAC,SAAS,EAAE,oBAAoB,CAAA;SAAE,CAAC,CAAC;QAC/H,gBAAgB,EAAE,UAAU,CAAC;YAAE,WAAW,EAAE,qBAAqB,CAAC;YAAC,YAAY,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC1G,aAAa,EAAE,UAAU,CAAC;YAAE,QAAQ,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC/D,cAAc,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,aAAa,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACzE,CAAC;IACF,OAAO,EAAE;QACP,QAAQ,EAAE;YAAE,aAAa,EAAE,qBAAqB,CAAC;YAAC,kBAAkB,EAAE,qBAAqB,CAAC;YAAC,0BAA0B,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QAC/L,OAAO,EAAE,oBAAoB,CAAC;QAC9B,MAAM,EAAE,oBAAoB,GAAG;YAAE,mBAAmB,EAAE,qBAAqB,CAAC;YAAC,sBAAsB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QAC9H,SAAS,EAAE,oBAAoB,GAAG;YAAE,wBAAwB,EAAE,qBAAqB,CAAC;YAAC,mBAAmB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QACnI,OAAO,EAAE;YAAE,oBAAoB,EAAE,qBAAqB,CAAC;YAAC,sBAAsB,EAAE,qBAAqB,CAAC;YAAC,uBAAuB,EAAE,qBAAqB,CAAC;YAAC,uBAAuB,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;YAAC,aAAa,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;SAAE,CAAC;KAC1U,CAAC;IACF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IAC/C,KAAK,EAAE,MAAM,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAC3C,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,0BAA0B,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAC;CACtD;AAED,UAAU,UAAU,CAAC,OAAO;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,UAAU,oBAAoB;IAC5B,eAAe,EAAE,+BAA+B,CAAC;IACjD,mBAAmB,EAAE,+BAA+B,CAAC;IACrD,YAAY,EAAE,+BAA+B,CAAC;IAC9C,SAAS,EAAE,+BAA+B,CAAC;IAC3C,sBAAsB,EAAE,+BAA+B,CAAC;IACxD,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,OAAO,EAAE,oBAAoB,CAAC;IAAC,QAAQ,EAAE,qBAAqB,CAAC;CAChE;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE;QAAE,YAAY,EAAE,+BAA+B,CAAC;QAAC,SAAS,EAAE,+BAA+B,CAAC;KAAE,CAAC;CACzG;AAKD,KAAK,kBAAkB,GAAG,iBAAiB,GAAG,6BAA6B,GAAG,8BAA8B,GACxG,yBAAyB,GAAG,uBAAuB,GAAG,oBAAoB,GAAG,4BAA4B,GACzG,kBAAkB,GAAG,mBAAmB,GAAG,2BAA2B,GAAG,wBAAwB,GACjG,sBAAsB,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,2BAA2B,GACrH,oBAAoB,GAAG,6BAA6B,GAAG,qBAAqB,GAAG,wBAAwB,GACvG,wBAAwB,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC;AASnF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,oBAAoB,EAC5B,WAAW,EAAE,2BAA2B,EACxC,cAAc,GAAE,wBAA8C,EAC9D,cAAc,CAAC,EAAE,0BAA0B,GAC1C,8BAA8B,CA+LhC"}
|
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
import { aggregateEvidenceQualityMetrics, candidateRecallAt20, eligibilitySurvival, evidenceQualityMetric, forbiddenContaminationAt5, precisionAt5, recallAt5, selectBestGoldSet, } from './evidence-quality-metrics.js';
|
|
2
|
+
import { labelHash, stableRuntimeHash } from '@devflow-tools/sdk';
|
|
3
|
+
const THRESHOLDS = {
|
|
4
|
+
semanticFieldAccuracy: 0.90, semanticAbstention: 0.90, contextRecallAt5: 0.85, contextPrecisionAt5: 0.75,
|
|
5
|
+
memoryRecallAt5: 0.80, memoryPrecisionAt5: 0.80, memoryContamination: 0.10, memoryTemporalCorrectness: 0.90,
|
|
6
|
+
knowledgeRecallAt5: 0.85, knowledgeVersionCorrectness: 1, knowledgeAbstentionPrecision: 0.90,
|
|
7
|
+
explicitSaveDurability: 1, attributionKnownOutcome: 0.90, domainActionAccuracy: 1,
|
|
8
|
+
};
|
|
9
|
+
export function evaluateUnifiedRuntimeCorpus(corpus, observation, currentRuntime = observation.runtime, bundleEvidence) {
|
|
10
|
+
const runtimeIdentityValid = validateObservation(corpus, observation, currentRuntime);
|
|
11
|
+
const semantic = joinSemantic(corpus.semantic, observation.semantic);
|
|
12
|
+
const context = joinRetrieval(corpus.context, observation.context);
|
|
13
|
+
const memoryCases = joinRetrieval(corpus.memory, observation.memory);
|
|
14
|
+
const knowledgeCases = joinRetrieval(corpus.knowledge, observation.knowledge);
|
|
15
|
+
const safetySemanticMetrics = evaluateSemantic(semantic);
|
|
16
|
+
const semanticExecuted = observation.suiteIdentity.semanticCapability.status === 'executed';
|
|
17
|
+
const retrievalCases = [...context, ...memoryCases, ...knowledgeCases];
|
|
18
|
+
const labelExpectations = validLabelExpectations(bundleEvidence, retrievalCases, observation.suiteIdentity.candidateGeneration);
|
|
19
|
+
const candidateExecuted = observation.suiteIdentity.candidateGeneration.status === 'executed'
|
|
20
|
+
&& labelExpectations !== undefined;
|
|
21
|
+
const externalExecuted = observation.suiteIdentity.externalReplay.status === 'executed'
|
|
22
|
+
&& Boolean(observation.suiteIdentity.externalReplay.replayId)
|
|
23
|
+
&& Boolean(observation.suiteIdentity.externalReplay.host)
|
|
24
|
+
&& Boolean(observation.suiteIdentity.externalReplay.modelProfileHash);
|
|
25
|
+
const semanticQualityMetrics = evaluateSemantic(semanticExecuted ? semantic : []);
|
|
26
|
+
const semanticMetrics = {
|
|
27
|
+
...semanticQualityMetrics,
|
|
28
|
+
mutationFalseAuthorization: safetySemanticMetrics.mutationFalseAuthorization,
|
|
29
|
+
};
|
|
30
|
+
const mechanicsExpectations = observation.suiteIdentity.retrievalExecution.source === 'fixture_seeded'
|
|
31
|
+
? undefined : new Map();
|
|
32
|
+
const positiveExecuted = (cases) => cases.filter(item => item.caseClass === 'positive' && item.observed.executionStatus === 'executed');
|
|
33
|
+
const mechanicsContext = evaluateRetrieval(positiveExecuted(context), mechanicsExpectations);
|
|
34
|
+
const mechanicsMemoryBase = evaluateRetrieval(positiveExecuted(memoryCases), mechanicsExpectations);
|
|
35
|
+
const mechanicsKnowledgeBase = evaluateRetrieval(positiveExecuted(knowledgeCases), mechanicsExpectations);
|
|
36
|
+
const candidateContext = labelExpectations ? evaluateRetrieval(positiveExecuted(context), labelExpectations) : undefined;
|
|
37
|
+
const candidateMemoryBase = labelExpectations ? evaluateRetrieval(positiveExecuted(memoryCases), labelExpectations) : undefined;
|
|
38
|
+
const candidateKnowledgeBase = labelExpectations ? evaluateRetrieval(positiveExecuted(knowledgeCases), labelExpectations) : undefined;
|
|
39
|
+
const allObserved = [...semantic.map(item => item.observed), ...retrievalCases.map(item => item.observed)];
|
|
40
|
+
const semanticChannelHashes = semantic.flatMap(item => Object.values(item.observed.channelContextHashes)
|
|
41
|
+
.map(hash => hash === item.observed.runtimeContextHash));
|
|
42
|
+
const hostProofCases = retrievalCases.filter(item => item.observed.attributionSource === 'host_ledger');
|
|
43
|
+
const attributionOutcomes = hostProofCases.flatMap(item => item.observed.attributionOutcomes);
|
|
44
|
+
const verificationOutcomes = hostProofCases.flatMap(item => item.observed.verificationOutcomes);
|
|
45
|
+
const cyclesRequiringStop = retrievalCases.filter(item => item.observed.cycles > 1);
|
|
46
|
+
const planningCoverage = booleanMetric(retrievalCases.map(item => item.observed.planRequirement === 'prohibited'
|
|
47
|
+
|| !['channel_plan_empty', 'original_leg_missing'].includes(item.observed.executionReasonCode ?? '')));
|
|
48
|
+
const executableCases = retrievalCases.filter(item => item.observed.planRequirement !== 'prohibited');
|
|
49
|
+
const producerExecution = booleanMetric(executableCases.map(item => item.observed.executionStatus === 'executed'));
|
|
50
|
+
const runtime = {
|
|
51
|
+
contextHashAgreement: booleanMetric(semanticChannelHashes),
|
|
52
|
+
attributionConsumption: evidenceQualityMetric(attributionOutcomes.filter(item => item === 'consumed').length, attributionOutcomes.length),
|
|
53
|
+
attributionKnownOutcome: evidenceQualityMetric(attributionOutcomes.filter(item => item !== 'unknown').length, attributionOutcomes.length),
|
|
54
|
+
attributionVerification: evidenceQualityMetric(verificationOutcomes.filter(item => item === 'verified').length, verificationOutcomes.length),
|
|
55
|
+
terminalFalseSuccess: evidenceQualityMetric(allObserved.filter(item => item.terminalFalseSuccess).length, allObserved.length),
|
|
56
|
+
boundedCycles: booleanMetric(retrievalCases.map(item => item.observed.cycles <= item.expected.maxCycles)),
|
|
57
|
+
noGainStopObservable: booleanMetric(cyclesRequiringStop.map(item => item.observed.duplicateOrNoGainStopObserved)),
|
|
58
|
+
};
|
|
59
|
+
const memoryExtras = {
|
|
60
|
+
temporalCorrectness: booleanMetric(memoryCases.filter(item => item.observed.executionStatus === 'executed'
|
|
61
|
+
&& item.expected.temporalCorrect !== undefined)
|
|
62
|
+
.map(item => item.expected.temporalCorrect === item.observed.temporalCorrect)),
|
|
63
|
+
explicitSaveDurability: booleanMetric(memoryCases.filter(item => item.observed.executionStatus === 'executed'
|
|
64
|
+
&& item.caseClass === 'lifecycle' && item.expected.explicitSave)
|
|
65
|
+
.map(item => item.observed.episodePersisted === true)),
|
|
66
|
+
};
|
|
67
|
+
const knowledgeExtras = {
|
|
68
|
+
sourceVersionCorrectness: booleanMetric(knowledgeCases.filter(item => item.observed.executionStatus === 'executed'
|
|
69
|
+
&& item.caseClass === 'positive' && item.expected.sourceVersion !== undefined)
|
|
70
|
+
.map(item => item.expected.sourceVersion === item.observed.sourceVersion)),
|
|
71
|
+
};
|
|
72
|
+
const mechanicsMemory = { ...mechanicsMemoryBase, ...memoryExtras };
|
|
73
|
+
const mechanicsKnowledge = {
|
|
74
|
+
...mechanicsKnowledgeBase, ...knowledgeExtras,
|
|
75
|
+
abstentionPrecision: abstentionPrecision(knowledgeCases),
|
|
76
|
+
};
|
|
77
|
+
const candidateMemory = candidateMemoryBase ? { ...candidateMemoryBase, ...memoryExtras } : undefined;
|
|
78
|
+
const candidateKnowledge = candidateKnowledgeBase ? {
|
|
79
|
+
...candidateKnowledgeBase, ...knowledgeExtras,
|
|
80
|
+
abstentionPrecision: abstentionPrecision(knowledgeCases),
|
|
81
|
+
} : undefined;
|
|
82
|
+
const contextMetrics = candidateContext ?? mechanicsContext;
|
|
83
|
+
const memory = candidateMemory ?? mechanicsMemory;
|
|
84
|
+
const knowledge = candidateKnowledge ?? mechanicsKnowledge;
|
|
85
|
+
const gates = {
|
|
86
|
+
runtimeIdentity: runtimeIdentityValid,
|
|
87
|
+
semanticCapabilityAvailable: semanticExecuted,
|
|
88
|
+
candidateGenerationAvailable: candidateExecuted,
|
|
89
|
+
externalReplayAvailable: externalExecuted,
|
|
90
|
+
planningCoverage: atLeast(planningCoverage, 1),
|
|
91
|
+
producerExecution: atLeast(producerExecution, 1),
|
|
92
|
+
consumptionProofAvailable: atLeast(runtime.attributionKnownOutcome, THRESHOLDS.attributionKnownOutcome),
|
|
93
|
+
terminalProofAvailable: atLeast(runtime.attributionVerification, 1),
|
|
94
|
+
semanticFieldAccuracy: semanticExecuted && atLeast(semanticMetrics.fieldAccuracy, THRESHOLDS.semanticFieldAccuracy),
|
|
95
|
+
semanticAbstention: semanticExecuted && atLeast(semanticMetrics.abstentionAccuracy, THRESHOLDS.semanticAbstention),
|
|
96
|
+
mutationFalseAuthorization: semanticMetrics.mutationFalseAuthorization.numerator === 0,
|
|
97
|
+
domainActionAccuracy: semanticExecuted && atLeast(semanticMetrics.domainActionAccuracy, THRESHOLDS.domainActionAccuracy),
|
|
98
|
+
contextRetrieval: candidateExecuted && atLeast(contextMetrics.recallAt5.refMicro, THRESHOLDS.contextRecallAt5) && atLeast(contextMetrics.precisionAt5.refMicro, THRESHOLDS.contextPrecisionAt5),
|
|
99
|
+
memoryRetrieval: candidateExecuted && atLeast(memory.recallAt5.refMicro, THRESHOLDS.memoryRecallAt5) && atLeast(memory.precisionAt5.refMicro, THRESHOLDS.memoryPrecisionAt5),
|
|
100
|
+
memoryContamination: candidateExecuted && atMost(memory.forbiddenContamination.refMicro, THRESHOLDS.memoryContamination),
|
|
101
|
+
memoryTemporalCorrectness: atLeast(memory.temporalCorrectness, THRESHOLDS.memoryTemporalCorrectness),
|
|
102
|
+
knowledgeRetrieval: candidateExecuted && atLeast(knowledge.recallAt5.refMicro, THRESHOLDS.knowledgeRecallAt5),
|
|
103
|
+
knowledgeVersionCorrectness: atLeast(knowledge.sourceVersionCorrectness, THRESHOLDS.knowledgeVersionCorrectness),
|
|
104
|
+
knowledgeAbstention: atLeast(knowledge.abstentionPrecision, THRESHOLDS.knowledgeAbstentionPrecision),
|
|
105
|
+
explicitSaveDurability: atLeast(memory.explicitSaveDurability, THRESHOLDS.explicitSaveDurability),
|
|
106
|
+
attributionConsumption: atLeast(runtime.attributionKnownOutcome, THRESHOLDS.attributionKnownOutcome),
|
|
107
|
+
hashAgreement: atLeast(runtime.contextHashAgreement, 1),
|
|
108
|
+
terminalTruth: runtime.terminalFalseSuccess.numerator === 0,
|
|
109
|
+
boundedCycles: atLeast(runtime.boundedCycles, 1)
|
|
110
|
+
&& (runtime.noGainStopObservable.denominator === 0 || atLeast(runtime.noGainStopObservable, 1)),
|
|
111
|
+
};
|
|
112
|
+
const safetyPassed = gates.runtimeIdentity && gates.mutationFalseAuthorization
|
|
113
|
+
&& gates.hashAgreement && gates.terminalTruth && gates.boundedCycles;
|
|
114
|
+
const semanticPassed = gates.semanticFieldAccuracy && gates.semanticAbstention && gates.domainActionAccuracy;
|
|
115
|
+
const candidateQualityPassed = candidateContext !== undefined && candidateMemory !== undefined
|
|
116
|
+
&& candidateKnowledge !== undefined
|
|
117
|
+
&& atLeast(candidateContext.candidateRecall.refMicro, THRESHOLDS.contextRecallAt5)
|
|
118
|
+
&& atLeast(candidateMemory.candidateRecall.refMicro, THRESHOLDS.memoryRecallAt5)
|
|
119
|
+
&& atLeast(candidateKnowledge.candidateRecall.refMicro, THRESHOLDS.knowledgeRecallAt5);
|
|
120
|
+
const selectionPassed = gates.contextRetrieval && gates.memoryRetrieval
|
|
121
|
+
&& gates.memoryContamination && gates.memoryTemporalCorrectness
|
|
122
|
+
&& gates.knowledgeRetrieval && gates.knowledgeVersionCorrectness
|
|
123
|
+
&& gates.knowledgeAbstention && gates.explicitSaveDurability;
|
|
124
|
+
const suites = {
|
|
125
|
+
safety: {
|
|
126
|
+
status: safetyPassed ? 'passed' : 'failed',
|
|
127
|
+
metrics: {
|
|
128
|
+
mutationFalseAuthorization: semanticMetrics.mutationFalseAuthorization,
|
|
129
|
+
contextHashAgreement: runtime.contextHashAgreement,
|
|
130
|
+
terminalFalseSuccess: runtime.terminalFalseSuccess,
|
|
131
|
+
boundedCycles: runtime.boundedCycles,
|
|
132
|
+
},
|
|
133
|
+
},
|
|
134
|
+
semanticCapability: {
|
|
135
|
+
status: !semanticExecuted ? 'unavailable' : semanticPassed ? 'passed' : 'failed',
|
|
136
|
+
metrics: {
|
|
137
|
+
fieldAccuracy: semanticMetrics.fieldAccuracy,
|
|
138
|
+
abstentionAccuracy: semanticMetrics.abstentionAccuracy,
|
|
139
|
+
domainActionAccuracy: semanticMetrics.domainActionAccuracy,
|
|
140
|
+
},
|
|
141
|
+
},
|
|
142
|
+
planningCoverage: {
|
|
143
|
+
status: planningCoverage.denominator === 0 ? 'unavailable' : gates.planningCoverage ? 'passed' : 'failed',
|
|
144
|
+
...(planningCoverage.denominator > 0 ? { metrics: { validPlans: planningCoverage } } : {}),
|
|
145
|
+
},
|
|
146
|
+
producerExecution: {
|
|
147
|
+
status: producerExecution.denominator === 0 ? 'unavailable' : gates.producerExecution ? 'passed' : 'failed',
|
|
148
|
+
...(producerExecution.denominator > 0 ? { metrics: { originalLegOutcomes: producerExecution } } : {}),
|
|
149
|
+
},
|
|
150
|
+
candidateQuality: {
|
|
151
|
+
status: !candidateExecuted ? 'unavailable' : candidateQualityPassed ? 'passed' : 'failed',
|
|
152
|
+
...(candidateExecuted && candidateContext && candidateMemory && candidateKnowledge
|
|
153
|
+
? { metrics: { context: candidateContext, memory: candidateMemory, knowledge: candidateKnowledge } } : {}),
|
|
154
|
+
},
|
|
155
|
+
selectionQuality: {
|
|
156
|
+
status: !candidateExecuted ? 'unavailable' : selectionPassed ? 'passed' : 'failed',
|
|
157
|
+
...(candidateExecuted && candidateContext && candidateMemory && candidateKnowledge
|
|
158
|
+
? { metrics: { context: candidateContext, memory: candidateMemory, knowledge: candidateKnowledge } } : {}),
|
|
159
|
+
},
|
|
160
|
+
consumptionProof: {
|
|
161
|
+
status: runtime.attributionKnownOutcome.denominator === 0 ? 'unavailable'
|
|
162
|
+
: gates.consumptionProofAvailable ? 'passed' : 'failed',
|
|
163
|
+
...(runtime.attributionKnownOutcome.denominator > 0 ? { metrics: {
|
|
164
|
+
consumption: runtime.attributionConsumption,
|
|
165
|
+
knownOutcome: runtime.attributionKnownOutcome,
|
|
166
|
+
} } : {}),
|
|
167
|
+
},
|
|
168
|
+
terminalProof: {
|
|
169
|
+
status: runtime.attributionVerification.denominator === 0 ? 'unavailable'
|
|
170
|
+
: gates.terminalProofAvailable ? 'passed' : 'failed',
|
|
171
|
+
...(runtime.attributionVerification.denominator > 0
|
|
172
|
+
? { metrics: { verified: runtime.attributionVerification } } : {}),
|
|
173
|
+
},
|
|
174
|
+
externalReplay: {
|
|
175
|
+
status: externalExecuted ? 'passed' : 'unavailable',
|
|
176
|
+
...(observation.suiteIdentity.externalReplay.replayId
|
|
177
|
+
? { replayId: observation.suiteIdentity.externalReplay.replayId } : {}),
|
|
178
|
+
},
|
|
179
|
+
};
|
|
180
|
+
const diagnostics = Object.entries(gates).filter(([, passed]) => !passed).map(([gate]) => `release_gate_failed:${gate}`);
|
|
181
|
+
return {
|
|
182
|
+
schemaVersion: 'devflow.unified-runtime-evaluation.v4', createdAt: Date.now(), status: diagnostics.length === 0 ? 'passed' : 'failed',
|
|
183
|
+
evaluationPolicy: { labelAuthority: 'stable_ids', llmJudge: 'diagnostic_only', productionLogsWithoutLabelsProveRecall: false, observationsProducedByRuntime: true },
|
|
184
|
+
datasetIds: { ...corpus.datasetIds }, runtime: observation.runtime,
|
|
185
|
+
samples: { semantic: semantic.length, context: context.length, memory: memoryCases.length, knowledge: knowledgeCases.length },
|
|
186
|
+
suites, metrics: { semantic: semanticMetrics, context: contextMetrics, memory, knowledge, runtime },
|
|
187
|
+
slices: buildSlices(retrievalCases, labelExpectations ?? mechanicsExpectations, labelExpectations !== undefined || observation.suiteIdentity.retrievalExecution.source === 'fixture_seeded'),
|
|
188
|
+
gates,
|
|
189
|
+
diagnostics,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function validateObservation(corpus, observation, current) {
|
|
193
|
+
if (observation.schemaVersion !== 'devflow.unified-runtime-observation.v4')
|
|
194
|
+
throw new Error('Unified runtime observation schema mismatch');
|
|
195
|
+
if (JSON.stringify(observation.goldDatasetIds) !== JSON.stringify(corpus.datasetIds))
|
|
196
|
+
throw new Error('Unified runtime observation dataset mismatch');
|
|
197
|
+
const expected = new Set([...corpus.semantic, ...corpus.context, ...corpus.memory, ...corpus.knowledge].map(item => item.id));
|
|
198
|
+
const observed = [...observation.semantic, ...observation.context, ...observation.memory, ...observation.knowledge].map(item => item.caseId);
|
|
199
|
+
if (new Set(observed).size !== observed.length || observed.length !== expected.size || observed.some(id => !expected.has(id)))
|
|
200
|
+
throw new Error('Unified runtime observation case mismatch');
|
|
201
|
+
return sameRuntime(observation.runtime, current)
|
|
202
|
+
&& observation.suiteIdentity.deterministicSafety.runtimeSourceHash === observation.runtime.runtimeSourceHash
|
|
203
|
+
&& observation.suiteIdentity.deterministicSafety.runnerVersion === observation.runtime.runnerVersion;
|
|
204
|
+
}
|
|
205
|
+
function sameRuntime(left, right) {
|
|
206
|
+
return left.packageVersion === right.packageVersion && left.pluginRegistryHash === right.pluginRegistryHash
|
|
207
|
+
&& left.runtimeSourceHash === right.runtimeSourceHash && left.runnerVersion === right.runnerVersion;
|
|
208
|
+
}
|
|
209
|
+
function joinSemantic(gold, observed) {
|
|
210
|
+
const map = new Map(observed.map(item => [item.caseId, item]));
|
|
211
|
+
return gold.map(item => ({ ...item, observed: map.get(item.id) }));
|
|
212
|
+
}
|
|
213
|
+
function joinRetrieval(gold, observed) {
|
|
214
|
+
const map = new Map(observed.map(item => [item.caseId, item]));
|
|
215
|
+
return gold.map(item => ({ ...item, observed: map.get(item.id) }));
|
|
216
|
+
}
|
|
217
|
+
function evaluateSemantic(cases) {
|
|
218
|
+
const fieldResults = cases.flatMap(semanticFieldResults);
|
|
219
|
+
const falseAuthorizationPopulation = cases.filter(item => !item.expected.mutationAuthorized);
|
|
220
|
+
const actionCases = cases.filter(item => item.expected.domainAction !== undefined);
|
|
221
|
+
return {
|
|
222
|
+
fieldAccuracy: booleanMetric(fieldResults),
|
|
223
|
+
abstentionAccuracy: booleanMetric(cases.map(item => item.expected.abstain === item.observed.abstain)),
|
|
224
|
+
mutationFalseAuthorization: evidenceQualityMetric(falseAuthorizationPopulation.filter(item => item.observed.mutationAuthorized).length, falseAuthorizationPopulation.length),
|
|
225
|
+
domainActionAccuracy: booleanMetric(actionCases.map(item => {
|
|
226
|
+
const expected = item.expected.domainAction;
|
|
227
|
+
const observed = item.observed.domainAction;
|
|
228
|
+
if (!observed || observed.disposition !== expected.expectedDisposition)
|
|
229
|
+
return false;
|
|
230
|
+
return expected.expectedDisposition !== 'selected' || observed.tool === expected.tool;
|
|
231
|
+
})),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
function semanticFieldResults(item) {
|
|
235
|
+
const namespaces = new Set([...item.expected.fieldRefs, ...item.observed.fieldRefs].map(ref => ref.slice(0, ref.indexOf(':'))));
|
|
236
|
+
return [...namespaces].map(namespace => {
|
|
237
|
+
const expected = item.expected.fieldRefs.filter(ref => ref.startsWith(`${namespace}:`));
|
|
238
|
+
const observed = item.observed.fieldRefs.filter(ref => ref.startsWith(`${namespace}:`));
|
|
239
|
+
return expected.length === observed.length && expected.every(ref => observed.includes(ref));
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
function evaluateRetrieval(cases, expectations) {
|
|
243
|
+
const evaluated = cases.flatMap(item => {
|
|
244
|
+
const expectation = expectationFor(item, expectations);
|
|
245
|
+
if (!expectation)
|
|
246
|
+
return [];
|
|
247
|
+
const goldSets = expectation.goldSets.filter(set => set.length > 0);
|
|
248
|
+
const goldRefs = new Set(selectBestGoldSet({
|
|
249
|
+
goldSets,
|
|
250
|
+
candidateRefs: item.observed.candidateEvidenceIds,
|
|
251
|
+
selectedRefs: item.observed.rankedEvidenceIds,
|
|
252
|
+
}));
|
|
253
|
+
return [{
|
|
254
|
+
item,
|
|
255
|
+
goldRefs,
|
|
256
|
+
forbiddenRefs: expectation.forbiddenRefs,
|
|
257
|
+
}];
|
|
258
|
+
});
|
|
259
|
+
const scored = evaluated.filter(item => item.goldRefs.size > 0);
|
|
260
|
+
return {
|
|
261
|
+
candidateRecall: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => candidateRecallAt20(item.observed.candidateEvidenceIds, goldRefs))),
|
|
262
|
+
eligibilitySurvival: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => eligibilitySurvival(item.observed.candidateEvidenceIds, item.observed.eligibilityRejectedEvidenceIds, goldRefs))),
|
|
263
|
+
precisionAt5: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => precisionAt5(item.observed.rankedEvidenceIds, goldRefs))),
|
|
264
|
+
recallAt5: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => recallAt5(item.observed.rankedEvidenceIds, goldRefs))),
|
|
265
|
+
forbiddenContamination: aggregateEvidenceQualityMetrics(evaluated.map(({ item, forbiddenRefs }) => forbiddenContaminationAt5(item.observed.rankedEvidenceIds, forbiddenRefs))),
|
|
266
|
+
rejectionReasonCounts: mergeReasonCounts(cases.map(item => item.observed.rejectionReasonCounts)),
|
|
267
|
+
latency: latencyMetric(cases.map(item => item.observed.latencyMs)),
|
|
268
|
+
reranker: rerankerMetric(scored.map(({ item, goldRefs }) => ({
|
|
269
|
+
item,
|
|
270
|
+
expected: [...goldRefs],
|
|
271
|
+
}))),
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
function abstentionPrecision(cases) {
|
|
275
|
+
const executed = cases.filter(item => item.caseClass === 'abstention'
|
|
276
|
+
&& item.observed.executionStatus === 'executed');
|
|
277
|
+
return booleanMetric(executed.map(item => item.observed.rankedEvidenceIds.length === 0));
|
|
278
|
+
}
|
|
279
|
+
function rerankerMetric(cases) {
|
|
280
|
+
if (cases.length === 0)
|
|
281
|
+
return { sampleCount: 0, meanRelevantAt5Delta: null };
|
|
282
|
+
const deltas = cases.map(({ item, expected }) => relevantAt5(item.observed.rankedEvidenceIds, expected)
|
|
283
|
+
- relevantAt5(item.observed.rerankerBaselineIds, expected));
|
|
284
|
+
return { sampleCount: deltas.length, meanRelevantAt5Delta: average(deltas) };
|
|
285
|
+
}
|
|
286
|
+
function relevantAt5(ranked, expected) {
|
|
287
|
+
return [...new Set(ranked)].slice(0, 5).filter(id => expected.includes(id)).length;
|
|
288
|
+
}
|
|
289
|
+
function latencyMetric(values) {
|
|
290
|
+
if (values.length === 0)
|
|
291
|
+
return { sampleCount: 0, meanMs: null, p95Ms: null };
|
|
292
|
+
const ordered = [...values].sort((a, b) => a - b);
|
|
293
|
+
return { sampleCount: values.length, meanMs: average(values), p95Ms: ordered[Math.max(0, Math.ceil(ordered.length * 0.95) - 1)] };
|
|
294
|
+
}
|
|
295
|
+
function buildSlices(retrieval, expectations, positiveLabelsAvailable = false) {
|
|
296
|
+
const groups = new Map();
|
|
297
|
+
for (const item of retrieval) {
|
|
298
|
+
const key = `${item.projectId}/${item.channel}/${item.caseClass}`;
|
|
299
|
+
const group = groups.get(key) ?? { caseClass: item.caseClass, cases: [] };
|
|
300
|
+
group.cases.push(item);
|
|
301
|
+
groups.set(key, group);
|
|
302
|
+
}
|
|
303
|
+
return Object.fromEntries([...groups.entries()].sort(([left], [right]) => left.localeCompare(right))
|
|
304
|
+
.map(([key, group]) => {
|
|
305
|
+
const executed = group.cases.filter(item => item.observed.executionStatus === 'executed');
|
|
306
|
+
const metrics = evaluateRetrieval(group.caseClass === 'positive' ? executed : [], expectations);
|
|
307
|
+
return [key, {
|
|
308
|
+
status: sliceStatus(group.caseClass, group.cases, metrics, positiveLabelsAvailable),
|
|
309
|
+
sampleCount: group.cases.length,
|
|
310
|
+
metrics: { precisionAt5: metrics.precisionAt5, recallAt5: metrics.recallAt5 },
|
|
311
|
+
}];
|
|
312
|
+
}));
|
|
313
|
+
}
|
|
314
|
+
function sliceStatus(caseClass, cases, metrics, positiveLabelsAvailable) {
|
|
315
|
+
if (caseClass !== 'coverage_gap' && cases.some(item => item.observed.executionStatus !== 'executed')) {
|
|
316
|
+
return 'unavailable';
|
|
317
|
+
}
|
|
318
|
+
if (caseClass === 'positive') {
|
|
319
|
+
if (!positiveLabelsAvailable)
|
|
320
|
+
return 'unavailable';
|
|
321
|
+
return metrics.recallAt5.refMicro.denominator > 0
|
|
322
|
+
&& metrics.recallAt5.refMicro.numerator === metrics.recallAt5.refMicro.denominator ? 'passed' : 'failed';
|
|
323
|
+
}
|
|
324
|
+
if (caseClass === 'abstention') {
|
|
325
|
+
return cases.every(item => item.observed.rankedEvidenceIds.length === 0) ? 'passed' : 'failed';
|
|
326
|
+
}
|
|
327
|
+
if (caseClass === 'lifecycle') {
|
|
328
|
+
return cases.every(item => !item.expected.explicitSave || item.observed.episodePersisted === true)
|
|
329
|
+
? 'passed' : 'failed';
|
|
330
|
+
}
|
|
331
|
+
if (cases.some(item => item.observed.executionStatus === 'unavailable'))
|
|
332
|
+
return 'unavailable';
|
|
333
|
+
return cases.every(item => item.observed.executionStatus === 'executed'
|
|
334
|
+
&& item.observed.executionReasonCode === item.expected.reasonCode
|
|
335
|
+
&& item.observed.rankedEvidenceIds.length === 0) ? 'passed' : 'failed';
|
|
336
|
+
}
|
|
337
|
+
function expectationFor(item, expectations) {
|
|
338
|
+
if (expectations)
|
|
339
|
+
return expectations.get(item.id);
|
|
340
|
+
return { goldSets: [item.expected.evidenceIds], forbiddenRefs: new Set() };
|
|
341
|
+
}
|
|
342
|
+
function validLabelExpectations(evidence, cases, candidateIdentity) {
|
|
343
|
+
if (!evidence || candidateIdentity.status !== 'executed')
|
|
344
|
+
return undefined;
|
|
345
|
+
const manifest = evidence.manifest;
|
|
346
|
+
if (candidateIdentity.bundleId !== manifest.bundleId
|
|
347
|
+
|| candidateIdentity.projectSnapshotId !== manifest.projectSnapshotId
|
|
348
|
+
|| candidateIdentity.corpusHash !== manifest.corpusHash
|
|
349
|
+
|| stableRuntimeHash(candidateIdentity.labelHashes) !== stableRuntimeHash(manifest.labelHashes)
|
|
350
|
+
|| stableRuntimeHash(candidateIdentity.channelSnapshotHashes) !== stableRuntimeHash(manifest.channelSnapshotHashes)
|
|
351
|
+
|| candidateIdentity.bundleProfilesHash !== manifest.profilesHash)
|
|
352
|
+
return undefined;
|
|
353
|
+
const expectedCases = new Map(cases.map(item => [item.id, item.channel]));
|
|
354
|
+
const entries = new Map();
|
|
355
|
+
for (const channel of ['context', 'memory', 'knowledge']) {
|
|
356
|
+
const labelSet = evidence.labels[channel];
|
|
357
|
+
if (labelSet.schemaVersion !== 'devflow.evaluation-labels.v1'
|
|
358
|
+
|| labelSet.channel !== channel
|
|
359
|
+
|| labelSet.projectSnapshotId !== manifest.projectSnapshotId
|
|
360
|
+
|| labelSet.channelSnapshotHash !== manifest.channelSnapshotHashes[channel]
|
|
361
|
+
|| labelSet.datasetHash !== manifest.corpusHash
|
|
362
|
+
|| labelHash(labelSet) !== manifest.labelHashes[channel])
|
|
363
|
+
return undefined;
|
|
364
|
+
for (const label of labelSet.labels) {
|
|
365
|
+
if (expectedCases.get(label.caseId) !== channel || entries.has(label.caseId)
|
|
366
|
+
|| !label.reviewer.trim() || !label.labelledAt.trim() || label.sourceEvidence.length === 0)
|
|
367
|
+
return undefined;
|
|
368
|
+
entries.set(label.caseId, {
|
|
369
|
+
goldSets: label.goldSets.map(set => [...new Set(set)]),
|
|
370
|
+
forbiddenRefs: new Set(label.forbiddenRefs),
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
return entries.size === expectedCases.size ? entries : undefined;
|
|
375
|
+
}
|
|
376
|
+
function booleanMetric(values) { return evidenceQualityMetric(values.filter(Boolean).length, values.length); }
|
|
377
|
+
function atLeast(metric, threshold) { return metric.value !== null && metric.value >= threshold; }
|
|
378
|
+
function atMost(metric, threshold) { return metric.value !== null && metric.value <= threshold; }
|
|
379
|
+
function average(values) { return values.reduce((total, value) => total + value, 0) / values.length; }
|
|
380
|
+
function mergeReasonCounts(values) {
|
|
381
|
+
const counts = {};
|
|
382
|
+
for (const value of values) {
|
|
383
|
+
for (const [reason, count] of Object.entries(value))
|
|
384
|
+
counts[reason] = (counts[reason] ?? 0) + count;
|
|
385
|
+
}
|
|
386
|
+
return counts;
|
|
387
|
+
}
|
|
388
|
+
//# sourceMappingURL=unified-runtime-evaluation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unified-runtime-evaluation.js","sourceRoot":"","sources":["../src/unified-runtime-evaluation.ts"],"names":[],"mappings":"AACA,OAAO,EACL,+BAA+B,EAC/B,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,EACzB,YAAY,EACZ,SAAS,EACT,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAmC,MAAM,oBAAoB,CAAC;AAsFnG,MAAM,UAAU,GAAG;IACjB,qBAAqB,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI;IACxG,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,yBAAyB,EAAE,IAAI;IAC3G,kBAAkB,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,EAAE,4BAA4B,EAAE,IAAI;IAC5F,sBAAsB,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;CACzE,CAAC;AAEX,MAAM,UAAU,4BAA4B,CAC1C,MAA4B,EAC5B,WAAwC,EACxC,iBAA2C,WAAW,CAAC,OAAO,EAC9D,cAA2C;IAE3C,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAC9E,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,UAAU,CAAC;IAC5F,MAAM,cAAc,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,CAAC;IACvE,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,cAAc,EACd,cAAc,EACd,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAC9C,CAAC;IACF,MAAM,iBAAiB,GAAG,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,KAAK,UAAU;WACxF,iBAAiB,KAAK,SAAS,CAAC;IACrC,MAAM,gBAAgB,GAAG,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,KAAK,UAAU;WAClF,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC;WAC1D,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC;WACtD,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IACxE,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG;QACtB,GAAG,sBAAsB;QACzB,0BAA0B,EAAE,qBAAqB,CAAC,0BAA0B;KAC7E,CAAC;IACF,MAAM,qBAAqB,GAAG,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,gBAAgB;QACpG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAgC,CAAC;IACxD,MAAM,gBAAgB,GAAG,CAAC,KAA+B,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAChF,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;IACjF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC7F,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACpG,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC1G,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzH,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChI,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtI,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3G,MAAM,qBAAqB,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC;SACrG,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,KAAK,aAAa,CAAC,CAAC;IACxG,MAAM,mBAAmB,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC9F,MAAM,oBAAoB,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAChG,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpF,MAAM,gBAAgB,GAAG,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,YAAY;WAC3G,CAAC,CAAC,oBAAoB,EAAE,sBAAsB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,YAAY,CAAC,CAAC;IACtG,MAAM,iBAAiB,GAAG,aAAa,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC,CAAC;IACnH,MAAM,OAAO,GAAG;QACd,oBAAoB,EAAE,aAAa,CAAC,qBAAqB,CAAC;QAC1D,sBAAsB,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzI,uBAAuB,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzI,uBAAuB,EAAE,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC;QAC5I,oBAAoB,EAAE,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;QAC7H,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACzG,oBAAoB,EAAE,aAAa,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;KAClH,CAAC;IACF,MAAM,YAAY,GAAG;QACnB,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eACrG,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,CAAC;aAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAChF,sBAAsB,EAAE,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eACxG,IAAI,CAAC,SAAS,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;aAC/D,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC;KACzD,CAAC;IACF,MAAM,eAAe,GAAG;QACtB,wBAAwB,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eAC7G,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC;aAC7E,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;KAC7E,CAAC;IACF,MAAM,eAAe,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC;IACpE,MAAM,kBAAkB,GAAG;QACzB,GAAG,sBAAsB,EAAE,GAAG,eAAe;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,cAAc,CAAC;KACzD,CAAC;IACF,MAAM,eAAe,GAAG,mBAAmB,CAAC,CAAC,CAAC,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACtG,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,CAAC,CAAC;QAClD,GAAG,sBAAsB,EAAE,GAAG,eAAe;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,cAAc,CAAC;KACzD,CAAC,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,GAAG,gBAAgB,IAAI,gBAAgB,CAAC;IAC5D,MAAM,MAAM,GAAG,eAAe,IAAI,eAAe,CAAC;IAClD,MAAM,SAAS,GAAG,kBAAkB,IAAI,kBAAkB,CAAC;IAC3D,MAAM,KAAK,GAA4C;QACrD,eAAe,EAAE,oBAAoB;QACrC,2BAA2B,EAAE,gBAAgB;QAC7C,4BAA4B,EAAE,iBAAiB;QAC/C,uBAAuB,EAAE,gBAAgB;QACzC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC9C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAChD,yBAAyB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,uBAAuB,CAAC;QACvG,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;QACnE,qBAAqB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,aAAa,EAAE,UAAU,CAAC,qBAAqB,CAAC;QACnH,kBAAkB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAClH,0BAA0B,EAAE,eAAe,CAAC,0BAA0B,CAAC,SAAS,KAAK,CAAC;QACtF,oBAAoB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,CAAC;QACxH,gBAAgB,EAAE,iBAAiB,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,mBAAmB,CAAC;QAC/L,eAAe,EAAE,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAC5K,mBAAmB,EAAE,iBAAiB,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,CAAC,mBAAmB,CAAC;QACxH,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,UAAU,CAAC,yBAAyB,CAAC;QACpG,kBAAkB,EAAE,iBAAiB,IAAI,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAC7G,2BAA2B,EAAE,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAAE,UAAU,CAAC,2BAA2B,CAAC;QAChH,mBAAmB,EAAE,OAAO,CAAC,SAAS,CAAC,mBAAmB,EAAE,UAAU,CAAC,4BAA4B,CAAC;QACpG,sBAAsB,EAAE,OAAO,CAAC,MAAM,CAAC,sBAAsB,EAAE,UAAU,CAAC,sBAAsB,CAAC;QACjG,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,uBAAuB,CAAC;QACpG,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACvD,aAAa,EAAE,OAAO,CAAC,oBAAoB,CAAC,SAAS,KAAK,CAAC;QAC3D,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;eAC3C,CAAC,OAAO,CAAC,oBAAoB,CAAC,WAAW,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;KAClG,CAAC;IACF,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,0BAA0B;WACzE,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;IACvE,MAAM,cAAc,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,oBAAoB,CAAC;IAC7G,MAAM,sBAAsB,GAAG,gBAAgB,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS;WACzF,kBAAkB,KAAK,SAAS;WAChC,OAAO,CAAC,gBAAgB,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,gBAAgB,CAAC;WAC/E,OAAO,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,eAAe,CAAC;WAC7E,OAAO,CAAC,kBAAkB,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC;IACzF,MAAM,eAAe,GAAG,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,eAAe;WAClE,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,yBAAyB;WAC5D,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,2BAA2B;WAC7D,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,sBAAsB,CAAC;IAC/D,MAAM,MAAM,GAA6C;QACvD,MAAM,EAAE;YACN,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC1C,OAAO,EAAE;gBACP,0BAA0B,EAAE,eAAe,CAAC,0BAA0B;gBACtE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC;SACF;QACD,kBAAkB,EAAE;YAClB,MAAM,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAChF,OAAO,EAAE;gBACP,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;gBACtD,oBAAoB,EAAE,eAAe,CAAC,oBAAoB;aAC3D;SACF;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,gBAAgB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzG,GAAG,CAAC,gBAAgB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3F;QACD,iBAAiB,EAAE;YACjB,MAAM,EAAE,iBAAiB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC3G,GAAG,CAAC,iBAAiB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzF,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,IAAI,eAAe,IAAI,kBAAkB;gBAChF,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7G;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAClF,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,IAAI,eAAe,IAAI,kBAAkB;gBAChF,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7G;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa;gBACvE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzD,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE;oBAC/D,WAAW,EAAE,OAAO,CAAC,sBAAsB;oBAC3C,YAAY,EAAE,OAAO,CAAC,uBAAuB;iBAC9C,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACV;QACD,aAAa,EAAE;YACb,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa;gBACvE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACtD,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,WAAW,GAAG,CAAC;gBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE;QACD,cAAc,EAAE;YACd,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa;YACnD,GAAG,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ;gBACnD,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1E;KACF,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACzH,OAAO;QACL,aAAa,EAAE,uCAAuC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACrI,gBAAgB,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,iBAAiB,EAAE,sCAAsC,EAAE,KAAK,EAAE,6BAA6B,EAAE,IAAI,EAAE;QACnK,UAAU,EAAE,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO;QAClE,OAAO,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;QAC7H,MAAM,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE;QACnG,MAAM,EAAE,WAAW,CACjB,cAAc,EACd,iBAAiB,IAAI,qBAAqB,EAC1C,iBAAiB,KAAK,SAAS,IAAI,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,gBAAgB,CAC5G;QACD,KAAK;QACL,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAA4B,EAAE,WAAwC,EAAE,OAAiC;IACpI,IAAI,WAAW,CAAC,aAAa,KAAK,wCAAwC;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC3I,IAAI,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACtJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9H,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,WAAW,CAAC,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7I,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC5L,OAAO,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;WAC3C,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,iBAAiB,KAAK,WAAW,CAAC,OAAO,CAAC,iBAAiB;WACzG,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,aAAa,KAAK,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC;AACzG,CAAC;AAED,SAAS,WAAW,CAAC,IAA8B,EAAE,KAA+B;IAClF,OAAO,IAAI,CAAC,cAAc,KAAK,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,kBAAkB,KAAK,KAAK,CAAC,kBAAkB;WACtG,IAAI,CAAC,iBAAiB,KAAK,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa,CAAC;AACxG,CAAC;AAED,SAAS,YAAY,CAAC,IAA2B,EAAE,QAAwC;IACzF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,IAA4B,EAAE,QAAyC;IAC5F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAA8B;IACtD,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACzD,MAAM,4BAA4B,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO;QACL,aAAa,EAAE,aAAa,CAAC,YAAY,CAAC;QAC1C,kBAAkB,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrG,0BAA0B,EAAE,qBAAqB,CAAC,4BAA4B,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,4BAA4B,CAAC,MAAM,CAAC;QAC5K,oBAAoB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAa,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC5C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,mBAAmB;gBAAE,OAAO,KAAK,CAAC;YACrF,OAAO,QAAQ,CAAC,mBAAmB,KAAK,UAAU,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC;QACxF,CAAC,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,IAA2B;IACvD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChI,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QACxF,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CACxB,KAA+B,EAC/B,YAAgD;IAEhD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QACrC,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC;YACzC,QAAQ;YACR,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,oBAAoB;YACjD,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB;SAC9C,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC;gBACN,IAAI;gBACJ,QAAQ;gBACR,aAAa,EAAE,WAAW,CAAC,aAAa;aACzC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAChE,OAAO;QACL,eAAe,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjF,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC;QACrE,mBAAmB,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACrF,mBAAmB,CACjB,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAClC,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAC5C,QAAQ,CACT,CAAC,CAAC;QACL,YAAY,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC9E,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC3D,SAAS,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC3E,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC,CAAC;QACxD,sBAAsB,EAAE,+BAA+B,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAChG,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC;QAC7E,qBAAqB,EAAE,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC;QAChG,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAClE,QAAQ,EAAE,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI;YACJ,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC;SACxB,CAAC,CAAC,CAAC;KACL,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAA+B;IAE/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,YAAY;WAChE,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;IACnD,OAAO,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,SAAS,cAAc,CAAC,KAAkE;IACxF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC;IAC9E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC;UACnG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/E,CAAC;AAED,SAAS,WAAW,CAAC,MAAgB,EAAE,QAAkB;IACvD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrF,CAAC;AACD,SAAS,aAAa,CAAC,MAAgB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC9E,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,CAAC;AACrI,CAAC;AAED,SAAS,WAAW,CAClB,SAAmC,EACnC,YAAgD,EAChD,uBAAuB,GAAG,KAAK;IAE/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAGlB,CAAC;IACL,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAC1E,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;QAC1F,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,EAAE;gBACX,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,uBAAuB,CAAC;gBACnF,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;gBAC/B,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;aAC7C,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,WAAW,CAClB,SAA2B,EAC3B,KAA+B,EAC/B,OAA6B,EAC7B,uBAAgC;IAEhC,IAAI,SAAS,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,EAAE,CAAC;QACrG,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,uBAAuB;YAAE,OAAO,aAAa,CAAC;QACnD,OAAO,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,GAAG,CAAC;eAC5C,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7G,CAAC;IACD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACjG,CAAC;IACD,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,KAAK,IAAI,CAAC;YAChG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,aAAa,CAAC;QAAE,OAAO,aAAa,CAAC;IAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;WAClE,IAAI,CAAC,QAAQ,CAAC,mBAAmB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU;WAC9D,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC3E,CAAC;AAED,SAAS,cAAc,CACrB,IAA4B,EAC5B,YAAgD;IAEhD,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnD,OAAO,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,aAAa,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AAC7E,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAgD,EAChD,KAA+B,EAC/B,iBAAsF;IAEtF,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACnC,IAAI,iBAAiB,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;WAC/C,iBAAiB,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB;WAClE,iBAAiB,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU;WACpD,iBAAiB,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC;WAC5F,iBAAiB,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,KAAK,iBAAiB,CAAC,QAAQ,CAAC,qBAAqB,CAAC;WAChH,iBAAiB,CAAC,kBAAkB,KAAK,QAAQ,CAAC,YAAY;QAAE,OAAO,SAAS,CAAC;IACtF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;IACxD,KAAK,MAAM,OAAO,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAU,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,QAAQ,CAAC,aAAa,KAAK,8BAA8B;eACxD,QAAQ,CAAC,OAAO,KAAK,OAAO;eAC5B,QAAQ,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB;eACzD,QAAQ,CAAC,mBAAmB,KAAK,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC;eACxE,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,UAAU;eAC5C,SAAS,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC;YAAE,OAAO,SAAS,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;mBACvE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC/G,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtD,aAAa,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,aAAa,CAAC,MAAiB,IAA2B,OAAO,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAChJ,SAAS,OAAO,CAAC,MAA6B,EAAE,SAAiB,IAAa,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC;AAC1I,SAAS,MAAM,CAAC,MAA6B,EAAE,SAAiB,IAAa,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC;AACzI,SAAS,OAAO,CAAC,MAAgB,IAAY,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAExH,SAAS,iBAAiB,CAAC,MAAqC;IAC9D,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACtG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|