@devflow-tools/benchmark 0.17.8 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/LICENSE +21 -0
  2. package/dist/accuracy-command-options.d.ts +3 -0
  3. package/dist/accuracy-command-options.d.ts.map +1 -0
  4. package/dist/accuracy-command-options.js +17 -0
  5. package/dist/accuracy-command-options.js.map +1 -0
  6. package/dist/context-symbol-quality-command.d.ts +59 -0
  7. package/dist/context-symbol-quality-command.d.ts.map +1 -0
  8. package/dist/context-symbol-quality-command.js +482 -0
  9. package/dist/context-symbol-quality-command.js.map +1 -0
  10. package/dist/evaluation-bundle-builder.d.ts +48 -0
  11. package/dist/evaluation-bundle-builder.d.ts.map +1 -0
  12. package/dist/evaluation-bundle-builder.js +496 -0
  13. package/dist/evaluation-bundle-builder.js.map +1 -0
  14. package/dist/evaluation-bundle-command.d.ts +2 -0
  15. package/dist/evaluation-bundle-command.d.ts.map +1 -0
  16. package/dist/evaluation-bundle-command.js +15 -0
  17. package/dist/evaluation-bundle-command.js.map +1 -0
  18. package/dist/evaluation-bundle.d.ts +55 -0
  19. package/dist/evaluation-bundle.d.ts.map +1 -0
  20. package/dist/evaluation-bundle.js +272 -0
  21. package/dist/evaluation-bundle.js.map +1 -0
  22. package/dist/evaluation-run-state.d.ts +23 -0
  23. package/dist/evaluation-run-state.d.ts.map +1 -0
  24. package/dist/evaluation-run-state.js +115 -0
  25. package/dist/evaluation-run-state.js.map +1 -0
  26. package/dist/evidence-quality-metrics.d.ts +15 -0
  27. package/dist/evidence-quality-metrics.d.ts.map +1 -1
  28. package/dist/evidence-quality-metrics.js +51 -1
  29. package/dist/evidence-quality-metrics.js.map +1 -1
  30. package/dist/graders/deterministic.d.ts.map +1 -1
  31. package/dist/graders/deterministic.js +5 -3
  32. package/dist/graders/deterministic.js.map +1 -1
  33. package/dist/index.d.ts +15 -2
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +12 -1
  36. package/dist/index.js.map +1 -1
  37. package/dist/release-quality-bundle-command.d.ts +3 -0
  38. package/dist/release-quality-bundle-command.d.ts.map +1 -0
  39. package/dist/release-quality-bundle-command.js +72 -0
  40. package/dist/release-quality-bundle-command.js.map +1 -0
  41. package/dist/release-quality-bundle-evaluator.d.ts +61 -0
  42. package/dist/release-quality-bundle-evaluator.d.ts.map +1 -0
  43. package/dist/release-quality-bundle-evaluator.js +321 -0
  44. package/dist/release-quality-bundle-evaluator.js.map +1 -0
  45. package/dist/release-quality-bundle-materializer.d.ts +79 -0
  46. package/dist/release-quality-bundle-materializer.d.ts.map +1 -0
  47. package/dist/release-quality-bundle-materializer.js +395 -0
  48. package/dist/release-quality-bundle-materializer.js.map +1 -0
  49. package/dist/release-quality-comparison.d.ts +34 -0
  50. package/dist/release-quality-comparison.d.ts.map +1 -0
  51. package/dist/release-quality-comparison.js +60 -0
  52. package/dist/release-quality-comparison.js.map +1 -0
  53. package/dist/release-quality-evaluation-command.d.ts +42 -0
  54. package/dist/release-quality-evaluation-command.d.ts.map +1 -0
  55. package/dist/release-quality-evaluation-command.js +91 -0
  56. package/dist/release-quality-evaluation-command.js.map +1 -0
  57. package/dist/release-quality-host-proof.d.ts +30 -0
  58. package/dist/release-quality-host-proof.d.ts.map +1 -0
  59. package/dist/release-quality-host-proof.js +109 -0
  60. package/dist/release-quality-host-proof.js.map +1 -0
  61. package/dist/release-quality-output-cleanup.d.ts +2 -0
  62. package/dist/release-quality-output-cleanup.d.ts.map +1 -0
  63. package/dist/release-quality-output-cleanup.js +18 -0
  64. package/dist/release-quality-output-cleanup.js.map +1 -0
  65. package/dist/release-quality-policy.d.ts +87 -0
  66. package/dist/release-quality-policy.d.ts.map +1 -0
  67. package/dist/release-quality-policy.js +179 -0
  68. package/dist/release-quality-policy.js.map +1 -0
  69. package/dist/release-quality-query-sampling.d.ts +36 -0
  70. package/dist/release-quality-query-sampling.d.ts.map +1 -0
  71. package/dist/release-quality-query-sampling.js +87 -0
  72. package/dist/release-quality-query-sampling.js.map +1 -0
  73. package/dist/release-quality-source-snapshot.d.ts +18 -0
  74. package/dist/release-quality-source-snapshot.d.ts.map +1 -0
  75. package/dist/release-quality-source-snapshot.js +46 -0
  76. package/dist/release-quality-source-snapshot.js.map +1 -0
  77. package/dist/retrieval-quality-corpus.d.ts +0 -1
  78. package/dist/retrieval-quality-corpus.d.ts.map +1 -1
  79. package/dist/retrieval-quality-corpus.js +1 -1
  80. package/dist/retrieval-quality-corpus.js.map +1 -1
  81. package/dist/retrieval-quality.d.ts.map +1 -1
  82. package/dist/retrieval-quality.js +29 -26
  83. package/dist/retrieval-quality.js.map +1 -1
  84. package/dist/run-accuracy-tests.js +82 -2
  85. package/dist/run-accuracy-tests.js.map +1 -1
  86. package/dist/suite-runner.js +1 -1
  87. package/dist/suite-runner.js.map +1 -1
  88. package/dist/unified-runtime-corpus.d.ts +64 -0
  89. package/dist/unified-runtime-corpus.d.ts.map +1 -0
  90. package/dist/unified-runtime-corpus.js +161 -0
  91. package/dist/unified-runtime-corpus.js.map +1 -0
  92. package/dist/unified-runtime-evaluation.d.ts +133 -0
  93. package/dist/unified-runtime-evaluation.d.ts.map +1 -0
  94. package/dist/unified-runtime-evaluation.js +388 -0
  95. package/dist/unified-runtime-evaluation.js.map +1 -0
  96. package/dist/unified-runtime-observation.d.ts +101 -0
  97. package/dist/unified-runtime-observation.d.ts.map +1 -0
  98. package/dist/unified-runtime-observation.js +2 -0
  99. package/dist/unified-runtime-observation.js.map +1 -0
  100. package/dist/unified-runtime-runner.d.ts +73 -0
  101. package/dist/unified-runtime-runner.d.ts.map +1 -0
  102. package/dist/unified-runtime-runner.js +620 -0
  103. package/dist/unified-runtime-runner.js.map +1 -0
  104. package/fixtures/evidence-runtime-corpus.json +0 -100
  105. package/fixtures/release-quality/bundles.json +1 -0
  106. package/fixtures/release-quality/calibration.json +602 -0
  107. package/fixtures/release-quality/ecc/corpus.json +257 -0
  108. package/fixtures/release-quality/ecc/input.json +62 -0
  109. package/fixtures/release-quality/llm-wiki/corpus.json +263 -0
  110. package/fixtures/release-quality/llm-wiki/input.json +72 -0
  111. package/fixtures/release-quality/mem0/corpus.json +288 -0
  112. package/fixtures/release-quality/mem0/input.json +68 -0
  113. package/fixtures/release-quality/projects.json +29 -0
  114. package/fixtures/release-quality/release-quality-policy.json +232 -0
  115. package/fixtures/release-quality/rsbuild-react/corpus.json +263 -0
  116. package/fixtures/release-quality/rsbuild-react/input.json +62 -0
  117. package/fixtures/retrieval-quality-corpus.json +7 -7
  118. package/fixtures/semantic-regex-inventory.json +3 -10
  119. package/fixtures/unified-runtime/baselines/2026-08-08.json +37 -0
  120. package/fixtures/unified-runtime/context.json +206 -0
  121. package/fixtures/unified-runtime/knowledge.json +224 -0
  122. package/fixtures/unified-runtime/memory.json +224 -0
  123. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/context.json +32 -0
  124. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/knowledge.json +32 -0
  125. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/memory.json +32 -0
  126. package/fixtures/unified-runtime/real-projects/rsbuild-react-2026-08-07/semantic.json +409 -0
  127. package/fixtures/unified-runtime/runs/rsbuild-react-2026-08-05.json +54 -0
  128. package/fixtures/unified-runtime/semantic.json +409 -0
  129. package/package.json +24 -6
@@ -0,0 +1 @@
1
+ {"version":3,"file":"unified-runtime-corpus.js","sourceRoot":"","sources":["../src/unified-runtime-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA0HpC,MAAM,SAAS,GAAG,gDAAgD,CAAC;AACnE,MAAM,iBAAiB,GAAqB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;AAC5H,MAAM,YAAY,GAAuB,CAAC,UAAU,EAAE,YAAY,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;AACjG,MAAM,0BAA0B,GAAG,IAAI,GAAG,CAAC;IACzC,UAAU,EAAE,UAAU,EAAE,wBAAwB,EAAE,mBAAmB,EAAE,gBAAgB;IACvF,+BAA+B,EAAE,WAAW,EAAE,qBAAqB;IACnE,qBAAqB,EAAE,uBAAuB,EAAE,kBAAkB;CACnE,CAAC,CAAC;AAEH,MAAM,UAAU,wBAAwB,CAAC,gBAAgB,GAAG,uBAAuB,EAAE;IACnF,MAAM,eAAe,GAAG,QAAQ,CAAkB,OAAO,CAAC,gBAAgB,EAAE,eAAe,CAAC,CAAC,CAAC;IAC9F,MAAM,iBAAiB,GAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CACpF,QAAQ,CAAmB,OAAO,CAAC,gBAAgB,EAAE,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC;IAC5E,qBAAqB,CAAC,eAAe,EAAE,qCAAqC,CAAC,CAAC;IAC9E,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,qBAAqB,CAAC,OAAO,EAAE,sCAAsC,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,UAAU,CAAC,WAAW,OAAO,CAAC,OAAO,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACvJ,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACrG,IAAI,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1J,IAAI,OAAO,CAAC,eAAe,CAAC,cAAc,KAAK,eAAe,CAAC,eAAe,CAAC,cAAc;eACxF,OAAO,CAAC,eAAe,CAAC,MAAM,KAAK,eAAe,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC;YAC/E,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAmD,CAAC;IACtK,MAAM,MAAM,GAAyB;QACnC,aAAa,EAAE,iCAAiC,EAAE,UAAU,EAAE;YAC5D,QAAQ,EAAE,eAAe,CAAC,SAAS;YACnC,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,CAAE,CAAC,SAAS;YAC9E,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAE,CAAC,SAAS;YAC5E,SAAS,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,WAAW,CAAE,CAAC,SAAS;SACnF,EAAE,eAAe,EAAE,eAAe,CAAC,eAAe;QACnD,QAAQ,EAAE,cAAc,CAAC,eAAe,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS;KAChI,CAAC;IACF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,OAAwB;IAC9C,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtD,OAAO,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;QAC1E,MAAM,aAAa,GAAG,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;QACjE,OAAO;YACL,EAAE,EAAE,YAAY,QAAQ,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM;YAClG,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;YAC9C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,WAAoB,EAAE;YAClH,QAAQ,EAAE;gBACR,SAAS,EAAE,oBAAoB,CAAC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;gBAClE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAClC,kBAAkB,EAAE,aAAa,KAAK,uBAAuB;gBAC7D,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE;wBAC1C,GAAG,QAAQ,CAAC,YAAY;wBACxB,mBAAmB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,aAAa,CAAC;qBAClF,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACT,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7E;SACF,CAAC;IACJ,CAAC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CACxB,IAA8B,EAC9B,aAAqB;IAErB,IAAI,IAAI,KAAK,WAAW;QAAE,OAAO,UAAU,CAAC;IAC5C,IAAI,aAAa,KAAK,uBAAuB;QAAE,OAAO,UAAU,CAAC;IACjE,IAAI,aAAa,KAAK,wBAAwB;QAAE,OAAO,sBAAsB,CAAC;IAC9E,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IAChD,gBAAgB,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtD,OAAO,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5E,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,IAAI,QAAQ,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS;QACnF,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS;QACrF,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE;QACnK,QAAQ,EAAE;YACR,WAAW,EAAE,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS;YACrE,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,GAAG,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY,IAAI,CAAC,OAAO,CAAC,EAAE,KAAK,QAAQ,IAAI,OAAO,CAAC,EAAE,KAAK,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnJ,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE;KACF,CAAC,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA0B,EAAE,OAAuB;IAC/E,IAAI,OAAO,KAAK,UAAU;QAAE,OAAO,uBAAuB,CAAC;IAC3D,IAAI,OAAO,KAAK,aAAa;QAAE,OAAO,wBAAwB,CAAC;IAC/D,IAAI,OAAO,KAAK,WAAW,IAAI,OAAO,KAAK,aAAa;QAAE,OAAO,0BAA0B,CAAC;IAC5F,OAAO,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,0BAA0B,CAAC;AAC5F,CAAC;AAED,SAAS,oBAAoB,CAAC,SAAmB,EAAE,aAAqB;IACtE,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,YAAY,CAAC,MAAqB,EAAE,OAAuB;IAClE,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;AAC/D,CAAC;AAED,SAAS,qBAAqB,CAAC,OAA2C,EAAE,aAAqB;IAC/F,cAAc,CAAC,OAAO,CAAC,CAAC;IACxB,IAAI,OAAO,CAAC,aAAa,KAAK,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACrH,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACrG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC7O,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;AAC7F,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,QAA0B;IACrE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1C,IAAI,GAAG,CAAC,MAAM,KAAK,iBAAiB,CAAC,MAAM,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,SAAS,qDAAqD,CAAC,CAAC;AACjO,CAAC;AAED,SAAS,sBAAsB,CAAC,MAA4B;IAC1D,MAAM,cAAc,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAW,CAAC;IAC1F,KAAK,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,MAAM,CAAC,OAAsC,CAAC,CAAC;QAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,6BAA6B,OAAO,QAAQ,CAAC,CAAC;IACtG,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3F,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACjG,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QACzI,MAAM,IAAI,GAAG,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAChG,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,SAAS,KAAK,cAAc,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU;gBAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;YACrI,IAAI,IAAI,CAAC,SAAS,KAAK,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU;gBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAChJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,IAAI,GAAG,CAAC,IAAa,EAAQ,EAAE;QACnC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO;QAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAChD,IAAI,0BAA0B,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,GAAG,EAAE,CAAC,CAAC;YACpH,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC,CAAC;IACF,IAAI,CAAC,KAAK,CAAC,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,KAAa,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,6BAA6B,CAAC,CAAC,CAAC,CAAC;AAClH,SAAS,QAAQ,CAAI,IAAY,IAAO,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAM,CAAC,CAAC,CAAC"}
@@ -0,0 +1,133 @@
1
+ import type { AggregatedEvidenceQualityMetric, EvidenceQualityMetric } from './evidence-quality-metrics.js';
2
+ import { type EvaluationBundleManifestV1 } from '@devflow-tools/sdk';
3
+ import type { EvaluationLabelSetV1 } from './evaluation-bundle.js';
4
+ import type { UnifiedChannel, UnifiedRuntimeCorpus } from './unified-runtime-corpus.js';
5
+ import type { UnifiedRuntimeIdentityV2, UnifiedRuntimeObservationV4 } from './unified-runtime-observation.js';
6
+ export interface UnifiedLatencyMetric {
7
+ sampleCount: number;
8
+ meanMs: number | null;
9
+ p95Ms: number | null;
10
+ }
11
+ export interface UnifiedRerankerMetric {
12
+ sampleCount: number;
13
+ meanRelevantAt5Delta: number | null;
14
+ }
15
+ export interface UnifiedRuntimeEvaluationReport {
16
+ schemaVersion: 'devflow.unified-runtime-evaluation.v4';
17
+ createdAt: number;
18
+ status: 'passed' | 'failed';
19
+ evaluationPolicy: {
20
+ labelAuthority: 'stable_ids';
21
+ llmJudge: 'diagnostic_only';
22
+ productionLogsWithoutLabelsProveRecall: false;
23
+ observationsProducedByRuntime: true;
24
+ };
25
+ datasetIds: UnifiedRuntimeCorpus['datasetIds'];
26
+ runtime: UnifiedRuntimeIdentityV2;
27
+ samples: Record<'semantic' | 'context' | 'memory' | 'knowledge', number>;
28
+ suites: {
29
+ safety: {
30
+ status: 'passed' | 'failed';
31
+ metrics: {
32
+ mutationFalseAuthorization: EvidenceQualityMetric;
33
+ contextHashAgreement: EvidenceQualityMetric;
34
+ terminalFalseSuccess: EvidenceQualityMetric;
35
+ boundedCycles: EvidenceQualityMetric;
36
+ };
37
+ };
38
+ semanticCapability: {
39
+ status: 'passed' | 'failed' | 'unavailable';
40
+ metrics: {
41
+ fieldAccuracy: EvidenceQualityMetric;
42
+ abstentionAccuracy: EvidenceQualityMetric;
43
+ domainActionAccuracy: EvidenceQualityMetric;
44
+ };
45
+ };
46
+ planningCoverage: StageSuite<{
47
+ validPlans: EvidenceQualityMetric;
48
+ }>;
49
+ producerExecution: StageSuite<{
50
+ originalLegOutcomes: EvidenceQualityMetric;
51
+ }>;
52
+ candidateQuality: StageSuite<{
53
+ context: RetrievalMetricGroup;
54
+ memory: RetrievalMetricGroup;
55
+ knowledge: RetrievalMetricGroup;
56
+ }>;
57
+ selectionQuality: StageSuite<{
58
+ context: RetrievalMetricGroup;
59
+ memory: RetrievalMetricGroup;
60
+ knowledge: RetrievalMetricGroup;
61
+ }>;
62
+ consumptionProof: StageSuite<{
63
+ consumption: EvidenceQualityMetric;
64
+ knownOutcome: EvidenceQualityMetric;
65
+ }>;
66
+ terminalProof: StageSuite<{
67
+ verified: EvidenceQualityMetric;
68
+ }>;
69
+ externalReplay: {
70
+ status: 'passed' | 'unavailable';
71
+ replayId?: string;
72
+ };
73
+ };
74
+ metrics: {
75
+ semantic: {
76
+ fieldAccuracy: EvidenceQualityMetric;
77
+ abstentionAccuracy: EvidenceQualityMetric;
78
+ mutationFalseAuthorization: EvidenceQualityMetric;
79
+ domainActionAccuracy: EvidenceQualityMetric;
80
+ };
81
+ context: RetrievalMetricGroup;
82
+ memory: RetrievalMetricGroup & {
83
+ temporalCorrectness: EvidenceQualityMetric;
84
+ explicitSaveDurability: EvidenceQualityMetric;
85
+ };
86
+ knowledge: RetrievalMetricGroup & {
87
+ sourceVersionCorrectness: EvidenceQualityMetric;
88
+ abstentionPrecision: EvidenceQualityMetric;
89
+ };
90
+ runtime: {
91
+ contextHashAgreement: EvidenceQualityMetric;
92
+ attributionConsumption: EvidenceQualityMetric;
93
+ attributionKnownOutcome: EvidenceQualityMetric;
94
+ attributionVerification: EvidenceQualityMetric;
95
+ terminalFalseSuccess: EvidenceQualityMetric;
96
+ boundedCycles: EvidenceQualityMetric;
97
+ noGainStopObservable: EvidenceQualityMetric;
98
+ };
99
+ };
100
+ slices: Record<string, UnifiedEvaluationSlice>;
101
+ gates: Record<UnifiedReleaseGate, boolean>;
102
+ diagnostics: string[];
103
+ }
104
+ export interface EvaluationBundleEvidenceV1 {
105
+ manifest: EvaluationBundleManifestV1;
106
+ labels: Record<UnifiedChannel, EvaluationLabelSetV1>;
107
+ }
108
+ interface StageSuite<Metrics> {
109
+ status: 'passed' | 'failed' | 'unavailable';
110
+ metrics?: Metrics;
111
+ }
112
+ interface RetrievalMetricGroup {
113
+ candidateRecall: AggregatedEvidenceQualityMetric;
114
+ eligibilitySurvival: AggregatedEvidenceQualityMetric;
115
+ precisionAt5: AggregatedEvidenceQualityMetric;
116
+ recallAt5: AggregatedEvidenceQualityMetric;
117
+ forbiddenContamination: AggregatedEvidenceQualityMetric;
118
+ rejectionReasonCounts: Record<string, number>;
119
+ latency: UnifiedLatencyMetric;
120
+ reranker: UnifiedRerankerMetric;
121
+ }
122
+ export interface UnifiedEvaluationSlice {
123
+ status: 'passed' | 'failed' | 'unavailable';
124
+ sampleCount: number;
125
+ metrics: {
126
+ precisionAt5: AggregatedEvidenceQualityMetric;
127
+ recallAt5: AggregatedEvidenceQualityMetric;
128
+ };
129
+ }
130
+ type UnifiedReleaseGate = 'runtimeIdentity' | 'semanticCapabilityAvailable' | 'candidateGenerationAvailable' | 'externalReplayAvailable' | 'semanticFieldAccuracy' | 'semanticAbstention' | 'mutationFalseAuthorization' | 'planningCoverage' | 'producerExecution' | 'consumptionProofAvailable' | 'terminalProofAvailable' | 'domainActionAccuracy' | 'contextRetrieval' | 'memoryRetrieval' | 'memoryContamination' | 'memoryTemporalCorrectness' | 'knowledgeRetrieval' | 'knowledgeVersionCorrectness' | 'knowledgeAbstention' | 'explicitSaveDurability' | 'attributionConsumption' | 'hashAgreement' | 'terminalTruth' | 'boundedCycles';
131
+ export declare function evaluateUnifiedRuntimeCorpus(corpus: UnifiedRuntimeCorpus, observation: UnifiedRuntimeObservationV4, currentRuntime?: UnifiedRuntimeIdentityV2, bundleEvidence?: EvaluationBundleEvidenceV1): UnifiedRuntimeEvaluationReport;
132
+ export {};
133
+ //# sourceMappingURL=unified-runtime-evaluation.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"unified-runtime-evaluation.d.ts","sourceRoot":"","sources":["../src/unified-runtime-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,+BAA+B,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAW5G,OAAO,EAAgC,KAAK,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AACnG,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAEV,cAAc,EAEd,oBAAoB,EAErB,MAAM,6BAA6B,CAAC;AACrC,OAAO,KAAK,EAEV,wBAAwB,EACxB,2BAA2B,EAE5B,MAAM,kCAAkC,CAAC;AAE1C,MAAM,WAAW,oBAAoB;IAAG,WAAW,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAC3G,MAAM,WAAW,qBAAqB;IAAG,WAAW,EAAE,MAAM,CAAC;IAAC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CAAE;AAEpG,MAAM,WAAW,8BAA8B;IAC7C,aAAa,EAAE,uCAAuC,CAAC;IACvD,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,gBAAgB,EAAE;QAAE,cAAc,EAAE,YAAY,CAAC;QAAC,QAAQ,EAAE,iBAAiB,CAAC;QAAC,sCAAsC,EAAE,KAAK,CAAC;QAAC,6BAA6B,EAAE,IAAI,CAAC;KAAE,CAAC;IACrK,UAAU,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAC/C,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,WAAW,EAAE,MAAM,CAAC,CAAC;IACzE,MAAM,EAAE;QACN,MAAM,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;YAAC,OAAO,EAAE;gBAAE,0BAA0B,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;gBAAC,aAAa,EAAE,qBAAqB,CAAC;aAAE,CAAA;SAAE,CAAC;QACzO,kBAAkB,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;YAAC,OAAO,EAAE;gBAAE,aAAa,EAAE,qBAAqB,CAAC;gBAAC,kBAAkB,EAAE,qBAAqB,CAAC;gBAAC,oBAAoB,EAAE,qBAAqB,CAAC;aAAE,CAAA;SAAE,CAAC;QAChN,gBAAgB,EAAE,UAAU,CAAC;YAAE,UAAU,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QACpE,iBAAiB,EAAE,UAAU,CAAC;YAAE,mBAAmB,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC9E,gBAAgB,EAAE,UAAU,CAAC;YAAE,OAAO,EAAE,oBAAoB,CAAC;YAAC,MAAM,EAAE,oBAAoB,CAAC;YAAC,SAAS,EAAE,oBAAoB,CAAA;SAAE,CAAC,CAAC;QAC/H,gBAAgB,EAAE,UAAU,CAAC;YAAE,OAAO,EAAE,oBAAoB,CAAC;YAAC,MAAM,EAAE,oBAAoB,CAAC;YAAC,SAAS,EAAE,oBAAoB,CAAA;SAAE,CAAC,CAAC;QAC/H,gBAAgB,EAAE,UAAU,CAAC;YAAE,WAAW,EAAE,qBAAqB,CAAC;YAAC,YAAY,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC1G,aAAa,EAAE,UAAU,CAAC;YAAE,QAAQ,EAAE,qBAAqB,CAAA;SAAE,CAAC,CAAC;QAC/D,cAAc,EAAE;YAAE,MAAM,EAAE,QAAQ,GAAG,aAAa,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KACzE,CAAC;IACF,OAAO,EAAE;QACP,QAAQ,EAAE;YAAE,aAAa,EAAE,qBAAqB,CAAC;YAAC,kBAAkB,EAAE,qBAAqB,CAAC;YAAC,0BAA0B,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QAC/L,OAAO,EAAE,oBAAoB,CAAC;QAC9B,MAAM,EAAE,oBAAoB,GAAG;YAAE,mBAAmB,EAAE,qBAAqB,CAAC;YAAC,sBAAsB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QAC9H,SAAS,EAAE,oBAAoB,GAAG;YAAE,wBAAwB,EAAE,qBAAqB,CAAC;YAAC,mBAAmB,EAAE,qBAAqB,CAAC;SAAE,CAAC;QACnI,OAAO,EAAE;YAAE,oBAAoB,EAAE,qBAAqB,CAAC;YAAC,sBAAsB,EAAE,qBAAqB,CAAC;YAAC,uBAAuB,EAAE,qBAAqB,CAAC;YAAC,uBAAuB,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;YAAC,aAAa,EAAE,qBAAqB,CAAC;YAAC,oBAAoB,EAAE,qBAAqB,CAAC;SAAE,CAAC;KAC1U,CAAC;IACF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC;IAC/C,KAAK,EAAE,MAAM,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAC3C,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,0BAA0B,CAAC;IACrC,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAC;CACtD;AAED,UAAU,UAAU,CAAC,OAAO;IAC1B,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,UAAU,oBAAoB;IAC5B,eAAe,EAAE,+BAA+B,CAAC;IACjD,mBAAmB,EAAE,+BAA+B,CAAC;IACrD,YAAY,EAAE,+BAA+B,CAAC;IAC9C,SAAS,EAAE,+BAA+B,CAAC;IAC3C,sBAAsB,EAAE,+BAA+B,CAAC;IACxD,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,OAAO,EAAE,oBAAoB,CAAC;IAAC,QAAQ,EAAE,qBAAqB,CAAC;CAChE;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,CAAC;IAC5C,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE;QAAE,YAAY,EAAE,+BAA+B,CAAC;QAAC,SAAS,EAAE,+BAA+B,CAAC;KAAE,CAAC;CACzG;AAKD,KAAK,kBAAkB,GAAG,iBAAiB,GAAG,6BAA6B,GAAG,8BAA8B,GACxG,yBAAyB,GAAG,uBAAuB,GAAG,oBAAoB,GAAG,4BAA4B,GACzG,kBAAkB,GAAG,mBAAmB,GAAG,2BAA2B,GAAG,wBAAwB,GACjG,sBAAsB,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,qBAAqB,GAAG,2BAA2B,GACrH,oBAAoB,GAAG,6BAA6B,GAAG,qBAAqB,GAAG,wBAAwB,GACvG,wBAAwB,GAAG,eAAe,GAAG,eAAe,GAAG,eAAe,CAAC;AASnF,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,oBAAoB,EAC5B,WAAW,EAAE,2BAA2B,EACxC,cAAc,GAAE,wBAA8C,EAC9D,cAAc,CAAC,EAAE,0BAA0B,GAC1C,8BAA8B,CA+LhC"}
@@ -0,0 +1,388 @@
1
+ import { aggregateEvidenceQualityMetrics, candidateRecallAt20, eligibilitySurvival, evidenceQualityMetric, forbiddenContaminationAt5, precisionAt5, recallAt5, selectBestGoldSet, } from './evidence-quality-metrics.js';
2
+ import { labelHash, stableRuntimeHash } from '@devflow-tools/sdk';
3
+ const THRESHOLDS = {
4
+ semanticFieldAccuracy: 0.90, semanticAbstention: 0.90, contextRecallAt5: 0.85, contextPrecisionAt5: 0.75,
5
+ memoryRecallAt5: 0.80, memoryPrecisionAt5: 0.80, memoryContamination: 0.10, memoryTemporalCorrectness: 0.90,
6
+ knowledgeRecallAt5: 0.85, knowledgeVersionCorrectness: 1, knowledgeAbstentionPrecision: 0.90,
7
+ explicitSaveDurability: 1, attributionKnownOutcome: 0.90, domainActionAccuracy: 1,
8
+ };
9
+ export function evaluateUnifiedRuntimeCorpus(corpus, observation, currentRuntime = observation.runtime, bundleEvidence) {
10
+ const runtimeIdentityValid = validateObservation(corpus, observation, currentRuntime);
11
+ const semantic = joinSemantic(corpus.semantic, observation.semantic);
12
+ const context = joinRetrieval(corpus.context, observation.context);
13
+ const memoryCases = joinRetrieval(corpus.memory, observation.memory);
14
+ const knowledgeCases = joinRetrieval(corpus.knowledge, observation.knowledge);
15
+ const safetySemanticMetrics = evaluateSemantic(semantic);
16
+ const semanticExecuted = observation.suiteIdentity.semanticCapability.status === 'executed';
17
+ const retrievalCases = [...context, ...memoryCases, ...knowledgeCases];
18
+ const labelExpectations = validLabelExpectations(bundleEvidence, retrievalCases, observation.suiteIdentity.candidateGeneration);
19
+ const candidateExecuted = observation.suiteIdentity.candidateGeneration.status === 'executed'
20
+ && labelExpectations !== undefined;
21
+ const externalExecuted = observation.suiteIdentity.externalReplay.status === 'executed'
22
+ && Boolean(observation.suiteIdentity.externalReplay.replayId)
23
+ && Boolean(observation.suiteIdentity.externalReplay.host)
24
+ && Boolean(observation.suiteIdentity.externalReplay.modelProfileHash);
25
+ const semanticQualityMetrics = evaluateSemantic(semanticExecuted ? semantic : []);
26
+ const semanticMetrics = {
27
+ ...semanticQualityMetrics,
28
+ mutationFalseAuthorization: safetySemanticMetrics.mutationFalseAuthorization,
29
+ };
30
+ const mechanicsExpectations = observation.suiteIdentity.retrievalExecution.source === 'fixture_seeded'
31
+ ? undefined : new Map();
32
+ const positiveExecuted = (cases) => cases.filter(item => item.caseClass === 'positive' && item.observed.executionStatus === 'executed');
33
+ const mechanicsContext = evaluateRetrieval(positiveExecuted(context), mechanicsExpectations);
34
+ const mechanicsMemoryBase = evaluateRetrieval(positiveExecuted(memoryCases), mechanicsExpectations);
35
+ const mechanicsKnowledgeBase = evaluateRetrieval(positiveExecuted(knowledgeCases), mechanicsExpectations);
36
+ const candidateContext = labelExpectations ? evaluateRetrieval(positiveExecuted(context), labelExpectations) : undefined;
37
+ const candidateMemoryBase = labelExpectations ? evaluateRetrieval(positiveExecuted(memoryCases), labelExpectations) : undefined;
38
+ const candidateKnowledgeBase = labelExpectations ? evaluateRetrieval(positiveExecuted(knowledgeCases), labelExpectations) : undefined;
39
+ const allObserved = [...semantic.map(item => item.observed), ...retrievalCases.map(item => item.observed)];
40
+ const semanticChannelHashes = semantic.flatMap(item => Object.values(item.observed.channelContextHashes)
41
+ .map(hash => hash === item.observed.runtimeContextHash));
42
+ const hostProofCases = retrievalCases.filter(item => item.observed.attributionSource === 'host_ledger');
43
+ const attributionOutcomes = hostProofCases.flatMap(item => item.observed.attributionOutcomes);
44
+ const verificationOutcomes = hostProofCases.flatMap(item => item.observed.verificationOutcomes);
45
+ const cyclesRequiringStop = retrievalCases.filter(item => item.observed.cycles > 1);
46
+ const planningCoverage = booleanMetric(retrievalCases.map(item => item.observed.planRequirement === 'prohibited'
47
+ || !['channel_plan_empty', 'original_leg_missing'].includes(item.observed.executionReasonCode ?? '')));
48
+ const executableCases = retrievalCases.filter(item => item.observed.planRequirement !== 'prohibited');
49
+ const producerExecution = booleanMetric(executableCases.map(item => item.observed.executionStatus === 'executed'));
50
+ const runtime = {
51
+ contextHashAgreement: booleanMetric(semanticChannelHashes),
52
+ attributionConsumption: evidenceQualityMetric(attributionOutcomes.filter(item => item === 'consumed').length, attributionOutcomes.length),
53
+ attributionKnownOutcome: evidenceQualityMetric(attributionOutcomes.filter(item => item !== 'unknown').length, attributionOutcomes.length),
54
+ attributionVerification: evidenceQualityMetric(verificationOutcomes.filter(item => item === 'verified').length, verificationOutcomes.length),
55
+ terminalFalseSuccess: evidenceQualityMetric(allObserved.filter(item => item.terminalFalseSuccess).length, allObserved.length),
56
+ boundedCycles: booleanMetric(retrievalCases.map(item => item.observed.cycles <= item.expected.maxCycles)),
57
+ noGainStopObservable: booleanMetric(cyclesRequiringStop.map(item => item.observed.duplicateOrNoGainStopObserved)),
58
+ };
59
+ const memoryExtras = {
60
+ temporalCorrectness: booleanMetric(memoryCases.filter(item => item.observed.executionStatus === 'executed'
61
+ && item.expected.temporalCorrect !== undefined)
62
+ .map(item => item.expected.temporalCorrect === item.observed.temporalCorrect)),
63
+ explicitSaveDurability: booleanMetric(memoryCases.filter(item => item.observed.executionStatus === 'executed'
64
+ && item.caseClass === 'lifecycle' && item.expected.explicitSave)
65
+ .map(item => item.observed.episodePersisted === true)),
66
+ };
67
+ const knowledgeExtras = {
68
+ sourceVersionCorrectness: booleanMetric(knowledgeCases.filter(item => item.observed.executionStatus === 'executed'
69
+ && item.caseClass === 'positive' && item.expected.sourceVersion !== undefined)
70
+ .map(item => item.expected.sourceVersion === item.observed.sourceVersion)),
71
+ };
72
+ const mechanicsMemory = { ...mechanicsMemoryBase, ...memoryExtras };
73
+ const mechanicsKnowledge = {
74
+ ...mechanicsKnowledgeBase, ...knowledgeExtras,
75
+ abstentionPrecision: abstentionPrecision(knowledgeCases),
76
+ };
77
+ const candidateMemory = candidateMemoryBase ? { ...candidateMemoryBase, ...memoryExtras } : undefined;
78
+ const candidateKnowledge = candidateKnowledgeBase ? {
79
+ ...candidateKnowledgeBase, ...knowledgeExtras,
80
+ abstentionPrecision: abstentionPrecision(knowledgeCases),
81
+ } : undefined;
82
+ const contextMetrics = candidateContext ?? mechanicsContext;
83
+ const memory = candidateMemory ?? mechanicsMemory;
84
+ const knowledge = candidateKnowledge ?? mechanicsKnowledge;
85
+ const gates = {
86
+ runtimeIdentity: runtimeIdentityValid,
87
+ semanticCapabilityAvailable: semanticExecuted,
88
+ candidateGenerationAvailable: candidateExecuted,
89
+ externalReplayAvailable: externalExecuted,
90
+ planningCoverage: atLeast(planningCoverage, 1),
91
+ producerExecution: atLeast(producerExecution, 1),
92
+ consumptionProofAvailable: atLeast(runtime.attributionKnownOutcome, THRESHOLDS.attributionKnownOutcome),
93
+ terminalProofAvailable: atLeast(runtime.attributionVerification, 1),
94
+ semanticFieldAccuracy: semanticExecuted && atLeast(semanticMetrics.fieldAccuracy, THRESHOLDS.semanticFieldAccuracy),
95
+ semanticAbstention: semanticExecuted && atLeast(semanticMetrics.abstentionAccuracy, THRESHOLDS.semanticAbstention),
96
+ mutationFalseAuthorization: semanticMetrics.mutationFalseAuthorization.numerator === 0,
97
+ domainActionAccuracy: semanticExecuted && atLeast(semanticMetrics.domainActionAccuracy, THRESHOLDS.domainActionAccuracy),
98
+ contextRetrieval: candidateExecuted && atLeast(contextMetrics.recallAt5.refMicro, THRESHOLDS.contextRecallAt5) && atLeast(contextMetrics.precisionAt5.refMicro, THRESHOLDS.contextPrecisionAt5),
99
+ memoryRetrieval: candidateExecuted && atLeast(memory.recallAt5.refMicro, THRESHOLDS.memoryRecallAt5) && atLeast(memory.precisionAt5.refMicro, THRESHOLDS.memoryPrecisionAt5),
100
+ memoryContamination: candidateExecuted && atMost(memory.forbiddenContamination.refMicro, THRESHOLDS.memoryContamination),
101
+ memoryTemporalCorrectness: atLeast(memory.temporalCorrectness, THRESHOLDS.memoryTemporalCorrectness),
102
+ knowledgeRetrieval: candidateExecuted && atLeast(knowledge.recallAt5.refMicro, THRESHOLDS.knowledgeRecallAt5),
103
+ knowledgeVersionCorrectness: atLeast(knowledge.sourceVersionCorrectness, THRESHOLDS.knowledgeVersionCorrectness),
104
+ knowledgeAbstention: atLeast(knowledge.abstentionPrecision, THRESHOLDS.knowledgeAbstentionPrecision),
105
+ explicitSaveDurability: atLeast(memory.explicitSaveDurability, THRESHOLDS.explicitSaveDurability),
106
+ attributionConsumption: atLeast(runtime.attributionKnownOutcome, THRESHOLDS.attributionKnownOutcome),
107
+ hashAgreement: atLeast(runtime.contextHashAgreement, 1),
108
+ terminalTruth: runtime.terminalFalseSuccess.numerator === 0,
109
+ boundedCycles: atLeast(runtime.boundedCycles, 1)
110
+ && (runtime.noGainStopObservable.denominator === 0 || atLeast(runtime.noGainStopObservable, 1)),
111
+ };
112
+ const safetyPassed = gates.runtimeIdentity && gates.mutationFalseAuthorization
113
+ && gates.hashAgreement && gates.terminalTruth && gates.boundedCycles;
114
+ const semanticPassed = gates.semanticFieldAccuracy && gates.semanticAbstention && gates.domainActionAccuracy;
115
+ const candidateQualityPassed = candidateContext !== undefined && candidateMemory !== undefined
116
+ && candidateKnowledge !== undefined
117
+ && atLeast(candidateContext.candidateRecall.refMicro, THRESHOLDS.contextRecallAt5)
118
+ && atLeast(candidateMemory.candidateRecall.refMicro, THRESHOLDS.memoryRecallAt5)
119
+ && atLeast(candidateKnowledge.candidateRecall.refMicro, THRESHOLDS.knowledgeRecallAt5);
120
+ const selectionPassed = gates.contextRetrieval && gates.memoryRetrieval
121
+ && gates.memoryContamination && gates.memoryTemporalCorrectness
122
+ && gates.knowledgeRetrieval && gates.knowledgeVersionCorrectness
123
+ && gates.knowledgeAbstention && gates.explicitSaveDurability;
124
+ const suites = {
125
+ safety: {
126
+ status: safetyPassed ? 'passed' : 'failed',
127
+ metrics: {
128
+ mutationFalseAuthorization: semanticMetrics.mutationFalseAuthorization,
129
+ contextHashAgreement: runtime.contextHashAgreement,
130
+ terminalFalseSuccess: runtime.terminalFalseSuccess,
131
+ boundedCycles: runtime.boundedCycles,
132
+ },
133
+ },
134
+ semanticCapability: {
135
+ status: !semanticExecuted ? 'unavailable' : semanticPassed ? 'passed' : 'failed',
136
+ metrics: {
137
+ fieldAccuracy: semanticMetrics.fieldAccuracy,
138
+ abstentionAccuracy: semanticMetrics.abstentionAccuracy,
139
+ domainActionAccuracy: semanticMetrics.domainActionAccuracy,
140
+ },
141
+ },
142
+ planningCoverage: {
143
+ status: planningCoverage.denominator === 0 ? 'unavailable' : gates.planningCoverage ? 'passed' : 'failed',
144
+ ...(planningCoverage.denominator > 0 ? { metrics: { validPlans: planningCoverage } } : {}),
145
+ },
146
+ producerExecution: {
147
+ status: producerExecution.denominator === 0 ? 'unavailable' : gates.producerExecution ? 'passed' : 'failed',
148
+ ...(producerExecution.denominator > 0 ? { metrics: { originalLegOutcomes: producerExecution } } : {}),
149
+ },
150
+ candidateQuality: {
151
+ status: !candidateExecuted ? 'unavailable' : candidateQualityPassed ? 'passed' : 'failed',
152
+ ...(candidateExecuted && candidateContext && candidateMemory && candidateKnowledge
153
+ ? { metrics: { context: candidateContext, memory: candidateMemory, knowledge: candidateKnowledge } } : {}),
154
+ },
155
+ selectionQuality: {
156
+ status: !candidateExecuted ? 'unavailable' : selectionPassed ? 'passed' : 'failed',
157
+ ...(candidateExecuted && candidateContext && candidateMemory && candidateKnowledge
158
+ ? { metrics: { context: candidateContext, memory: candidateMemory, knowledge: candidateKnowledge } } : {}),
159
+ },
160
+ consumptionProof: {
161
+ status: runtime.attributionKnownOutcome.denominator === 0 ? 'unavailable'
162
+ : gates.consumptionProofAvailable ? 'passed' : 'failed',
163
+ ...(runtime.attributionKnownOutcome.denominator > 0 ? { metrics: {
164
+ consumption: runtime.attributionConsumption,
165
+ knownOutcome: runtime.attributionKnownOutcome,
166
+ } } : {}),
167
+ },
168
+ terminalProof: {
169
+ status: runtime.attributionVerification.denominator === 0 ? 'unavailable'
170
+ : gates.terminalProofAvailable ? 'passed' : 'failed',
171
+ ...(runtime.attributionVerification.denominator > 0
172
+ ? { metrics: { verified: runtime.attributionVerification } } : {}),
173
+ },
174
+ externalReplay: {
175
+ status: externalExecuted ? 'passed' : 'unavailable',
176
+ ...(observation.suiteIdentity.externalReplay.replayId
177
+ ? { replayId: observation.suiteIdentity.externalReplay.replayId } : {}),
178
+ },
179
+ };
180
+ const diagnostics = Object.entries(gates).filter(([, passed]) => !passed).map(([gate]) => `release_gate_failed:${gate}`);
181
+ return {
182
+ schemaVersion: 'devflow.unified-runtime-evaluation.v4', createdAt: Date.now(), status: diagnostics.length === 0 ? 'passed' : 'failed',
183
+ evaluationPolicy: { labelAuthority: 'stable_ids', llmJudge: 'diagnostic_only', productionLogsWithoutLabelsProveRecall: false, observationsProducedByRuntime: true },
184
+ datasetIds: { ...corpus.datasetIds }, runtime: observation.runtime,
185
+ samples: { semantic: semantic.length, context: context.length, memory: memoryCases.length, knowledge: knowledgeCases.length },
186
+ suites, metrics: { semantic: semanticMetrics, context: contextMetrics, memory, knowledge, runtime },
187
+ slices: buildSlices(retrievalCases, labelExpectations ?? mechanicsExpectations, labelExpectations !== undefined || observation.suiteIdentity.retrievalExecution.source === 'fixture_seeded'),
188
+ gates,
189
+ diagnostics,
190
+ };
191
+ }
192
+ function validateObservation(corpus, observation, current) {
193
+ if (observation.schemaVersion !== 'devflow.unified-runtime-observation.v4')
194
+ throw new Error('Unified runtime observation schema mismatch');
195
+ if (JSON.stringify(observation.goldDatasetIds) !== JSON.stringify(corpus.datasetIds))
196
+ throw new Error('Unified runtime observation dataset mismatch');
197
+ const expected = new Set([...corpus.semantic, ...corpus.context, ...corpus.memory, ...corpus.knowledge].map(item => item.id));
198
+ const observed = [...observation.semantic, ...observation.context, ...observation.memory, ...observation.knowledge].map(item => item.caseId);
199
+ if (new Set(observed).size !== observed.length || observed.length !== expected.size || observed.some(id => !expected.has(id)))
200
+ throw new Error('Unified runtime observation case mismatch');
201
+ return sameRuntime(observation.runtime, current)
202
+ && observation.suiteIdentity.deterministicSafety.runtimeSourceHash === observation.runtime.runtimeSourceHash
203
+ && observation.suiteIdentity.deterministicSafety.runnerVersion === observation.runtime.runnerVersion;
204
+ }
205
+ function sameRuntime(left, right) {
206
+ return left.packageVersion === right.packageVersion && left.pluginRegistryHash === right.pluginRegistryHash
207
+ && left.runtimeSourceHash === right.runtimeSourceHash && left.runnerVersion === right.runnerVersion;
208
+ }
209
+ function joinSemantic(gold, observed) {
210
+ const map = new Map(observed.map(item => [item.caseId, item]));
211
+ return gold.map(item => ({ ...item, observed: map.get(item.id) }));
212
+ }
213
+ function joinRetrieval(gold, observed) {
214
+ const map = new Map(observed.map(item => [item.caseId, item]));
215
+ return gold.map(item => ({ ...item, observed: map.get(item.id) }));
216
+ }
217
+ function evaluateSemantic(cases) {
218
+ const fieldResults = cases.flatMap(semanticFieldResults);
219
+ const falseAuthorizationPopulation = cases.filter(item => !item.expected.mutationAuthorized);
220
+ const actionCases = cases.filter(item => item.expected.domainAction !== undefined);
221
+ return {
222
+ fieldAccuracy: booleanMetric(fieldResults),
223
+ abstentionAccuracy: booleanMetric(cases.map(item => item.expected.abstain === item.observed.abstain)),
224
+ mutationFalseAuthorization: evidenceQualityMetric(falseAuthorizationPopulation.filter(item => item.observed.mutationAuthorized).length, falseAuthorizationPopulation.length),
225
+ domainActionAccuracy: booleanMetric(actionCases.map(item => {
226
+ const expected = item.expected.domainAction;
227
+ const observed = item.observed.domainAction;
228
+ if (!observed || observed.disposition !== expected.expectedDisposition)
229
+ return false;
230
+ return expected.expectedDisposition !== 'selected' || observed.tool === expected.tool;
231
+ })),
232
+ };
233
+ }
234
+ function semanticFieldResults(item) {
235
+ const namespaces = new Set([...item.expected.fieldRefs, ...item.observed.fieldRefs].map(ref => ref.slice(0, ref.indexOf(':'))));
236
+ return [...namespaces].map(namespace => {
237
+ const expected = item.expected.fieldRefs.filter(ref => ref.startsWith(`${namespace}:`));
238
+ const observed = item.observed.fieldRefs.filter(ref => ref.startsWith(`${namespace}:`));
239
+ return expected.length === observed.length && expected.every(ref => observed.includes(ref));
240
+ });
241
+ }
242
+ function evaluateRetrieval(cases, expectations) {
243
+ const evaluated = cases.flatMap(item => {
244
+ const expectation = expectationFor(item, expectations);
245
+ if (!expectation)
246
+ return [];
247
+ const goldSets = expectation.goldSets.filter(set => set.length > 0);
248
+ const goldRefs = new Set(selectBestGoldSet({
249
+ goldSets,
250
+ candidateRefs: item.observed.candidateEvidenceIds,
251
+ selectedRefs: item.observed.rankedEvidenceIds,
252
+ }));
253
+ return [{
254
+ item,
255
+ goldRefs,
256
+ forbiddenRefs: expectation.forbiddenRefs,
257
+ }];
258
+ });
259
+ const scored = evaluated.filter(item => item.goldRefs.size > 0);
260
+ return {
261
+ candidateRecall: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => candidateRecallAt20(item.observed.candidateEvidenceIds, goldRefs))),
262
+ eligibilitySurvival: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => eligibilitySurvival(item.observed.candidateEvidenceIds, item.observed.eligibilityRejectedEvidenceIds, goldRefs))),
263
+ precisionAt5: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => precisionAt5(item.observed.rankedEvidenceIds, goldRefs))),
264
+ recallAt5: aggregateEvidenceQualityMetrics(scored.map(({ item, goldRefs }) => recallAt5(item.observed.rankedEvidenceIds, goldRefs))),
265
+ forbiddenContamination: aggregateEvidenceQualityMetrics(evaluated.map(({ item, forbiddenRefs }) => forbiddenContaminationAt5(item.observed.rankedEvidenceIds, forbiddenRefs))),
266
+ rejectionReasonCounts: mergeReasonCounts(cases.map(item => item.observed.rejectionReasonCounts)),
267
+ latency: latencyMetric(cases.map(item => item.observed.latencyMs)),
268
+ reranker: rerankerMetric(scored.map(({ item, goldRefs }) => ({
269
+ item,
270
+ expected: [...goldRefs],
271
+ }))),
272
+ };
273
+ }
274
+ function abstentionPrecision(cases) {
275
+ const executed = cases.filter(item => item.caseClass === 'abstention'
276
+ && item.observed.executionStatus === 'executed');
277
+ return booleanMetric(executed.map(item => item.observed.rankedEvidenceIds.length === 0));
278
+ }
279
+ function rerankerMetric(cases) {
280
+ if (cases.length === 0)
281
+ return { sampleCount: 0, meanRelevantAt5Delta: null };
282
+ const deltas = cases.map(({ item, expected }) => relevantAt5(item.observed.rankedEvidenceIds, expected)
283
+ - relevantAt5(item.observed.rerankerBaselineIds, expected));
284
+ return { sampleCount: deltas.length, meanRelevantAt5Delta: average(deltas) };
285
+ }
286
+ function relevantAt5(ranked, expected) {
287
+ return [...new Set(ranked)].slice(0, 5).filter(id => expected.includes(id)).length;
288
+ }
289
+ function latencyMetric(values) {
290
+ if (values.length === 0)
291
+ return { sampleCount: 0, meanMs: null, p95Ms: null };
292
+ const ordered = [...values].sort((a, b) => a - b);
293
+ return { sampleCount: values.length, meanMs: average(values), p95Ms: ordered[Math.max(0, Math.ceil(ordered.length * 0.95) - 1)] };
294
+ }
295
+ function buildSlices(retrieval, expectations, positiveLabelsAvailable = false) {
296
+ const groups = new Map();
297
+ for (const item of retrieval) {
298
+ const key = `${item.projectId}/${item.channel}/${item.caseClass}`;
299
+ const group = groups.get(key) ?? { caseClass: item.caseClass, cases: [] };
300
+ group.cases.push(item);
301
+ groups.set(key, group);
302
+ }
303
+ return Object.fromEntries([...groups.entries()].sort(([left], [right]) => left.localeCompare(right))
304
+ .map(([key, group]) => {
305
+ const executed = group.cases.filter(item => item.observed.executionStatus === 'executed');
306
+ const metrics = evaluateRetrieval(group.caseClass === 'positive' ? executed : [], expectations);
307
+ return [key, {
308
+ status: sliceStatus(group.caseClass, group.cases, metrics, positiveLabelsAvailable),
309
+ sampleCount: group.cases.length,
310
+ metrics: { precisionAt5: metrics.precisionAt5, recallAt5: metrics.recallAt5 },
311
+ }];
312
+ }));
313
+ }
314
+ function sliceStatus(caseClass, cases, metrics, positiveLabelsAvailable) {
315
+ if (caseClass !== 'coverage_gap' && cases.some(item => item.observed.executionStatus !== 'executed')) {
316
+ return 'unavailable';
317
+ }
318
+ if (caseClass === 'positive') {
319
+ if (!positiveLabelsAvailable)
320
+ return 'unavailable';
321
+ return metrics.recallAt5.refMicro.denominator > 0
322
+ && metrics.recallAt5.refMicro.numerator === metrics.recallAt5.refMicro.denominator ? 'passed' : 'failed';
323
+ }
324
+ if (caseClass === 'abstention') {
325
+ return cases.every(item => item.observed.rankedEvidenceIds.length === 0) ? 'passed' : 'failed';
326
+ }
327
+ if (caseClass === 'lifecycle') {
328
+ return cases.every(item => !item.expected.explicitSave || item.observed.episodePersisted === true)
329
+ ? 'passed' : 'failed';
330
+ }
331
+ if (cases.some(item => item.observed.executionStatus === 'unavailable'))
332
+ return 'unavailable';
333
+ return cases.every(item => item.observed.executionStatus === 'executed'
334
+ && item.observed.executionReasonCode === item.expected.reasonCode
335
+ && item.observed.rankedEvidenceIds.length === 0) ? 'passed' : 'failed';
336
+ }
337
+ function expectationFor(item, expectations) {
338
+ if (expectations)
339
+ return expectations.get(item.id);
340
+ return { goldSets: [item.expected.evidenceIds], forbiddenRefs: new Set() };
341
+ }
342
+ function validLabelExpectations(evidence, cases, candidateIdentity) {
343
+ if (!evidence || candidateIdentity.status !== 'executed')
344
+ return undefined;
345
+ const manifest = evidence.manifest;
346
+ if (candidateIdentity.bundleId !== manifest.bundleId
347
+ || candidateIdentity.projectSnapshotId !== manifest.projectSnapshotId
348
+ || candidateIdentity.corpusHash !== manifest.corpusHash
349
+ || stableRuntimeHash(candidateIdentity.labelHashes) !== stableRuntimeHash(manifest.labelHashes)
350
+ || stableRuntimeHash(candidateIdentity.channelSnapshotHashes) !== stableRuntimeHash(manifest.channelSnapshotHashes)
351
+ || candidateIdentity.bundleProfilesHash !== manifest.profilesHash)
352
+ return undefined;
353
+ const expectedCases = new Map(cases.map(item => [item.id, item.channel]));
354
+ const entries = new Map();
355
+ for (const channel of ['context', 'memory', 'knowledge']) {
356
+ const labelSet = evidence.labels[channel];
357
+ if (labelSet.schemaVersion !== 'devflow.evaluation-labels.v1'
358
+ || labelSet.channel !== channel
359
+ || labelSet.projectSnapshotId !== manifest.projectSnapshotId
360
+ || labelSet.channelSnapshotHash !== manifest.channelSnapshotHashes[channel]
361
+ || labelSet.datasetHash !== manifest.corpusHash
362
+ || labelHash(labelSet) !== manifest.labelHashes[channel])
363
+ return undefined;
364
+ for (const label of labelSet.labels) {
365
+ if (expectedCases.get(label.caseId) !== channel || entries.has(label.caseId)
366
+ || !label.reviewer.trim() || !label.labelledAt.trim() || label.sourceEvidence.length === 0)
367
+ return undefined;
368
+ entries.set(label.caseId, {
369
+ goldSets: label.goldSets.map(set => [...new Set(set)]),
370
+ forbiddenRefs: new Set(label.forbiddenRefs),
371
+ });
372
+ }
373
+ }
374
+ return entries.size === expectedCases.size ? entries : undefined;
375
+ }
376
+ function booleanMetric(values) { return evidenceQualityMetric(values.filter(Boolean).length, values.length); }
377
+ function atLeast(metric, threshold) { return metric.value !== null && metric.value >= threshold; }
378
+ function atMost(metric, threshold) { return metric.value !== null && metric.value <= threshold; }
379
+ function average(values) { return values.reduce((total, value) => total + value, 0) / values.length; }
380
+ function mergeReasonCounts(values) {
381
+ const counts = {};
382
+ for (const value of values) {
383
+ for (const [reason, count] of Object.entries(value))
384
+ counts[reason] = (counts[reason] ?? 0) + count;
385
+ }
386
+ return counts;
387
+ }
388
+ //# sourceMappingURL=unified-runtime-evaluation.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"unified-runtime-evaluation.js","sourceRoot":"","sources":["../src/unified-runtime-evaluation.ts"],"names":[],"mappings":"AACA,OAAO,EACL,+BAA+B,EAC/B,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,yBAAyB,EACzB,YAAY,EACZ,SAAS,EACT,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAmC,MAAM,oBAAoB,CAAC;AAsFnG,MAAM,UAAU,GAAG;IACjB,qBAAqB,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI;IACxG,eAAe,EAAE,IAAI,EAAE,kBAAkB,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,yBAAyB,EAAE,IAAI;IAC3G,kBAAkB,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,EAAE,4BAA4B,EAAE,IAAI;IAC5F,sBAAsB,EAAE,CAAC,EAAE,uBAAuB,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;CACzE,CAAC;AAEX,MAAM,UAAU,4BAA4B,CAC1C,MAA4B,EAC5B,WAAwC,EACxC,iBAA2C,WAAW,CAAC,OAAO,EAC9D,cAA2C;IAE3C,MAAM,oBAAoB,GAAG,mBAAmB,CAAC,MAAM,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAC9E,MAAM,qBAAqB,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,UAAU,CAAC;IAC5F,MAAM,cAAc,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,CAAC;IACvE,MAAM,iBAAiB,GAAG,sBAAsB,CAC9C,cAAc,EACd,cAAc,EACd,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAC9C,CAAC;IACF,MAAM,iBAAiB,GAAG,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,MAAM,KAAK,UAAU;WACxF,iBAAiB,KAAK,SAAS,CAAC;IACrC,MAAM,gBAAgB,GAAG,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,KAAK,UAAU;WAClF,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC;WAC1D,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC;WACtD,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC;IACxE,MAAM,sBAAsB,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAClF,MAAM,eAAe,GAAG;QACtB,GAAG,sBAAsB;QACzB,0BAA0B,EAAE,qBAAqB,CAAC,0BAA0B;KAC7E,CAAC;IACF,MAAM,qBAAqB,GAAG,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,gBAAgB;QACpG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,EAAgC,CAAC;IACxD,MAAM,gBAAgB,GAAG,CAAC,KAA+B,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAChF,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;IACjF,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC7F,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACpG,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC1G,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACzH,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAChI,MAAM,sBAAsB,GAAG,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtI,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3G,MAAM,qBAAqB,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC;SACrG,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,KAAK,aAAa,CAAC,CAAC;IACxG,MAAM,mBAAmB,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAC9F,MAAM,oBAAoB,GAAG,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAChG,MAAM,mBAAmB,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpF,MAAM,gBAAgB,GAAG,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,YAAY;WAC3G,CAAC,CAAC,oBAAoB,EAAE,sBAAsB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,YAAY,CAAC,CAAC;IACtG,MAAM,iBAAiB,GAAG,aAAa,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC,CAAC;IACnH,MAAM,OAAO,GAAG;QACd,oBAAoB,EAAE,aAAa,CAAC,qBAAqB,CAAC;QAC1D,sBAAsB,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzI,uBAAuB,EAAE,qBAAqB,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,CAAC;QACzI,uBAAuB,EAAE,qBAAqB,CAAC,oBAAoB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC;QAC5I,oBAAoB,EAAE,qBAAqB,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC;QAC7H,aAAa,EAAE,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACzG,oBAAoB,EAAE,aAAa,CAAC,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC;KAClH,CAAC;IACF,MAAM,YAAY,GAAG;QACnB,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eACrG,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,SAAS,CAAC;aAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QAChF,sBAAsB,EAAE,aAAa,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eACxG,IAAI,CAAC,SAAS,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;aAC/D,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,KAAK,IAAI,CAAC,CAAC;KACzD,CAAC;IACF,MAAM,eAAe,GAAG;QACtB,wBAAwB,EAAE,aAAa,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;eAC7G,IAAI,CAAC,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC;aAC7E,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,KAAK,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;KAC7E,CAAC;IACF,MAAM,eAAe,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC;IACpE,MAAM,kBAAkB,GAAG;QACzB,GAAG,sBAAsB,EAAE,GAAG,eAAe;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,cAAc,CAAC;KACzD,CAAC;IACF,MAAM,eAAe,GAAG,mBAAmB,CAAC,CAAC,CAAC,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACtG,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,CAAC,CAAC;QAClD,GAAG,sBAAsB,EAAE,GAAG,eAAe;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,cAAc,CAAC;KACzD,CAAC,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,cAAc,GAAG,gBAAgB,IAAI,gBAAgB,CAAC;IAC5D,MAAM,MAAM,GAAG,eAAe,IAAI,eAAe,CAAC;IAClD,MAAM,SAAS,GAAG,kBAAkB,IAAI,kBAAkB,CAAC;IAC3D,MAAM,KAAK,GAA4C;QACrD,eAAe,EAAE,oBAAoB;QACrC,2BAA2B,EAAE,gBAAgB;QAC7C,4BAA4B,EAAE,iBAAiB;QAC/C,uBAAuB,EAAE,gBAAgB;QACzC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAC9C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAChD,yBAAyB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,uBAAuB,CAAC;QACvG,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAC;QACnE,qBAAqB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,aAAa,EAAE,UAAU,CAAC,qBAAqB,CAAC;QACnH,kBAAkB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,kBAAkB,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAClH,0BAA0B,EAAE,eAAe,CAAC,0BAA0B,CAAC,SAAS,KAAK,CAAC;QACtF,oBAAoB,EAAE,gBAAgB,IAAI,OAAO,CAAC,eAAe,CAAC,oBAAoB,EAAE,UAAU,CAAC,oBAAoB,CAAC;QACxH,gBAAgB,EAAE,iBAAiB,IAAI,OAAO,CAAC,cAAc,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,gBAAgB,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,mBAAmB,CAAC;QAC/L,eAAe,EAAE,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAC5K,mBAAmB,EAAE,iBAAiB,IAAI,MAAM,CAAC,MAAM,CAAC,sBAAsB,CAAC,QAAQ,EAAE,UAAU,CAAC,mBAAmB,CAAC;QACxH,yBAAyB,EAAE,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,UAAU,CAAC,yBAAyB,CAAC;QACpG,kBAAkB,EAAE,iBAAiB,IAAI,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC;QAC7G,2BAA2B,EAAE,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAAE,UAAU,CAAC,2BAA2B,CAAC;QAChH,mBAAmB,EAAE,OAAO,CAAC,SAAS,CAAC,mBAAmB,EAAE,UAAU,CAAC,4BAA4B,CAAC;QACpG,sBAAsB,EAAE,OAAO,CAAC,MAAM,CAAC,sBAAsB,EAAE,UAAU,CAAC,sBAAsB,CAAC;QACjG,sBAAsB,EAAE,OAAO,CAAC,OAAO,CAAC,uBAAuB,EAAE,UAAU,CAAC,uBAAuB,CAAC;QACpG,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;QACvD,aAAa,EAAE,OAAO,CAAC,oBAAoB,CAAC,SAAS,KAAK,CAAC;QAC3D,aAAa,EAAE,OAAO,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;eAC3C,CAAC,OAAO,CAAC,oBAAoB,CAAC,WAAW,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;KAClG,CAAC;IACF,MAAM,YAAY,GAAG,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,0BAA0B;WACzE,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,aAAa,CAAC;IACvE,MAAM,cAAc,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,oBAAoB,CAAC;IAC7G,MAAM,sBAAsB,GAAG,gBAAgB,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS;WACzF,kBAAkB,KAAK,SAAS;WAChC,OAAO,CAAC,gBAAgB,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,gBAAgB,CAAC;WAC/E,OAAO,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,eAAe,CAAC;WAC7E,OAAO,CAAC,kBAAkB,CAAC,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,kBAAkB,CAAC,CAAC;IACzF,MAAM,eAAe,GAAG,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC,eAAe;WAClE,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,yBAAyB;WAC5D,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,2BAA2B;WAC7D,KAAK,CAAC,mBAAmB,IAAI,KAAK,CAAC,sBAAsB,CAAC;IAC/D,MAAM,MAAM,GAA6C;QACvD,MAAM,EAAE;YACN,MAAM,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC1C,OAAO,EAAE;gBACP,0BAA0B,EAAE,eAAe,CAAC,0BAA0B;gBACtE,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;gBAClD,aAAa,EAAE,OAAO,CAAC,aAAa;aACrC;SACF;QACD,kBAAkB,EAAE;YAClB,MAAM,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAChF,OAAO,EAAE;gBACP,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,kBAAkB,EAAE,eAAe,CAAC,kBAAkB;gBACtD,oBAAoB,EAAE,eAAe,CAAC,oBAAoB;aAC3D;SACF;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,gBAAgB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzG,GAAG,CAAC,gBAAgB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3F;QACD,iBAAiB,EAAE;YACjB,MAAM,EAAE,iBAAiB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAC3G,GAAG,CAAC,iBAAiB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtG;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzF,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,IAAI,eAAe,IAAI,kBAAkB;gBAChF,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7G;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YAClF,GAAG,CAAC,iBAAiB,IAAI,gBAAgB,IAAI,eAAe,IAAI,kBAAkB;gBAChF,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,eAAe,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7G;QACD,gBAAgB,EAAE;YAChB,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa;gBACvE,CAAC,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACzD,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE;oBAC/D,WAAW,EAAE,OAAO,CAAC,sBAAsB;oBAC3C,YAAY,EAAE,OAAO,CAAC,uBAAuB;iBAC9C,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACV;QACD,aAAa,EAAE;YACb,MAAM,EAAE,OAAO,CAAC,uBAAuB,CAAC,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa;gBACvE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACtD,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,WAAW,GAAG,CAAC;gBACjD,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE;QACD,cAAc,EAAE;YACd,MAAM,EAAE,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,aAAa;YACnD,GAAG,CAAC,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ;gBACnD,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1E;KACF,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACzH,OAAO;QACL,aAAa,EAAE,uCAAuC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACrI,gBAAgB,EAAE,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,iBAAiB,EAAE,sCAAsC,EAAE,KAAK,EAAE,6BAA6B,EAAE,IAAI,EAAE;QACnK,UAAU,EAAE,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO;QAClE,OAAO,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE;QAC7H,MAAM,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE;QACnG,MAAM,EAAE,WAAW,CACjB,cAAc,EACd,iBAAiB,IAAI,qBAAqB,EAC1C,iBAAiB,KAAK,SAAS,IAAI,WAAW,CAAC,aAAa,CAAC,kBAAkB,CAAC,MAAM,KAAK,gBAAgB,CAC5G;QACD,KAAK;QACL,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAA4B,EAAE,WAAwC,EAAE,OAAiC;IACpI,IAAI,WAAW,CAAC,aAAa,KAAK,wCAAwC;QAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IAC3I,IAAI,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,cAAc,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACtJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9H,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,GAAG,WAAW,CAAC,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC7I,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC5L,OAAO,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC;WAC3C,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,iBAAiB,KAAK,WAAW,CAAC,OAAO,CAAC,iBAAiB;WACzG,WAAW,CAAC,aAAa,CAAC,mBAAmB,CAAC,aAAa,KAAK,WAAW,CAAC,OAAO,CAAC,aAAa,CAAC;AACzG,CAAC;AAED,SAAS,WAAW,CAAC,IAA8B,EAAE,KAA+B;IAClF,OAAO,IAAI,CAAC,cAAc,KAAK,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,kBAAkB,KAAK,KAAK,CAAC,kBAAkB;WACtG,IAAI,CAAC,iBAAiB,KAAK,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa,CAAC;AACxG,CAAC;AAED,SAAS,YAAY,CAAC,IAA2B,EAAE,QAAwC;IACzF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,aAAa,CAAC,IAA4B,EAAE,QAAyC;IAC5F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAE,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAA8B;IACtD,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACzD,MAAM,4BAA4B,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO;QACL,aAAa,EAAE,aAAa,CAAC,YAAY,CAAC;QAC1C,kBAAkB,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACrG,0BAA0B,EAAE,qBAAqB,CAAC,4BAA4B,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,4BAA4B,CAAC,MAAM,CAAC;QAC5K,oBAAoB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAa,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC5C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,mBAAmB;gBAAE,OAAO,KAAK,CAAC;YACrF,OAAO,QAAQ,CAAC,mBAAmB,KAAK,UAAU,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC;QACxF,CAAC,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,IAA2B;IACvD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChI,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;QACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC;QACxF,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CACxB,KAA+B,EAC/B,YAAgD;IAEhD,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QACrC,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW;YAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC;YACzC,QAAQ;YACR,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,oBAAoB;YACjD,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,iBAAiB;SAC9C,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC;gBACN,IAAI;gBACJ,QAAQ;gBACR,aAAa,EAAE,WAAW,CAAC,aAAa;aACzC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAChE,OAAO;QACL,eAAe,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACjF,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC;QACrE,mBAAmB,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CACrF,mBAAmB,CACjB,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAClC,IAAI,CAAC,QAAQ,CAAC,8BAA8B,EAC5C,QAAQ,CACT,CAAC,CAAC;QACL,YAAY,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC9E,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC3D,SAAS,EAAE,+BAA+B,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC3E,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC,CAAC,CAAC;QACxD,sBAAsB,EAAE,+BAA+B,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAChG,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,aAAa,CAAC,CAAC,CAAC;QAC7E,qBAAqB,EAAE,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC;QAChG,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QAClE,QAAQ,EAAE,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3D,IAAI;YACJ,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC;SACxB,CAAC,CAAC,CAAC;KACL,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAA+B;IAE/B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,KAAK,YAAY;WAChE,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;IACnD,OAAO,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;AAC3F,CAAC;AAED,SAAS,cAAc,CAAC,KAAkE;IACxF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,oBAAoB,EAAE,IAAI,EAAE,CAAC;IAC9E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,CAAC;UACnG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/E,CAAC;AAED,SAAS,WAAW,CAAC,MAAgB,EAAE,QAAkB;IACvD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;AACrF,CAAC;AACD,SAAS,aAAa,CAAC,MAAgB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAC9E,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,CAAC;AACrI,CAAC;AAED,SAAS,WAAW,CAClB,SAAmC,EACnC,YAAgD,EAChD,uBAAuB,GAAG,KAAK;IAE/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAGlB,CAAC;IACL,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QAClE,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAC1E,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;SACjG,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACpB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,CAAC;QAC1F,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,EAAE;gBACX,MAAM,EAAE,WAAW,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,uBAAuB,CAAC;gBACnF,WAAW,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;gBAC/B,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;aAC7C,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,SAAS,WAAW,CAClB,SAA2B,EAC3B,KAA+B,EAC/B,OAA6B,EAC7B,uBAAgC;IAEhC,IAAI,SAAS,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU,CAAC,EAAE,CAAC;QACrG,OAAO,aAAa,CAAC;IACvB,CAAC;IACD,IAAI,SAAS,KAAK,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,uBAAuB;YAAE,OAAO,aAAa,CAAC;QACnD,OAAO,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,GAAG,CAAC;eAC5C,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7G,CAAC;IACD,IAAI,SAAS,KAAK,YAAY,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACjG,CAAC;IACD,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,KAAK,IAAI,CAAC;YAChG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,aAAa,CAAC;QAAE,OAAO,aAAa,CAAC;IAC9F,OAAO,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,KAAK,UAAU;WAClE,IAAI,CAAC,QAAQ,CAAC,mBAAmB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU;WAC9D,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC3E,CAAC;AAED,SAAS,cAAc,CACrB,IAA4B,EAC5B,YAAgD;IAEhD,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnD,OAAO,EAAE,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,aAAa,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;AAC7E,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAAgD,EAChD,KAA+B,EAC/B,iBAAsF;IAEtF,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC;IACnC,IAAI,iBAAiB,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;WAC/C,iBAAiB,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB;WAClE,iBAAiB,CAAC,UAAU,KAAK,QAAQ,CAAC,UAAU;WACpD,iBAAiB,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC;WAC5F,iBAAiB,CAAC,iBAAiB,CAAC,qBAAqB,CAAC,KAAK,iBAAiB,CAAC,QAAQ,CAAC,qBAAqB,CAAC;WAChH,iBAAiB,CAAC,kBAAkB,KAAK,QAAQ,CAAC,YAAY;QAAE,OAAO,SAAS,CAAC;IACtF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;IACxD,KAAK,MAAM,OAAO,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,CAAU,EAAE,CAAC;QAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC1C,IAAI,QAAQ,CAAC,aAAa,KAAK,8BAA8B;eACxD,QAAQ,CAAC,OAAO,KAAK,OAAO;eAC5B,QAAQ,CAAC,iBAAiB,KAAK,QAAQ,CAAC,iBAAiB;eACzD,QAAQ,CAAC,mBAAmB,KAAK,QAAQ,CAAC,qBAAqB,CAAC,OAAO,CAAC;eACxE,QAAQ,CAAC,WAAW,KAAK,QAAQ,CAAC,UAAU;eAC5C,SAAS,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC;YAAE,OAAO,SAAS,CAAC;QAC7E,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpC,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC;mBACvE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,SAAS,CAAC;YAC/G,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtD,aAAa,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC;aAC5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACnE,CAAC;AAED,SAAS,aAAa,CAAC,MAAiB,IAA2B,OAAO,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AAChJ,SAAS,OAAO,CAAC,MAA6B,EAAE,SAAiB,IAAa,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC;AAC1I,SAAS,MAAM,CAAC,MAA6B,EAAE,SAAiB,IAAa,OAAO,MAAM,CAAC,KAAK,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,CAAC;AACzI,SAAS,OAAO,CAAC,MAAgB,IAAY,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAExH,SAAS,iBAAiB,CAAC,MAAqC;IAC9D,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC;IACtG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}