@dsh-enhanced/assistant-evaluation 0.1.23 → 0.1.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/README.md +109 -3
  2. package/bin/dsh-benchmark.js +21 -0
  3. package/lib/benchmark/cli.d.ts +6 -0
  4. package/lib/benchmark/cli.d.ts.map +1 -0
  5. package/lib/benchmark/cli.js +332 -0
  6. package/lib/benchmark/cli.js.map +1 -0
  7. package/lib/benchmark/corpus.d.ts +28 -0
  8. package/lib/benchmark/corpus.d.ts.map +1 -0
  9. package/lib/benchmark/corpus.js +173 -0
  10. package/lib/benchmark/corpus.js.map +1 -0
  11. package/lib/benchmark/deepseek.d.ts +15 -0
  12. package/lib/benchmark/deepseek.d.ts.map +1 -0
  13. package/lib/benchmark/deepseek.js +143 -0
  14. package/lib/benchmark/deepseek.js.map +1 -0
  15. package/lib/benchmark/holdout-evidence.d.ts +126 -0
  16. package/lib/benchmark/holdout-evidence.d.ts.map +1 -0
  17. package/lib/benchmark/holdout-evidence.js +466 -0
  18. package/lib/benchmark/holdout-evidence.js.map +1 -0
  19. package/lib/benchmark/holdout-protocol.d.ts +68 -0
  20. package/lib/benchmark/holdout-protocol.d.ts.map +1 -0
  21. package/lib/benchmark/holdout-protocol.js +271 -0
  22. package/lib/benchmark/holdout-protocol.js.map +1 -0
  23. package/lib/benchmark/holdout-provider.d.ts +26 -0
  24. package/lib/benchmark/holdout-provider.d.ts.map +1 -0
  25. package/lib/benchmark/holdout-provider.js +382 -0
  26. package/lib/benchmark/holdout-provider.js.map +1 -0
  27. package/lib/benchmark/holdout.d.ts +83 -0
  28. package/lib/benchmark/holdout.d.ts.map +1 -0
  29. package/lib/benchmark/holdout.js +391 -0
  30. package/lib/benchmark/holdout.js.map +1 -0
  31. package/lib/benchmark/index.d.ts +9 -0
  32. package/lib/benchmark/index.d.ts.map +1 -0
  33. package/lib/benchmark/index.js +7 -0
  34. package/lib/benchmark/index.js.map +1 -0
  35. package/lib/benchmark/memory-corpus.d.ts +55 -0
  36. package/lib/benchmark/memory-corpus.d.ts.map +1 -0
  37. package/lib/benchmark/memory-corpus.js +153 -0
  38. package/lib/benchmark/memory-corpus.js.map +1 -0
  39. package/lib/benchmark/memory-runtime.d.ts +63 -0
  40. package/lib/benchmark/memory-runtime.d.ts.map +1 -0
  41. package/lib/benchmark/memory-runtime.js +75 -0
  42. package/lib/benchmark/memory-runtime.js.map +1 -0
  43. package/lib/benchmark/native.d.ts +46 -0
  44. package/lib/benchmark/native.d.ts.map +1 -0
  45. package/lib/benchmark/native.js +300 -0
  46. package/lib/benchmark/native.js.map +1 -0
  47. package/lib/benchmark/report.d.ts +4 -0
  48. package/lib/benchmark/report.d.ts.map +1 -0
  49. package/lib/benchmark/report.js +153 -0
  50. package/lib/benchmark/report.js.map +1 -0
  51. package/lib/benchmark/runner.d.ts +25 -0
  52. package/lib/benchmark/runner.d.ts.map +1 -0
  53. package/lib/benchmark/runner.js +119 -0
  54. package/lib/benchmark/runner.js.map +1 -0
  55. package/lib/benchmark/schema.d.ts +20 -0
  56. package/lib/benchmark/schema.d.ts.map +1 -0
  57. package/lib/benchmark/schema.js +183 -0
  58. package/lib/benchmark/schema.js.map +1 -0
  59. package/lib/benchmark/store.d.ts +18 -0
  60. package/lib/benchmark/store.d.ts.map +1 -0
  61. package/lib/benchmark/store.js +321 -0
  62. package/lib/benchmark/store.js.map +1 -0
  63. package/lib/benchmark/strategy-capabilities.d.ts +154 -0
  64. package/lib/benchmark/strategy-capabilities.d.ts.map +1 -0
  65. package/lib/benchmark/strategy-capabilities.js +300 -0
  66. package/lib/benchmark/strategy-capabilities.js.map +1 -0
  67. package/lib/benchmark/strategy-config.d.ts +26 -0
  68. package/lib/benchmark/strategy-config.d.ts.map +1 -0
  69. package/lib/benchmark/strategy-config.js +80 -0
  70. package/lib/benchmark/strategy-config.js.map +1 -0
  71. package/lib/benchmark/strategy-corpus.d.ts +21 -0
  72. package/lib/benchmark/strategy-corpus.d.ts.map +1 -0
  73. package/lib/benchmark/strategy-corpus.js +95 -0
  74. package/lib/benchmark/strategy-corpus.js.map +1 -0
  75. package/lib/benchmark/strategy-evidence.d.ts +161 -0
  76. package/lib/benchmark/strategy-evidence.d.ts.map +1 -0
  77. package/lib/benchmark/strategy-evidence.js +520 -0
  78. package/lib/benchmark/strategy-evidence.js.map +1 -0
  79. package/lib/benchmark/strategy-executor.d.ts +27 -0
  80. package/lib/benchmark/strategy-executor.d.ts.map +1 -0
  81. package/lib/benchmark/strategy-executor.js +178 -0
  82. package/lib/benchmark/strategy-executor.js.map +1 -0
  83. package/lib/benchmark/strategy-goal-runtime.d.ts +78 -0
  84. package/lib/benchmark/strategy-goal-runtime.d.ts.map +1 -0
  85. package/lib/benchmark/strategy-goal-runtime.js +324 -0
  86. package/lib/benchmark/strategy-goal-runtime.js.map +1 -0
  87. package/lib/benchmark/strategy-meter.d.ts +62 -0
  88. package/lib/benchmark/strategy-meter.d.ts.map +1 -0
  89. package/lib/benchmark/strategy-meter.js +236 -0
  90. package/lib/benchmark/strategy-meter.js.map +1 -0
  91. package/lib/benchmark/strategy-owner.d.ts +71 -0
  92. package/lib/benchmark/strategy-owner.d.ts.map +1 -0
  93. package/lib/benchmark/strategy-owner.js +282 -0
  94. package/lib/benchmark/strategy-owner.js.map +1 -0
  95. package/lib/benchmark/strategy-plan.d.ts +51 -0
  96. package/lib/benchmark/strategy-plan.d.ts.map +1 -0
  97. package/lib/benchmark/strategy-plan.js +71 -0
  98. package/lib/benchmark/strategy-plan.js.map +1 -0
  99. package/lib/benchmark/strategy-policy.d.ts +16 -0
  100. package/lib/benchmark/strategy-policy.d.ts.map +1 -0
  101. package/lib/benchmark/strategy-policy.js +77 -0
  102. package/lib/benchmark/strategy-policy.js.map +1 -0
  103. package/lib/benchmark/strategy.d.ts +17 -0
  104. package/lib/benchmark/strategy.d.ts.map +1 -0
  105. package/lib/benchmark/strategy.js +21 -0
  106. package/lib/benchmark/strategy.js.map +1 -0
  107. package/lib/benchmark/types.d.ts +133 -0
  108. package/lib/benchmark/types.d.ts.map +1 -0
  109. package/lib/benchmark/types.js +3 -0
  110. package/lib/benchmark/types.js.map +1 -0
  111. package/lib/benchmark/usage.d.ts +36 -0
  112. package/lib/benchmark/usage.d.ts.map +1 -0
  113. package/lib/benchmark/usage.js +84 -0
  114. package/lib/benchmark/usage.js.map +1 -0
  115. package/lib/service.d.ts +50 -1
  116. package/lib/service.d.ts.map +1 -1
  117. package/lib/service.js +556 -9
  118. package/lib/service.js.map +1 -1
  119. package/lib/sqlite.d.ts +1 -1
  120. package/lib/sqlite.d.ts.map +1 -1
  121. package/lib/sqlite.js +94 -2
  122. package/lib/sqlite.js.map +1 -1
  123. package/lib/store.d.ts +27 -5
  124. package/lib/store.d.ts.map +1 -1
  125. package/lib/store.js +324 -25
  126. package/lib/store.js.map +1 -1
  127. package/lib/types.d.ts +114 -5
  128. package/lib/types.d.ts.map +1 -1
  129. package/lib/types.js.map +1 -1
  130. package/lib/version.d.ts +1 -1
  131. package/lib/version.js +1 -1
  132. package/package.json +125 -18
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy-owner.js","sourceRoot":"","sources":["../../src/benchmark/strategy-owner.ts"],"names":[],"mappings":";;;;;;;;AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,SAAS,MAAM,6BAA6B,CAAA;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,4CAA4C,CAAA;AACpF,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,qCAAqC,CAAA;AAC/E,OAAO,YAAY,MAAM,gCAAgC,CAAA;AACzD,OAAO,WAAW,MAAM,wBAAwB,CAAA;AAEhD,OAAO,EAAE,sBAAsB,EAAmB,MAAM,gCAAgC,CAAA;AACxF,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAc/C,MAAM,cAAc,GAAG,EAAE,GAAG,IAAI,CAAA;AAChC,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,KAAa,EAAU,EAAE;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,CAAA;IACxH,OAAO,KAAK,CAAA;AACd,CAAC,CAAA;AACD,MAAM,SAAS,GAAG,CAAC,KAAc,EAAE,KAAa,EAAU,EAAE;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,uCAAuC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,CAAA;IAC1H,OAAO,KAAK,CAAA;AACd,CAAC,CAAA;AACD,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAE,KAAa,EAAE,GAAW,EAAU,EAAE;IACtE,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,CAAA;IACrE,MAAM,MAAM,GAAG,KAAe,CAAA;IAC9B,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,KAAK,EAAE,CAAC,CAAA;IACnE,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AACD,MAAM,MAAM,GAAG,CAAC,MAAc,EAAE,KAAa,EAAW,EAAE,CAAC,KAAK,KAAK,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAA;AAC7G,KAAK,UAAU,yBAAyB,CAAC,IAAY,EAAE,KAAa;IAClE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,QAAQ,CAAC,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,0CAA0C,CAAC,CAAA;IACrE,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,GAAG,KAAK,OAAO,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,qCAAqC,CAAC,CAAA;IACrI,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;IACtC,IAAI,SAAS,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAA;IACrE,OAAO,SAAS,CAAA;AAClB,CAAC;AAyDD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,KAAoC;IAEpC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAA;IACrB,IAAI,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAa,CAAC,KAAK,SAAS,CAAC,EAAE,CAAC;QACnH,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;IACvE,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAC1C,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC;QACjI,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAA;IAC7F,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAA;IACxD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IACtD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,WAAW,EAAE,cAAc,CAAC,CAAA;IACvE,MAAM,eAAe,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,EAAE,mBAAmB,EAAE,SAAS,CAAC,CAAA;IACvF,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,cAAc;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAA;IAChJ,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAA;IAC7B,MAAM,WAAW,GAAG,iBAAiB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;IACxD,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAA;IAC1J,IAAI,IAAI,GAAG,CAAC,gBAAgB,CAAC,CAAC,IAAI,KAAK,gBAAgB,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC9G,MAAM,yBAAyB,GAAG,KAAK,CAAC,yBAAyB,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,yBAAyB,EAAE,2BAA2B,EAAE,OAAO,CAAC,CAAA;IACrK,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,KAAK,SAAS,KAAK,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACnH,MAAM,OAAO,GAAG,aAAa,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;IACvD,MAAM,OAAO,GAAG,QAAQ,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,CAAA;IAC/C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,WAAW,EAAE,EAAE,CAAC,CAAA;IACvG,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,IAAa,EAAE,IAAI,EAAE,WAAW,WAAW,EAAE,EAAE,CAAC,CAAA;IACjI,MAAM,eAAe,GAAG,kCAAkC,CAAA;IAC1D,MAAM,cAAc,GAAG,MAAM,MAAM,kCAAC,eAAe,EAAC,CAAA;IACpD,IAAI,OAAO,cAAc,CAAC,wBAAwB,KAAK,UAAU,IAAI,OAAO,cAAc,CAAC,mBAAmB,KAAK,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;IACpM,MAAM,WAAW,GAAW,cAAc,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,cAAc,CAAC,wBAAgF,CAAA;IAChH,IAAI,QAAwB,CAAA;IAC5B,MAAM,WAAW,GAAG,mBAAmB,WAAW,EAAE,CAAA;IACpD,MAAM,WAAW,GAAG,uBAAuB,WAAW,EAAE,CAAA;IACxD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,WAAW,EAAE,CAAC,CAAA;IAC3D,MAAM,IAAI,GAAqB,EAAE,CAAA;IACjC,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,WAAW,GAAG,KAAK,CAAA;IACvB,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,cAAyC,CAAA;IAC7C,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,aAAuC,CAAA;IAC3C,IAAI,aAAgD,CAAA;IAEpD,MAAM,yBAAyB,CAAC,SAAS,EAAE,qBAAqB,CAAC,CAAA;IACjE,MAAM,yBAAyB,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAA;IAClE,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;IAC3D,MAAM,yBAAyB,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAA;IACtE,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;QAC5B,MAAM,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;QAC9B,IAAI,yBAAyB,CAAC,GAAG,CAAC,CAAA;QAClC,MAAM,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,sBAAsB,EAAE,KAAK,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAA;QAC3G,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;QACzG,MAAM,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAA;QACjD,MAAM,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAA;QAC/B,MAAM,GAAG,CAAC,MAAM,CAAC,uBAAuB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,WAAW,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC,CAAA;QAChI,0EAA0E;QAC1E,uEAAuE;QACvE,uEAAuE;QACvE,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAA;QACtF,MAAM,GAAG,CAAC,MAAM,CAAC,sBAAsB,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,EAAE,iBAAiB,EAAE,MAAM,EAAE,KAAK,EAAE;gBAC7H,EAAE,EAAE,EAAE,sBAAsB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,SAAS,WAAW,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE;gBAC1N,EAAE,EAAE,EAAE,wBAAwB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE;gBACrN,EAAE,EAAE,EAAE,uBAAuB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE;gBAC9N,GAAG,WAAW;aACf,EAAE,CAAC,CAAA;QACJ,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE;YACpC,qEAAqE;YACrE,0EAA0E;YAC1E,sEAAsE;YACtE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAA;QACvD,CAAC,CAAC,CAAA;QACF,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;YACzB,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,EAAE,gBAAgB,EAAE,KAAK;YAClH,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK;YACjI,oBAAoB,EAAE,eAAe,EAAE,6BAA6B,EAAE,CAAC,EAAE,8BAA8B,EAAE,yBAAyB;YAClI,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAC;SAChJ,CAAC,CAAA;QACF,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,mBAA4B,CAA6B,CAAA;QAC5E,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAC7H,MAAM,GAAG,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAA;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YAAC,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QAAC,CAAC;QAAC,OAAO,OAAO,EAAE,CAAC;YACjD,MAAM,IAAI,cAAc,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,mDAAmD,CAAC,CAAA;QACjG,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IAED,MAAM,OAAO,GAAoB;QAC/B,OAAO,EAAE,OAAO;QAChB,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,oBAAoB,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAW,EAAE,OAAO,EAAE,CAAC,OAAO,CAAU,EAAE,CAAC;QAChH,KAAK,CAAC,KAAK,KAAI,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,MAAM;YACf,kEAAkE;YAClE,yEAAyE;YACzE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAA;YAClC,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,aAAa,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAA;QACjJ,CAAC;KACF,CAAA;IAED,MAAM,UAAU,GAAG,GAAS,EAAE,GAAG,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA,CAAC,CAAC,CAAA;IACtG,MAAM,gBAAgB,GAAG,GAAwC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC7I,MAAM,UAAU,GAAG,GAA0C,EAAE;QAC7D,UAAU,EAAE,CAAA;QACZ,MAAM,KAAK,GAAG,QAAQ,CAAC,kBAAkB,CAAC,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAA;QACvG,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;IACjL,CAAC,CAAA;IACD,MAAM,iBAAiB,GAAG,KAAK,EAAE,KAA+C,EAAiB,EAAE;QACjG,UAAU,EAAE,CAAA;QACZ,4EAA4E;QAC5E,yEAAyE;QACzE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,KAAK,EAAE,QAAQ,EAAE,CAAA;QACzE,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAAC,MAAM,KAAK,EAAE,QAAQ,EAAE,CAAA;QACzE,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAA;QAChC,IAAI,MAAM,CAAC,YAAY,KAAK,CAAC,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,IAAI,MAAM,CAAC,eAAe,KAAK,CAAC;eACtF,MAAM,CAAC,gBAAgB,KAAK,CAAC,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,IAAI,MAAM,CAAC,oBAAoB,KAAK,CAAC,IAAI,MAAM,CAAC,iBAAiB,KAAK,CAAC,EAAE,CAAC;YACxI,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;QACrE,CAAC;IACH,CAAC,CAAA;IACD,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAA;IACzD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YAAC,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QAAC,CAAC;QAAC,OAAO,OAAO,EAAE,CAAC;YACjD,MAAM,IAAI,cAAc,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,mDAAmD,CAAC,CAAA;QACjG,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY;QACvG,IAAI,QAAQ,KAAK,OAAO,gBAAgB,EAAE,CAAA,CAAC,CAAC;QAC5C,KAAK,CAAC,YAAY,CAAC,YAAwB;YACzC,UAAU,EAAE,CAAA;YACZ,IAAI,aAAa,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;YAChG,IAAI,CAAC,CAAC,YAAY,YAAY,UAAU,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;YAC3G,aAAa,GAAG,GAAG,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAC,CAAA;YACjE,IAAI,CAAC;gBAAC,MAAM,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAAC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,aAAa,EAAE,CAAC;gBAAC,aAAa,GAAG,SAAS,CAAC;gBAAC,MAAM,KAAK,CAAA;YAAC,CAAC;QACnI,CAAC;QACD,SAAS;YACP,UAAU,EAAE,CAAA;YACZ,IAAI,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAA;YAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;YAC7D,MAAM,SAAS,GAAG,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC/G,IAAI,SAAS,CAAC,IAAI,KAAK,OAAO,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;gBACvF,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAA;YAC9D,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,iBAAiB,EAAE,SAAS,CAAC,EAAE,EAAE,gBAAgB,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAA;YACrG,MAAM,GAAG,IAAI,CAAA;YACb,OAAO,KAAK,CAAA;QACd,CAAC;QACD,KAAK,CAAC,iBAAiB,CAAC,IAAY;YAClC,UAAU,EAAE,CAAA;YACZ,IAAI,CAAC,MAAM,IAAI,WAAW;gBAAE,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAA;YACnH,IAAI,aAAa,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;YAC5F,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,cAAc;gBAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;YACzJ,WAAW,GAAG,IAAI,CAAA;YAClB,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,qBAAqB,WAAW,IAAI,EAAE,WAAW,EAAE;gBAC5H,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;YACxE,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAChC,CAAC;QACD,iBAAiB;QACjB,UAAU;QACV,QAAQ;YACN,IAAI,cAAc,KAAK,SAAS;gBAAE,OAAO,cAAc,CAAA;YACvD,MAAM,GAAG,IAAI,CAAA;YACb,cAAc,GAAG,CAAC,KAAK,IAAI,EAAE;gBAC3B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;oBAAE,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC,CAAA;gBACzG,MAAM,MAAM,GAAc,EAAE,CAAA;gBAC5B,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;oBAC/H,IAAI,CAAC;wBAAC,MAAM,OAAO,EAAE,CAAA;oBAAC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAAC,CAAC;gBAC9D,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;oBAAE,MAAM,IAAI,cAAc,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAA;YAC5F,CAAC,CAAC,EAAE,CAAA;YACJ,OAAO,cAAc,CAAA;QACvB,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
@@ -0,0 +1,51 @@
1
+ import type { BenchmarkExecutionRequest } from './runner.js';
2
+ import type { BenchmarkPlan } from './types.js';
3
+ export declare const strategyBenchmarkProtocol = "dsh-native-goal-strategy-benchmark-v1";
4
+ /** Separate execution contract: schema-v1 benchmark budgets cannot express these limits. */
5
+ export interface StrategyBenchmarkExecutionLimits {
6
+ modelCalls: number;
7
+ maxOutputTokensPerCall: number;
8
+ maxGoalRounds: number;
9
+ }
10
+ export interface StrategyBenchmarkPlan {
11
+ schemaVersion: 1;
12
+ protocol: typeof strategyBenchmarkProtocol;
13
+ /** Unbound version manifest. Use strategyBenchmarkJournalPlan for scheduling/persistence. */
14
+ benchmark: BenchmarkPlan;
15
+ execution: StrategyBenchmarkExecutionLimits;
16
+ capabilities: StrategyBenchmarkCapabilities;
17
+ }
18
+ /** Operator-frozen manifests, to be compared with actual mounted capabilities by the executor. */
19
+ export interface StrategyBenchmarkCapabilities {
20
+ common: {
21
+ persona: string;
22
+ tools: string;
23
+ policy: string;
24
+ runtime: string;
25
+ };
26
+ strategy: {
27
+ guide: string;
28
+ tool: string;
29
+ policy: string;
30
+ runtime: string;
31
+ };
32
+ }
33
+ export declare function strategyBenchmarkCapabilityVersions(input: StrategyBenchmarkCapabilities, enabled: boolean): Readonly<{
34
+ prompt: string;
35
+ tools: string;
36
+ policy: string;
37
+ runtime: string;
38
+ }>;
39
+ /** Both arms use the same non-strategy feature configuration and one execution limit set. */
40
+ export declare function parseStrategyBenchmarkPlan(value: unknown): Readonly<StrategyBenchmarkPlan>;
41
+ export declare function strategyBenchmarkPlanDigest(input: StrategyBenchmarkPlan): string;
42
+ /**
43
+ * Bind the complete execution contract into the existing journal's frozen version manifest.
44
+ * Reusing a plan id with different call/round/output limits is rejected by BenchmarkStore.
45
+ * This binding is not enforcement: the native executor must install the outer meter and
46
+ * native round limit from the same parsed contract before starting any agent request.
47
+ */
48
+ export declare function strategyBenchmarkJournalPlan(input: StrategyBenchmarkPlan): Readonly<BenchmarkPlan>;
49
+ /** Called before constructing a runtime or invoking a trusted provider factory. */
50
+ export declare function strategyBenchmarkRequestLimits(input: StrategyBenchmarkPlan, request: BenchmarkExecutionRequest): Readonly<StrategyBenchmarkExecutionLimits>;
51
+ //# sourceMappingURL=strategy-plan.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy-plan.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-plan.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AAC5D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,eAAO,MAAM,yBAAyB,0CAA0C,CAAA;AAEhF,4FAA4F;AAC5F,MAAM,WAAW,gCAAgC;IAC/C,UAAU,EAAE,MAAM,CAAA;IAClB,sBAAsB,EAAE,MAAM,CAAA;IAC9B,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,CAAC,CAAA;IAChB,QAAQ,EAAE,OAAO,yBAAyB,CAAA;IAC1C,6FAA6F;IAC7F,SAAS,EAAE,aAAa,CAAA;IACxB,SAAS,EAAE,gCAAgC,CAAA;IAC3C,YAAY,EAAE,6BAA6B,CAAA;CAC5C;AAED,kGAAkG;AAClG,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3E,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;CAC3E;AAED,wBAAgB,mCAAmC,CAAC,KAAK,EAAE,6BAA6B,EAAE,OAAO,EAAE,OAAO;;;;;GAWzG;AAED,6FAA6F;AAC7F,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,QAAQ,CAAC,qBAAqB,CAAC,CAuB1F;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,qBAAqB,GAAG,MAAM,CAEhF;AAED;;;;;GAKG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,qBAAqB,GAAG,QAAQ,CAAC,aAAa,CAAC,CAMlG;AAED,mFAAmF;AACnF,wBAAgB,8BAA8B,CAAC,KAAK,EAAE,qBAAqB,EAAE,OAAO,EAAE,yBAAyB,GAAG,QAAQ,CAAC,gCAAgC,CAAC,CAW3J"}
@@ -0,0 +1,71 @@
1
+ import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
2
+ import { benchmarkAssert, benchmarkHash, benchmarkInteger, benchmarkObject, benchmarkSchedule, benchmarkSnapshot, parseBenchmarkPlan } from './schema.js';
3
+ export const strategyBenchmarkProtocol = 'dsh-native-goal-strategy-benchmark-v1';
4
+ export function strategyBenchmarkCapabilityVersions(input, enabled) {
5
+ const raw = benchmarkObject(benchmarkSnapshot(input), ['common', 'strategy']);
6
+ const common = benchmarkObject(raw.common, ['persona', 'tools', 'policy', 'runtime']);
7
+ const strategy = benchmarkObject(raw.strategy, ['guide', 'tool', 'policy', 'runtime']);
8
+ Object.values(common).forEach(benchmarkHash);
9
+ Object.values(strategy).forEach(benchmarkHash);
10
+ return Object.freeze({
11
+ prompt: acceptanceDigest({ commonPersona: common.persona, strategyGuide: enabled ? strategy.guide : null }),
12
+ tools: acceptanceDigest({ commonTools: common.tools, strategyTool: enabled ? strategy.tool : null }),
13
+ policy: acceptanceDigest({ commonPolicy: common.policy, strategyPolicy: enabled ? strategy.policy : null }),
14
+ runtime: acceptanceDigest({ commonRuntime: common.runtime, strategyRuntime: enabled ? strategy.runtime : null }),
15
+ });
16
+ }
17
+ /** Both arms use the same non-strategy feature configuration and one execution limit set. */
18
+ export function parseStrategyBenchmarkPlan(value) {
19
+ const copy = benchmarkSnapshot(value);
20
+ const raw = benchmarkObject(copy, ['schemaVersion', 'protocol', 'benchmark', 'execution', 'capabilities']);
21
+ benchmarkAssert(raw.schemaVersion === 1 && raw.protocol === strategyBenchmarkProtocol, 'unsupported strategy benchmark protocol');
22
+ const benchmark = parseBenchmarkPlan(raw.benchmark);
23
+ const execution = benchmarkObject(raw.execution, ['modelCalls', 'maxOutputTokensPerCall', 'maxGoalRounds']);
24
+ benchmarkInteger(execution.modelCalls, 1, 10_000);
25
+ benchmarkInteger(execution.maxOutputTokensPerCall, 1, benchmark.budget.outputTokens);
26
+ benchmarkInteger(execution.maxGoalRounds, 1, 100);
27
+ benchmarkAssert(benchmark.comparison === 'capability' && benchmark.variants.length === 2, 'strategy benchmark requires exactly two capability arms');
28
+ const direct = benchmark.variants.find(variant => variant.id === 'direct');
29
+ const adaptive = benchmark.variants.find(variant => variant.id === 'adaptive-strategy');
30
+ benchmarkAssert(direct?.role === 'baseline' && adaptive?.role === 'candidate', 'strategy benchmark requires direct baseline and adaptive-strategy candidate');
31
+ benchmarkAssert(acceptanceCanonicalJson(direct.features) === acceptanceCanonicalJson(adaptive.features), 'strategy arms must preserve non-strategy features');
32
+ benchmarkAssert(direct.versions.skills === adaptive.versions.skills, 'strategy arms must preserve common skills');
33
+ for (const variant of [direct, adaptive]) {
34
+ const expected = strategyBenchmarkCapabilityVersions(raw.capabilities, variant === adaptive);
35
+ benchmarkAssert(['prompt', 'tools', 'policy', 'runtime'].every(key => variant.versions[key] === expected[key]), 'strategy arm capability manifest drift');
36
+ }
37
+ // This establishes manifest consistency, not attestation of a loaded runtime.
38
+ // The native executor must compare these same common/strategy descriptors
39
+ // with actual persona, registered tools, Policy and loaded runtime versions.
40
+ return copy;
41
+ }
42
+ export function strategyBenchmarkPlanDigest(input) {
43
+ return acceptanceDigest(parseStrategyBenchmarkPlan(input));
44
+ }
45
+ /**
46
+ * Bind the complete execution contract into the existing journal's frozen version manifest.
47
+ * Reusing a plan id with different call/round/output limits is rejected by BenchmarkStore.
48
+ * This binding is not enforcement: the native executor must install the outer meter and
49
+ * native round limit from the same parsed contract before starting any agent request.
50
+ */
51
+ export function strategyBenchmarkJournalPlan(input) {
52
+ const parsed = parseStrategyBenchmarkPlan(input);
53
+ const digest = strategyBenchmarkPlanDigest(parsed);
54
+ return parseBenchmarkPlan({ ...parsed.benchmark, variants: parsed.benchmark.variants.map(variant => ({ ...variant,
55
+ versions: { ...variant.versions, runtime: acceptanceDigest({ protocol: parsed.protocol, strategyPlanDigest: digest, runtime: variant.versions.runtime }) },
56
+ })) });
57
+ }
58
+ /** Called before constructing a runtime or invoking a trusted provider factory. */
59
+ export function strategyBenchmarkRequestLimits(input, request) {
60
+ const parsed = parseStrategyBenchmarkPlan(input);
61
+ const plan = strategyBenchmarkJournalPlan(parsed);
62
+ const cell = benchmarkSchedule(plan).find(value => value.id === request.cell.id);
63
+ const same = (left, right) => acceptanceCanonicalJson(left) === acceptanceCanonicalJson(right);
64
+ benchmarkAssert(cell !== undefined && request.planId === plan.id && same(request.cell, cell)
65
+ && same(request.dataset, plan.dataset) && same(request.budget, plan.budget)
66
+ && same(request.task, plan.cases.find(value => value.id === cell.caseId))
67
+ && same(request.variant, plan.variants.find(value => value.id === cell.variantId)), 'strategy benchmark request drift');
68
+ benchmarkAssert(!request.signal.aborted, 'strategy benchmark aborted');
69
+ return parsed.execution;
70
+ }
71
+ //# sourceMappingURL=strategy-plan.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy-plan.js","sourceRoot":"","sources":["../../src/benchmark/strategy-plan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAClG,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAIzJ,MAAM,CAAC,MAAM,yBAAyB,GAAG,uCAAuC,CAAA;AAwBhF,MAAM,UAAU,mCAAmC,CAAC,KAAoC,EAAE,OAAgB;IACxG,MAAM,GAAG,GAAG,eAAe,CAAC,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAA;IAC7E,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;IACrF,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAA;IACtF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAA;IAC5F,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,MAAM,EAAE,gBAAgB,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3G,KAAK,EAAE,gBAAgB,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACpG,MAAM,EAAE,gBAAgB,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3G,OAAO,EAAE,gBAAgB,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;KACjH,CAAC,CAAA;AACJ,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,0BAA0B,CAAC,KAAc;IACvD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,eAAe,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAA;IAC1G,eAAe,CAAC,GAAG,CAAC,aAAa,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,KAAK,yBAAyB,EAAE,yCAAyC,CAAC,CAAA;IACjI,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACnD,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,YAAY,EAAE,wBAAwB,EAAE,eAAe,CAAC,CAAC,CAAA;IAC3G,gBAAgB,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IACjD,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,EAAE,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACpF,gBAAgB,CAAC,SAAS,CAAC,aAAa,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;IACjD,eAAe,CAAC,SAAS,CAAC,UAAU,KAAK,YAAY,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,yDAAyD,CAAC,CAAA;IACpJ,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAA;IAC1E,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,mBAAmB,CAAC,CAAA;IACvF,eAAe,CAAC,MAAM,EAAE,IAAI,KAAK,UAAU,IAAI,QAAQ,EAAE,IAAI,KAAK,WAAW,EAAE,6EAA6E,CAAC,CAAA;IAC7J,eAAe,CAAC,uBAAuB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,uBAAuB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,mDAAmD,CAAC,CAAA;IAC7J,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,2CAA2C,CAAC,CAAA;IACjH,KAAK,MAAM,OAAO,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,mCAAmC,CAAC,GAAG,CAAC,YAA6C,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAA;QAC7H,eAAe,CAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAW,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,wCAAwC,CAAC,CAAA;IACtK,CAAC;IACD,8EAA8E;IAC9E,0EAA0E;IAC1E,6EAA6E;IAC7E,OAAO,IAA6B,CAAA;AACtC,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,KAA4B;IACtE,OAAO,gBAAgB,CAAC,0BAA0B,CAAC,KAAK,CAAC,CAAC,CAAA;AAC5D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,4BAA4B,CAAC,KAA4B;IACvE,MAAM,MAAM,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,MAAM,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAA;IAClD,OAAO,kBAAkB,CAAC,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO;YAC/G,QAAQ,EAAE,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,gBAAgB,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE;SAC3J,CAAC,CAAC,EAAE,CAAC,CAAA;AACR,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,8BAA8B,CAAC,KAA4B,EAAE,OAAkC;IAC7G,MAAM,MAAM,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG,4BAA4B,CAAC,MAAM,CAAC,CAAA;IACjD,MAAM,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChF,MAAM,IAAI,GAAG,CAAC,IAAa,EAAE,KAAc,EAAW,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,CAAC,CAAA;IACzH,eAAe,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC;WACvF,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC;WACxE,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC;WACtE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,kCAAkC,CAAC,CAAA;IACzH,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,4BAA4B,CAAC,CAAA;IACtE,OAAO,MAAM,CAAC,SAAS,CAAA;AACzB,CAAC"}
@@ -0,0 +1,16 @@
1
+ export interface StrategyPolicyRecipe {
2
+ protocol: 'dsh-native-goal-strategy-policy-v1';
3
+ enabled: boolean;
4
+ /** Dynamic identity values are normalized out of this plan-bound recipe. */
5
+ rules: readonly Readonly<Record<string, unknown>>[];
6
+ }
7
+ /** Plan-bound rule shape. Bootstrap/principal/workspace values are normalized. */
8
+ export declare function strategyPolicyRecipe(enabled: boolean): Readonly<StrategyPolicyRecipe>;
9
+ /**
10
+ * Validate the exact public Policy configuration emitted by the benchmark
11
+ * owner. Dynamic bootstrap/principal IDs are accepted only in their owning
12
+ * rule; every agent rule must bind one identical benchmark workspace.
13
+ */
14
+ export declare function assertStrategyPolicyConfiguration(config: unknown, enabled: boolean, workspace?: string): Readonly<Record<string, unknown>>;
15
+ export declare function strategyPolicyConfigurationDigest(config: unknown, enabled: boolean, workspace?: string): string;
16
+ //# sourceMappingURL=strategy-policy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy-policy.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy-policy.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,oCAAoC,CAAA;IAC9C,OAAO,EAAE,OAAO,CAAA;IAChB,4EAA4E;IAC5E,KAAK,EAAE,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,CAAA;CACpD;AAYD,mFAAmF;AACnF,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CASrF;AAED;;;;GAIG;AACH,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CA2B1I;AAED,wBAAgB,iCAAiC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAE/G"}
@@ -0,0 +1,77 @@
1
+ import { acceptanceCanonicalJson, acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
2
+ function fail(message) { throw new Error(`strategy policy: ${message}`); }
3
+ const object = (value) => {
4
+ if (value === null || typeof value !== 'object' || Array.isArray(value) || Object.getPrototypeOf(value) !== Object.prototype)
5
+ fail('invalid configuration');
6
+ return value;
7
+ };
8
+ const same = (left, right) => acceptanceCanonicalJson(left) === acceptanceCanonicalJson(right);
9
+ const subject = (id, workspace) => ({ kind: 'agent', id, ...(workspace === undefined ? {} : { workspace }) });
10
+ const external = (id) => ({ kind: 'external', id });
11
+ const tool = (id) => ({ kind: 'tool', id });
12
+ const externalContext = { initiators: ['external'] };
13
+ /** Plan-bound rule shape. Bootstrap/principal/workspace values are normalized. */
14
+ export function strategyPolicyRecipe(enabled) {
15
+ const tools = ['goal_create', 'goal_context', 'goal_checkpoint', 'isolation_run', ...(enabled ? ['goal_strategy'] : []), 'isolation:benchmark-work'];
16
+ return Object.freeze({ protocol: 'dsh-native-goal-strategy-policy-v1', enabled, rules: Object.freeze([
17
+ { role: 'pair-issue', effect: 'allow', subject: external('$bootstrap'), actions: ['pair.issue'], resource: { kind: 'message', id: 'pairing' }, context: { initiators: ['foreground'] } },
18
+ { role: 'owner-ingest', effect: 'allow', subject: external('$principal'), actions: ['pair.confirm', 'ingest'], resource: { kind: 'message', id: '*' }, context: externalContext },
19
+ { role: 'owner-reply', effect: 'allow', subject: { ...subject('benchmark', '$workspace'), principal: '$principal' }, actions: ['reply'], resource: { kind: 'message', id: '*' }, context: externalContext },
20
+ { role: 'goal', effect: 'allow', subject: subject('benchmark', '$workspace'), actions: ['create', 'observe', 'inspect', 'focus', 'checkpoint', 'snapshot', 'execute', ...(enabled ? ['delegate'] : [])], resource: { kind: 'goal', id: 'business-context' }, context: externalContext },
21
+ ...tools.map(id => ({ role: `tool:${id}`, effect: 'allow', subject: subject('benchmark', '$workspace'), actions: ['execute'], resource: tool(id), context: externalContext })),
22
+ ]) });
23
+ }
24
+ /**
25
+ * Validate the exact public Policy configuration emitted by the benchmark
26
+ * owner. Dynamic bootstrap/principal IDs are accepted only in their owning
27
+ * rule; every agent rule must bind one identical benchmark workspace.
28
+ */
29
+ export function assertStrategyPolicyConfiguration(config, enabled, workspace) {
30
+ const value = object(config);
31
+ if (!same(Object.keys(value).sort(), ['autoReview', 'budgets', 'rules', 'toolDefaultEffect']) || !same(value.budgets, []) || value.autoReview !== null)
32
+ fail('unexpected policy configuration');
33
+ const rules = value.rules;
34
+ if (!Array.isArray(rules) || rules.length !== strategyPolicyRecipe(enabled).rules.length || value.toolDefaultEffect !== 'deny')
35
+ fail('unexpected policy rules or default effect');
36
+ const byId = new Map();
37
+ for (const item of rules) {
38
+ const rule = object(item);
39
+ if (typeof rule.id !== 'string' || byId.has(rule.id))
40
+ fail('invalid policy rule id');
41
+ byId.set(rule.id, rule);
42
+ }
43
+ const get = (id) => byId.get(id) ?? fail(`missing policy rule ${id}`);
44
+ const exact = (id, expected) => {
45
+ const rule = get(id);
46
+ const normalized = { ...rule, id: undefined };
47
+ delete normalized.id;
48
+ if (!same(normalized, expected))
49
+ fail(`policy rule drift: ${id}`);
50
+ };
51
+ const pair = get('benchmark-pair-issue');
52
+ const bootstrap = object(pair.subject).id;
53
+ if (typeof bootstrap !== 'string' || !bootstrap.startsWith('local:benchmark-bootstrap-') || bootstrap.includes('*'))
54
+ fail('invalid bootstrap scope');
55
+ exact('benchmark-pair-issue', { effect: 'allow', subject: external(bootstrap), actions: ['pair.issue'], resource: { kind: 'message', id: 'pairing' }, context: { initiators: ['foreground'] } });
56
+ const ingest = get('benchmark-owner-ingest');
57
+ const principal = object(ingest.subject).id;
58
+ if (typeof principal !== 'string' || principal === '' || principal.includes('*'))
59
+ fail('invalid principal scope');
60
+ exact('benchmark-owner-ingest', { effect: 'allow', subject: external(principal), actions: ['pair.confirm', 'ingest'], resource: { kind: 'message', id: '*' }, context: externalContext });
61
+ const owner = get('benchmark-owner-reply');
62
+ const ownerSubject = object(owner.subject);
63
+ const boundWorkspace = ownerSubject.workspace;
64
+ if (typeof boundWorkspace !== 'string' || boundWorkspace === '' || boundWorkspace.includes('*') || workspace !== undefined && boundWorkspace !== workspace)
65
+ fail('invalid workspace scope');
66
+ exact('benchmark-owner-reply', { effect: 'allow', subject: { ...subject('benchmark', boundWorkspace), principal }, actions: ['reply'], resource: { kind: 'message', id: '*' }, context: externalContext });
67
+ const actions = ['create', 'observe', 'inspect', 'focus', 'checkpoint', 'snapshot', 'execute', ...(enabled ? ['delegate'] : [])];
68
+ exact('benchmark-goal', { effect: 'allow', subject: subject('benchmark', boundWorkspace), actions, resource: { kind: 'goal', id: 'business-context' }, context: externalContext });
69
+ const tools = ['goal_create', 'goal_context', 'goal_checkpoint', 'isolation_run', ...(enabled ? ['goal_strategy'] : []), 'isolation:benchmark-work'];
70
+ for (const id of tools)
71
+ exact(`allow-${id.replace(':', '-')}`, { effect: 'allow', subject: subject('benchmark', boundWorkspace), actions: ['execute'], resource: tool(id), context: externalContext });
72
+ return Object.freeze(JSON.parse(JSON.stringify(value)));
73
+ }
74
+ export function strategyPolicyConfigurationDigest(config, enabled, workspace) {
75
+ return acceptanceDigest(assertStrategyPolicyConfiguration(config, enabled, workspace));
76
+ }
77
+ //# sourceMappingURL=strategy-policy.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy-policy.js","sourceRoot":"","sources":["../../src/benchmark/strategy-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAQlG,SAAS,IAAI,CAAC,OAAe,IAAW,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAA,CAAC,CAAC;AACxF,MAAM,MAAM,GAAG,CAAC,KAAc,EAA2B,EAAE;IACzD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS;QAAE,IAAI,CAAC,uBAAuB,CAAC,CAAA;IAC3J,OAAO,KAAgC,CAAA;AACzC,CAAC,CAAA;AACD,MAAM,IAAI,GAAG,CAAC,IAAa,EAAE,KAAc,EAAW,EAAE,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,uBAAuB,CAAC,KAAK,CAAC,CAAA;AACzH,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,SAAkB,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAA;AAC9H,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAA;AAC3D,MAAM,IAAI,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAA;AACnD,MAAM,eAAe,GAAG,EAAE,UAAU,EAAE,CAAC,UAAU,CAAC,EAAE,CAAA;AAEpD,mFAAmF;AACnF,MAAM,UAAU,oBAAoB,CAAC,OAAgB;IACnD,MAAM,KAAK,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACpJ,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,oCAAoC,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC;YACnG,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE;YACxL,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE;YACjL,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE;YAC3M,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,kBAAkB,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE;YACvR,GAAG,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC;SAC/K,CAAC,EAAE,CAAC,CAAA;AACP,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iCAAiC,CAAC,MAAe,EAAE,OAAgB,EAAE,SAAkB;IACrG,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI;QAAE,IAAI,CAAC,iCAAiC,CAAC,CAAA;IAC/L,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;IACzB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,oBAAoB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,MAAM;QAAE,IAAI,CAAC,2CAA2C,CAAC,CAAA;IACjL,MAAM,IAAI,GAAG,IAAI,GAAG,EAAmC,CAAA;IACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAAC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAAC,IAAI,OAAO,IAAI,CAAC,EAAE,KAAK,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,IAAI,CAAC,wBAAwB,CAAC,CAAC;QAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;IAAC,CAAC;IACtK,MAAM,GAAG,GAAG,CAAC,EAAU,EAA2B,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAA;IACtG,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,QAAiC,EAAQ,EAAE;QACpE,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;QAAC,MAAM,UAAU,GAAG,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,SAAS,EAAE,CAAA;QACnE,OAAQ,UAAsC,CAAC,EAAE,CAAA;QACjD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC;YAAE,IAAI,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAA;IACnE,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAAC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IACnF,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,4BAA4B,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACpJ,KAAK,CAAC,sBAAsB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,YAAY,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC,CAAA;IAChM,MAAM,MAAM,GAAG,GAAG,CAAC,wBAAwB,CAAC,CAAC;IAAC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAA;IACzF,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,KAAK,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,IAAI,CAAC,yBAAyB,CAAC,CAAA;IACjH,KAAK,CAAC,wBAAwB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;IACzL,MAAM,KAAK,GAAG,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAAC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAAC,MAAM,cAAc,GAAG,YAAY,CAAC,SAAS,CAAA;IACrI,IAAI,OAAO,cAAc,KAAK,QAAQ,IAAI,cAAc,KAAK,EAAE,IAAI,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,SAAS,KAAK,SAAS,IAAI,cAAc,KAAK,SAAS;QAAE,IAAI,CAAC,yBAAyB,CAAC,CAAA;IAC3L,KAAK,CAAC,uBAAuB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;IAC1M,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAChI,KAAK,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,kBAAkB,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;IAClL,MAAM,KAAK,GAAG,CAAC,aAAa,EAAE,cAAc,EAAE,iBAAiB,EAAE,eAAe,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACpJ,KAAK,MAAM,EAAE,IAAI,KAAK;QAAE,KAAK,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,OAAO,EAAE,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;IACtM,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAA4B,CAAC,CAAA;AACpF,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,MAAe,EAAE,OAAgB,EAAE,SAAkB;IACrG,OAAO,gBAAgB,CAAC,iCAAiC,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAA;AACxF,CAAC"}
@@ -0,0 +1,17 @@
1
+ /** Host-only strategy benchmark building blocks; no model-visible tools are registered. */
2
+ import type { Context } from '@deepseek-ai/cordis';
3
+ import type { NativeAdapterBinding, NativeModelConfig } from './native.js';
4
+ import type { BenchmarkExecutionRequest } from './runner.js';
5
+ import { type StrategyBenchmarkPlan } from './strategy-plan.js';
6
+ export * from './strategy-plan.js';
7
+ export * from './strategy-meter.js';
8
+ export * from './strategy-owner.js';
9
+ export * from './strategy-goal-runtime.js';
10
+ export * from './strategy-evidence.js';
11
+ export * from './strategy-capabilities.js';
12
+ export * from './strategy-config.js';
13
+ export * from './strategy-corpus.js';
14
+ export * from './strategy-executor.js';
15
+ /** Request identity and call/output budgets are checked before installing the cell meter. */
16
+ export declare function installStrategyBenchmarkRequestMeter(ctx: Context, plan: StrategyBenchmarkPlan, request: BenchmarkExecutionRequest, model: Readonly<NativeModelConfig>, binding: Readonly<NativeAdapterBinding>): Readonly<import("./strategy-meter.js").StrategyBenchmarkMeter>;
17
+ //# sourceMappingURL=strategy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy.d.ts","sourceRoot":"","sources":["../../src/benchmark/strategy.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAElD,OAAO,KAAK,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC1E,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AAE5D,OAAO,EAAkC,KAAK,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAE/F,cAAc,oBAAoB,CAAA;AAClC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,sBAAsB,CAAA;AACpC,cAAc,sBAAsB,CAAA;AACpC,cAAc,wBAAwB,CAAA;AAEtC,6FAA6F;AAC7F,wBAAgB,oCAAoC,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,qBAAqB,EAC5F,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,oBAAoB,CAAC,kEAKhH"}
@@ -0,0 +1,21 @@
1
+ import { acceptanceDigest } from '@dsh-enhanced/task-acceptance-contract';
2
+ import { installStrategyBenchmarkMeter } from './strategy-meter.js';
3
+ import { strategyBenchmarkRequestLimits } from './strategy-plan.js';
4
+ export * from './strategy-plan.js';
5
+ export * from './strategy-meter.js';
6
+ export * from './strategy-owner.js';
7
+ export * from './strategy-goal-runtime.js';
8
+ export * from './strategy-evidence.js';
9
+ export * from './strategy-capabilities.js';
10
+ export * from './strategy-config.js';
11
+ export * from './strategy-corpus.js';
12
+ export * from './strategy-executor.js';
13
+ /** Request identity and call/output budgets are checked before installing the cell meter. */
14
+ export function installStrategyBenchmarkRequestMeter(ctx, plan, request, model, binding) {
15
+ const limits = strategyBenchmarkRequestLimits(plan, request);
16
+ if (acceptanceDigest(model) !== request.variant.versions.model)
17
+ throw new Error('strategy benchmark model configuration drift');
18
+ return installStrategyBenchmarkMeter(ctx, { budget: request.budget, modelCalls: limits.modelCalls,
19
+ maxOutputTokens: limits.maxOutputTokensPerCall, model, binding, signal: request.signal });
20
+ }
21
+ //# sourceMappingURL=strategy.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"strategy.js","sourceRoot":"","sources":["../../src/benchmark/strategy.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,wCAAwC,CAAA;AAGzE,OAAO,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAA;AACnE,OAAO,EAAE,8BAA8B,EAA8B,MAAM,oBAAoB,CAAA;AAE/F,cAAc,oBAAoB,CAAA;AAClC,cAAc,qBAAqB,CAAA;AACnC,cAAc,qBAAqB,CAAA;AACnC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,wBAAwB,CAAA;AACtC,cAAc,4BAA4B,CAAA;AAC1C,cAAc,sBAAsB,CAAA;AACpC,cAAc,sBAAsB,CAAA;AACpC,cAAc,wBAAwB,CAAA;AAEtC,6FAA6F;AAC7F,MAAM,UAAU,oCAAoC,CAAC,GAAY,EAAE,IAA2B,EAC5F,OAAkC,EAAE,KAAkC,EAAE,OAAuC;IAC/G,MAAM,MAAM,GAAG,8BAA8B,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAC5D,IAAI,gBAAgB,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAA;IAC/H,OAAO,6BAA6B,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU;QAC/F,eAAe,EAAE,MAAM,CAAC,sBAAsB,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;AAC7F,CAAC"}
@@ -0,0 +1,133 @@
1
+ /** Host-side benchmark data. These APIs are deliberately not registered as model tools. */
2
+ export declare const benchmarkDomains: readonly ["code", "research", "cross-day", "proactivity", "injection", "revocation", "goal-change", "duplicate-action"];
3
+ export type BenchmarkDomain = typeof benchmarkDomains[number];
4
+ export type BenchmarkSplit = 'development' | 'holdout';
5
+ export type BenchmarkVerdict = 'achieved' | 'not-achieved' | 'unknown';
6
+ export type BenchmarkFeatures = Readonly<Record<'memory' | 'planning' | 'review' | 'growth', boolean>>;
7
+ export interface BenchmarkVersions {
8
+ /** Every value is a SHA-256 of the resolved configuration/content, including model sampling. */
9
+ model: string;
10
+ prompt: string;
11
+ skills: string;
12
+ tools: string;
13
+ policy: string;
14
+ runtime: string;
15
+ }
16
+ export interface BenchmarkBudget {
17
+ durationMs: number;
18
+ inputTokens: number;
19
+ outputTokens: number;
20
+ /** Null explicitly selects token-only budgeting for providers without a known tariff. */
21
+ costUsdMicros: number | null;
22
+ toolCalls: number;
23
+ }
24
+ export interface BenchmarkVariant {
25
+ id: string;
26
+ role: 'baseline' | 'candidate' | 'ablation';
27
+ versions: BenchmarkVersions;
28
+ features: BenchmarkFeatures;
29
+ }
30
+ /** No prompts, answers, judge programs, file paths or source documents in this manifest. */
31
+ export interface BenchmarkCase {
32
+ id: string;
33
+ domain: BenchmarkDomain;
34
+ inputDigest: string;
35
+ acceptanceDigest: string;
36
+ }
37
+ export interface BenchmarkPlan {
38
+ schemaVersion: 1;
39
+ id: string;
40
+ dataset: {
41
+ id: string;
42
+ version: string;
43
+ digest: string;
44
+ split: BenchmarkSplit;
45
+ };
46
+ comparison: 'capability' | 'model';
47
+ cases: readonly BenchmarkCase[];
48
+ variants: readonly BenchmarkVariant[];
49
+ budget: BenchmarkBudget;
50
+ repeats: number;
51
+ seed: number;
52
+ }
53
+ /** Measured by the Host, never extracted from model prose. Null means unavailable. */
54
+ export interface BenchmarkMetrics {
55
+ inputTokens: number | null;
56
+ outputTokens: number | null;
57
+ costUsdMicros: number | null;
58
+ toolCalls: number | null;
59
+ rework: number | null;
60
+ interventions: number | null;
61
+ latencyMs: number | null;
62
+ }
63
+ export interface BenchmarkCell {
64
+ id: string;
65
+ caseId: string;
66
+ variantId: string;
67
+ repeat: number;
68
+ seed: number;
69
+ }
70
+ export interface BenchmarkObservation {
71
+ /** The adapter must compare actual loaded versions, input and judge with the plan. */
72
+ versions: BenchmarkVersions;
73
+ inputDigest: string;
74
+ acceptanceDigest: string;
75
+ verdict: BenchmarkVerdict;
76
+ metrics: BenchmarkMetrics;
77
+ evidenceDigest: string;
78
+ /** True only after the original agent and its owned resources have stopped. */
79
+ quiescent: boolean;
80
+ }
81
+ export interface BenchmarkResult {
82
+ cell: BenchmarkCell;
83
+ status: 'completed' | 'unknown';
84
+ verdict: BenchmarkVerdict;
85
+ metrics: BenchmarkMetrics;
86
+ evidenceDigest: string | null;
87
+ reason: 'verified' | 'adapter-error' | 'interrupted' | 'timeout' | 'invalid-observation' | 'budget-exceeded' | 'not-quiescent';
88
+ startedAt: number;
89
+ completedAt: number;
90
+ }
91
+ export interface BenchmarkReport {
92
+ schemaVersion: 1;
93
+ planDigest: string;
94
+ complete: boolean;
95
+ expectedCells: number;
96
+ recordedCells: number;
97
+ variants: readonly {
98
+ id: string;
99
+ expected: number;
100
+ recorded: number;
101
+ achieved: number;
102
+ notAchieved: number;
103
+ unknown: number;
104
+ successRate: number;
105
+ /** Task-cluster bootstrap; unavailable with missing/unknown results or fewer than two tasks. */
106
+ successInterval95: readonly [number, number] | null;
107
+ metrics: Readonly<Record<keyof BenchmarkMetrics, {
108
+ measured: number;
109
+ missing: number;
110
+ mean: number | null;
111
+ median: number | null;
112
+ p95: number | null;
113
+ }>>;
114
+ }[];
115
+ comparisons: readonly {
116
+ /** Reference arm: baseline for candidates, the unique candidate for ablations. */
117
+ baselineId: string;
118
+ variantId: string;
119
+ paired: number;
120
+ missingPairs: number;
121
+ wins: number;
122
+ losses: number;
123
+ ties: number;
124
+ unknownPairs: number;
125
+ /** Unavailable if either arm has missing or unknown results: these cannot prove a gain. */
126
+ successRateDelta: number | null;
127
+ /** Clustered by task: repeats are not independent tasks. Null with fewer than two tasks. */
128
+ taskBootstrapInterval95: readonly [number, number] | null;
129
+ }[];
130
+ /** Measurement reports never grant promotion authority. */
131
+ promotionAuthorized: false;
132
+ }
133
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/benchmark/types.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,eAAO,MAAM,gBAAgB,yHAA0H,CAAA;AACvJ,MAAM,MAAM,eAAe,GAAG,OAAO,gBAAgB,CAAC,MAAM,CAAC,CAAA;AAC7D,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,SAAS,CAAA;AACtD,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,cAAc,GAAG,SAAS,CAAA;AACtE,MAAM,MAAM,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAA;AAEtG,MAAM,WAAW,iBAAiB;IAChC,gGAAgG;IAChG,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,yFAAyF;IACzF,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,UAAU,GAAG,WAAW,GAAG,UAAU,CAAA;IAC3C,QAAQ,EAAE,iBAAiB,CAAA;IAC3B,QAAQ,EAAE,iBAAiB,CAAA;CAC5B;AAED,4FAA4F;AAC5F,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,eAAe,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,CAAC,CAAA;IAChB,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,cAAc,CAAA;KAAE,CAAA;IAC/E,UAAU,EAAE,YAAY,GAAG,OAAO,CAAA;IAClC,KAAK,EAAE,SAAS,aAAa,EAAE,CAAA;IAC/B,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAA;IACrC,MAAM,EAAE,eAAe,CAAA;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;CACb;AAED,sFAAsF;AACtF,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,QAAQ,EAAE,iBAAiB,CAAA;IAC3B,WAAW,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,MAAM,CAAA;IACxB,OAAO,EAAE,gBAAgB,CAAA;IACzB,OAAO,EAAE,gBAAgB,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,+EAA+E;IAC/E,SAAS,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,aAAa,CAAA;IACnB,MAAM,EAAE,WAAW,GAAG,SAAS,CAAA;IAC/B,OAAO,EAAE,gBAAgB,CAAA;IACzB,OAAO,EAAE,gBAAgB,CAAA;IACzB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,MAAM,EAAE,UAAU,GAAG,eAAe,GAAG,aAAa,GAAG,SAAS,GAAG,qBAAqB,GAAG,iBAAiB,GAAG,eAAe,CAAA;IAC9H,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,CAAC,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,OAAO,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,SAAS;QACjB,EAAE,EAAE,MAAM,CAAA;QACV,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,WAAW,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,MAAM,CAAA;QACf,WAAW,EAAE,MAAM,CAAA;QACnB,gGAAgG;QAChG,iBAAiB,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;QACnD,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,gBAAgB,EAAE;YAC/C,QAAQ,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;YAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;YAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;SAClG,CAAC,CAAC,CAAA;KACJ,EAAE,CAAA;IACH,WAAW,EAAE,SAAS;QACpB,kFAAkF;QAClF,UAAU,EAAE,MAAM,CAAA;QAClB,SAAS,EAAE,MAAM,CAAA;QACjB,MAAM,EAAE,MAAM,CAAA;QACd,YAAY,EAAE,MAAM,CAAA;QACpB,IAAI,EAAE,MAAM,CAAA;QACZ,MAAM,EAAE,MAAM,CAAA;QACd,IAAI,EAAE,MAAM,CAAA;QACZ,YAAY,EAAE,MAAM,CAAA;QACpB,2FAA2F;QAC3F,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAA;QAC/B,4FAA4F;QAC5F,uBAAuB,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAA;KAC1D,EAAE,CAAA;IACH,2DAA2D;IAC3D,mBAAmB,EAAE,KAAK,CAAA;CAC3B"}
@@ -0,0 +1,3 @@
1
+ /** Host-side benchmark data. These APIs are deliberately not registered as model tools. */
2
+ export const benchmarkDomains = ['code', 'research', 'cross-day', 'proactivity', 'injection', 'revocation', 'goal-change', 'duplicate-action'];
3
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/benchmark/types.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,CAAU,CAAA"}
@@ -0,0 +1,36 @@
1
+ import type { TokenUsage } from '@deepseek-ai/dsh-llm';
2
+ export interface NormalizedTokenUsage {
3
+ /** Complete billed input: uncached input plus both cache classes. */
4
+ inputTokens: number;
5
+ /** The disjoint, non-cached input component reported by DSH. */
6
+ uncachedInputTokens: number;
7
+ outputTokens: number;
8
+ cacheReadTokens: number;
9
+ cacheWriteTokens: number;
10
+ /** Included in outputTokens, so never added to totals or cost separately. */
11
+ reasoningTokens: number;
12
+ totalTokens: number;
13
+ }
14
+ /** Per-million-token rates in USD micros for the disjoint DSH usage classes. */
15
+ export interface TokenUsageRates {
16
+ inputUsdMicrosPerMillionTokens: number | null;
17
+ outputUsdMicrosPerMillionTokens: number | null;
18
+ cacheReadUsdMicrosPerMillionTokens?: number | null;
19
+ cacheWriteUsdMicrosPerMillionTokens?: number | null;
20
+ }
21
+ /**
22
+ * Makes DSH's disjoint usage fields explicit for benchmark metering.
23
+ * `totalTokens`, when supplied, must be the provider's aggregate prompt plus output total.
24
+ */
25
+ export declare function normalizeTokenUsage(usage: TokenUsage): Readonly<NormalizedTokenUsage>;
26
+ /**
27
+ * Returns null when an observed billed class lacks an exact tariff. Costs round up once,
28
+ * after all priced classes are summed, to whole USD micros.
29
+ */
30
+ export declare function tokenUsageCost(usage: TokenUsage | Readonly<NormalizedTokenUsage>, rates: Readonly<TokenUsageRates>): number | null;
31
+ /**
32
+ * Conservative preflight cost for an input upper bound and output cap. Known cache rates
33
+ * participate in the worst-case input rate; absent cache rates do not invent a tariff.
34
+ */
35
+ export declare function tokenUsageReserveCost(inputUpperBound: number, maxOutputTokens: number, rates: Readonly<TokenUsageRates>): number | null;
36
+ //# sourceMappingURL=usage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"usage.d.ts","sourceRoot":"","sources":["../../src/benchmark/usage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAEtD,MAAM,WAAW,oBAAoB;IACnC,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,mBAAmB,EAAE,MAAM,CAAA;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,6EAA6E;IAC7E,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC9B,8BAA8B,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7C,+BAA+B,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9C,kCAAkC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAClD,mCAAmC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACpD;AAaD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,QAAQ,CAAC,oBAAoB,CAAC,CAcrF;AAQD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,QAAQ,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,IAAI,CAgBlI;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,eAAe,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,MAAM,GAAG,IAAI,CAYvI"}
@@ -0,0 +1,84 @@
1
+ const MAX = Number.MAX_SAFE_INTEGER;
2
+ const integer = (value) => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0;
3
+ const checkedSum = (...values) => {
4
+ let total = 0;
5
+ for (const value of values) {
6
+ if (total > MAX - value)
7
+ throw new Error('token usage total overflow');
8
+ total += value;
9
+ }
10
+ return total;
11
+ };
12
+ /**
13
+ * Makes DSH's disjoint usage fields explicit for benchmark metering.
14
+ * `totalTokens`, when supplied, must be the provider's aggregate prompt plus output total.
15
+ */
16
+ export function normalizeTokenUsage(usage) {
17
+ const uncachedInputTokens = usage.inputTokens;
18
+ const outputTokens = usage.outputTokens;
19
+ const cacheReadTokens = usage.cacheReadTokens ?? 0;
20
+ const cacheWriteTokens = usage.cacheWriteTokens ?? 0;
21
+ const reasoningTokens = usage.reasoningTokens ?? 0;
22
+ const reportedTotal = usage.totalTokens;
23
+ if (![uncachedInputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens].every(integer)
24
+ || (reportedTotal !== undefined && !integer(reportedTotal)))
25
+ throw new Error('invalid token usage');
26
+ if (reasoningTokens > outputTokens)
27
+ throw new Error('reasoning tokens exceed output tokens');
28
+ const inputTokens = checkedSum(uncachedInputTokens, cacheReadTokens, cacheWriteTokens);
29
+ const totalTokens = checkedSum(inputTokens, outputTokens);
30
+ if (reportedTotal !== undefined && reportedTotal !== totalTokens)
31
+ throw new Error('token usage total is inconsistent');
32
+ return Object.freeze({ inputTokens, uncachedInputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens, totalTokens });
33
+ }
34
+ const rate = (value) => {
35
+ if (value === null || value === undefined)
36
+ return null;
37
+ if (!integer(value))
38
+ throw new Error('invalid token rate');
39
+ return value;
40
+ };
41
+ /**
42
+ * Returns null when an observed billed class lacks an exact tariff. Costs round up once,
43
+ * after all priced classes are summed, to whole USD micros.
44
+ */
45
+ export function tokenUsageCost(usage, rates) {
46
+ const normalized = 'uncachedInputTokens' in usage ? usage : normalizeTokenUsage(usage);
47
+ const inputRate = rate(rates.inputUsdMicrosPerMillionTokens);
48
+ const outputRate = rate(rates.outputUsdMicrosPerMillionTokens);
49
+ const cacheReadRate = rate(rates.cacheReadUsdMicrosPerMillionTokens);
50
+ const cacheWriteRate = rate(rates.cacheWriteUsdMicrosPerMillionTokens);
51
+ if (inputRate === null || outputRate === null
52
+ || (normalized.cacheReadTokens !== 0 && cacheReadRate === null)
53
+ || (normalized.cacheWriteTokens !== 0 && cacheWriteRate === null))
54
+ return null;
55
+ const amount = BigInt(normalized.uncachedInputTokens) * BigInt(inputRate)
56
+ + BigInt(normalized.cacheReadTokens) * BigInt(cacheReadRate ?? 0)
57
+ + BigInt(normalized.cacheWriteTokens) * BigInt(cacheWriteRate ?? 0)
58
+ + BigInt(normalized.outputTokens) * BigInt(outputRate);
59
+ const rounded = (amount + 999999n) / 1000000n;
60
+ if (rounded > BigInt(MAX))
61
+ throw new Error('token usage cost overflow');
62
+ return Number(rounded);
63
+ }
64
+ /**
65
+ * Conservative preflight cost for an input upper bound and output cap. Known cache rates
66
+ * participate in the worst-case input rate; absent cache rates do not invent a tariff.
67
+ */
68
+ export function tokenUsageReserveCost(inputUpperBound, maxOutputTokens, rates) {
69
+ if (!integer(inputUpperBound) || !integer(maxOutputTokens))
70
+ throw new Error('invalid token reservation');
71
+ const inputRate = rate(rates.inputUsdMicrosPerMillionTokens);
72
+ const outputRate = rate(rates.outputUsdMicrosPerMillionTokens);
73
+ const cacheReadRate = rate(rates.cacheReadUsdMicrosPerMillionTokens);
74
+ const cacheWriteRate = rate(rates.cacheWriteUsdMicrosPerMillionTokens);
75
+ if (inputRate === null || outputRate === null)
76
+ return null;
77
+ const maximumInputRate = Math.max(inputRate, cacheReadRate ?? 0, cacheWriteRate ?? 0);
78
+ const amount = BigInt(inputUpperBound) * BigInt(maximumInputRate) + BigInt(maxOutputTokens) * BigInt(outputRate);
79
+ const rounded = (amount + 999999n) / 1000000n;
80
+ if (rounded > BigInt(MAX))
81
+ throw new Error('token usage cost overflow');
82
+ return Number(rounded);
83
+ }
84
+ //# sourceMappingURL=usage.js.map