oh-my-knowledge 1.0.0-beta.4 → 1.0.0-beta.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (276) hide show
  1. package/README.md +13 -1
  2. package/README.zh.md +13 -1
  3. package/dist/cli/commands/eval/index.js +4 -0
  4. package/dist/cli/commands/install.js +1 -1
  5. package/dist/cli/lib/evaluation-composition.d.ts +3 -0
  6. package/dist/cli/lib/evaluation-composition.js +96 -0
  7. package/dist/cli/lib/parse-run-config/variant-resolution.js +1 -1
  8. package/dist/cli/lib/run-core-evaluation.d.ts +1 -2
  9. package/dist/cli/lib/run-core-evaluation.js +46 -257
  10. package/dist/dsh-plugin/core-adapter.d.ts +2 -2
  11. package/dist/dsh-plugin/core-adapter.js +4 -4
  12. package/dist/dsh-plugin/core-command.d.ts +1 -1
  13. package/dist/dsh-plugin/core-command.js +28 -143
  14. package/dist/dsh-plugin/core-protocol.d.ts +1 -1
  15. package/dist/dsh-plugin/core-protocol.js +2 -2
  16. package/dist/eval-runtime/evaluate.d.ts +9 -569
  17. package/dist/eval-runtime/evaluate.js +5 -2717
  18. package/dist/eval-runtime/evaluation/capture-evaluators.d.ts +28 -0
  19. package/dist/eval-runtime/evaluation/capture-evaluators.js +516 -0
  20. package/dist/eval-runtime/evaluation/capture-input.d.ts +38 -0
  21. package/dist/eval-runtime/evaluation/capture-input.js +460 -0
  22. package/dist/eval-runtime/evaluation/conformance.d.ts +7 -0
  23. package/dist/eval-runtime/evaluation/conformance.js +108 -0
  24. package/dist/eval-runtime/evaluation/contracts.d.ts +517 -0
  25. package/dist/eval-runtime/evaluation/contracts.js +1 -0
  26. package/dist/eval-runtime/evaluation/definition.d.ts +12 -0
  27. package/dist/eval-runtime/evaluation/definition.js +659 -0
  28. package/dist/eval-runtime/evaluation/errors.d.ts +16 -0
  29. package/dist/eval-runtime/evaluation/errors.js +22 -0
  30. package/dist/eval-runtime/evaluation/ordering.d.ts +1 -0
  31. package/dist/eval-runtime/evaluation/ordering.js +3 -0
  32. package/dist/eval-runtime/evaluation/prepare.d.ts +12 -0
  33. package/dist/eval-runtime/evaluation/prepare.js +351 -0
  34. package/dist/eval-runtime/evaluation/result-state.d.ts +14 -0
  35. package/dist/eval-runtime/evaluation/result-state.js +58 -0
  36. package/dist/eval-runtime/evaluation/reuse.d.ts +21 -0
  37. package/dist/eval-runtime/evaluation/reuse.js +343 -0
  38. package/dist/eval-runtime/evaluation/schemas.d.ts +414 -0
  39. package/dist/eval-runtime/evaluation/schemas.js +260 -0
  40. package/dist/eval-runtime/evaluators/abstention.d.ts +54 -0
  41. package/dist/eval-runtime/evaluators/abstention.js +123 -0
  42. package/dist/eval-runtime/execution.d.ts +60 -0
  43. package/dist/eval-runtime/execution.js +183 -0
  44. package/dist/eval-runtime/index.d.ts +1 -0
  45. package/dist/eval-runtime/index.js +0 -1
  46. package/dist/eval-runtime/provider.d.ts +38 -0
  47. package/dist/eval-runtime/provider.js +13 -0
  48. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/api.js +1 -1
  49. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/protocol.js +1 -1
  50. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli-protocol.d.ts +1 -1
  51. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli-protocol.js +2 -2
  52. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli.js +5 -5
  53. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/resources.d.ts +3 -2
  54. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/resources.js +19 -7
  55. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-protocol.d.ts +1 -1
  56. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-protocol.js +2 -2
  57. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk.js +5 -5
  58. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-resources.d.ts +2 -2
  59. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-resources.js +3 -3
  60. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli.d.ts +1 -1
  61. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli.js +28 -33
  62. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/protocol-core.js +1 -1
  63. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/resources.d.ts +2 -1
  64. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/resources.js +12 -7
  65. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk.d.ts +1 -1
  66. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk.js +21 -11
  67. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/custom/command.js +55 -37
  68. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/index.d.ts +1 -1
  69. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/index.js +1 -1
  70. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/api.js +1 -1
  71. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/protocol.js +1 -1
  72. package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-protocol-core.js +1 -1
  73. package/dist/eval-workflows/hosts/adapters/shared/trial-workspace.d.ts +10 -0
  74. package/dist/eval-workflows/hosts/adapters/shared/trial-workspace.js +52 -0
  75. package/dist/eval-workflows/hosts/application.d.ts +82 -0
  76. package/dist/eval-workflows/hosts/application.js +121 -0
  77. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/assembly.d.ts +1 -1
  78. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/assembly.js +9 -10
  79. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/builtins.d.ts +1 -1
  80. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/builtins.js +13 -13
  81. package/dist/eval-workflows/{production-host → hosts/composition}/judge-provider-identity.d.ts +2 -2
  82. package/dist/eval-workflows/{production-host → hosts/composition}/judge-provider-identity.js +1 -1
  83. package/dist/eval-workflows/hosts/composition/node-preflight.d.ts +3 -0
  84. package/dist/eval-workflows/hosts/composition/node-preflight.js +163 -0
  85. package/dist/eval-workflows/hosts/composition/node-runtime.d.ts +22 -0
  86. package/dist/eval-workflows/hosts/composition/node-runtime.js +218 -0
  87. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/preflight.d.ts +2 -2
  88. package/dist/eval-workflows/{runtime-adapter → hosts/composition}/preflight.js +1 -1
  89. package/dist/eval-workflows/hosts/composition/registered-runtime.d.ts +20 -0
  90. package/dist/eval-workflows/hosts/composition/registered-runtime.js +19 -0
  91. package/dist/eval-workflows/{production-host → hosts/composition}/runtime-registry.d.ts +12 -6
  92. package/dist/eval-workflows/{production-host → hosts/composition}/runtime-registry.js +15 -8
  93. package/dist/eval-workflows/{runtime-adapter/composition.d.ts → hosts/composition/runtime.d.ts} +14 -10
  94. package/dist/eval-workflows/{runtime-adapter/composition.js → hosts/composition/runtime.js} +163 -187
  95. package/dist/eval-workflows/hosts/evaluators/index.d.ts +6 -0
  96. package/dist/eval-workflows/hosts/evaluators/index.js +6 -0
  97. package/dist/eval-workflows/hosts/evaluators/llm-assertion-factory.d.ts +6 -0
  98. package/dist/eval-workflows/hosts/evaluators/llm-assertion-factory.js +43 -0
  99. package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/rubric-judge.d.ts +1 -1
  100. package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/rubric-judge.js +1 -1
  101. package/dist/eval-workflows/{runtime-adapter → hosts}/index.d.ts +7 -7
  102. package/dist/eval-workflows/hosts/index.js +11 -0
  103. package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-cli-evaluation-resolver.d.ts +2 -2
  104. package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-cli-evaluation-resolver.js +9 -9
  105. package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-sample-content-resolver.d.ts +1 -1
  106. package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-sample-content-resolver.js +1 -1
  107. package/dist/eval-workflows/{production-host → hosts/input-resolution}/safe-http-content-resolver.d.ts +1 -1
  108. package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/access.js +1 -46
  109. package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/node.d.ts +1 -1
  110. package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/node.js +39 -39
  111. package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/types.d.ts +2 -1
  112. package/dist/eval-workflows/inputs/batch-discovery.d.ts +5 -0
  113. package/dist/eval-workflows/inputs/batch-discovery.js +28 -0
  114. package/dist/eval-workflows/inputs/contracts/config.d.ts +1 -1
  115. package/dist/eval-workflows/inputs/contracts/variant.d.ts +1 -8
  116. package/dist/eval-workflows/{runtime-adapter → measurement/analysis}/series-variance.d.ts +2 -2
  117. package/dist/eval-workflows/{runtime-adapter → measurement/analysis}/series-variance.js +2 -2
  118. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/execution-assertions.d.ts +3 -3
  119. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/execution-assertions.js +1 -1
  120. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/index.d.ts +1 -2
  121. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/index.js +1 -2
  122. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/llm-assertions.d.ts +12 -8
  123. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/llm-assertions.js +3 -40
  124. package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/output-assertions.d.ts +2 -2
  125. package/dist/eval-workflows/orchestration/evaluation-service.d.ts +26 -0
  126. package/dist/eval-workflows/orchestration/evaluation-service.js +81 -0
  127. package/dist/eval-workflows/orchestration/index.d.ts +5 -0
  128. package/dist/eval-workflows/orchestration/index.js +5 -0
  129. package/dist/eval-workflows/{production-host → orchestration}/measurement-design.js +11 -2
  130. package/dist/eval-workflows/{production-host → orchestration}/orchestration.d.ts +3 -3
  131. package/dist/eval-workflows/{production-host → orchestration}/orchestration.js +40 -45
  132. package/dist/eval-workflows/{production-host → orchestration}/workflow.d.ts +16 -18
  133. package/dist/eval-workflows/{production-host → orchestration}/workflow.js +42 -29
  134. package/dist/executors/openai/codex/cli-arguments.d.ts +12 -0
  135. package/dist/executors/openai/codex/cli-arguments.js +24 -0
  136. package/dist/executors/openai/codex/cli.d.ts +1 -0
  137. package/dist/executors/openai/codex/cli.js +9 -24
  138. package/dist/knowledge-artifacts/authoring/core-evolver.js +1 -1
  139. package/dist/knowledge-artifacts/contracts.d.ts +7 -0
  140. package/dist/knowledge-artifacts/doctor/index.js +1 -1
  141. package/dist/knowledge-artifacts/governance/source-probe.js +1 -1
  142. package/dist/{eval-workflows/inputs/skill-loader.d.ts → knowledge-artifacts/sources/artifact-resolution.d.ts} +2 -9
  143. package/dist/{eval-workflows/inputs/skill-loader.js → knowledge-artifacts/sources/artifact-resolution.js} +3 -33
  144. package/dist/knowledge-artifacts/sources/content-hash.js +2 -6
  145. package/dist/{eval-workflows/inputs/source-resolver.d.ts → knowledge-artifacts/sources/install-source.d.ts} +1 -1
  146. package/dist/{eval-workflows/inputs/source-resolver.js → knowledge-artifacts/sources/install-source.js} +3 -4
  147. package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.js +1 -1
  148. package/package.json +1 -1
  149. package/dist/cli/lib/source-probe.d.ts +0 -15
  150. package/dist/cli/lib/source-probe.js +0 -129
  151. package/dist/eval-workflows/production-host/index.d.ts +0 -10
  152. package/dist/eval-workflows/production-host/index.js +0 -10
  153. package/dist/eval-workflows/production-host/node-cli-composition.d.ts +0 -26
  154. package/dist/eval-workflows/production-host/node-cli-composition.js +0 -465
  155. package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.d.ts +0 -15
  156. package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.js +0 -7
  157. package/dist/eval-workflows/runtime-adapter/index.js +0 -11
  158. package/dist/eval-workflows/runtime-adapter/source-neutral-trace.d.ts +0 -1
  159. package/dist/eval-workflows/runtime-adapter/source-neutral-trace.js +0 -1
  160. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/api.d.ts +0 -0
  161. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/protocol.d.ts +0 -0
  162. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli.d.ts +0 -0
  163. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-runtime.d.ts +0 -0
  164. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-runtime.js +0 -0
  165. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk.d.ts +0 -0
  166. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-protocol.d.ts +0 -0
  167. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-protocol.js +0 -0
  168. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/content-identity.d.ts +0 -0
  169. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/content-identity.js +0 -0
  170. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/environment.d.ts +0 -0
  171. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/environment.js +0 -0
  172. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/protocol-core.d.ts +0 -0
  173. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-protocol.d.ts +0 -0
  174. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-protocol.js +0 -0
  175. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-runtime.d.ts +0 -0
  176. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-runtime.js +0 -0
  177. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/custom/command.d.ts +0 -0
  178. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/api.d.ts +0 -0
  179. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/protocol.d.ts +0 -0
  180. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-http.d.ts +0 -0
  181. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-http.js +0 -0
  182. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-protocol-core.d.ts +0 -0
  183. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/classified-environment.d.ts +0 -0
  184. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/classified-environment.js +0 -0
  185. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/content-identity.d.ts +0 -0
  186. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/content-identity.js +0 -0
  187. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/stateless-api-resources.d.ts +0 -0
  188. /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/stateless-api-resources.js +0 -0
  189. /package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/llm-judge-invocation.d.ts +0 -0
  190. /package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/llm-judge-invocation.js +0 -0
  191. /package/dist/eval-workflows/{production-host → hosts/input-resolution}/safe-http-content-resolver.js +0 -0
  192. /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/access.d.ts +0 -0
  193. /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/index.d.ts +0 -0
  194. /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/index.js +0 -0
  195. /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/types.js +0 -0
  196. /package/dist/eval-workflows/{runtime-adapter → hosts}/types.d.ts +0 -0
  197. /package/dist/eval-workflows/{runtime-adapter → hosts}/types.js +0 -0
  198. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node-contract.d.ts +0 -0
  199. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node-contract.js +0 -0
  200. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node.d.ts +0 -0
  201. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node.js +0 -0
  202. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-parameters.d.ts +0 -0
  203. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-parameters.js +0 -0
  204. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter-v1.d.ts +0 -0
  205. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter-v1.js +0 -0
  206. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter.d.ts +0 -0
  207. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter.js +0 -0
  208. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-table.d.ts +0 -0
  209. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-table.js +0 -0
  210. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/analysis-support.d.ts +0 -0
  211. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/analysis-support.js +0 -0
  212. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-node.d.ts +0 -0
  213. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-node.js +0 -0
  214. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-parameters.d.ts +0 -0
  215. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-parameters.js +0 -0
  216. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer.d.ts +0 -0
  217. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer.js +0 -0
  218. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-contract.d.ts +0 -0
  219. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-contract.js +0 -0
  220. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-support.d.ts +0 -0
  221. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-support.js +0 -0
  222. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-v2.d.ts +0 -0
  223. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-v2.js +0 -0
  224. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node.d.ts +0 -0
  225. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node.js +0 -0
  226. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-parameters.d.ts +0 -0
  227. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-parameters.js +0 -0
  228. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-source-adapter.d.ts +0 -0
  229. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-source-adapter.js +0 -0
  230. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table-v2.d.ts +0 -0
  231. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table-v2.js +0 -0
  232. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table.d.ts +0 -0
  233. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table.js +0 -0
  234. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node-contract.d.ts +0 -0
  235. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node-contract.js +0 -0
  236. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node.d.ts +0 -0
  237. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node.js +0 -0
  238. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-parameters.d.ts +0 -0
  239. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-parameters.js +0 -0
  240. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-source-adapter.d.ts +0 -0
  241. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-source-adapter.js +0 -0
  242. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-table.d.ts +0 -0
  243. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-table.js +0 -0
  244. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-node.d.ts +0 -0
  245. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-node.js +0 -0
  246. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-parameters.d.ts +0 -0
  247. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-parameters.js +0 -0
  248. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table-v1.d.ts +0 -0
  249. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table-v1.js +0 -0
  250. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table.d.ts +0 -0
  251. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table.js +0 -0
  252. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/index.d.ts +0 -0
  253. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/index.js +0 -0
  254. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-aggregation.d.ts +0 -0
  255. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-aggregation.js +0 -0
  256. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-agreement.d.ts +0 -0
  257. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-agreement.js +0 -0
  258. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision-parameters.d.ts +0 -0
  259. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision-parameters.js +0 -0
  260. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision.d.ts +0 -0
  261. /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision.js +0 -0
  262. /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/assertion-common.d.ts +0 -0
  263. /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/assertion-common.js +0 -0
  264. /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/output-assertions.js +0 -0
  265. /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.d.ts +0 -0
  266. /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.js +0 -0
  267. /package/dist/eval-workflows/{production-host → orchestration}/environment.d.ts +0 -0
  268. /package/dist/eval-workflows/{production-host → orchestration}/environment.js +0 -0
  269. /package/dist/eval-workflows/{production-host → orchestration}/holdout.d.ts +0 -0
  270. /package/dist/eval-workflows/{production-host → orchestration}/holdout.js +0 -0
  271. /package/dist/eval-workflows/{production-host → orchestration}/measurement-design.d.ts +0 -0
  272. /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.d.ts +0 -0
  273. /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.js +0 -0
  274. /package/dist/eval-workflows/{runtime-adapter/event-projection.d.ts → projections/runtime-progress.d.ts} +0 -0
  275. /package/dist/eval-workflows/{runtime-adapter/event-projection.js → projections/runtime-progress.js} +0 -0
  276. /package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.d.ts +0 -0
package/README.md CHANGED
@@ -22,6 +22,16 @@ Observe real-world performance, measure version differences, and determine wheth
22
22
 
23
23
  📖 **Full documentation: [oh-my-knowledge.pages.dev](https://oh-my-knowledge.pages.dev)** (searchable, English / 简体中文)
24
24
 
25
+ ## What knowledge means in OMK
26
+
27
+ **Entities are the things knowledge describes; knowledge expresses their states, relationships, or actions in a specific context.**
28
+
29
+ **How knowledge is expressed:**
30
+
31
+ > **Time scope + scenario + conditions + entity A + relation or action + entity B (if any)**
32
+
33
+ See [How OMK understands knowledge](docs/explanation/knowledge.md) for entity roles, shared relationships, and the distinction between knowledge and its carriers. Entity-based retrieval remains a design direction.
34
+
25
35
  ## What OMK helps you know
26
36
 
27
37
  | Decision | Command | Evidence you get |
@@ -35,6 +45,8 @@ Observe real-world performance, measure version differences, and determine wheth
35
45
 
36
46
  ## Quick start
37
47
 
48
+ Already have a Node.js service, retriever, or Agent? Start with [Use OMK in your service](docs/guides/eval-runtime.md): run an example without model credentials, choose a scorer, connect your service, and read the results. The command-line quickstart follows below.
49
+
38
50
  ```bash
39
51
  npm i -g oh-my-knowledge
40
52
  omk init demo && cd demo
@@ -228,7 +240,7 @@ The full docs are published at **[oh-my-knowledge.pages.dev](https://oh-my-knowl
228
240
  - **[CLI reference](docs/reference/cli.md)** — all top-level commands with bash examples and flag tables
229
241
  - **[Migrate to the 1.0 preview](docs/guides/v1-preview-migration.md)** — install channel, storage reset, sample protocol, CLI automation, and embedded API changes since 0.54
230
242
  - **[Evaluation Core cutover](docs/guides/eval-core-cutover.md)** — `BREAKING-SCHEMA` storage, resume, Studio, Gold, managed-evidence, and evolve migration
231
- - **[Embed OMK in a service](docs/guides/eval-runtime.md)** — canonical `evaluate()` plus repeat-run `evaluateSeries()` for Node.js and FaaS hosts
243
+ - **[Embed OMK in a service](docs/guides/eval-runtime.md)** — choose a scorer, connect a Node.js service, and interpret scores and failures
232
244
  - **[Storage layout v2](docs/specs/storage-layout-spec.md)** — project/global domains, compatibility boundary, and Git policy
233
245
  - **[Executors](docs/reference/executors.md)** & **[artifact layout](docs/reference/artifact-layout.md)** — built-in / custom executors; how `variant` resolves to an artifact + runtime context
234
246
  - **[How-to guides](docs/guides/agent-eval.md)** — [evaluate an agent](docs/guides/agent-eval.md) (project runtime context) and [use non-Claude models](docs/guides/non-claude-models.md) (GLM / Qwen / DeepSeek / Moonshot / Ollama)
package/README.zh.md CHANGED
@@ -22,6 +22,16 @@
22
22
 
23
23
  📖 **完整文档:[oh-my-knowledge.pages.dev/zh](https://oh-my-knowledge.pages.dev/zh/)**(可搜索,可切换英文)
24
24
 
25
+ ## OMK 中的知识是什么
26
+
27
+ **实体是知识所描述的事物;知识表达实体在特定上下文中的状态、关系或行为。**
28
+
29
+ **知识的表达形式:**
30
+
31
+ > **时间范围+场景+条件+实体 A+关系或行为+实体 B(如有)**
32
+
33
+ 实体角色、多实体关联及知识与载体的区别,见[OMK 如何理解知识](docs/zh/explanation/knowledge.md)。实体关联检索仍是设计方向。
34
+
25
35
  ## OMK 让你知道什么
26
36
 
27
37
  | 决策问题 | 命令 | 你会得到的证据 |
@@ -35,6 +45,8 @@
35
45
 
36
46
  ## 快速开始
37
47
 
48
+ 已有 Node.js 服务、检索系统或 Agent,希望接入评分与版本对比?从[在服务中使用 OMK](docs/zh/guides/eval-runtime.md)开始:先运行无需模型凭证的示例,再选择评分方法、接入自己的服务并解读结果。下面是命令行上手流程。
49
+
38
50
  ```bash
39
51
  npm i -g oh-my-knowledge
40
52
  omk init demo && cd demo
@@ -228,7 +240,7 @@ omk-mcp
228
240
  - **[CLI 参考](docs/zh/reference/cli.md)** —— 顶层命令的 bash 示例和 flag 表
229
241
  - **[迁移到 1.0 预览版](docs/zh/guides/v1-preview-migration.md)** —— 从 `0.54` 升级时的安装渠道、存储重建、用例协议、CLI 自动化与嵌入式 API 变化
230
242
  - **[Evaluation Core 生产切换](docs/zh/guides/eval-core-cutover.md)** —— `BREAKING-SCHEMA` 存储、resume、Studio、Gold、受管证据与 evolve 迁移
231
- - **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS 宿主的 canonical `evaluate()` 与重复运行 `evaluateSeries()`
243
+ - **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 选择评分方法、接入 Node.js 服务、解读分数与失败原因
232
244
  - **[存储布局 v2](docs/zh/specs/storage-layout-spec.md)** —— 项目/全局领域、迁移兼容与 Git 策略
233
245
  - **[执行器](docs/zh/reference/executors.md)** & **[知识载体布局](docs/zh/reference/artifact-layout.md)** —— 内置 / 自定义执行器;variant 如何解析为 artifact + runtime context
234
246
  - **[操作指南](docs/zh/guides/agent-eval.md)** —— [评测 agent](docs/zh/guides/agent-eval.md)(项目 runtime context)与[使用非 Claude 模型](docs/zh/guides/non-claude-models.md)(GLM / 通义 / DeepSeek / Moonshot / Ollama)
@@ -160,6 +160,10 @@ async function runEval(_args, flags, lang) {
160
160
  process.stdout.write(`Core: ${outcome.status?.runStatus ?? 'prepared'}/${outcome.status?.evidenceStatus ?? 'n/a'}/${outcome.status?.conclusionStatus ?? 'n/a'}\n`
161
161
  + `Gate: ${outcome.gate?.gateStatus ?? 'not-applicable'}${outcome.gate?.reasonCodes?.length ? `(${outcome.gate.reasonCodes.join(', ')})` : ''}\n`);
162
162
  }
163
+ // A blocked gate exits immediately through oclif; flush a piped JSON report first.
164
+ await new Promise((resolve, reject) => {
165
+ process.stdout.write('', (error) => error ? reject(error) : resolve());
166
+ });
163
167
  throw new CliExit(result.exitCode);
164
168
  }
165
169
  catch (err) {
@@ -6,7 +6,7 @@ import { Args, Flags } from '@oclif/core';
6
6
  import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
7
7
  import { BaseCommand } from '../oclif/base-command.js';
8
8
  import { tCli } from '../lib/i18n.js';
9
- import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../eval-workflows/inputs/source-resolver.js';
9
+ import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../knowledge-artifacts/sources/install-source.js';
10
10
  import { buildManagedArtifactRecord, hashArtifactSource, distributableCopyFilter, managedDir, recordManagedArtifact } from '../../knowledge-artifacts/governance/index.js';
11
11
  const BUILTIN_OMK_AGENT_SKILL_ID = 'omk-agent-skill';
12
12
  const INSTALLABLE_KINDS = ['skill', 'prompt', 'agent', 'workflow'];
@@ -0,0 +1,3 @@
1
+ import { type NodeEvaluationEnvironment, type ClassifiedEnvironmentEntry } from '../../eval-workflows/hosts/application.js';
2
+ export declare function classifyNodeCliEnvironment(environment: NodeJS.ProcessEnv): Readonly<Record<string, ClassifiedEnvironmentEntry>>;
3
+ export declare function captureNodeCliEvaluationEnvironment(environment?: NodeJS.ProcessEnv): NodeEvaluationEnvironment;
@@ -0,0 +1,96 @@
1
+ import { access, realpath } from 'node:fs/promises';
2
+ import { constants } from 'node:fs';
3
+ import { delimiter, isAbsolute, join } from 'node:path';
4
+ import { NodeCliProductionCompositionError, } from '../../eval-workflows/hosts/application.js';
5
+ const CREDENTIAL_ENVIRONMENT = new Set([
6
+ 'ANTHROPIC_API_KEY',
7
+ 'CLAUDE_CODE_OAUTH_TOKEN',
8
+ 'OPENAI_API_KEY',
9
+ ]);
10
+ const SECRET_TAINT_ENVIRONMENT = new Set([
11
+ 'ALL_PROXY',
12
+ 'HTTPS_PROXY',
13
+ 'HTTP_PROXY',
14
+ 'all_proxy',
15
+ 'http_proxy',
16
+ 'https_proxy',
17
+ ]);
18
+ const EFFECT_LOCATOR_ENVIRONMENT = new Set([
19
+ 'ALL_PROXY',
20
+ 'CODEX_HOME',
21
+ 'CURL_CA_BUNDLE',
22
+ 'HOME',
23
+ 'HTTPS_PROXY',
24
+ 'HTTP_PROXY',
25
+ 'NODE_EXTRA_CA_CERTS',
26
+ 'NO_PROXY',
27
+ 'PATH',
28
+ 'REQUESTS_CA_BUNDLE',
29
+ 'SSL_CERT_DIR',
30
+ 'SSL_CERT_FILE',
31
+ 'TMPDIR',
32
+ 'XDG_CACHE_HOME',
33
+ 'XDG_CONFIG_HOME',
34
+ 'XDG_DATA_HOME',
35
+ 'all_proxy',
36
+ 'http_proxy',
37
+ 'https_proxy',
38
+ 'no_proxy',
39
+ ]);
40
+ const BEHAVIOR_ENVIRONMENT = new Set([
41
+ 'LANG',
42
+ 'LC_ALL',
43
+ 'NO_COLOR',
44
+ ]);
45
+ export function classifyNodeCliEnvironment(environment) {
46
+ const keys = new Set([
47
+ ...CREDENTIAL_ENVIRONMENT,
48
+ ...EFFECT_LOCATOR_ENVIRONMENT,
49
+ ...BEHAVIOR_ENVIRONMENT,
50
+ ]);
51
+ return Object.freeze(Object.fromEntries([...keys].sort().flatMap((key) => {
52
+ const value = environment[key];
53
+ if (value === undefined)
54
+ return [];
55
+ const identity = CREDENTIAL_ENVIRONMENT.has(key)
56
+ ? { identityKind: 'credential' }
57
+ : EFFECT_LOCATOR_ENVIRONMENT.has(key)
58
+ ? { identityKind: 'effect-locator' }
59
+ : { identityKind: 'behavior', value };
60
+ return [[key, Object.freeze({
61
+ value,
62
+ identity,
63
+ ...(SECRET_TAINT_ENVIRONMENT.has(key) ? { outputTaint: 'secret' } : {}),
64
+ })]];
65
+ })));
66
+ }
67
+ async function resolveExecutable(command, environment) {
68
+ const candidates = isAbsolute(command)
69
+ ? [command]
70
+ : (environment.PATH ?? '').split(delimiter).filter(Boolean).map((directory) => (join(directory, command)));
71
+ for (const candidate of candidates) {
72
+ try {
73
+ await access(candidate, constants.X_OK);
74
+ return await realpath(candidate);
75
+ }
76
+ catch {
77
+ // Continue through the explicit PATH snapshot.
78
+ }
79
+ }
80
+ throw new NodeCliProductionCompositionError('NODE_CLI_EXECUTABLE_UNAVAILABLE', `执行器「${command}」在当前 PATH 中不可用。`);
81
+ }
82
+ function requiredCredential(environment, key) {
83
+ const value = environment[key]?.trim();
84
+ if (value !== undefined && value !== '')
85
+ return value;
86
+ throw new NodeCliProductionCompositionError('NODE_CLI_CREDENTIAL_MISSING', `执行器需要环境变量 ${key}。`);
87
+ }
88
+ export function captureNodeCliEvaluationEnvironment(environment = process.env) {
89
+ const captured = Object.freeze({ ...environment });
90
+ return Object.freeze({
91
+ environment: captured,
92
+ classifiedEnvironment: classifyNodeCliEnvironment(captured),
93
+ resolveExecutable: (command) => resolveExecutable(command, captured),
94
+ requiredCredential: (key) => requiredCredential(captured, key),
95
+ });
96
+ }
@@ -18,7 +18,7 @@
18
18
  * 否则会被解析成两个 artifact 后用 `x` / `x#2` 消歧,变成同一份 skill 自比,
19
19
  * 悄悄废掉「control 不能等于 treatment」的测量保护。
20
20
  */
21
- import { discoverVariants, variantExprToSkillName, variantIdentity, parseVariantCwd } from '../../../eval-workflows/inputs/skill-loader.js';
21
+ import { discoverVariants, variantExprToSkillName, variantIdentity, parseVariantCwd } from '../../../knowledge-artifacts/sources/artifact-resolution.js';
22
22
  import { configVariantsToSpecs } from '../../../eval-workflows/inputs/eval-config.js';
23
23
  /** CLI 的 --control / --treatment 只收 artifact 身份。`name@cwd` 语法已移除:撞到就抛迁移错误,
24
24
  * 引导用户改用 --control-cwd / --treatment-cwd 或 eval.yaml 的 variant.cwd。git 修订语法 `@{...}`
@@ -1,5 +1,5 @@
1
1
  import type { EvalConfig } from '../../eval-workflows/inputs/contracts/config.js';
2
- import { type CoreRunArtifactStore, type StoredCoreRunArtifacts } from '../../eval-workflows/artifact-store/index.js';
2
+ import { type CoreRunArtifactStore, type StoredCoreRunArtifacts } from '../../eval-workflows/hosts/application.js';
3
3
  import type { RunConfig } from './parse-run-config.js';
4
4
  import type { CliLang } from './i18n.js';
5
5
  export interface RunCoreEvaluationCommandInput {
@@ -17,5 +17,4 @@ export interface RunCoreEvaluationCommandResult {
17
17
  readonly stored?: StoredCoreRunArtifacts;
18
18
  readonly outputDirectory: string;
19
19
  }
20
- /** Executes the single authoritative Node CLI → Evaluation Core production path. */
21
20
  export declare function runCoreEvaluationCommand(input: Readonly<RunCoreEvaluationCommandInput>): Promise<RunCoreEvaluationCommandResult>;
@@ -1,33 +1,10 @@
1
1
  import { existsSync } from 'node:fs';
2
2
  import { join, resolve } from 'node:path';
3
- import { compileCliEvaluationInput, parseCliEvaluationRequest, } from '../../eval-workflows/input-compilation/index.js';
4
- import { createNodeCoreBatchArtifactStore, createNodeCoreContentStore, createNodeCoreRunArtifactStore, createOverlayCoreRunArtifactStore, } from '../../eval-workflows/artifact-store/index.js';
5
- import { createNodeCliProductionComposition, createProductionEvaluationHost, executeProductionEvaluationSeries, persistCoreArtifactSidecars, resolveNodeCliEvaluationRequest, } from '../../eval-workflows/production-host/index.js';
6
- import { projectCoreCliDryRun, projectCoreCliBatchOutcome, projectCoreCliRunOutcome, projectCoreCliSeriesOutcome, } from '../../eval-workflows/projections/cli.js';
7
- import { projectCoreManagedEvidence } from '../../eval-workflows/projections/managed.js';
8
- import { discoverBatchSkills } from '../../eval-workflows/inputs/skill-loader.js';
3
+ import { createNodeEvaluationApplication, parseCliEvaluationRequest, } from '../../eval-workflows/hosts/application.js';
4
+ import { captureNodeCliEvaluationEnvironment } from './evaluation-composition.js';
9
5
  import { withLocalizedSampleDiscovery } from './localized-sample-discovery.js';
10
6
  import { projectReportsDir, globalReportsDir } from '../../evidence/storage/directories.js';
11
- import { generateRunId } from '../../evidence/storage/run-id.js';
12
- import { globalLayout, projectLayout } from '../../evidence/storage/layout.js';
13
- function runStoreForOutput(outputDirectory, primaryContentResolver) {
14
- const primary = createNodeCoreRunArtifactStore(outputDirectory, {
15
- contentResolver: primaryContentResolver,
16
- });
17
- const resolved = resolve(outputDirectory);
18
- const project = projectLayout();
19
- const global = globalLayout();
20
- const fallbackDirs = resolved === resolve(project.evalDir)
21
- ? [global.evalDir]
22
- : [];
23
- const unique = [...new Set(fallbackDirs.map((dir) => resolve(dir)))]
24
- .filter((dir) => dir !== resolved);
25
- if (unique.length === 0)
26
- return primary;
27
- return createOverlayCoreRunArtifactStore(primary, unique.map((dir) => (createNodeCoreRunArtifactStore(dir, {
28
- contentResolver: createNodeCoreContentStore(join(dir, 'content')),
29
- }))));
30
- }
7
+ import { globalLayout } from '../../evidence/storage/layout.js';
31
8
  function requestFor(input, projectRoot) {
32
9
  const projectMcpConfig = join(projectRoot, '.mcp.json');
33
10
  return parseCliEvaluationRequest({
@@ -98,246 +75,58 @@ async function announceCoreReport(artifacts, store, outputDirectory, serve, lang
98
75
  const { openWorkbench } = await import('./open-workbench.js');
99
76
  await openWorkbench(reportUrl, lang);
100
77
  }
101
- async function appendManagedEvidenceSafely(stored, enabled, lang) {
102
- if (!enabled)
103
- return;
104
- try {
105
- const { recordCoreEvalEvidence } = await import('../../knowledge-artifacts/governance/evidence.js');
106
- const written = recordCoreEvalEvidence(projectCoreManagedEvidence(stored));
107
- if (written.length > 0)
78
+ function renderNotice(notice, lang) {
79
+ switch (notice.noticeKind) {
80
+ case 'doctor-skipped':
108
81
  process.stderr.write(lang === 'zh'
109
- ? `已写入 ${written.length} 条 Core 受管证据。\n`
110
- : `Recorded ${written.length} Core managed evidence reference(s).\n`);
111
- }
112
- catch (error) {
113
- const message = error instanceof Error ? error.message : String(error);
114
- process.stderr.write(lang === 'zh'
115
- ? `警告:Core 受管证据写入失败:${message}\n`
116
- : `Warning: failed to record Core managed evidence: ${message}\n`);
117
- }
118
- }
119
- async function persistArtifactGraph(stored, outputDirectory, cwd) {
120
- await persistCoreArtifactSidecars({ source: stored, outputDirectory, cwd });
121
- }
122
- function resumeRunId(request) {
123
- const locator = request.values.orchestration.resumeSourceLocator?.trim();
124
- if (locator === undefined || locator === '')
125
- return undefined;
126
- if (!/^[a-z0-9]+(?:[._-][a-z0-9]+)*$/i.test(locator)) {
127
- throw new TypeError('--resume 只接受 Core runId,不接受旧报告路径。');
82
+ ? '警告:--skip-doctor 已开启,Core 静态健康检查已跳过;依赖正确性由用户负责。\n'
83
+ : 'Warning: --skip-doctor is enabled; Core static health checks were skipped and dependency correctness is user-owned.\n');
84
+ break;
85
+ case 'batch-item':
86
+ process.stderr.write(lang === 'zh' ? `\nCore Batch:${notice.name}\n` : `\nCore Batch: ${notice.name}\n`);
87
+ break;
88
+ case 'managed-evidence-recorded':
89
+ process.stderr.write(lang === 'zh' ? `已写入 ${notice.count} 条 Core 受管证据。\n` : `Recorded ${notice.count} Core managed evidence reference(s).\n`);
90
+ break;
91
+ case 'managed-evidence-failed': {
92
+ const message = notice.error instanceof Error ? notice.error.message : String(notice.error);
93
+ process.stderr.write(lang === 'zh' ? `警告:Core 受管证据写入失败:${message}\n` : `Warning: failed to record Core managed evidence: ${message}\n`);
94
+ break;
95
+ }
96
+ case 'series-managed-evidence-skipped':
97
+ process.stderr.write(lang === 'zh'
98
+ ? 'Core Series 不写入单次 member 受管证据;需由预注册的 Series 总体决定投影后再纳入生命周期。\n'
99
+ : 'Core Series does not write single-member managed evidence; lifecycle admission requires a preregistered Series-level decision projection.\n');
128
100
  }
129
- return locator;
130
101
  }
131
- /** Executes the single authoritative Node CLI → Evaluation Core production path. */
132
102
  export async function runCoreEvaluationCommand(input) {
133
103
  const projectRoot = resolve(input.projectRoot ?? process.cwd());
134
104
  const machineLayout = globalLayout(input.environment?.OMK_HOME);
135
- const request = requestFor(input, projectRoot);
136
- if (request.values.orchestration.preflight.doctor === 'skip') {
137
- process.stderr.write(input.lang === 'zh'
138
- ? '警告:--skip-doctor 已开启,Core 静态健康检查已跳过;依赖正确性由用户负责。\n'
139
- : 'Warning: --skip-doctor is enabled; Core static health checks were skipped and dependency correctness is user-owned.\n');
140
- }
141
- if (request.values.orchestration.batch) {
142
- const entries = withLocalizedSampleDiscovery(() => discoverBatchSkills(resolve(projectRoot, request.values.locators.skillDirectory)), input.lang);
143
- if (entries.length === 0) {
144
- throw new TypeError(input.lang === 'zh'
145
- ? `没有找到带 canonical 私有用例的目录 skill:${request.values.locators.skillDirectory}。每个 skill 应使用 <skill>/.omk/eval-samples.json 或 eval-samples.yaml。`
146
- : `No directory skill with canonical private samples found in ${request.values.locators.skillDirectory}. Use <skill>/.omk/eval-samples.json or eval-samples.yaml for each skill.`);
147
- }
148
- if (request.values.orchestration.resumeSourceLocator !== undefined) {
149
- throw new TypeError('Batch resume 必须按 child runId 显式执行,不能复用旧聚合报告。');
150
- }
151
- const children = [];
152
- for (const entry of entries) {
153
- process.stderr.write(input.lang === 'zh'
154
- ? `\nCore Batch:${entry.name}\n`
155
- : `\nCore Batch: ${entry.name}\n`);
156
- children.push(await runCoreEvaluationCommand({
157
- ...input,
158
- flags: {
159
- ...input.flags,
160
- batch: undefined,
161
- control: 'baseline',
162
- treatment: entry.skillPath,
163
- samples: entry.samplesPath,
164
- 'no-serve': true,
165
- },
166
- }));
167
- }
168
- const outputDirectory = children[0].outputDirectory;
169
- if (request.values.orchestration.dryRun) {
170
- return {
171
- exitCode: 0,
172
- output: Object.freeze({
173
- projectionKind: 'core-cli-batch-dry-run',
174
- children: Object.freeze(entries.map((entry, index) => Object.freeze({
175
- itemId: entry.name,
176
- plan: children[index].output,
177
- }))),
178
- }),
179
- outputDirectory,
180
- };
181
- }
182
- const storedChildren = children.map((child) => {
183
- if (child.stored === undefined)
184
- throw new Error('Core Batch child 缺少持久化产物。');
185
- return child.stored;
186
- });
187
- const contentResolver = createNodeCoreContentStore(join(outputDirectory, 'content'));
188
- const runStore = input.store ?? runStoreForOutput(outputDirectory, contentResolver);
189
- const batchId = generateRunId(['batch']);
190
- const batch = await createNodeCoreBatchArtifactStore(outputDirectory, runStore).save({
191
- batchId,
192
- createdAt: new Date().toISOString(),
193
- children: storedChildren.map((child, index) => ({
194
- itemId: entries[index].name,
195
- runId: child.manifest.runId,
196
- })),
197
- });
198
- const output = projectCoreCliBatchOutcome({
199
- batch,
200
- children: storedChildren,
201
- exitMode: request.values.presentation.exitMode,
202
- diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
203
- ? 'enabled'
204
- : 'disabled',
105
+ const application = createNodeEvaluationApplication(captureNodeCliEvaluationEnvironment(input.environment));
106
+ try {
107
+ const result = await application.run({
108
+ request: requestFor(input, projectRoot), projectRoot,
109
+ materializationRoot: machineLayout.resolvedInputsDir, resourceLeaseRoot: machineLayout.resourceLeasesDir,
110
+ store: input.store,
111
+ createProgressSink: () => emitProgress(input.lang),
112
+ onNotice: (notice) => renderNotice(notice, input.lang),
113
+ requestForBatchItem: (entry) => requestFor({ ...input, flags: { ...input.flags, batch: undefined, control: 'baseline', treatment: entry.skillPath, samples: entry.samplesPath, 'no-serve': true } }, projectRoot),
114
+ async onCompleted(completed, request) {
115
+ if (completed.outcomeKind === 'run')
116
+ await announceCoreReport(completed.artifacts, completed.store, completed.outputDirectory, request.values.presentation.serve, input.lang);
117
+ if (completed.outcomeKind === 'series')
118
+ process.stderr.write(input.lang === 'zh'
119
+ ? `Core Series 已完成:${completed.outcome.seriesId}(${completed.outcome.members.length} 个独立 run)\n`
120
+ : `Core Series completed: ${completed.outcome.seriesId} (${completed.outcome.members.length} independent runs)\n`);
121
+ },
205
122
  });
206
123
  return {
207
- exitCode: output.gate.exitCode,
208
- output,
209
- outputDirectory,
124
+ exitCode: result.outcomeKind === 'dry-run' || result.outcomeKind === 'batch-dry-run' ? 0 : result.outcome.gate.exitCode,
125
+ output: result.outcome, outputDirectory: result.outputDirectory,
126
+ ...(result.outcomeKind === 'run' ? { stored: result.artifacts } : {}),
210
127
  };
211
128
  }
212
- const outputDirectory = resolve(projectRoot, request.values.presentation.outputDirectoryLocator);
213
- const resolved = await resolveNodeCliEvaluationRequest(request, {
214
- projectRoot,
215
- materializationRoot: machineLayout.resolvedInputsDir,
216
- ...(input.environment === undefined ? {} : { environment: input.environment }),
217
- ...(request.values.orchestration.repeatCount > 1
218
- ? { seriesInstanceId: generateRunId(['series']) }
219
- : {}),
220
- });
221
- const compiled = compileCliEvaluationInput(resolved);
222
- const composition = await createNodeCliProductionComposition({
223
- compiled,
224
- projectRoot,
225
- outputDirectory,
226
- resourceLeaseRoot: machineLayout.resourceLeasesDir,
227
- environment: input.environment,
228
- });
229
- const store = input.store ?? runStoreForOutput(outputDirectory, composition.support.contentResolver);
230
- const host = {
231
- compiled,
232
- ...composition,
233
- artifactStore: store,
234
- };
235
- if (compiled.orchestration.dryRun) {
236
- const prepared = await createProductionEvaluationHost(host).prepare();
237
- const output = projectCoreCliDryRun({ plan: prepared.plan, preflight: prepared.preflight });
238
- return { exitCode: 0, output, outputDirectory };
239
- }
240
- const sourceRunId = resumeRunId(request);
241
- const independentSeries = compiled.orchestration.independentSeries;
242
- if (independentSeries !== undefined) {
243
- if (sourceRunId !== undefined) {
244
- throw new TypeError('Independent Series resume 必须按 member runId 单独执行。');
245
- }
246
- const createdAt = new Date().toISOString();
247
- const series = await executeProductionEvaluationSeries({
248
- host,
249
- members: independentSeries.memberships.map((membership) => ({
250
- runId: generateRunId([membership.memberId]),
251
- createdAt,
252
- progressSink: emitProgress(input.lang),
253
- })),
254
- bundleId: generateRunId(['series-analysis']),
255
- reportId: generateRunId(['series-report']),
256
- });
257
- await series.result;
258
- const evolution = await series.evolution;
259
- if (evolution === undefined)
260
- throw new Error('Core Series 未完成,无法生成 evolution evidence。');
261
- const memberArtifacts = await Promise.all(series.members.map(async (member) => {
262
- if (member.executionStatus !== 'started')
263
- throw member.error;
264
- await member.run.result;
265
- const persistence = await member.run.persistence;
266
- if (persistence.persistenceStatus !== 'stored') {
267
- if (persistence.persistenceStatus === 'failed')
268
- throw persistence.error;
269
- throw new Error(`Core Series member 产物未保存:${persistence.reasonCode}`);
270
- }
271
- return persistence.artifacts;
272
- }));
273
- const output = projectCoreCliSeriesOutcome({
274
- evolution,
275
- members: memberArtifacts,
276
- exitMode: request.values.presentation.exitMode,
277
- diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
278
- ? 'enabled'
279
- : 'disabled',
280
- });
281
- for (const artifacts of memberArtifacts) {
282
- await persistArtifactGraph(artifacts, outputDirectory, projectRoot);
283
- }
284
- if (compiled.orchestration.managedEvidence === 'append') {
285
- process.stderr.write(input.lang === 'zh'
286
- ? 'Core Series 不写入单次 member 受管证据;需由预注册的 Series 总体决定投影后再纳入生命周期。\n'
287
- : 'Core Series does not write single-member managed evidence; lifecycle admission requires a preregistered Series-level decision projection.\n');
288
- }
289
- process.stderr.write(input.lang === 'zh'
290
- ? `Core Series 已完成:${output.seriesId}(${output.members.length} 个独立 run)\n`
291
- : `Core Series completed: ${output.seriesId} (${output.members.length} independent runs)\n`);
292
- return { exitCode: output.gate.exitCode, output, outputDirectory };
293
- }
294
- const prepared = await createProductionEvaluationHost(host).prepare();
295
- let stored;
296
- if (sourceRunId !== undefined) {
297
- const admission = await prepared.admitResume({
298
- locator: { locatorKind: 'core-run', runId: sourceRunId },
299
- policy: {
300
- rejectionMode: 'fail-closed',
301
- minimumSourceTrust: 'unknown',
302
- cacheReceiptMode: 'allow-indeterminate',
303
- budgetVerificationMode: 'allow-indeterminate',
304
- },
305
- });
306
- if (admission.disposition !== 'reuse') {
307
- throw new Error(`Core resume 被拒绝:${admission.reasonCode}`);
308
- }
309
- stored = admission.artifacts;
310
- }
311
- else {
312
- const targetIds = prepared.plan.execution.targets.map((target) => target.targetId);
313
- const runId = generateRunId(targetIds);
314
- const run = await prepared.execute({
315
- runId,
316
- createdAt: new Date().toISOString(),
317
- progressSink: emitProgress(input.lang),
318
- });
319
- await run.result;
320
- const persistence = await run.persistence;
321
- if (persistence.persistenceStatus !== 'stored') {
322
- if (persistence.persistenceStatus === 'failed')
323
- throw persistence.error;
324
- throw new Error(`Core 产物未保存:${persistence.reasonCode}`);
325
- }
326
- stored = persistence.artifacts;
129
+ catch (error) {
130
+ return withLocalizedSampleDiscovery(() => { throw error; }, input.lang);
327
131
  }
328
- const output = projectCoreCliRunOutcome(stored, {
329
- exitMode: request.values.presentation.exitMode,
330
- diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
331
- ? 'enabled'
332
- : 'disabled',
333
- });
334
- await persistArtifactGraph(stored, outputDirectory, projectRoot);
335
- await appendManagedEvidenceSafely(stored, compiled.orchestration.managedEvidence === 'append', input.lang);
336
- await announceCoreReport(stored, store, outputDirectory, request.values.presentation.serve, input.lang);
337
- return {
338
- exitCode: output.gate.exitCode,
339
- output,
340
- stored,
341
- outputDirectory,
342
- };
343
132
  }
@@ -1,7 +1,7 @@
1
1
  import { type EvaluationDefinition } from '../eval-core/contracts/index.js';
2
2
  import { type ExecutionExecutor } from '../eval-core/execution/index.js';
3
- import type { OmkBindingResourceLeaseAccess } from '../eval-workflows/runtime-adapter/resource-leases/types.js';
4
- import type { RuntimeBindingOf } from '../eval-workflows/runtime-adapter/types.js';
3
+ import type { OmkBindingResourceLeaseAccess } from '../eval-workflows/hosts/resource-leases/types.js';
4
+ import type { RuntimeBindingOf } from '../eval-workflows/hosts/types.js';
5
5
  import type { DshAgentLike, DshHostContextLike } from './host-executor.js';
6
6
  export { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, createDshHostCoreSchemaValidators, } from './core-protocol.js';
7
7
  export declare const DEFAULT_DSH_HOST_CORE_MAX_INPUT_BYTES: number;
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
2
2
  import { JsonValueSchema, RuntimeIdentitySchema, canonicalizeJson, deepFreezeCanonicalJson, digestCanonicalJson, } from '../eval-core/contracts/index.js';
3
3
  import { ExecutionPortFailure, } from '../eval-core/execution/index.js';
4
4
  import { createDshHostRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
5
- import { captureClaudeCliRunState, captureClaudeCliTarget, disposeClaudeCliTrial, openClaudeCliTrial, } from '../eval-workflows/runtime-adapter/adapters/claude/resources.js';
5
+ import { captureClaudeCliRunState, captureClaudeCliTarget, disposeClaudeCliTrial, openClaudeCliTrial, } from '../eval-workflows/hosts/adapters/claude/resources.js';
6
6
  import { createSameProcessExecutorAdapter } from '../eval-runtime/adapters/same-process.js';
7
7
  import { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, dshHostCoreExecutorCapabilities, parseDshHostCoreResult, } from './core-protocol.js';
8
8
  import { supportsDshTraceEventType } from './trace-adapter.js';
@@ -716,7 +716,7 @@ export async function createDshHostCoreExecutorAdapter(input) {
716
716
  }
717
717
  runState.acquireTrial();
718
718
  try {
719
- return openClaudeCliTrial(trial, runState, host.maxInputBytes, RESOURCE_PROFILE);
719
+ return await openClaudeCliTrial(trial, runState, host.maxInputBytes, RESOURCE_PROFILE);
720
720
  }
721
721
  catch (error) {
722
722
  await runState.releaseTrial();
@@ -726,8 +726,8 @@ export async function createDshHostCoreExecutorAdapter(input) {
726
726
  execute({ runState, trialState, attempt, scope }) {
727
727
  return executeDshHost(host, target, runState, trialState, attempt, scope.operationIsolationKey);
728
728
  },
729
- disposeTrial({ runState }) {
730
- return disposeClaudeCliTrial(runState, RESOURCE_PROFILE);
729
+ disposeTrial({ runState, trialState }) {
730
+ return disposeClaudeCliTrial(runState, trialState, RESOURCE_PROFILE);
731
731
  },
732
732
  disposeRun({ runState }) {
733
733
  return runState.requestDispose();
@@ -1,4 +1,4 @@
1
- import { type StoredCoreRunArtifacts } from '../eval-workflows/artifact-store/index.js';
1
+ import { type StoredCoreRunArtifacts } from '../eval-workflows/hosts/application.js';
2
2
  import type { CoreCliRunOutcome, CoreCliSeriesOutcome } from '../eval-workflows/projections/contracts.js';
3
3
  import type { EvalConfig } from '../eval-workflows/inputs/contracts/config.js';
4
4
  import { type DshAgentLike, type DshHostContextLike } from './host-executor.js';