oh-my-knowledge 1.0.0-beta.4 → 1.0.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/README.zh.md +13 -1
- package/dist/cli/commands/eval/index.js +4 -0
- package/dist/cli/commands/install.js +1 -1
- package/dist/cli/lib/evaluation-composition.d.ts +3 -0
- package/dist/cli/lib/evaluation-composition.js +96 -0
- package/dist/cli/lib/parse-run-config/variant-resolution.js +1 -1
- package/dist/cli/lib/run-core-evaluation.d.ts +1 -2
- package/dist/cli/lib/run-core-evaluation.js +46 -257
- package/dist/dsh-plugin/core-adapter.d.ts +2 -2
- package/dist/dsh-plugin/core-adapter.js +4 -4
- package/dist/dsh-plugin/core-command.d.ts +1 -1
- package/dist/dsh-plugin/core-command.js +28 -143
- package/dist/dsh-plugin/core-protocol.d.ts +1 -1
- package/dist/dsh-plugin/core-protocol.js +2 -2
- package/dist/eval-runtime/evaluate.d.ts +9 -569
- package/dist/eval-runtime/evaluate.js +5 -2717
- package/dist/eval-runtime/evaluation/capture-evaluators.d.ts +28 -0
- package/dist/eval-runtime/evaluation/capture-evaluators.js +516 -0
- package/dist/eval-runtime/evaluation/capture-input.d.ts +38 -0
- package/dist/eval-runtime/evaluation/capture-input.js +460 -0
- package/dist/eval-runtime/evaluation/conformance.d.ts +7 -0
- package/dist/eval-runtime/evaluation/conformance.js +108 -0
- package/dist/eval-runtime/evaluation/contracts.d.ts +517 -0
- package/dist/eval-runtime/evaluation/contracts.js +1 -0
- package/dist/eval-runtime/evaluation/definition.d.ts +12 -0
- package/dist/eval-runtime/evaluation/definition.js +659 -0
- package/dist/eval-runtime/evaluation/errors.d.ts +16 -0
- package/dist/eval-runtime/evaluation/errors.js +22 -0
- package/dist/eval-runtime/evaluation/ordering.d.ts +1 -0
- package/dist/eval-runtime/evaluation/ordering.js +3 -0
- package/dist/eval-runtime/evaluation/prepare.d.ts +12 -0
- package/dist/eval-runtime/evaluation/prepare.js +351 -0
- package/dist/eval-runtime/evaluation/result-state.d.ts +14 -0
- package/dist/eval-runtime/evaluation/result-state.js +58 -0
- package/dist/eval-runtime/evaluation/reuse.d.ts +21 -0
- package/dist/eval-runtime/evaluation/reuse.js +343 -0
- package/dist/eval-runtime/evaluation/schemas.d.ts +414 -0
- package/dist/eval-runtime/evaluation/schemas.js +260 -0
- package/dist/eval-runtime/evaluators/abstention.d.ts +54 -0
- package/dist/eval-runtime/evaluators/abstention.js +123 -0
- package/dist/eval-runtime/execution.d.ts +60 -0
- package/dist/eval-runtime/execution.js +183 -0
- package/dist/eval-runtime/index.d.ts +1 -0
- package/dist/eval-runtime/index.js +0 -1
- package/dist/eval-runtime/provider.d.ts +38 -0
- package/dist/eval-runtime/provider.js +13 -0
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/api.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/protocol.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli-protocol.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli-protocol.js +2 -2
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli.js +5 -5
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/resources.d.ts +3 -2
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/resources.js +19 -7
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-protocol.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-protocol.js +2 -2
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk.js +5 -5
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-resources.d.ts +2 -2
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-resources.js +3 -3
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli.js +28 -33
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/protocol-core.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/resources.d.ts +2 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/resources.js +12 -7
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk.js +21 -11
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/custom/command.js +55 -37
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/index.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/index.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/api.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/protocol.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-protocol-core.js +1 -1
- package/dist/eval-workflows/hosts/adapters/shared/trial-workspace.d.ts +10 -0
- package/dist/eval-workflows/hosts/adapters/shared/trial-workspace.js +52 -0
- package/dist/eval-workflows/hosts/application.d.ts +82 -0
- package/dist/eval-workflows/hosts/application.js +121 -0
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/assembly.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/assembly.js +9 -10
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/builtins.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/builtins.js +13 -13
- package/dist/eval-workflows/{production-host → hosts/composition}/judge-provider-identity.d.ts +2 -2
- package/dist/eval-workflows/{production-host → hosts/composition}/judge-provider-identity.js +1 -1
- package/dist/eval-workflows/hosts/composition/node-preflight.d.ts +3 -0
- package/dist/eval-workflows/hosts/composition/node-preflight.js +163 -0
- package/dist/eval-workflows/hosts/composition/node-runtime.d.ts +22 -0
- package/dist/eval-workflows/hosts/composition/node-runtime.js +218 -0
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/preflight.d.ts +2 -2
- package/dist/eval-workflows/{runtime-adapter → hosts/composition}/preflight.js +1 -1
- package/dist/eval-workflows/hosts/composition/registered-runtime.d.ts +20 -0
- package/dist/eval-workflows/hosts/composition/registered-runtime.js +19 -0
- package/dist/eval-workflows/{production-host → hosts/composition}/runtime-registry.d.ts +12 -6
- package/dist/eval-workflows/{production-host → hosts/composition}/runtime-registry.js +15 -8
- package/dist/eval-workflows/{runtime-adapter/composition.d.ts → hosts/composition/runtime.d.ts} +14 -10
- package/dist/eval-workflows/{runtime-adapter/composition.js → hosts/composition/runtime.js} +163 -187
- package/dist/eval-workflows/hosts/evaluators/index.d.ts +6 -0
- package/dist/eval-workflows/hosts/evaluators/index.js +6 -0
- package/dist/eval-workflows/hosts/evaluators/llm-assertion-factory.d.ts +6 -0
- package/dist/eval-workflows/hosts/evaluators/llm-assertion-factory.js +43 -0
- package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/rubric-judge.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/rubric-judge.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/index.d.ts +7 -7
- package/dist/eval-workflows/hosts/index.js +11 -0
- package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-cli-evaluation-resolver.d.ts +2 -2
- package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-cli-evaluation-resolver.js +9 -9
- package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-sample-content-resolver.d.ts +1 -1
- package/dist/eval-workflows/{production-host → hosts/input-resolution}/node-sample-content-resolver.js +1 -1
- package/dist/eval-workflows/{production-host → hosts/input-resolution}/safe-http-content-resolver.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/access.js +1 -46
- package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/node.d.ts +1 -1
- package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/node.js +39 -39
- package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/types.d.ts +2 -1
- package/dist/eval-workflows/inputs/batch-discovery.d.ts +5 -0
- package/dist/eval-workflows/inputs/batch-discovery.js +28 -0
- package/dist/eval-workflows/inputs/contracts/config.d.ts +1 -1
- package/dist/eval-workflows/inputs/contracts/variant.d.ts +1 -8
- package/dist/eval-workflows/{runtime-adapter → measurement/analysis}/series-variance.d.ts +2 -2
- package/dist/eval-workflows/{runtime-adapter → measurement/analysis}/series-variance.js +2 -2
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/execution-assertions.d.ts +3 -3
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/execution-assertions.js +1 -1
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/index.d.ts +1 -2
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/index.js +1 -2
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/llm-assertions.d.ts +12 -8
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/llm-assertions.js +3 -40
- package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/output-assertions.d.ts +2 -2
- package/dist/eval-workflows/orchestration/evaluation-service.d.ts +26 -0
- package/dist/eval-workflows/orchestration/evaluation-service.js +81 -0
- package/dist/eval-workflows/orchestration/index.d.ts +5 -0
- package/dist/eval-workflows/orchestration/index.js +5 -0
- package/dist/eval-workflows/{production-host → orchestration}/measurement-design.js +11 -2
- package/dist/eval-workflows/{production-host → orchestration}/orchestration.d.ts +3 -3
- package/dist/eval-workflows/{production-host → orchestration}/orchestration.js +40 -45
- package/dist/eval-workflows/{production-host → orchestration}/workflow.d.ts +16 -18
- package/dist/eval-workflows/{production-host → orchestration}/workflow.js +42 -29
- package/dist/executors/openai/codex/cli-arguments.d.ts +12 -0
- package/dist/executors/openai/codex/cli-arguments.js +24 -0
- package/dist/executors/openai/codex/cli.d.ts +1 -0
- package/dist/executors/openai/codex/cli.js +9 -24
- package/dist/knowledge-artifacts/authoring/core-evolver.js +1 -1
- package/dist/knowledge-artifacts/contracts.d.ts +7 -0
- package/dist/knowledge-artifacts/doctor/index.js +1 -1
- package/dist/knowledge-artifacts/governance/source-probe.js +1 -1
- package/dist/{eval-workflows/inputs/skill-loader.d.ts → knowledge-artifacts/sources/artifact-resolution.d.ts} +2 -9
- package/dist/{eval-workflows/inputs/skill-loader.js → knowledge-artifacts/sources/artifact-resolution.js} +3 -33
- package/dist/knowledge-artifacts/sources/content-hash.js +2 -6
- package/dist/{eval-workflows/inputs/source-resolver.d.ts → knowledge-artifacts/sources/install-source.d.ts} +1 -1
- package/dist/{eval-workflows/inputs/source-resolver.js → knowledge-artifacts/sources/install-source.js} +3 -4
- package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.js +1 -1
- package/package.json +1 -1
- package/dist/cli/lib/source-probe.d.ts +0 -15
- package/dist/cli/lib/source-probe.js +0 -129
- package/dist/eval-workflows/production-host/index.d.ts +0 -10
- package/dist/eval-workflows/production-host/index.js +0 -10
- package/dist/eval-workflows/production-host/node-cli-composition.d.ts +0 -26
- package/dist/eval-workflows/production-host/node-cli-composition.js +0 -465
- package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.d.ts +0 -15
- package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.js +0 -7
- package/dist/eval-workflows/runtime-adapter/index.js +0 -11
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.d.ts +0 -1
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.js +0 -1
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/api.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/anthropic/protocol.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/cli.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-runtime.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk-runtime.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/claude/sdk.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-protocol.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/cli-protocol.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/content-identity.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/content-identity.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/environment.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/environment.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/protocol-core.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-protocol.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-protocol.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-runtime.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/codex/sdk-runtime.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/custom/command.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/api.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/openai/protocol.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-http.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-http.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/api-protocol-core.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/classified-environment.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/classified-environment.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/content-identity.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/content-identity.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/stateless-api-resources.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/adapters/shared/stateless-api-resources.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/llm-judge-invocation.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/evaluators/llm-judge-invocation.js +0 -0
- /package/dist/eval-workflows/{production-host → hosts/input-resolution}/safe-http-content-resolver.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/access.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/index.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/index.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/resource-leases/types.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/types.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → hosts}/types.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node-contract.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node-contract.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-node.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter-v1.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter-v1.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-source-adapter.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-table.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/agreement-table.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/analysis-support.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/analysis-support.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-node.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-node.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/assertion-layer.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-contract.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-contract.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-support.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-support.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-v2.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node-v2.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-node.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-source-adapter.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-source-adapter.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table-v2.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table-v2.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/bootstrap-family-table.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node-contract.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node-contract.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-node.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-source-adapter.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-source-adapter.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-table.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/composite-table.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-node.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-node.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table-v1.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table-v1.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/dimension-table.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/index.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/index.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-aggregation.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-aggregation.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-agreement.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/judge-agreement.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision-parameters.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision-parameters.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/analysis/release-decision.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/assertion-common.d.ts +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/assertion-common.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter → measurement}/evaluators/output-assertions.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/environment.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/environment.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/holdout.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/holdout.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/measurement-design.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.js +0 -0
- /package/dist/eval-workflows/{runtime-adapter/event-projection.d.ts → projections/runtime-progress.d.ts} +0 -0
- /package/dist/eval-workflows/{runtime-adapter/event-projection.js → projections/runtime-progress.js} +0 -0
- /package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.d.ts +0 -0
package/README.md
CHANGED
|
@@ -22,6 +22,16 @@ Observe real-world performance, measure version differences, and determine wheth
|
|
|
22
22
|
|
|
23
23
|
📖 **Full documentation: [oh-my-knowledge.pages.dev](https://oh-my-knowledge.pages.dev)** (searchable, English / 简体中文)
|
|
24
24
|
|
|
25
|
+
## What knowledge means in OMK
|
|
26
|
+
|
|
27
|
+
**Entities are the things knowledge describes; knowledge expresses their states, relationships, or actions in a specific context.**
|
|
28
|
+
|
|
29
|
+
**How knowledge is expressed:**
|
|
30
|
+
|
|
31
|
+
> **Time scope + scenario + conditions + entity A + relation or action + entity B (if any)**
|
|
32
|
+
|
|
33
|
+
See [How OMK understands knowledge](docs/explanation/knowledge.md) for entity roles, shared relationships, and the distinction between knowledge and its carriers. Entity-based retrieval remains a design direction.
|
|
34
|
+
|
|
25
35
|
## What OMK helps you know
|
|
26
36
|
|
|
27
37
|
| Decision | Command | Evidence you get |
|
|
@@ -35,6 +45,8 @@ Observe real-world performance, measure version differences, and determine wheth
|
|
|
35
45
|
|
|
36
46
|
## Quick start
|
|
37
47
|
|
|
48
|
+
Already have a Node.js service, retriever, or Agent? Start with [Use OMK in your service](docs/guides/eval-runtime.md): run an example without model credentials, choose a scorer, connect your service, and read the results. The command-line quickstart follows below.
|
|
49
|
+
|
|
38
50
|
```bash
|
|
39
51
|
npm i -g oh-my-knowledge
|
|
40
52
|
omk init demo && cd demo
|
|
@@ -228,7 +240,7 @@ The full docs are published at **[oh-my-knowledge.pages.dev](https://oh-my-knowl
|
|
|
228
240
|
- **[CLI reference](docs/reference/cli.md)** — all top-level commands with bash examples and flag tables
|
|
229
241
|
- **[Migrate to the 1.0 preview](docs/guides/v1-preview-migration.md)** — install channel, storage reset, sample protocol, CLI automation, and embedded API changes since 0.54
|
|
230
242
|
- **[Evaluation Core cutover](docs/guides/eval-core-cutover.md)** — `BREAKING-SCHEMA` storage, resume, Studio, Gold, managed-evidence, and evolve migration
|
|
231
|
-
- **[Embed OMK in a service](docs/guides/eval-runtime.md)** —
|
|
243
|
+
- **[Embed OMK in a service](docs/guides/eval-runtime.md)** — choose a scorer, connect a Node.js service, and interpret scores and failures
|
|
232
244
|
- **[Storage layout v2](docs/specs/storage-layout-spec.md)** — project/global domains, compatibility boundary, and Git policy
|
|
233
245
|
- **[Executors](docs/reference/executors.md)** & **[artifact layout](docs/reference/artifact-layout.md)** — built-in / custom executors; how `variant` resolves to an artifact + runtime context
|
|
234
246
|
- **[How-to guides](docs/guides/agent-eval.md)** — [evaluate an agent](docs/guides/agent-eval.md) (project runtime context) and [use non-Claude models](docs/guides/non-claude-models.md) (GLM / Qwen / DeepSeek / Moonshot / Ollama)
|
package/README.zh.md
CHANGED
|
@@ -22,6 +22,16 @@
|
|
|
22
22
|
|
|
23
23
|
📖 **完整文档:[oh-my-knowledge.pages.dev/zh](https://oh-my-knowledge.pages.dev/zh/)**(可搜索,可切换英文)
|
|
24
24
|
|
|
25
|
+
## OMK 中的知识是什么
|
|
26
|
+
|
|
27
|
+
**实体是知识所描述的事物;知识表达实体在特定上下文中的状态、关系或行为。**
|
|
28
|
+
|
|
29
|
+
**知识的表达形式:**
|
|
30
|
+
|
|
31
|
+
> **时间范围+场景+条件+实体 A+关系或行为+实体 B(如有)**
|
|
32
|
+
|
|
33
|
+
实体角色、多实体关联及知识与载体的区别,见[OMK 如何理解知识](docs/zh/explanation/knowledge.md)。实体关联检索仍是设计方向。
|
|
34
|
+
|
|
25
35
|
## OMK 让你知道什么
|
|
26
36
|
|
|
27
37
|
| 决策问题 | 命令 | 你会得到的证据 |
|
|
@@ -35,6 +45,8 @@
|
|
|
35
45
|
|
|
36
46
|
## 快速开始
|
|
37
47
|
|
|
48
|
+
已有 Node.js 服务、检索系统或 Agent,希望接入评分与版本对比?从[在服务中使用 OMK](docs/zh/guides/eval-runtime.md)开始:先运行无需模型凭证的示例,再选择评分方法、接入自己的服务并解读结果。下面是命令行上手流程。
|
|
49
|
+
|
|
38
50
|
```bash
|
|
39
51
|
npm i -g oh-my-knowledge
|
|
40
52
|
omk init demo && cd demo
|
|
@@ -228,7 +240,7 @@ omk-mcp
|
|
|
228
240
|
- **[CLI 参考](docs/zh/reference/cli.md)** —— 顶层命令的 bash 示例和 flag 表
|
|
229
241
|
- **[迁移到 1.0 预览版](docs/zh/guides/v1-preview-migration.md)** —— 从 `0.54` 升级时的安装渠道、存储重建、用例协议、CLI 自动化与嵌入式 API 变化
|
|
230
242
|
- **[Evaluation Core 生产切换](docs/zh/guides/eval-core-cutover.md)** —— `BREAKING-SCHEMA` 存储、resume、Studio、Gold、受管证据与 evolve 迁移
|
|
231
|
-
- **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** ——
|
|
243
|
+
- **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 选择评分方法、接入 Node.js 服务、解读分数与失败原因
|
|
232
244
|
- **[存储布局 v2](docs/zh/specs/storage-layout-spec.md)** —— 项目/全局领域、迁移兼容与 Git 策略
|
|
233
245
|
- **[执行器](docs/zh/reference/executors.md)** & **[知识载体布局](docs/zh/reference/artifact-layout.md)** —— 内置 / 自定义执行器;variant 如何解析为 artifact + runtime context
|
|
234
246
|
- **[操作指南](docs/zh/guides/agent-eval.md)** —— [评测 agent](docs/zh/guides/agent-eval.md)(项目 runtime context)与[使用非 Claude 模型](docs/zh/guides/non-claude-models.md)(GLM / 通义 / DeepSeek / Moonshot / Ollama)
|
|
@@ -160,6 +160,10 @@ async function runEval(_args, flags, lang) {
|
|
|
160
160
|
process.stdout.write(`Core: ${outcome.status?.runStatus ?? 'prepared'}/${outcome.status?.evidenceStatus ?? 'n/a'}/${outcome.status?.conclusionStatus ?? 'n/a'}\n`
|
|
161
161
|
+ `Gate: ${outcome.gate?.gateStatus ?? 'not-applicable'}${outcome.gate?.reasonCodes?.length ? `(${outcome.gate.reasonCodes.join(', ')})` : ''}\n`);
|
|
162
162
|
}
|
|
163
|
+
// A blocked gate exits immediately through oclif; flush a piped JSON report first.
|
|
164
|
+
await new Promise((resolve, reject) => {
|
|
165
|
+
process.stdout.write('', (error) => error ? reject(error) : resolve());
|
|
166
|
+
});
|
|
163
167
|
throw new CliExit(result.exitCode);
|
|
164
168
|
}
|
|
165
169
|
catch (err) {
|
|
@@ -6,7 +6,7 @@ import { Args, Flags } from '@oclif/core';
|
|
|
6
6
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
7
7
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
8
8
|
import { tCli } from '../lib/i18n.js';
|
|
9
|
-
import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../
|
|
9
|
+
import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../knowledge-artifacts/sources/install-source.js';
|
|
10
10
|
import { buildManagedArtifactRecord, hashArtifactSource, distributableCopyFilter, managedDir, recordManagedArtifact } from '../../knowledge-artifacts/governance/index.js';
|
|
11
11
|
const BUILTIN_OMK_AGENT_SKILL_ID = 'omk-agent-skill';
|
|
12
12
|
const INSTALLABLE_KINDS = ['skill', 'prompt', 'agent', 'workflow'];
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { type NodeEvaluationEnvironment, type ClassifiedEnvironmentEntry } from '../../eval-workflows/hosts/application.js';
|
|
2
|
+
export declare function classifyNodeCliEnvironment(environment: NodeJS.ProcessEnv): Readonly<Record<string, ClassifiedEnvironmentEntry>>;
|
|
3
|
+
export declare function captureNodeCliEvaluationEnvironment(environment?: NodeJS.ProcessEnv): NodeEvaluationEnvironment;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { access, realpath } from 'node:fs/promises';
|
|
2
|
+
import { constants } from 'node:fs';
|
|
3
|
+
import { delimiter, isAbsolute, join } from 'node:path';
|
|
4
|
+
import { NodeCliProductionCompositionError, } from '../../eval-workflows/hosts/application.js';
|
|
5
|
+
const CREDENTIAL_ENVIRONMENT = new Set([
|
|
6
|
+
'ANTHROPIC_API_KEY',
|
|
7
|
+
'CLAUDE_CODE_OAUTH_TOKEN',
|
|
8
|
+
'OPENAI_API_KEY',
|
|
9
|
+
]);
|
|
10
|
+
const SECRET_TAINT_ENVIRONMENT = new Set([
|
|
11
|
+
'ALL_PROXY',
|
|
12
|
+
'HTTPS_PROXY',
|
|
13
|
+
'HTTP_PROXY',
|
|
14
|
+
'all_proxy',
|
|
15
|
+
'http_proxy',
|
|
16
|
+
'https_proxy',
|
|
17
|
+
]);
|
|
18
|
+
const EFFECT_LOCATOR_ENVIRONMENT = new Set([
|
|
19
|
+
'ALL_PROXY',
|
|
20
|
+
'CODEX_HOME',
|
|
21
|
+
'CURL_CA_BUNDLE',
|
|
22
|
+
'HOME',
|
|
23
|
+
'HTTPS_PROXY',
|
|
24
|
+
'HTTP_PROXY',
|
|
25
|
+
'NODE_EXTRA_CA_CERTS',
|
|
26
|
+
'NO_PROXY',
|
|
27
|
+
'PATH',
|
|
28
|
+
'REQUESTS_CA_BUNDLE',
|
|
29
|
+
'SSL_CERT_DIR',
|
|
30
|
+
'SSL_CERT_FILE',
|
|
31
|
+
'TMPDIR',
|
|
32
|
+
'XDG_CACHE_HOME',
|
|
33
|
+
'XDG_CONFIG_HOME',
|
|
34
|
+
'XDG_DATA_HOME',
|
|
35
|
+
'all_proxy',
|
|
36
|
+
'http_proxy',
|
|
37
|
+
'https_proxy',
|
|
38
|
+
'no_proxy',
|
|
39
|
+
]);
|
|
40
|
+
const BEHAVIOR_ENVIRONMENT = new Set([
|
|
41
|
+
'LANG',
|
|
42
|
+
'LC_ALL',
|
|
43
|
+
'NO_COLOR',
|
|
44
|
+
]);
|
|
45
|
+
export function classifyNodeCliEnvironment(environment) {
|
|
46
|
+
const keys = new Set([
|
|
47
|
+
...CREDENTIAL_ENVIRONMENT,
|
|
48
|
+
...EFFECT_LOCATOR_ENVIRONMENT,
|
|
49
|
+
...BEHAVIOR_ENVIRONMENT,
|
|
50
|
+
]);
|
|
51
|
+
return Object.freeze(Object.fromEntries([...keys].sort().flatMap((key) => {
|
|
52
|
+
const value = environment[key];
|
|
53
|
+
if (value === undefined)
|
|
54
|
+
return [];
|
|
55
|
+
const identity = CREDENTIAL_ENVIRONMENT.has(key)
|
|
56
|
+
? { identityKind: 'credential' }
|
|
57
|
+
: EFFECT_LOCATOR_ENVIRONMENT.has(key)
|
|
58
|
+
? { identityKind: 'effect-locator' }
|
|
59
|
+
: { identityKind: 'behavior', value };
|
|
60
|
+
return [[key, Object.freeze({
|
|
61
|
+
value,
|
|
62
|
+
identity,
|
|
63
|
+
...(SECRET_TAINT_ENVIRONMENT.has(key) ? { outputTaint: 'secret' } : {}),
|
|
64
|
+
})]];
|
|
65
|
+
})));
|
|
66
|
+
}
|
|
67
|
+
async function resolveExecutable(command, environment) {
|
|
68
|
+
const candidates = isAbsolute(command)
|
|
69
|
+
? [command]
|
|
70
|
+
: (environment.PATH ?? '').split(delimiter).filter(Boolean).map((directory) => (join(directory, command)));
|
|
71
|
+
for (const candidate of candidates) {
|
|
72
|
+
try {
|
|
73
|
+
await access(candidate, constants.X_OK);
|
|
74
|
+
return await realpath(candidate);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
// Continue through the explicit PATH snapshot.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
throw new NodeCliProductionCompositionError('NODE_CLI_EXECUTABLE_UNAVAILABLE', `执行器「${command}」在当前 PATH 中不可用。`);
|
|
81
|
+
}
|
|
82
|
+
function requiredCredential(environment, key) {
|
|
83
|
+
const value = environment[key]?.trim();
|
|
84
|
+
if (value !== undefined && value !== '')
|
|
85
|
+
return value;
|
|
86
|
+
throw new NodeCliProductionCompositionError('NODE_CLI_CREDENTIAL_MISSING', `执行器需要环境变量 ${key}。`);
|
|
87
|
+
}
|
|
88
|
+
export function captureNodeCliEvaluationEnvironment(environment = process.env) {
|
|
89
|
+
const captured = Object.freeze({ ...environment });
|
|
90
|
+
return Object.freeze({
|
|
91
|
+
environment: captured,
|
|
92
|
+
classifiedEnvironment: classifyNodeCliEnvironment(captured),
|
|
93
|
+
resolveExecutable: (command) => resolveExecutable(command, captured),
|
|
94
|
+
requiredCredential: (key) => requiredCredential(captured, key),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
* 否则会被解析成两个 artifact 后用 `x` / `x#2` 消歧,变成同一份 skill 自比,
|
|
19
19
|
* 悄悄废掉「control 不能等于 treatment」的测量保护。
|
|
20
20
|
*/
|
|
21
|
-
import { discoverVariants, variantExprToSkillName, variantIdentity, parseVariantCwd } from '../../../
|
|
21
|
+
import { discoverVariants, variantExprToSkillName, variantIdentity, parseVariantCwd } from '../../../knowledge-artifacts/sources/artifact-resolution.js';
|
|
22
22
|
import { configVariantsToSpecs } from '../../../eval-workflows/inputs/eval-config.js';
|
|
23
23
|
/** CLI 的 --control / --treatment 只收 artifact 身份。`name@cwd` 语法已移除:撞到就抛迁移错误,
|
|
24
24
|
* 引导用户改用 --control-cwd / --treatment-cwd 或 eval.yaml 的 variant.cwd。git 修订语法 `@{...}`
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { EvalConfig } from '../../eval-workflows/inputs/contracts/config.js';
|
|
2
|
-
import { type CoreRunArtifactStore, type StoredCoreRunArtifacts } from '../../eval-workflows/
|
|
2
|
+
import { type CoreRunArtifactStore, type StoredCoreRunArtifacts } from '../../eval-workflows/hosts/application.js';
|
|
3
3
|
import type { RunConfig } from './parse-run-config.js';
|
|
4
4
|
import type { CliLang } from './i18n.js';
|
|
5
5
|
export interface RunCoreEvaluationCommandInput {
|
|
@@ -17,5 +17,4 @@ export interface RunCoreEvaluationCommandResult {
|
|
|
17
17
|
readonly stored?: StoredCoreRunArtifacts;
|
|
18
18
|
readonly outputDirectory: string;
|
|
19
19
|
}
|
|
20
|
-
/** Executes the single authoritative Node CLI → Evaluation Core production path. */
|
|
21
20
|
export declare function runCoreEvaluationCommand(input: Readonly<RunCoreEvaluationCommandInput>): Promise<RunCoreEvaluationCommandResult>;
|
|
@@ -1,33 +1,10 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { createNodeCliProductionComposition, createProductionEvaluationHost, executeProductionEvaluationSeries, persistCoreArtifactSidecars, resolveNodeCliEvaluationRequest, } from '../../eval-workflows/production-host/index.js';
|
|
6
|
-
import { projectCoreCliDryRun, projectCoreCliBatchOutcome, projectCoreCliRunOutcome, projectCoreCliSeriesOutcome, } from '../../eval-workflows/projections/cli.js';
|
|
7
|
-
import { projectCoreManagedEvidence } from '../../eval-workflows/projections/managed.js';
|
|
8
|
-
import { discoverBatchSkills } from '../../eval-workflows/inputs/skill-loader.js';
|
|
3
|
+
import { createNodeEvaluationApplication, parseCliEvaluationRequest, } from '../../eval-workflows/hosts/application.js';
|
|
4
|
+
import { captureNodeCliEvaluationEnvironment } from './evaluation-composition.js';
|
|
9
5
|
import { withLocalizedSampleDiscovery } from './localized-sample-discovery.js';
|
|
10
6
|
import { projectReportsDir, globalReportsDir } from '../../evidence/storage/directories.js';
|
|
11
|
-
import {
|
|
12
|
-
import { globalLayout, projectLayout } from '../../evidence/storage/layout.js';
|
|
13
|
-
function runStoreForOutput(outputDirectory, primaryContentResolver) {
|
|
14
|
-
const primary = createNodeCoreRunArtifactStore(outputDirectory, {
|
|
15
|
-
contentResolver: primaryContentResolver,
|
|
16
|
-
});
|
|
17
|
-
const resolved = resolve(outputDirectory);
|
|
18
|
-
const project = projectLayout();
|
|
19
|
-
const global = globalLayout();
|
|
20
|
-
const fallbackDirs = resolved === resolve(project.evalDir)
|
|
21
|
-
? [global.evalDir]
|
|
22
|
-
: [];
|
|
23
|
-
const unique = [...new Set(fallbackDirs.map((dir) => resolve(dir)))]
|
|
24
|
-
.filter((dir) => dir !== resolved);
|
|
25
|
-
if (unique.length === 0)
|
|
26
|
-
return primary;
|
|
27
|
-
return createOverlayCoreRunArtifactStore(primary, unique.map((dir) => (createNodeCoreRunArtifactStore(dir, {
|
|
28
|
-
contentResolver: createNodeCoreContentStore(join(dir, 'content')),
|
|
29
|
-
}))));
|
|
30
|
-
}
|
|
7
|
+
import { globalLayout } from '../../evidence/storage/layout.js';
|
|
31
8
|
function requestFor(input, projectRoot) {
|
|
32
9
|
const projectMcpConfig = join(projectRoot, '.mcp.json');
|
|
33
10
|
return parseCliEvaluationRequest({
|
|
@@ -98,246 +75,58 @@ async function announceCoreReport(artifacts, store, outputDirectory, serve, lang
|
|
|
98
75
|
const { openWorkbench } = await import('./open-workbench.js');
|
|
99
76
|
await openWorkbench(reportUrl, lang);
|
|
100
77
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
try {
|
|
105
|
-
const { recordCoreEvalEvidence } = await import('../../knowledge-artifacts/governance/evidence.js');
|
|
106
|
-
const written = recordCoreEvalEvidence(projectCoreManagedEvidence(stored));
|
|
107
|
-
if (written.length > 0)
|
|
78
|
+
function renderNotice(notice, lang) {
|
|
79
|
+
switch (notice.noticeKind) {
|
|
80
|
+
case 'doctor-skipped':
|
|
108
81
|
process.stderr.write(lang === 'zh'
|
|
109
|
-
?
|
|
110
|
-
:
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
throw new TypeError('--resume 只接受 Core runId,不接受旧报告路径。');
|
|
82
|
+
? '警告:--skip-doctor 已开启,Core 静态健康检查已跳过;依赖正确性由用户负责。\n'
|
|
83
|
+
: 'Warning: --skip-doctor is enabled; Core static health checks were skipped and dependency correctness is user-owned.\n');
|
|
84
|
+
break;
|
|
85
|
+
case 'batch-item':
|
|
86
|
+
process.stderr.write(lang === 'zh' ? `\nCore Batch:${notice.name}\n` : `\nCore Batch: ${notice.name}\n`);
|
|
87
|
+
break;
|
|
88
|
+
case 'managed-evidence-recorded':
|
|
89
|
+
process.stderr.write(lang === 'zh' ? `已写入 ${notice.count} 条 Core 受管证据。\n` : `Recorded ${notice.count} Core managed evidence reference(s).\n`);
|
|
90
|
+
break;
|
|
91
|
+
case 'managed-evidence-failed': {
|
|
92
|
+
const message = notice.error instanceof Error ? notice.error.message : String(notice.error);
|
|
93
|
+
process.stderr.write(lang === 'zh' ? `警告:Core 受管证据写入失败:${message}\n` : `Warning: failed to record Core managed evidence: ${message}\n`);
|
|
94
|
+
break;
|
|
95
|
+
}
|
|
96
|
+
case 'series-managed-evidence-skipped':
|
|
97
|
+
process.stderr.write(lang === 'zh'
|
|
98
|
+
? 'Core Series 不写入单次 member 受管证据;需由预注册的 Series 总体决定投影后再纳入生命周期。\n'
|
|
99
|
+
: 'Core Series does not write single-member managed evidence; lifecycle admission requires a preregistered Series-level decision projection.\n');
|
|
128
100
|
}
|
|
129
|
-
return locator;
|
|
130
101
|
}
|
|
131
|
-
/** Executes the single authoritative Node CLI → Evaluation Core production path. */
|
|
132
102
|
export async function runCoreEvaluationCommand(input) {
|
|
133
103
|
const projectRoot = resolve(input.projectRoot ?? process.cwd());
|
|
134
104
|
const machineLayout = globalLayout(input.environment?.OMK_HOME);
|
|
135
|
-
const
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
:
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
for (const entry of entries) {
|
|
153
|
-
process.stderr.write(input.lang === 'zh'
|
|
154
|
-
? `\nCore Batch:${entry.name}\n`
|
|
155
|
-
: `\nCore Batch: ${entry.name}\n`);
|
|
156
|
-
children.push(await runCoreEvaluationCommand({
|
|
157
|
-
...input,
|
|
158
|
-
flags: {
|
|
159
|
-
...input.flags,
|
|
160
|
-
batch: undefined,
|
|
161
|
-
control: 'baseline',
|
|
162
|
-
treatment: entry.skillPath,
|
|
163
|
-
samples: entry.samplesPath,
|
|
164
|
-
'no-serve': true,
|
|
165
|
-
},
|
|
166
|
-
}));
|
|
167
|
-
}
|
|
168
|
-
const outputDirectory = children[0].outputDirectory;
|
|
169
|
-
if (request.values.orchestration.dryRun) {
|
|
170
|
-
return {
|
|
171
|
-
exitCode: 0,
|
|
172
|
-
output: Object.freeze({
|
|
173
|
-
projectionKind: 'core-cli-batch-dry-run',
|
|
174
|
-
children: Object.freeze(entries.map((entry, index) => Object.freeze({
|
|
175
|
-
itemId: entry.name,
|
|
176
|
-
plan: children[index].output,
|
|
177
|
-
}))),
|
|
178
|
-
}),
|
|
179
|
-
outputDirectory,
|
|
180
|
-
};
|
|
181
|
-
}
|
|
182
|
-
const storedChildren = children.map((child) => {
|
|
183
|
-
if (child.stored === undefined)
|
|
184
|
-
throw new Error('Core Batch child 缺少持久化产物。');
|
|
185
|
-
return child.stored;
|
|
186
|
-
});
|
|
187
|
-
const contentResolver = createNodeCoreContentStore(join(outputDirectory, 'content'));
|
|
188
|
-
const runStore = input.store ?? runStoreForOutput(outputDirectory, contentResolver);
|
|
189
|
-
const batchId = generateRunId(['batch']);
|
|
190
|
-
const batch = await createNodeCoreBatchArtifactStore(outputDirectory, runStore).save({
|
|
191
|
-
batchId,
|
|
192
|
-
createdAt: new Date().toISOString(),
|
|
193
|
-
children: storedChildren.map((child, index) => ({
|
|
194
|
-
itemId: entries[index].name,
|
|
195
|
-
runId: child.manifest.runId,
|
|
196
|
-
})),
|
|
197
|
-
});
|
|
198
|
-
const output = projectCoreCliBatchOutcome({
|
|
199
|
-
batch,
|
|
200
|
-
children: storedChildren,
|
|
201
|
-
exitMode: request.values.presentation.exitMode,
|
|
202
|
-
diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
|
|
203
|
-
? 'enabled'
|
|
204
|
-
: 'disabled',
|
|
105
|
+
const application = createNodeEvaluationApplication(captureNodeCliEvaluationEnvironment(input.environment));
|
|
106
|
+
try {
|
|
107
|
+
const result = await application.run({
|
|
108
|
+
request: requestFor(input, projectRoot), projectRoot,
|
|
109
|
+
materializationRoot: machineLayout.resolvedInputsDir, resourceLeaseRoot: machineLayout.resourceLeasesDir,
|
|
110
|
+
store: input.store,
|
|
111
|
+
createProgressSink: () => emitProgress(input.lang),
|
|
112
|
+
onNotice: (notice) => renderNotice(notice, input.lang),
|
|
113
|
+
requestForBatchItem: (entry) => requestFor({ ...input, flags: { ...input.flags, batch: undefined, control: 'baseline', treatment: entry.skillPath, samples: entry.samplesPath, 'no-serve': true } }, projectRoot),
|
|
114
|
+
async onCompleted(completed, request) {
|
|
115
|
+
if (completed.outcomeKind === 'run')
|
|
116
|
+
await announceCoreReport(completed.artifacts, completed.store, completed.outputDirectory, request.values.presentation.serve, input.lang);
|
|
117
|
+
if (completed.outcomeKind === 'series')
|
|
118
|
+
process.stderr.write(input.lang === 'zh'
|
|
119
|
+
? `Core Series 已完成:${completed.outcome.seriesId}(${completed.outcome.members.length} 个独立 run)\n`
|
|
120
|
+
: `Core Series completed: ${completed.outcome.seriesId} (${completed.outcome.members.length} independent runs)\n`);
|
|
121
|
+
},
|
|
205
122
|
});
|
|
206
123
|
return {
|
|
207
|
-
exitCode:
|
|
208
|
-
output,
|
|
209
|
-
|
|
124
|
+
exitCode: result.outcomeKind === 'dry-run' || result.outcomeKind === 'batch-dry-run' ? 0 : result.outcome.gate.exitCode,
|
|
125
|
+
output: result.outcome, outputDirectory: result.outputDirectory,
|
|
126
|
+
...(result.outcomeKind === 'run' ? { stored: result.artifacts } : {}),
|
|
210
127
|
};
|
|
211
128
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
projectRoot,
|
|
215
|
-
materializationRoot: machineLayout.resolvedInputsDir,
|
|
216
|
-
...(input.environment === undefined ? {} : { environment: input.environment }),
|
|
217
|
-
...(request.values.orchestration.repeatCount > 1
|
|
218
|
-
? { seriesInstanceId: generateRunId(['series']) }
|
|
219
|
-
: {}),
|
|
220
|
-
});
|
|
221
|
-
const compiled = compileCliEvaluationInput(resolved);
|
|
222
|
-
const composition = await createNodeCliProductionComposition({
|
|
223
|
-
compiled,
|
|
224
|
-
projectRoot,
|
|
225
|
-
outputDirectory,
|
|
226
|
-
resourceLeaseRoot: machineLayout.resourceLeasesDir,
|
|
227
|
-
environment: input.environment,
|
|
228
|
-
});
|
|
229
|
-
const store = input.store ?? runStoreForOutput(outputDirectory, composition.support.contentResolver);
|
|
230
|
-
const host = {
|
|
231
|
-
compiled,
|
|
232
|
-
...composition,
|
|
233
|
-
artifactStore: store,
|
|
234
|
-
};
|
|
235
|
-
if (compiled.orchestration.dryRun) {
|
|
236
|
-
const prepared = await createProductionEvaluationHost(host).prepare();
|
|
237
|
-
const output = projectCoreCliDryRun({ plan: prepared.plan, preflight: prepared.preflight });
|
|
238
|
-
return { exitCode: 0, output, outputDirectory };
|
|
239
|
-
}
|
|
240
|
-
const sourceRunId = resumeRunId(request);
|
|
241
|
-
const independentSeries = compiled.orchestration.independentSeries;
|
|
242
|
-
if (independentSeries !== undefined) {
|
|
243
|
-
if (sourceRunId !== undefined) {
|
|
244
|
-
throw new TypeError('Independent Series resume 必须按 member runId 单独执行。');
|
|
245
|
-
}
|
|
246
|
-
const createdAt = new Date().toISOString();
|
|
247
|
-
const series = await executeProductionEvaluationSeries({
|
|
248
|
-
host,
|
|
249
|
-
members: independentSeries.memberships.map((membership) => ({
|
|
250
|
-
runId: generateRunId([membership.memberId]),
|
|
251
|
-
createdAt,
|
|
252
|
-
progressSink: emitProgress(input.lang),
|
|
253
|
-
})),
|
|
254
|
-
bundleId: generateRunId(['series-analysis']),
|
|
255
|
-
reportId: generateRunId(['series-report']),
|
|
256
|
-
});
|
|
257
|
-
await series.result;
|
|
258
|
-
const evolution = await series.evolution;
|
|
259
|
-
if (evolution === undefined)
|
|
260
|
-
throw new Error('Core Series 未完成,无法生成 evolution evidence。');
|
|
261
|
-
const memberArtifacts = await Promise.all(series.members.map(async (member) => {
|
|
262
|
-
if (member.executionStatus !== 'started')
|
|
263
|
-
throw member.error;
|
|
264
|
-
await member.run.result;
|
|
265
|
-
const persistence = await member.run.persistence;
|
|
266
|
-
if (persistence.persistenceStatus !== 'stored') {
|
|
267
|
-
if (persistence.persistenceStatus === 'failed')
|
|
268
|
-
throw persistence.error;
|
|
269
|
-
throw new Error(`Core Series member 产物未保存:${persistence.reasonCode}`);
|
|
270
|
-
}
|
|
271
|
-
return persistence.artifacts;
|
|
272
|
-
}));
|
|
273
|
-
const output = projectCoreCliSeriesOutcome({
|
|
274
|
-
evolution,
|
|
275
|
-
members: memberArtifacts,
|
|
276
|
-
exitMode: request.values.presentation.exitMode,
|
|
277
|
-
diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
|
|
278
|
-
? 'enabled'
|
|
279
|
-
: 'disabled',
|
|
280
|
-
});
|
|
281
|
-
for (const artifacts of memberArtifacts) {
|
|
282
|
-
await persistArtifactGraph(artifacts, outputDirectory, projectRoot);
|
|
283
|
-
}
|
|
284
|
-
if (compiled.orchestration.managedEvidence === 'append') {
|
|
285
|
-
process.stderr.write(input.lang === 'zh'
|
|
286
|
-
? 'Core Series 不写入单次 member 受管证据;需由预注册的 Series 总体决定投影后再纳入生命周期。\n'
|
|
287
|
-
: 'Core Series does not write single-member managed evidence; lifecycle admission requires a preregistered Series-level decision projection.\n');
|
|
288
|
-
}
|
|
289
|
-
process.stderr.write(input.lang === 'zh'
|
|
290
|
-
? `Core Series 已完成:${output.seriesId}(${output.members.length} 个独立 run)\n`
|
|
291
|
-
: `Core Series completed: ${output.seriesId} (${output.members.length} independent runs)\n`);
|
|
292
|
-
return { exitCode: output.gate.exitCode, output, outputDirectory };
|
|
293
|
-
}
|
|
294
|
-
const prepared = await createProductionEvaluationHost(host).prepare();
|
|
295
|
-
let stored;
|
|
296
|
-
if (sourceRunId !== undefined) {
|
|
297
|
-
const admission = await prepared.admitResume({
|
|
298
|
-
locator: { locatorKind: 'core-run', runId: sourceRunId },
|
|
299
|
-
policy: {
|
|
300
|
-
rejectionMode: 'fail-closed',
|
|
301
|
-
minimumSourceTrust: 'unknown',
|
|
302
|
-
cacheReceiptMode: 'allow-indeterminate',
|
|
303
|
-
budgetVerificationMode: 'allow-indeterminate',
|
|
304
|
-
},
|
|
305
|
-
});
|
|
306
|
-
if (admission.disposition !== 'reuse') {
|
|
307
|
-
throw new Error(`Core resume 被拒绝:${admission.reasonCode}`);
|
|
308
|
-
}
|
|
309
|
-
stored = admission.artifacts;
|
|
310
|
-
}
|
|
311
|
-
else {
|
|
312
|
-
const targetIds = prepared.plan.execution.targets.map((target) => target.targetId);
|
|
313
|
-
const runId = generateRunId(targetIds);
|
|
314
|
-
const run = await prepared.execute({
|
|
315
|
-
runId,
|
|
316
|
-
createdAt: new Date().toISOString(),
|
|
317
|
-
progressSink: emitProgress(input.lang),
|
|
318
|
-
});
|
|
319
|
-
await run.result;
|
|
320
|
-
const persistence = await run.persistence;
|
|
321
|
-
if (persistence.persistenceStatus !== 'stored') {
|
|
322
|
-
if (persistence.persistenceStatus === 'failed')
|
|
323
|
-
throw persistence.error;
|
|
324
|
-
throw new Error(`Core 产物未保存:${persistence.reasonCode}`);
|
|
325
|
-
}
|
|
326
|
-
stored = persistence.artifacts;
|
|
129
|
+
catch (error) {
|
|
130
|
+
return withLocalizedSampleDiscovery(() => { throw error; }, input.lang);
|
|
327
131
|
}
|
|
328
|
-
const output = projectCoreCliRunOutcome(stored, {
|
|
329
|
-
exitMode: request.values.presentation.exitMode,
|
|
330
|
-
diagnosticMode: request.values.orchestration.diagnostic === 'enabled-outside-core'
|
|
331
|
-
? 'enabled'
|
|
332
|
-
: 'disabled',
|
|
333
|
-
});
|
|
334
|
-
await persistArtifactGraph(stored, outputDirectory, projectRoot);
|
|
335
|
-
await appendManagedEvidenceSafely(stored, compiled.orchestration.managedEvidence === 'append', input.lang);
|
|
336
|
-
await announceCoreReport(stored, store, outputDirectory, request.values.presentation.serve, input.lang);
|
|
337
|
-
return {
|
|
338
|
-
exitCode: output.gate.exitCode,
|
|
339
|
-
output,
|
|
340
|
-
stored,
|
|
341
|
-
outputDirectory,
|
|
342
|
-
};
|
|
343
132
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type EvaluationDefinition } from '../eval-core/contracts/index.js';
|
|
2
2
|
import { type ExecutionExecutor } from '../eval-core/execution/index.js';
|
|
3
|
-
import type { OmkBindingResourceLeaseAccess } from '../eval-workflows/
|
|
4
|
-
import type { RuntimeBindingOf } from '../eval-workflows/
|
|
3
|
+
import type { OmkBindingResourceLeaseAccess } from '../eval-workflows/hosts/resource-leases/types.js';
|
|
4
|
+
import type { RuntimeBindingOf } from '../eval-workflows/hosts/types.js';
|
|
5
5
|
import type { DshAgentLike, DshHostContextLike } from './host-executor.js';
|
|
6
6
|
export { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, createDshHostCoreSchemaValidators, } from './core-protocol.js';
|
|
7
7
|
export declare const DEFAULT_DSH_HOST_CORE_MAX_INPUT_BYTES: number;
|
|
@@ -2,7 +2,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { JsonValueSchema, RuntimeIdentitySchema, canonicalizeJson, deepFreezeCanonicalJson, digestCanonicalJson, } from '../eval-core/contracts/index.js';
|
|
3
3
|
import { ExecutionPortFailure, } from '../eval-core/execution/index.js';
|
|
4
4
|
import { createDshHostRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
|
|
5
|
-
import { captureClaudeCliRunState, captureClaudeCliTarget, disposeClaudeCliTrial, openClaudeCliTrial, } from '../eval-workflows/
|
|
5
|
+
import { captureClaudeCliRunState, captureClaudeCliTarget, disposeClaudeCliTrial, openClaudeCliTrial, } from '../eval-workflows/hosts/adapters/claude/resources.js';
|
|
6
6
|
import { createSameProcessExecutorAdapter } from '../eval-runtime/adapters/same-process.js';
|
|
7
7
|
import { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, dshHostCoreExecutorCapabilities, parseDshHostCoreResult, } from './core-protocol.js';
|
|
8
8
|
import { supportsDshTraceEventType } from './trace-adapter.js';
|
|
@@ -716,7 +716,7 @@ export async function createDshHostCoreExecutorAdapter(input) {
|
|
|
716
716
|
}
|
|
717
717
|
runState.acquireTrial();
|
|
718
718
|
try {
|
|
719
|
-
return openClaudeCliTrial(trial, runState, host.maxInputBytes, RESOURCE_PROFILE);
|
|
719
|
+
return await openClaudeCliTrial(trial, runState, host.maxInputBytes, RESOURCE_PROFILE);
|
|
720
720
|
}
|
|
721
721
|
catch (error) {
|
|
722
722
|
await runState.releaseTrial();
|
|
@@ -726,8 +726,8 @@ export async function createDshHostCoreExecutorAdapter(input) {
|
|
|
726
726
|
execute({ runState, trialState, attempt, scope }) {
|
|
727
727
|
return executeDshHost(host, target, runState, trialState, attempt, scope.operationIsolationKey);
|
|
728
728
|
},
|
|
729
|
-
disposeTrial({ runState }) {
|
|
730
|
-
return disposeClaudeCliTrial(runState, RESOURCE_PROFILE);
|
|
729
|
+
disposeTrial({ runState, trialState }) {
|
|
730
|
+
return disposeClaudeCliTrial(runState, trialState, RESOURCE_PROFILE);
|
|
731
731
|
},
|
|
732
732
|
disposeRun({ runState }) {
|
|
733
733
|
return runState.requestDispose();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type StoredCoreRunArtifacts } from '../eval-workflows/
|
|
1
|
+
import { type StoredCoreRunArtifacts } from '../eval-workflows/hosts/application.js';
|
|
2
2
|
import type { CoreCliRunOutcome, CoreCliSeriesOutcome } from '../eval-workflows/projections/contracts.js';
|
|
3
3
|
import type { EvalConfig } from '../eval-workflows/inputs/contracts/config.js';
|
|
4
4
|
import { type DshAgentLike, type DshHostContextLike } from './host-executor.js';
|