oh-my-knowledge 1.0.0-beta.5 → 1.0.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/agent-skills/omk/references/commands.md +1 -2
- package/dist/cli/commands/eval/gold/index.js +2 -8
- package/dist/cli/commands/eval/index.d.ts +11 -3
- package/dist/cli/commands/eval/index.js +31 -19
- package/dist/cli/commands/evolve.js +5 -13
- package/dist/cli/commands/init.js +1 -2
- package/dist/cli/commands/install.js +1 -1
- package/dist/cli/commands/observe/inbox.js +3 -1
- package/dist/cli/commands/promote.js +1 -13
- package/dist/cli/commands/rollback.js +1 -13
- package/dist/cli/lib/actor.d.ts +2 -0
- package/dist/cli/lib/actor.js +13 -0
- package/dist/cli/lib/cmd-flags.d.ts +1 -2
- package/dist/cli/lib/cmd-flags.js +1 -1
- package/dist/cli/lib/evaluation-composition.d.ts +3 -0
- package/dist/cli/lib/evaluation-composition.js +96 -0
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +0 -12
- package/dist/cli/lib/i18n-dict/help.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/help.js +0 -226
- package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/run.js +0 -156
- package/dist/cli/lib/i18n.d.ts +0 -10
- package/dist/cli/lib/i18n.js +0 -12
- package/dist/cli/lib/parse-run-config/judge-models.d.ts +0 -21
- package/dist/cli/lib/parse-run-config/judge-models.js +1 -43
- package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +1 -1
- package/dist/cli/lib/parse-run-config/samples-discovery.js +3 -3
- package/dist/cli/lib/prepare-evaluation.d.ts +15 -0
- package/dist/cli/lib/prepare-evaluation.js +45 -0
- package/dist/cli/lib/progress.d.ts +1 -24
- package/dist/cli/lib/progress.js +1 -61
- package/dist/cli/lib/run-core-evaluation.d.ts +3 -11
- package/dist/cli/lib/run-core-evaluation.js +51 -295
- package/dist/cli/oclif/i18n.js +4 -7
- package/dist/diagnosis/observe-mapper.d.ts +1 -2
- package/dist/diagnosis/observe-mapper.js +0 -9
- package/dist/dsh-plugin/core-adapter.d.ts +2 -2
- package/dist/dsh-plugin/core-adapter.js +1 -1
- package/dist/dsh-plugin/core-command.d.ts +1 -1
- package/dist/dsh-plugin/core-command.js +29 -204
- package/dist/eval-core/analysis/decision.d.ts +1 -3
- package/dist/eval-core/analysis/decision.js +0 -6
- package/dist/eval-core/analysis/runtime.d.ts +1 -3
- package/dist/eval-core/analysis/runtime.js +0 -6
- package/dist/eval-core/contracts/comparability.d.ts +0 -1
- package/dist/eval-core/contracts/comparability.js +0 -9
- package/dist/eval-core/contracts/json.d.ts +0 -1
- package/dist/eval-core/contracts/json.js +0 -9
- package/dist/eval-core/evaluation/runtime.d.ts +1 -3
- package/dist/eval-core/evaluation/runtime.js +0 -6
- package/dist/eval-core/execution/runtime.d.ts +0 -3
- package/dist/eval-core/execution/runtime.js +4 -10
- package/dist/eval-runtime/evaluate.d.ts +9 -570
- package/dist/eval-runtime/evaluate.js +5 -2744
- package/dist/eval-runtime/evaluation/capture-evaluators.d.ts +28 -0
- package/dist/eval-runtime/evaluation/capture-evaluators.js +516 -0
- package/dist/eval-runtime/evaluation/capture-input.d.ts +38 -0
- package/dist/eval-runtime/evaluation/capture-input.js +460 -0
- package/dist/eval-runtime/evaluation/conformance.d.ts +7 -0
- package/dist/eval-runtime/evaluation/conformance.js +108 -0
- package/dist/eval-runtime/evaluation/contracts.d.ts +517 -0
- package/dist/eval-runtime/evaluation/definition.d.ts +12 -0
- package/dist/eval-runtime/evaluation/definition.js +659 -0
- package/dist/eval-runtime/evaluation/errors.d.ts +16 -0
- package/dist/eval-runtime/evaluation/errors.js +22 -0
- package/dist/eval-runtime/evaluation/ordering.d.ts +1 -0
- package/dist/eval-runtime/evaluation/ordering.js +3 -0
- package/dist/eval-runtime/evaluation/prepare.d.ts +12 -0
- package/dist/eval-runtime/evaluation/prepare.js +351 -0
- package/dist/eval-runtime/evaluation/result-state.d.ts +14 -0
- package/dist/eval-runtime/evaluation/result-state.js +58 -0
- package/dist/eval-runtime/evaluation/reuse.d.ts +21 -0
- package/dist/eval-runtime/evaluation/reuse.js +343 -0
- package/dist/eval-runtime/evaluation/schemas.d.ts +414 -0
- package/dist/eval-runtime/evaluation/schemas.js +260 -0
- package/dist/eval-workflows/analysis/bootstrap.d.ts +0 -7
- package/dist/eval-workflows/analysis/bootstrap.js +0 -10
- package/dist/eval-workflows/assertions/deterministic.d.ts +0 -4
- package/dist/eval-workflows/assertions/deterministic.js +0 -19
- package/dist/eval-workflows/gold/human.d.ts +0 -28
- package/dist/eval-workflows/gold/human.js +1 -29
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/cli-protocol.js +4 -52
- package/dist/eval-workflows/hosts/adapters/claude/core-protocol-contract.d.ts +5 -0
- package/dist/{eval-hosts/runtime-adapter/adapters/claude/sdk-protocol.js → eval-workflows/hosts/adapters/claude/core-protocol-contract.js} +4 -31
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/resources.js +1 -16
- package/dist/eval-workflows/hosts/adapters/claude/sdk-protocol.js +36 -0
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli-protocol.d.ts +0 -2
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli-protocol.js +1 -3
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli.js +8 -23
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk-protocol.d.ts +0 -2
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk-protocol.js +1 -3
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk.js +1 -1
- package/dist/eval-workflows/hosts/adapters/shared/readonly-map-snapshot.d.ts +2 -0
- package/dist/eval-workflows/hosts/adapters/shared/readonly-map-snapshot.js +17 -0
- package/dist/eval-workflows/hosts/application.d.ts +83 -0
- package/dist/eval-workflows/hosts/application.js +126 -0
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/assembly.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/assembly.js +5 -20
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/builtins.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/builtins.js +13 -13
- package/dist/{eval-hosts/node → eval-workflows/hosts/composition}/judge-provider-identity.d.ts +2 -2
- package/dist/{eval-hosts/node → eval-workflows/hosts/composition}/judge-provider-identity.js +1 -1
- package/dist/eval-workflows/hosts/composition/node-preflight.d.ts +3 -0
- package/dist/eval-workflows/hosts/composition/node-preflight.js +163 -0
- package/dist/eval-workflows/hosts/composition/node-runtime.d.ts +22 -0
- package/dist/eval-workflows/hosts/composition/node-runtime.js +165 -0
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/preflight.d.ts +2 -2
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts/composition}/preflight.js +1 -1
- package/dist/eval-workflows/hosts/composition/registered-runtime.d.ts +20 -0
- package/dist/eval-workflows/hosts/composition/registered-runtime.js +19 -0
- package/dist/{eval-hosts/node → eval-workflows/hosts/composition}/runtime-registry.d.ts +12 -12
- package/dist/{eval-hosts/node → eval-workflows/hosts/composition}/runtime-registry.js +16 -31
- package/dist/{eval-hosts/runtime-adapter/composition.d.ts → eval-workflows/hosts/composition/runtime.d.ts} +11 -18
- package/dist/{eval-hosts/runtime-adapter/composition.js → eval-workflows/hosts/composition/runtime.js} +23 -69
- package/dist/eval-workflows/hosts/evaluators/executor-judge-invocation.d.ts +5 -0
- package/dist/eval-workflows/hosts/evaluators/executor-judge-invocation.js +57 -0
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/llm-assertion-factory.js +1 -1
- package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/node-cli-evaluation-resolver.d.ts +2 -2
- package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/node-cli-evaluation-resolver.js +71 -36
- package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/node-sample-content-resolver.d.ts +1 -1
- package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/node-sample-content-resolver.js +1 -1
- package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/safe-http-content-resolver.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/access.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/node.d.ts +0 -5
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/node.js +2 -21
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/types.d.ts +1 -1
- package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/types.d.ts +1 -1
- package/dist/eval-workflows/input-compilation/error.d.ts +1 -1
- package/dist/eval-workflows/input-compilation/parse.d.ts +2 -0
- package/dist/eval-workflows/input-compilation/parse.js +79 -94
- package/dist/eval-workflows/input-compilation/registry.d.ts +1 -1
- package/dist/eval-workflows/input-compilation/registry.js +24 -40
- package/dist/eval-workflows/inputs/assertion-types.d.ts +0 -1
- package/dist/eval-workflows/inputs/assertion-types.js +0 -1
- package/dist/eval-workflows/inputs/batch-discovery.d.ts +5 -0
- package/dist/eval-workflows/inputs/batch-discovery.js +28 -0
- package/dist/eval-workflows/inputs/contracts/config-schema.d.ts +375 -0
- package/dist/eval-workflows/inputs/contracts/config-schema.js +173 -0
- package/dist/eval-workflows/inputs/contracts/config.d.ts +1 -82
- package/dist/eval-workflows/inputs/contracts/variant.d.ts +1 -8
- package/dist/eval-workflows/inputs/eval-config.d.ts +1 -1
- package/dist/eval-workflows/inputs/eval-config.js +22 -384
- package/dist/eval-workflows/inputs/judge-models.d.ts +2 -0
- package/dist/eval-workflows/inputs/judge-models.js +23 -0
- package/dist/eval-workflows/inputs/load-samples.d.ts +2 -8
- package/dist/eval-workflows/inputs/load-samples.js +12 -30
- package/dist/eval-workflows/inputs/sample-document.d.ts +0 -6
- package/dist/eval-workflows/inputs/sample-document.js +2 -70
- package/dist/eval-workflows/inputs/sample-locator.d.ts +0 -1
- package/dist/eval-workflows/inputs/sample-locator.js +0 -51
- package/dist/eval-workflows/measurement/analysis/agreement-source-adapter-v1.js +2 -69
- package/dist/eval-workflows/measurement/analysis/agreement-source-adapter.js +2 -69
- package/dist/eval-workflows/measurement/analysis/agreement-source-pairs.d.ts +9 -0
- package/dist/eval-workflows/measurement/analysis/agreement-source-pairs.js +72 -0
- package/dist/eval-workflows/measurement/analysis/bootstrap-family-table.d.ts +0 -1
- package/dist/eval-workflows/measurement/analysis/bootstrap-family-table.js +0 -3
- package/dist/eval-workflows/measurement/analysis/dimension-binding.d.ts +7 -0
- package/dist/eval-workflows/measurement/analysis/dimension-binding.js +17 -0
- package/dist/eval-workflows/measurement/analysis/dimension-table-v1.d.ts +0 -4
- package/dist/eval-workflows/measurement/analysis/dimension-table-v1.js +3 -28
- package/dist/eval-workflows/measurement/analysis/dimension-table.js +1 -16
- package/dist/eval-workflows/orchestration/evaluation-service.d.ts +26 -0
- package/dist/eval-workflows/orchestration/evaluation-service.js +82 -0
- package/dist/eval-workflows/{production-host → orchestration}/orchestration.d.ts +2 -42
- package/dist/eval-workflows/{production-host → orchestration}/orchestration.js +0 -84
- package/dist/eval-workflows/{production-host → orchestration}/workflow.d.ts +4 -4
- package/dist/eval-workflows/{production-host → orchestration}/workflow.js +1 -1
- package/dist/evidence/storage/default-dirs.d.ts +0 -2
- package/dist/evidence/storage/default-dirs.js +0 -2
- package/dist/evidence/storage/discovery-index.d.ts +0 -2
- package/dist/evidence/storage/discovery-index.js +0 -4
- package/dist/evidence/storage/file-names.d.ts +0 -2
- package/dist/evidence/storage/file-names.js +0 -6
- package/dist/executors/contracts/tool-call-status-schema.d.ts +7 -0
- package/dist/executors/contracts/tool-call-status-schema.js +7 -0
- package/dist/executors/contracts/trace-source-schema.d.ts +9 -0
- package/dist/executors/contracts/trace-source-schema.js +9 -0
- package/dist/executors/contracts/trace-source.d.ts +3 -1
- package/dist/executors/contracts/trace.d.ts +3 -1
- package/dist/executors/core/capabilities.d.ts +0 -5
- package/dist/executors/core/capabilities.js +3 -26
- package/dist/executors/core/registry.d.ts +11 -4
- package/dist/executors/core/registry.js +0 -9
- package/dist/executors/core/trace-source-kind.js +2 -10
- package/dist/executors/index.d.ts +1 -1
- package/dist/executors/index.js +1 -1
- package/dist/executors/mock-runtime/runtime.d.ts +0 -3
- package/dist/executors/mock-runtime/runtime.js +1 -9
- package/dist/executors/openai/codex/cli-arguments.d.ts +12 -0
- package/dist/executors/openai/codex/cli-arguments.js +24 -0
- package/dist/executors/openai/codex/cli.d.ts +1 -0
- package/dist/executors/openai/codex/cli.js +9 -24
- package/dist/executors/result-validation.d.ts +0 -1
- package/dist/executors/result-validation.js +0 -5
- package/dist/executors/tool-call-status.d.ts +0 -1
- package/dist/executors/tool-call-status.js +2 -10
- package/dist/knowledge-artifacts/contracts.d.ts +7 -0
- package/dist/knowledge-artifacts/doctor/index.js +1 -1
- package/dist/knowledge-artifacts/governance/source-probe.js +1 -1
- package/dist/{eval-workflows/inputs/skill-loader.d.ts → knowledge-artifacts/sources/artifact-resolution.d.ts} +2 -9
- package/dist/{eval-workflows/inputs/skill-loader.js → knowledge-artifacts/sources/artifact-resolution.js} +3 -33
- package/dist/knowledge-artifacts/sources/content-hash.js +2 -6
- package/dist/{eval-workflows/inputs/source-resolver.d.ts → knowledge-artifacts/sources/install-source.d.ts} +1 -1
- package/dist/{eval-workflows/inputs/source-resolver.js → knowledge-artifacts/sources/install-source.js} +3 -4
- package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.js +1 -1
- package/dist/observability/analysis/coverage-analyzer.d.ts +0 -1
- package/dist/observability/analysis/coverage-analyzer.js +0 -48
- package/dist/observability/analysis/gap-analyzer.d.ts +0 -8
- package/dist/observability/analysis/gap-analyzer.js +0 -15
- package/dist/observability/contracts/experience-enums.d.ts +227 -0
- package/dist/observability/contracts/experience-enums.js +227 -0
- package/dist/observability/contracts/experience-evidence-schema.d.ts +33130 -0
- package/dist/observability/contracts/experience-evidence-schema.js +704 -0
- package/dist/observability/contracts/experience.d.ts +75 -646
- package/dist/observability/contracts/inbox.d.ts +4 -4
- package/dist/observability/contracts/problem-patterns.d.ts +6 -30
- package/dist/observability/contracts/trace-metadata-schema.d.ts +10 -0
- package/dist/observability/contracts/trace-metadata-schema.js +10 -0
- package/dist/observability/contracts/trace.d.ts +3 -12
- package/dist/observability/conversation/catalog.d.ts +0 -1
- package/dist/observability/conversation/catalog.js +1 -4
- package/dist/observability/experience/report-codec.d.ts +8 -22
- package/dist/observability/experience/report-codec.js +22 -33
- package/dist/observability/experience/report-reference-validator.d.ts +2 -2
- package/dist/observability/experience/report-reference-validator.js +45 -88
- package/dist/observability/experience/report-structure.d.ts +1 -1
- package/dist/observability/experience/report-structure.js +2 -1
- package/dist/observability/experience/report-value-guards.d.ts +8 -21
- package/dist/observability/experience/report-value-guards.js +230 -1003
- package/dist/observability/experience/review-checklist.d.ts +1 -0
- package/dist/observability/experience/review-checklist.js +9 -5
- package/dist/observability/experience/reviewer-report.d.ts +1 -2
- package/dist/observability/experience/reviewer-report.js +2 -2
- package/dist/observability/experience/session-story.js +0 -2
- package/dist/observability/experience.d.ts +19 -0
- package/dist/observability/experience.js +8 -3
- package/dist/observability/inbox/effective-review.d.ts +25 -0
- package/dist/observability/inbox/effective-review.js +141 -0
- package/dist/observability/inbox/index.d.ts +3 -4
- package/dist/observability/inbox/index.js +5 -43
- package/dist/observability/inbox/resolved-review.d.ts +0 -1
- package/dist/observability/inbox/resolved-review.js +3 -38
- package/dist/observability/inbox/session-time-range.d.ts +2 -0
- package/dist/observability/inbox/session-time-range.js +10 -0
- package/dist/observability/inbox/view-model.d.ts +6 -3
- package/dist/observability/inbox/view-model.js +8 -24
- package/dist/observability/skill-health/analyzer.d.ts +8 -9
- package/dist/observability/skill-health/analyzer.js +22 -33
- package/dist/observability/soft-standards/index.d.ts +1 -12
- package/dist/observability/soft-standards/index.js +1 -12
- package/dist/observability/soft-standards/skill-standards-store.d.ts +1 -2
- package/dist/observability/soft-standards/skill-standards-store.js +0 -22
- package/dist/observability/trace/attribution.d.ts +0 -33
- package/dist/observability/trace/attribution.js +0 -92
- package/dist/observability/trace/index.d.ts +4 -4
- package/dist/observability/trace/index.js +3 -3
- package/dist/observability/trace/segmentation.d.ts +0 -3
- package/dist/observability/trace/segmentation.js +2 -7
- package/dist/observability/trace/source.d.ts +1 -32
- package/dist/observability/trace/source.js +2 -36
- package/dist/observability/trace/trace-ir.d.ts +0 -2
- package/dist/observability/trace/trace-ir.js +0 -6
- package/dist/observability/view-models/conversation.d.ts +2 -2
- package/dist/observability/view-models/knowledge-debugger.d.ts +2 -2
- package/dist/shared/atomic-json.d.ts +0 -4
- package/dist/shared/atomic-json.js +0 -15
- package/dist/studio/application/skill-index.d.ts +1 -2
- package/dist/studio/application/skill-index.js +4 -16
- package/dist/studio/application/skill-insights.d.ts +1 -2
- package/dist/studio/application/skill-insights.js +0 -13
- package/dist/studio/http/routes/observations.js +6 -0
- package/dist/studio/presentation/layout.d.ts +0 -3
- package/dist/studio/presentation/layout.js +0 -18
- package/dist/studio/presentation/observation-inbox/experience-workspace-renderer.js +22 -102
- package/dist/studio/presentation/observation-inbox/metric-renderer.d.ts +124 -11
- package/dist/studio/presentation/observation-inbox/metric-renderer.js +18 -377
- package/dist/studio/presentation/observation-inbox/process-workspace-renderer.d.ts +40 -2
- package/dist/studio/presentation/observation-inbox/review-renderer.js +0 -32
- package/dist/studio/presentation/observation-inbox-renderer.js +2 -2
- package/dist/studio/presentation/skill-detail-renderer.js +4 -13
- package/dist/studio/view-models/skill-index.d.ts +2 -0
- package/package.json +2 -3
- package/dist/cli/lib/parse-run-config/variant-resolution.d.ts +0 -23
- package/dist/cli/lib/parse-run-config/variant-resolution.js +0 -98
- package/dist/cli/lib/parse-run-config.d.ts +0 -93
- package/dist/cli/lib/parse-run-config.js +0 -163
- package/dist/cli/lib/source-probe.d.ts +0 -15
- package/dist/cli/lib/source-probe.js +0 -129
- package/dist/eval-hosts/node/index.d.ts +0 -6
- package/dist/eval-hosts/node/index.js +0 -6
- package/dist/eval-hosts/node/node-cli-composition.d.ts +0 -26
- package/dist/eval-hosts/node/node-cli-composition.js +0 -474
- package/dist/eval-hosts/runtime-adapter/adapters/index.d.ts +0 -8
- package/dist/eval-hosts/runtime-adapter/adapters/index.js +0 -8
- package/dist/eval-hosts/runtime-adapter/evaluators/index.d.ts +0 -6
- package/dist/eval-hosts/runtime-adapter/evaluators/index.js +0 -6
- package/dist/eval-hosts/runtime-adapter/index.d.ts +0 -11
- package/dist/eval-hosts/runtime-adapter/index.js +0 -11
- package/dist/eval-hosts/runtime-adapter/resource-leases/index.d.ts +0 -3
- package/dist/eval-hosts/runtime-adapter/resource-leases/index.js +0 -3
- package/dist/eval-workflows/inputs/contracts/index.d.ts +0 -5
- package/dist/eval-workflows/instruments/contracts/index.d.ts +0 -1
- package/dist/eval-workflows/instruments/contracts/index.js +0 -1
- package/dist/eval-workflows/measurement/analysis/index.d.ts +0 -23
- package/dist/eval-workflows/measurement/analysis/index.js +0 -23
- package/dist/eval-workflows/measurement/evaluators/index.d.ts +0 -4
- package/dist/eval-workflows/measurement/evaluators/index.js +0 -4
- package/dist/eval-workflows/production-host/index.d.ts +0 -5
- package/dist/eval-workflows/production-host/index.js +0 -5
- package/dist/executors/contracts/index.d.ts +0 -4
- package/dist/executors/contracts/index.js +0 -1
- package/dist/knowledge-artifacts/doctor/preflight.d.ts +0 -32
- package/dist/knowledge-artifacts/doctor/preflight.js +0 -31
- package/dist/observability/contracts/index.d.ts +0 -7
- package/dist/observability/contracts/index.js +0 -1
- package/dist/observability/conversation/view-model.d.ts +0 -5
- package/dist/observability/conversation/view-model.js +0 -100
- /package/dist/{eval-workflows/inputs/contracts/index.js → eval-runtime/evaluation/contracts.js} +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/anthropic/api.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/anthropic/api.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/anthropic/protocol.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/anthropic/protocol.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/cli-protocol.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/cli.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/cli.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/resources.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/sdk-protocol.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/sdk-runtime.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/sdk-runtime.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/sdk.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/claude/sdk.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli-resources.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/cli-resources.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/content-identity.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/content-identity.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/environment.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/environment.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/protocol-core.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/protocol-core.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/resources.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/resources.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk-runtime.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/codex/sdk-runtime.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/custom/command.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/custom/command.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/openai/api.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/openai/api.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/openai/protocol.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/openai/protocol.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/api-http.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/api-http.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/api-protocol-core.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/api-protocol-core.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/classified-environment.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/classified-environment.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/content-identity.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/content-identity.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/stateless-api-resources.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/stateless-api-resources.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/trial-workspace.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/adapters/shared/trial-workspace.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/llm-assertion-factory.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/llm-judge-invocation.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/llm-judge-invocation.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/rubric-judge.d.ts +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/evaluators/rubric-judge.js +0 -0
- /package/dist/{eval-hosts/node → eval-workflows/hosts/input-resolution}/safe-http-content-resolver.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/access.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/resource-leases/types.js +0 -0
- /package/dist/{eval-hosts/runtime-adapter → eval-workflows/hosts}/types.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/artifact-graph-persistence.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/environment.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/environment.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/holdout.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/holdout.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/measurement-design.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/measurement-design.js +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.d.ts +0 -0
- /package/dist/eval-workflows/{production-host → orchestration}/sample-content-resolution.js +0 -0
- /package/dist/{eval-workflows/inputs → knowledge-artifacts/sources}/materialize-copy.d.ts +0 -0
|
@@ -73,7 +73,7 @@ omk eval [flags]
|
|
|
73
73
|
|
|
74
74
|
**Flags:**
|
|
75
75
|
|
|
76
|
-
- `--batch` `boolean
|
|
76
|
+
- `--batch` `boolean`:批量评测:baseline 作为对照组,逐个 skill 作为实验组;不支持 repeat 大于 1
|
|
77
77
|
- `--bootstrap` `boolean`:加 bootstrap CI
|
|
78
78
|
- `--bootstrap-samples` `option`:bootstrap 重采样次数,默认 1000
|
|
79
79
|
- `--budget-per-sample-ms` `option`:单用例时长上限 ms(必须 > 0,不传则无上限)
|
|
@@ -95,7 +95,6 @@ omk eval [flags]
|
|
|
95
95
|
- `--layered-stats` `boolean`:输出分层统计
|
|
96
96
|
- `--mcp-config` `option`:MCP 配置文件路径
|
|
97
97
|
- `--model` `option`:被测模型
|
|
98
|
-
- `--no-cache` `boolean`:跳过 executor cache
|
|
99
98
|
- `--no-debias-length` `boolean`:关 length-debias(默认开)
|
|
100
99
|
- `--no-diagnostic` `boolean`:关闭基于 Core 失败、缺失、排除与稳定 reason code 的诊断投影。
|
|
101
100
|
- `--no-evidence` `boolean`:不把本次评测写成证据追加进受管记录(默认会为已 install 的 skill 自动写)。
|
|
@@ -1,11 +1,5 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
// sub-sub」)。
|
|
4
|
-
//
|
|
5
|
-
// 不再依赖 legacy hand-written usage() 字符串(已删):oclif 默认没内建 help command,
|
|
6
|
-
// 但 helpClass 是可实例化的 Help 子类。直接 new HelpClass(config).showCommandHelp(
|
|
7
|
-
// config.findCommand('eval:gold')) 让 LangAwareHelp 按 --lang 渲染当前 topic 的
|
|
8
|
-
// description + sub-sub 列表(init / validate / compare)。
|
|
1
|
+
// 裸 `omk eval gold` 显示帮助并退出 1,让自动化能识别缺少子命令。
|
|
2
|
+
// 复用当前 HelpClass,保持语言选择和子命令列表与 CLI 一致。
|
|
9
3
|
import { BaseCommand } from '../../../oclif/base-command.js';
|
|
10
4
|
import { LANG_FLAG, bilingual } from '../../../oclif/i18n.js';
|
|
11
5
|
export default class EvalGold extends BaseCommand {
|
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import { BaseCommand } from '../../oclif/base-command.js';
|
|
2
2
|
import { type CliLang } from '../../lib/i18n.js';
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
interface ConnectivityFailureContext {
|
|
4
|
+
executorName: string;
|
|
5
|
+
model: string;
|
|
6
|
+
judgeModels: readonly {
|
|
7
|
+
executor: string;
|
|
8
|
+
model: string;
|
|
9
|
+
}[];
|
|
10
|
+
noJudge: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function formatConnectivityFailureHint(message: string, config: Pick<ConnectivityFailureContext, 'executorName' | 'model' | 'judgeModels' | 'noJudge'>, lang: CliLang, env?: NodeJS.ProcessEnv): string;
|
|
5
13
|
export default class Eval extends BaseCommand {
|
|
6
14
|
static description: string;
|
|
7
15
|
static examples: {
|
|
@@ -23,7 +31,6 @@ export default class Eval extends BaseCommand {
|
|
|
23
31
|
'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
24
32
|
global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
25
33
|
'no-judge': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
26
|
-
'no-cache': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
27
34
|
'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
28
35
|
concurrency: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
29
36
|
timeout: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
@@ -58,3 +65,4 @@ export default class Eval extends BaseCommand {
|
|
|
58
65
|
};
|
|
59
66
|
run(): Promise<void>;
|
|
60
67
|
}
|
|
68
|
+
export {};
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
|
-
import { join, relative, sep } from 'node:path';
|
|
2
|
+
import { join, relative, resolve, sep } from 'node:path';
|
|
3
3
|
import { Flags } from '@oclif/core';
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
|
|
5
5
|
import { BaseCommand } from '../../oclif/base-command.js';
|
|
6
6
|
import { enumStringParser, integerStringParser, numberStringParser } from '../../oclif/parsers.js';
|
|
7
7
|
import { CliExit } from '../../lib/cli-exit.js';
|
|
8
8
|
import { tCli } from '../../lib/i18n.js';
|
|
9
|
-
import {
|
|
9
|
+
import { prepareCliEvaluation } from '../../lib/prepare-evaluation.js';
|
|
10
|
+
import { CliEvaluationInputError } from '../../../eval-workflows/hosts/application.js';
|
|
10
11
|
import { codexModelFlagValue, codexModelHint } from '../../lib/codex-model-hint.js';
|
|
11
12
|
import { looksLikeModelUnavailableFailure } from '../../lib/llm-failure-classifier.js';
|
|
12
13
|
import { DEFAULT_EVALUATION_GATE_THRESHOLD as DEFAULT_GATE_THRESHOLD } from '../../../eval-workflows/evaluation-defaults.js';
|
|
@@ -15,7 +16,7 @@ import { shellQuoteArg } from '../../../shared/shell-quote.js';
|
|
|
15
16
|
import { executorNamesForFamily } from '../../../executors/core/registry.js';
|
|
16
17
|
import { DEFAULT_EVALUATION_TIMEOUT_MS } from '../../../eval-workflows/evaluation-defaults.js';
|
|
17
18
|
// oclif 版 eval(默认 = run 模式) — 单次 typed parse 之后业务 inline。flag schema
|
|
18
|
-
//
|
|
19
|
+
// 参数语义与约束由 Workflow 的 parseCliEvaluationRequest 维护。
|
|
19
20
|
//
|
|
20
21
|
// `omk eval gold ...` 由 src/cli/commands/eval/gold/{init,validate,compare}.ts
|
|
21
22
|
// 处理,oclif 文件目录路由自动接管,不进 eval.ts。
|
|
@@ -127,15 +128,29 @@ export function formatConnectivityFailureHint(message, config, lang, env = proce
|
|
|
127
128
|
return '';
|
|
128
129
|
}
|
|
129
130
|
async function runEval(_args, flags, lang) {
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
let prepared;
|
|
132
|
+
try {
|
|
133
|
+
prepared = prepareCliEvaluation({ ...flags }, { lang });
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
if (!(error instanceof CliEvaluationInputError))
|
|
137
|
+
throw error;
|
|
138
|
+
console.error(`error: ${error.message}`);
|
|
139
|
+
throw new CliExit(2);
|
|
140
|
+
}
|
|
141
|
+
const { request } = prepared;
|
|
142
|
+
const values = flags;
|
|
143
|
+
const evalConfig = prepared.parseInput.evalConfig;
|
|
144
|
+
const samplesPath = resolve(prepared.projectRoot, request.values.locators.samples);
|
|
145
|
+
const skillDir = resolve(prepared.projectRoot, request.values.locators.skillDirectory);
|
|
146
|
+
if (!values.batch && !hasUsableSamplesPath(samplesPath)) {
|
|
132
147
|
const treatmentRaw = typeof values.treatment === 'string' ? values.treatment : '';
|
|
133
148
|
const treatments = treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean);
|
|
134
149
|
const sampleCommand = !values.samples && !evalConfig?.samples && treatments.length === 1
|
|
135
|
-
? sampleCommandForSingleTreatment(treatments[0],
|
|
150
|
+
? sampleCommandForSingleTreatment(treatments[0], skillDir)
|
|
136
151
|
: null;
|
|
137
152
|
const missingSamplesMessage = [
|
|
138
|
-
tCli('cli.common.samples_not_found', lang, { path:
|
|
153
|
+
tCli('cli.common.samples_not_found', lang, { path: samplesPath }),
|
|
139
154
|
sampleCommand ? tCli('cli.common.samples_not_found_hint', lang, { command: sampleCommand }) : '',
|
|
140
155
|
].filter(Boolean).join('\n');
|
|
141
156
|
console.error(tCli('cli.common.error_prefix', lang, {
|
|
@@ -145,12 +160,7 @@ async function runEval(_args, flags, lang) {
|
|
|
145
160
|
}
|
|
146
161
|
try {
|
|
147
162
|
const { runCoreEvaluationCommand } = await import('../../lib/run-core-evaluation.js');
|
|
148
|
-
const result = await runCoreEvaluationCommand({
|
|
149
|
-
flags: { ...flags },
|
|
150
|
-
config,
|
|
151
|
-
evalConfig,
|
|
152
|
-
lang,
|
|
153
|
-
});
|
|
163
|
+
const result = await runCoreEvaluationCommand({ prepared });
|
|
154
164
|
if (!process.stdout.isTTY || result.output && typeof result.output === 'object'
|
|
155
165
|
&& result.output.projectionKind === 'core-cli-dry-run') {
|
|
156
166
|
console.log(JSON.stringify(result.output, null, 2));
|
|
@@ -171,7 +181,12 @@ async function runEval(_args, flags, lang) {
|
|
|
171
181
|
throw err;
|
|
172
182
|
const message = err instanceof Error ? err.message : String(err);
|
|
173
183
|
console.error(tCli('cli.common.error_prefix', lang, {
|
|
174
|
-
message: `${message}${formatConnectivityFailureHint(message,
|
|
184
|
+
message: `${message}${formatConnectivityFailureHint(message, {
|
|
185
|
+
executorName: request.values.targetRuntime.executorId,
|
|
186
|
+
model: request.values.targetRuntime.model,
|
|
187
|
+
noJudge: !request.values.judges.enabled,
|
|
188
|
+
judgeModels: request.values.judges.members.map((member) => ({ executor: member.executorId, model: member.model })),
|
|
189
|
+
}, lang, prepared.environment.environment)}`,
|
|
175
190
|
}));
|
|
176
191
|
throw new CliExit(1);
|
|
177
192
|
}
|
|
@@ -262,9 +277,6 @@ export default class Eval extends BaseCommand {
|
|
|
262
277
|
'no-judge': Flags.boolean({
|
|
263
278
|
description: bilingual({ zh: '跳过 LLM 评委', en: 'Skip LLM judge' }),
|
|
264
279
|
}),
|
|
265
|
-
'no-cache': Flags.boolean({
|
|
266
|
-
description: bilingual({ zh: '跳过 executor cache', en: 'Skip executor cache' }),
|
|
267
|
-
}),
|
|
268
280
|
'dry-run': Flags.boolean({
|
|
269
281
|
description: bilingual({ zh: '只 plan 不实跑', en: 'Plan only, no real exec' }),
|
|
270
282
|
}),
|
|
@@ -281,8 +293,8 @@ export default class Eval extends BaseCommand {
|
|
|
281
293
|
}),
|
|
282
294
|
batch: Flags.boolean({
|
|
283
295
|
description: bilingual({
|
|
284
|
-
zh: '
|
|
285
|
-
en: 'Batch mode: baseline vs each skill',
|
|
296
|
+
zh: '批量评测:baseline 作为对照组,逐个 skill 作为实验组;不支持 repeat 大于 1',
|
|
297
|
+
en: 'Batch mode: baseline vs each skill; repeat must be 1',
|
|
286
298
|
}),
|
|
287
299
|
}),
|
|
288
300
|
'skip-connectivity': Flags.boolean({
|
|
@@ -92,7 +92,7 @@ export async function runEvolve(args, flags, lang) {
|
|
|
92
92
|
samplesFile = resolvedInput.samplesPath;
|
|
93
93
|
}
|
|
94
94
|
// 参数校验必须早于任何昂贵副作用(自动生成用例 / LLM 调用)。
|
|
95
|
-
const { parseJudgeModelsArgOrExit } = await import('../lib/parse-run-config.js');
|
|
95
|
+
const { parseJudgeModelsArgOrExit } = await import('../lib/parse-run-config/judge-models.js');
|
|
96
96
|
const evolveJudges = parseJudgeModelsArgOrExit(flags['judge-models']);
|
|
97
97
|
if (evolveJudges.length > 1) {
|
|
98
98
|
console.error(tCli('cli.common.judge_models_single_only', lang, { cmd: 'omk evolve' }));
|
|
@@ -168,13 +168,15 @@ export async function runEvolve(args, flags, lang) {
|
|
|
168
168
|
}
|
|
169
169
|
const { evolveSkillCore } = await import('../../knowledge-artifacts/authoring/core-evolver.js');
|
|
170
170
|
const { runCoreEvaluationCommand } = await import('../lib/run-core-evaluation.js');
|
|
171
|
+
const { prepareCliEvaluation } = await import('../lib/prepare-evaluation.js');
|
|
171
172
|
const evolveEffort = flags.effort ? validateEvolveEffort(flags.effort, lang) : undefined;
|
|
172
173
|
const evaluatePair = async (control, treatment) => {
|
|
173
174
|
const evaluation = await runCoreEvaluationCommand({
|
|
174
|
-
|
|
175
|
+
prepared: prepareCliEvaluation({
|
|
175
176
|
control,
|
|
176
177
|
treatment,
|
|
177
178
|
samples: resolve(samplesFile),
|
|
179
|
+
'skill-dir': dirname(resolve(skillPath)),
|
|
178
180
|
executor: flags.executor,
|
|
179
181
|
model: flags.model,
|
|
180
182
|
'judge-models': evolveJudges.map((judge) => `${judge.executor}:${judge.model}`).join(','),
|
|
@@ -185,17 +187,7 @@ export async function runEvolve(args, flags, lang) {
|
|
|
185
187
|
'no-evidence': true,
|
|
186
188
|
'no-serve': true,
|
|
187
189
|
'report-only': true,
|
|
188
|
-
},
|
|
189
|
-
config: {
|
|
190
|
-
samplesPath: resolve(samplesFile),
|
|
191
|
-
skillDir: dirname(resolve(skillPath)),
|
|
192
|
-
executorName: flags.executor,
|
|
193
|
-
model: flags.model,
|
|
194
|
-
effort: evolveEffort,
|
|
195
|
-
judgeModels: evolveJudges,
|
|
196
|
-
},
|
|
197
|
-
evalConfig: null,
|
|
198
|
-
lang: 'zh',
|
|
190
|
+
}, { lang: 'zh' }),
|
|
199
191
|
});
|
|
200
192
|
if (evaluation.stored === undefined) {
|
|
201
193
|
throw new Error('Core evolve evaluation 未持久化 artifact chain。');
|
|
@@ -91,8 +91,7 @@ export default class Init extends BaseCommand {
|
|
|
91
91
|
}),
|
|
92
92
|
required: false,
|
|
93
93
|
parse: async (input) => {
|
|
94
|
-
//
|
|
95
|
-
// 创建名为 `--weird` 的目录,新人一头雾水。在 oclif Args 这层拦住更友好。
|
|
94
|
+
// 在参数解析阶段拒绝疑似遗漏的 flag,避免误建名为 `--weird` 的目录。
|
|
96
95
|
if (input.startsWith('--')) {
|
|
97
96
|
const lang = resolveLang();
|
|
98
97
|
const msg = lang === 'zh'
|
|
@@ -6,7 +6,7 @@ import { Args, Flags } from '@oclif/core';
|
|
|
6
6
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
7
7
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
8
8
|
import { tCli } from '../lib/i18n.js';
|
|
9
|
-
import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../
|
|
9
|
+
import { resolveInstallSource, resolveRemoteGitSource, SourceResolveError } from '../../knowledge-artifacts/sources/install-source.js';
|
|
10
10
|
import { buildManagedArtifactRecord, hashArtifactSource, distributableCopyFilter, managedDir, recordManagedArtifact } from '../../knowledge-artifacts/governance/index.js';
|
|
11
11
|
const BUILTIN_OMK_AGENT_SKILL_ID = 'omk-agent-skill';
|
|
12
12
|
const INSTALLABLE_KINDS = ['skill', 'prompt', 'agent', 'workflow'];
|
|
@@ -114,7 +114,9 @@ export async function runObserveInbox(_args, flags, lang) {
|
|
|
114
114
|
items = items.slice(0, limit);
|
|
115
115
|
}
|
|
116
116
|
if (flags.json) {
|
|
117
|
-
|
|
117
|
+
const { buildObservationInboxViewModel } = await import('../../../observability/inbox/view-model.js');
|
|
118
|
+
const { effectiveExperienceReports, resolvedReviewSessions, unappliedMetricAnnotations } = buildObservationInboxViewModel(dir, { skill: flags.skill });
|
|
119
|
+
console.log(JSON.stringify({ kind: 'observe-inbox-query', items, effectiveExperienceReports, resolvedReviewSessions, unappliedMetricAnnotations }, null, 2));
|
|
118
120
|
return;
|
|
119
121
|
}
|
|
120
122
|
if (items.length === 0) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveActor } from '../lib/actor.js';
|
|
2
2
|
import { Args, Flags } from '@oclif/core';
|
|
3
3
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
4
4
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
@@ -7,18 +7,6 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
7
7
|
import { sanitizeCell } from '../lib/cell-format.js';
|
|
8
8
|
import { appendManagedDecision, evaluatePromoteGate, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, probeSourceState, managedDir, managedRecordId, resolveManagedDir, } from '../../knowledge-artifacts/governance/index.js';
|
|
9
9
|
const SUPPORTED_KINDS = ['skill'];
|
|
10
|
-
/** 决定的 actor:--actor > git config user.name > $USER / $LOGNAME > unknown。git 参数无用户输入,安全。 */
|
|
11
|
-
function resolveActor(flagActor) {
|
|
12
|
-
if (flagActor && flagActor.trim())
|
|
13
|
-
return flagActor.trim();
|
|
14
|
-
try {
|
|
15
|
-
const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
16
|
-
if (name)
|
|
17
|
-
return name;
|
|
18
|
-
}
|
|
19
|
-
catch { /* git 缺失 / 无配置 → 回退环境 */ }
|
|
20
|
-
return process.env.USER || process.env.LOGNAME || 'unknown';
|
|
21
|
-
}
|
|
22
10
|
/** 洗 detail 里来自受管 JSON 的不可信值(verdict / judgePromptHash),文本输出防 ANSI/OSC 注入。 */
|
|
23
11
|
function safeDetail(detail) {
|
|
24
12
|
if (!detail)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { resolveActor } from '../lib/actor.js';
|
|
2
2
|
import { Args, Flags } from '@oclif/core';
|
|
3
3
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
4
4
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
@@ -7,18 +7,6 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
7
7
|
import { sanitizeCell } from '../lib/cell-format.js';
|
|
8
8
|
import { appendManagedDecision, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, managedDir, managedRecordId, resolveManagedDir, } from '../../knowledge-artifacts/governance/index.js';
|
|
9
9
|
const SUPPORTED_KINDS = ['skill'];
|
|
10
|
-
/** 决定的 actor:--actor > git config user.name > $USER / $LOGNAME > unknown。git 参数无用户输入,安全。 */
|
|
11
|
-
function resolveActor(flagActor) {
|
|
12
|
-
if (flagActor && flagActor.trim())
|
|
13
|
-
return flagActor.trim();
|
|
14
|
-
try {
|
|
15
|
-
const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
16
|
-
if (name)
|
|
17
|
-
return name;
|
|
18
|
-
}
|
|
19
|
-
catch { /* git 缺失 / 无配置 → 回退环境 */ }
|
|
20
|
-
return process.env.USER || process.env.LOGNAME || 'unknown';
|
|
21
|
-
}
|
|
22
10
|
export default class Rollback extends BaseCommand {
|
|
23
11
|
static description = bilingual({
|
|
24
12
|
zh: '回退受管 skill 当前版本的 promoted 接受:撤销最近一次 promote,在记录里追加一条 rollback 决定(源未漂移则状态回到 measurable,源已漂移则仍 stale)。',
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { execFileSync } from 'node:child_process';
|
|
2
|
+
/** Resolve a decision actor from the flag, Git config, environment, then fallback. */
|
|
3
|
+
export function resolveActor(flagActor) {
|
|
4
|
+
if (flagActor && flagActor.trim())
|
|
5
|
+
return flagActor.trim();
|
|
6
|
+
try {
|
|
7
|
+
const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
8
|
+
if (name)
|
|
9
|
+
return name;
|
|
10
|
+
}
|
|
11
|
+
catch { /* git 缺失 / 无配置 → 回退环境 */ }
|
|
12
|
+
return process.env.USER || process.env.LOGNAME || 'unknown';
|
|
13
|
+
}
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
* 优先级 `--lang CLI flag > OMK_LANG env > zh`。
|
|
37
37
|
*
|
|
38
38
|
* 三态 boolean 注意:oclif 对 `Flags.boolean({})`(没 `default`)运行时不会塞 `false`,
|
|
39
|
-
* 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `
|
|
39
|
+
* 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseCliEvaluationRequest()` fallback
|
|
40
40
|
* 关键 — `undefined`(CLI 没传)走 `eval.yaml` config / 内置 default;`false`(显式
|
|
41
41
|
* `--no-xxx`)真的关。如果在 interface 里把它标成 `boolean`,后续维护者按类型读
|
|
42
42
|
* `flags.bootstrap` 会静默把「没传」当成「关」,绕开 config fallback。所以未设
|
|
@@ -80,7 +80,6 @@ export interface EvalFlags {
|
|
|
80
80
|
/** 报告写全局而非项目 .omk/eval(写入侧 escape;读取侧 studio / 复用走 overlay 兜底)。 */
|
|
81
81
|
global?: boolean;
|
|
82
82
|
'no-judge'?: boolean;
|
|
83
|
-
'no-cache'?: boolean;
|
|
84
83
|
'dry-run'?: boolean;
|
|
85
84
|
concurrency?: string;
|
|
86
85
|
timeout?: string;
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
* 优先级 `--lang CLI flag > OMK_LANG env > zh`。
|
|
37
37
|
*
|
|
38
38
|
* 三态 boolean 注意:oclif 对 `Flags.boolean({})`(没 `default`)运行时不会塞 `false`,
|
|
39
|
-
* 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `
|
|
39
|
+
* 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseCliEvaluationRequest()` fallback
|
|
40
40
|
* 关键 — `undefined`(CLI 没传)走 `eval.yaml` config / 内置 default;`false`(显式
|
|
41
41
|
* `--no-xxx`)真的关。如果在 interface 里把它标成 `boolean`,后续维护者按类型读
|
|
42
42
|
* `flags.bootstrap` 会静默把「没传」当成「关」,绕开 config fallback。所以未设
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import { type NodeEvaluationEnvironment, type ClassifiedEnvironmentEntry } from '../../eval-workflows/hosts/application.js';
|
|
2
|
+
export declare function classifyNodeCliEnvironment(environment: NodeJS.ProcessEnv): Readonly<Record<string, ClassifiedEnvironmentEntry>>;
|
|
3
|
+
export declare function captureNodeCliEvaluationEnvironment(environment?: NodeJS.ProcessEnv): NodeEvaluationEnvironment;
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { access, realpath } from 'node:fs/promises';
|
|
2
|
+
import { constants } from 'node:fs';
|
|
3
|
+
import { delimiter, isAbsolute, join } from 'node:path';
|
|
4
|
+
import { NodeCliProductionCompositionError, } from '../../eval-workflows/hosts/application.js';
|
|
5
|
+
const CREDENTIAL_ENVIRONMENT = new Set([
|
|
6
|
+
'ANTHROPIC_API_KEY',
|
|
7
|
+
'CLAUDE_CODE_OAUTH_TOKEN',
|
|
8
|
+
'OPENAI_API_KEY',
|
|
9
|
+
]);
|
|
10
|
+
const SECRET_TAINT_ENVIRONMENT = new Set([
|
|
11
|
+
'ALL_PROXY',
|
|
12
|
+
'HTTPS_PROXY',
|
|
13
|
+
'HTTP_PROXY',
|
|
14
|
+
'all_proxy',
|
|
15
|
+
'http_proxy',
|
|
16
|
+
'https_proxy',
|
|
17
|
+
]);
|
|
18
|
+
const EFFECT_LOCATOR_ENVIRONMENT = new Set([
|
|
19
|
+
'ALL_PROXY',
|
|
20
|
+
'CODEX_HOME',
|
|
21
|
+
'CURL_CA_BUNDLE',
|
|
22
|
+
'HOME',
|
|
23
|
+
'HTTPS_PROXY',
|
|
24
|
+
'HTTP_PROXY',
|
|
25
|
+
'NODE_EXTRA_CA_CERTS',
|
|
26
|
+
'NO_PROXY',
|
|
27
|
+
'PATH',
|
|
28
|
+
'REQUESTS_CA_BUNDLE',
|
|
29
|
+
'SSL_CERT_DIR',
|
|
30
|
+
'SSL_CERT_FILE',
|
|
31
|
+
'TMPDIR',
|
|
32
|
+
'XDG_CACHE_HOME',
|
|
33
|
+
'XDG_CONFIG_HOME',
|
|
34
|
+
'XDG_DATA_HOME',
|
|
35
|
+
'all_proxy',
|
|
36
|
+
'http_proxy',
|
|
37
|
+
'https_proxy',
|
|
38
|
+
'no_proxy',
|
|
39
|
+
]);
|
|
40
|
+
const BEHAVIOR_ENVIRONMENT = new Set([
|
|
41
|
+
'LANG',
|
|
42
|
+
'LC_ALL',
|
|
43
|
+
'NO_COLOR',
|
|
44
|
+
]);
|
|
45
|
+
export function classifyNodeCliEnvironment(environment) {
|
|
46
|
+
const keys = new Set([
|
|
47
|
+
...CREDENTIAL_ENVIRONMENT,
|
|
48
|
+
...EFFECT_LOCATOR_ENVIRONMENT,
|
|
49
|
+
...BEHAVIOR_ENVIRONMENT,
|
|
50
|
+
]);
|
|
51
|
+
return Object.freeze(Object.fromEntries([...keys].sort().flatMap((key) => {
|
|
52
|
+
const value = environment[key];
|
|
53
|
+
if (value === undefined)
|
|
54
|
+
return [];
|
|
55
|
+
const identity = CREDENTIAL_ENVIRONMENT.has(key)
|
|
56
|
+
? { identityKind: 'credential' }
|
|
57
|
+
: EFFECT_LOCATOR_ENVIRONMENT.has(key)
|
|
58
|
+
? { identityKind: 'effect-locator' }
|
|
59
|
+
: { identityKind: 'behavior', value };
|
|
60
|
+
return [[key, Object.freeze({
|
|
61
|
+
value,
|
|
62
|
+
identity,
|
|
63
|
+
...(SECRET_TAINT_ENVIRONMENT.has(key) ? { outputTaint: 'secret' } : {}),
|
|
64
|
+
})]];
|
|
65
|
+
})));
|
|
66
|
+
}
|
|
67
|
+
async function resolveExecutable(command, environment) {
|
|
68
|
+
const candidates = isAbsolute(command)
|
|
69
|
+
? [command]
|
|
70
|
+
: (environment.PATH ?? '').split(delimiter).filter(Boolean).map((directory) => (join(directory, command)));
|
|
71
|
+
for (const candidate of candidates) {
|
|
72
|
+
try {
|
|
73
|
+
await access(candidate, constants.X_OK);
|
|
74
|
+
return await realpath(candidate);
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
// Continue through the explicit PATH snapshot.
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
throw new NodeCliProductionCompositionError('NODE_CLI_EXECUTABLE_UNAVAILABLE', `执行器「${command}」在当前 PATH 中不可用。`);
|
|
81
|
+
}
|
|
82
|
+
function requiredCredential(environment, key) {
|
|
83
|
+
const value = environment[key]?.trim();
|
|
84
|
+
if (value !== undefined && value !== '')
|
|
85
|
+
return value;
|
|
86
|
+
throw new NodeCliProductionCompositionError('NODE_CLI_CREDENTIAL_MISSING', `执行器需要环境变量 ${key}。`);
|
|
87
|
+
}
|
|
88
|
+
export function captureNodeCliEvaluationEnvironment(environment = process.env) {
|
|
89
|
+
const captured = Object.freeze({ ...environment });
|
|
90
|
+
return Object.freeze({
|
|
91
|
+
environment: captured,
|
|
92
|
+
classifiedEnvironment: classifyNodeCliEnvironment(captured),
|
|
93
|
+
resolveExecutable: (command) => resolveExecutable(command, captured),
|
|
94
|
+
requiredCredential: (key) => requiredCredential(captured, key),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.
|
|
2
|
+
export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.judge_models_single_only' | 'cli.common.ambiguous_sample_files' | 'cli.common.samples_not_found' | 'cli.common.samples_not_found_hint' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
|
|
3
3
|
export declare const commonDict: Record<CommonMessageKey, CliMessage>;
|
|
@@ -19,22 +19,10 @@ export const commonDict = {
|
|
|
19
19
|
zh: '目录下未找到 SKILL.md: {path}',
|
|
20
20
|
en: 'SKILL.md not found in directory: {path}',
|
|
21
21
|
},
|
|
22
|
-
'cli.common.report_not_found': {
|
|
23
|
-
zh: '未找到 report: {id}',
|
|
24
|
-
en: 'Report not found: {id}',
|
|
25
|
-
},
|
|
26
|
-
'cli.common.no_judge_model': {
|
|
27
|
-
zh: '未指定评委。请加 --judge-models <executor:model>, 或确保 report.meta.judgeModels 已写。',
|
|
28
|
-
en: 'No judge configured. Pass --judge-models <executor:model> or ensure the report has meta.judgeModels.',
|
|
29
|
-
},
|
|
30
22
|
'cli.common.judge_models_single_only': {
|
|
31
23
|
zh: '{cmd} 仅支持单评委。--judge-models 只能传一个 executor:model entry。',
|
|
32
24
|
en: '{cmd} only supports a single judge. --judge-models accepts exactly one executor:model entry.',
|
|
33
25
|
},
|
|
34
|
-
'cli.common.warn_load_samples_failed': {
|
|
35
|
-
zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
|
|
36
|
-
en: '⚠ Failed to load samples file ({path}): {message}\n',
|
|
37
|
-
},
|
|
38
26
|
'cli.common.ambiguous_sample_files': {
|
|
39
27
|
zh: '评测用例文件存在歧义:{paths}。同一作用域只能保留 eval-samples.json 或 eval-samples.yaml 其中一个。',
|
|
40
28
|
en: 'Ambiguous eval sample files: {paths}. Keep only one of eval-samples.json or eval-samples.yaml in the same scope.',
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CliMessage } from './types.js';
|
|
2
|
-
export type HelpMessageKey = 'cli.help.observe'
|
|
2
|
+
export type HelpMessageKey = 'cli.help.observe';
|
|
3
3
|
export declare const helpDict: Record<HelpMessageKey, CliMessage>;
|