oh-my-knowledge 0.53.0 → 1.0.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -15
- package/README.zh.md +49 -15
- package/dist/assets/agent-skills/omk/SKILL.md +35 -12
- package/dist/assets/agent-skills/omk/references/commands.md +45 -53
- package/dist/cli/commands/doctor.js +39 -36
- package/dist/cli/commands/eval/gold/compare.d.ts +5 -2
- package/dist/cli/commands/eval/gold/compare.js +57 -28
- package/dist/cli/commands/eval/gold/init.js +1 -1
- package/dist/cli/commands/eval/gold/validate.js +1 -1
- package/dist/cli/commands/eval/index.js +38 -360
- package/dist/cli/commands/evolve.d.ts +2 -12
- package/dist/cli/commands/evolve.js +67 -142
- package/dist/cli/commands/init.d.ts +2 -0
- package/dist/cli/commands/init.js +55 -65
- package/dist/cli/commands/install.js +2 -2
- package/dist/cli/commands/list.d.ts +1 -1
- package/dist/cli/commands/list.js +2 -2
- package/dist/cli/commands/observe/inbox.js +6 -6
- package/dist/cli/commands/observe/index.d.ts +4 -4
- package/dist/cli/commands/observe/index.js +22 -20
- package/dist/cli/commands/observe/ingest.d.ts +1 -1
- package/dist/cli/commands/observe/ingest.js +7 -7
- package/dist/cli/commands/observe/show.js +3 -3
- package/dist/cli/commands/promote.js +10 -18
- package/dist/cli/commands/rollback.js +2 -2
- package/dist/cli/commands/sample.d.ts +3 -20
- package/dist/cli/commands/sample.js +38 -306
- package/dist/cli/commands/studio.js +32 -42
- package/dist/cli/lib/cli-exit.d.ts +1 -1
- package/dist/cli/lib/cli-exit.js +1 -1
- package/dist/cli/lib/cmd-flags.d.ts +1 -14
- package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/common.js +5 -5
- package/dist/cli/lib/i18n-dict/evolve.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/evolve.js +3 -27
- package/dist/cli/lib/i18n-dict/gen.js +4 -4
- package/dist/cli/lib/i18n-dict/help.js +26 -26
- package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/init.js +15 -3
- package/dist/cli/lib/i18n-dict/promote.d.ts +1 -1
- package/dist/cli/lib/i18n-dict/promote.js +2 -10
- package/dist/cli/lib/i18n-dict.d.ts +1 -1
- package/dist/cli/lib/i18n-dict.js +1 -1
- package/dist/cli/lib/i18n.d.ts +1 -1
- package/dist/cli/lib/localized-sample-discovery.d.ts +3 -0
- package/dist/cli/lib/localized-sample-discovery.js +16 -0
- package/dist/cli/lib/open-workbench.d.ts +2 -0
- package/dist/cli/lib/open-workbench.js +19 -0
- package/dist/cli/lib/parse-run-config/judge-models.d.ts +1 -1
- package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +6 -7
- package/dist/cli/lib/parse-run-config/samples-discovery.js +19 -18
- package/dist/cli/lib/parse-run-config/variant-resolution.d.ts +2 -1
- package/dist/cli/lib/parse-run-config/variant-resolution.js +2 -2
- package/dist/cli/lib/parse-run-config.d.ts +5 -6
- package/dist/cli/lib/parse-run-config.js +10 -10
- package/dist/cli/lib/progress.d.ts +1 -1
- package/dist/cli/lib/record-evolve-outcome.d.ts +10 -25
- package/dist/cli/lib/record-evolve-outcome.js +26 -103
- package/dist/cli/lib/resolve-skill-input.js +8 -9
- package/dist/cli/lib/run-core-evaluation.d.ts +21 -0
- package/dist/cli/lib/run-core-evaluation.js +340 -0
- package/dist/cli/lib/shared.d.ts +0 -7
- package/dist/cli/lib/shared.js +0 -15
- package/dist/cli/lib/source-probe.d.ts +2 -2
- package/dist/cli/lib/source-probe.js +2 -2
- package/dist/cli/lib/update-check.d.ts +2 -3
- package/dist/cli/lib/update-check.js +4 -5
- package/dist/cli/lib/update-fetch-worker.js +2 -1
- package/dist/cli/templates/init-samples.d.ts +4 -0
- package/dist/cli/templates/init-samples.js +328 -0
- package/dist/diagnosis/contracts/parser.d.ts +9 -0
- package/dist/diagnosis/contracts/parser.js +181 -0
- package/dist/diagnosis/lifecycle.d.ts +4 -0
- package/dist/diagnosis/observe-mapper.d.ts +1 -1
- package/dist/diagnosis/observe-mapper.js +1 -1
- package/dist/diagnosis/observe-producer.d.ts +3 -3
- package/dist/diagnosis/observe-producer.js +2 -2
- package/dist/diagnosis/studio-projection.d.ts +1 -1
- package/dist/diagnosis/studio-projection.js +1 -1
- package/dist/dsh-plugin/cordis.patch.yml +1 -1
- package/dist/dsh-plugin/core-adapter.d.ts +26 -0
- package/dist/dsh-plugin/core-adapter.js +737 -0
- package/dist/dsh-plugin/core-command.d.ts +22 -0
- package/dist/dsh-plugin/core-command.js +305 -0
- package/dist/dsh-plugin/core-protocol.d.ts +16 -0
- package/dist/dsh-plugin/core-protocol.js +250 -0
- package/dist/dsh-plugin/host-executor.d.ts +7 -2
- package/dist/dsh-plugin/host-executor.js +1 -1
- package/dist/dsh-plugin/index.d.ts +5 -3
- package/dist/dsh-plugin/index.js +158 -100
- package/dist/dsh-plugin/observe.d.ts +47 -0
- package/dist/dsh-plugin/observe.js +359 -0
- package/dist/dsh-plugin/protocol.d.ts +1 -1
- package/dist/dsh-plugin/protocol.js +3 -3
- package/dist/dsh-plugin/trace-adapter.d.ts +49 -0
- package/dist/dsh-plugin/trace-adapter.js +590 -0
- package/dist/eval-core/analysis/builtins.d.ts +88 -0
- package/dist/eval-core/analysis/builtins.js +771 -0
- package/dist/eval-core/analysis/decision.d.ts +8 -0
- package/dist/eval-core/analysis/decision.js +488 -0
- package/dist/eval-core/analysis/index.d.ts +4 -0
- package/dist/eval-core/analysis/index.js +4 -0
- package/dist/eval-core/analysis/runtime.d.ts +6 -0
- package/dist/eval-core/analysis/runtime.js +908 -0
- package/dist/eval-core/analysis/types.d.ts +254 -0
- package/dist/eval-core/analysis/types.js +65 -0
- package/dist/eval-core/budget/index.d.ts +44 -0
- package/dist/eval-core/budget/index.js +577 -0
- package/dist/eval-core/compiler/errors.d.ts +24 -0
- package/dist/eval-core/compiler/errors.js +35 -0
- package/dist/eval-core/compiler/immutability.d.ts +2 -0
- package/dist/eval-core/compiler/immutability.js +17 -0
- package/dist/eval-core/compiler/index.d.ts +9 -0
- package/dist/eval-core/compiler/index.js +1013 -0
- package/dist/eval-core/compiler/types.d.ts +334 -0
- package/dist/eval-core/compiler/types.js +110 -0
- package/dist/eval-core/compiler/validation.d.ts +4 -0
- package/dist/eval-core/compiler/validation.js +566 -0
- package/dist/eval-core/contracts/analysis-bundle.d.ts +83 -0
- package/dist/eval-core/contracts/analysis-bundle.js +513 -0
- package/dist/eval-core/contracts/analysis-identities.d.ts +10 -0
- package/dist/eval-core/contracts/analysis-identities.js +24 -0
- package/dist/eval-core/contracts/artifacts.d.ts +9168 -0
- package/dist/eval-core/contracts/artifacts.js +468 -0
- package/dist/eval-core/contracts/budget.d.ts +299 -0
- package/dist/eval-core/contracts/budget.js +323 -0
- package/dist/eval-core/contracts/common.d.ts +191 -0
- package/dist/eval-core/contracts/common.js +132 -0
- package/dist/eval-core/contracts/comparability.d.ts +595 -0
- package/dist/eval-core/contracts/comparability.js +1208 -0
- package/dist/eval-core/contracts/definition.d.ts +1415 -0
- package/dist/eval-core/contracts/definition.js +335 -0
- package/dist/eval-core/contracts/digests.d.ts +118 -0
- package/dist/eval-core/contracts/digests.js +401 -0
- package/dist/eval-core/contracts/evaluation-bundle.d.ts +180 -0
- package/dist/eval-core/contracts/evaluation-bundle.js +862 -0
- package/dist/eval-core/contracts/evaluation-identities.d.ts +46 -0
- package/dist/eval-core/contracts/evaluation-identities.js +82 -0
- package/dist/eval-core/contracts/evaluation-report.d.ts +62 -0
- package/dist/eval-core/contracts/evaluation-report.js +233 -0
- package/dist/eval-core/contracts/execution-bundle.d.ts +150 -0
- package/dist/eval-core/contracts/execution-bundle.js +657 -0
- package/dist/eval-core/contracts/execution-controls.d.ts +147 -0
- package/dist/eval-core/contracts/execution-controls.js +77 -0
- package/dist/eval-core/contracts/execution-facts.d.ts +191 -0
- package/dist/eval-core/contracts/execution-facts.js +349 -0
- package/dist/eval-core/contracts/execution-identities.d.ts +96 -0
- package/dist/eval-core/contracts/execution-identities.js +374 -0
- package/dist/eval-core/contracts/index.d.ts +20 -0
- package/dist/eval-core/contracts/index.js +20 -0
- package/dist/eval-core/contracts/json-schema.d.ts +13 -0
- package/dist/eval-core/contracts/json-schema.js +183 -0
- package/dist/eval-core/contracts/json.d.ts +18 -0
- package/dist/eval-core/contracts/json.js +154 -0
- package/dist/eval-core/contracts/plans.d.ts +2562 -0
- package/dist/eval-core/contracts/plans.js +137 -0
- package/dist/eval-core/contracts/runtime-capabilities.d.ts +203 -0
- package/dist/eval-core/contracts/runtime-capabilities.js +48 -0
- package/dist/eval-core/contracts/schemas/v1/analysis-bundle.schema.json +991 -0
- package/dist/eval-core/contracts/schemas/v1/analysis-plan.schema.json +889 -0
- package/dist/eval-core/contracts/schemas/v1/budget-summary.schema.json +462 -0
- package/dist/eval-core/contracts/schemas/v1/comparability-assessment.schema.json +452 -0
- package/dist/eval-core/contracts/schemas/v1/comparability-policy.schema.json +84 -0
- package/dist/eval-core/contracts/schemas/v1/decision-plan.schema.json +389 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-bundle.schema.json +1793 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-definition.schema.json +1243 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-event.schema.json +144 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-plan.schema.json +789 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-report.schema.json +1041 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-definition.schema.json +333 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-plan.schema.json +573 -0
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-report.schema.json +513 -0
- package/dist/eval-core/contracts/schemas/v1/execution-bundle.schema.json +1512 -0
- package/dist/eval-core/contracts/schemas/v1/execution-facts.schema.json +460 -0
- package/dist/eval-core/contracts/schemas/v1/execution-plan.schema.json +1007 -0
- package/dist/eval-core/contracts/schemas/v1/executor-capabilities.schema.json +307 -0
- package/dist/eval-core/contracts/schemas/v1/measurement-policy.schema.json +473 -0
- package/dist/eval-core/contracts/schemas/v1/run-plan.schema.json +2514 -0
- package/dist/eval-core/contracts/schemas/v1/series-analysis-bundle.schema.json +1299 -0
- package/dist/eval-core/contracts/series.d.ts +1924 -0
- package/dist/eval-core/contracts/series.js +686 -0
- package/dist/eval-core/engine/index.d.ts +7 -0
- package/dist/eval-core/engine/index.js +570 -0
- package/dist/eval-core/engine/types.d.ts +148 -0
- package/dist/eval-core/evaluation/index.d.ts +3 -0
- package/dist/eval-core/evaluation/index.js +3 -0
- package/dist/eval-core/evaluation/runtime.d.ts +6 -0
- package/dist/eval-core/evaluation/runtime.js +1224 -0
- package/dist/eval-core/evaluation/types.d.ts +163 -0
- package/dist/eval-core/evaluation/types.js +33 -0
- package/dist/eval-core/execution/index.d.ts +4 -0
- package/dist/eval-core/execution/index.js +4 -0
- package/dist/eval-core/execution/runtime.d.ts +6 -0
- package/dist/eval-core/execution/runtime.js +1140 -0
- package/dist/eval-core/execution/scheduler.d.ts +7 -0
- package/dist/eval-core/execution/scheduler.js +102 -0
- package/dist/eval-core/execution/semaphore.d.ts +6 -0
- package/dist/eval-core/execution/semaphore.js +54 -0
- package/dist/eval-core/execution/types.d.ts +115 -0
- package/dist/eval-core/execution/types.js +34 -0
- package/dist/eval-core/facade.d.ts +8 -0
- package/dist/eval-core/facade.js +2 -0
- package/dist/eval-core/index.d.ts +14 -0
- package/dist/eval-core/index.js +6 -0
- package/dist/eval-core/internal/sealed-run-plan.d.ts +14 -0
- package/dist/eval-core/internal/sealed-run-plan.js +12 -0
- package/dist/eval-core/runtime/event-stream.d.ts +8 -0
- package/dist/eval-core/runtime/event-stream.js +51 -0
- package/dist/eval-core/runtime/events.d.ts +30 -0
- package/dist/eval-core/runtime/events.js +102 -0
- package/dist/eval-core/runtime/snapshot.d.ts +6 -0
- package/dist/eval-core/runtime/snapshot.js +23 -0
- package/dist/eval-core/schema-url.d.ts +4 -0
- package/dist/eval-core/schema-url.js +10 -0
- package/dist/eval-core/schemas.d.ts +2 -0
- package/dist/eval-core/schemas.js +23 -0
- package/dist/eval-core/series/index.d.ts +1 -0
- package/dist/eval-core/series/index.js +1 -0
- package/dist/eval-core/series/runtime.d.ts +146 -0
- package/dist/eval-core/series/runtime.js +758 -0
- package/dist/eval-workflows/analysis/bootstrap.d.ts +145 -0
- package/dist/eval-workflows/analysis/bootstrap.js +274 -0
- package/dist/eval-workflows/artifact-store/batch-contracts.d.ts +117 -0
- package/dist/eval-workflows/artifact-store/batch-contracts.js +70 -0
- package/dist/eval-workflows/artifact-store/contracts.d.ts +161 -0
- package/dist/eval-workflows/artifact-store/contracts.js +115 -0
- package/dist/eval-workflows/artifact-store/index.d.ts +6 -0
- package/dist/eval-workflows/artifact-store/index.js +6 -0
- package/dist/eval-workflows/artifact-store/node-batch-store.d.ts +8 -0
- package/dist/eval-workflows/artifact-store/node-batch-store.js +187 -0
- package/dist/eval-workflows/artifact-store/node-content-store.d.ts +13 -0
- package/dist/eval-workflows/artifact-store/node-content-store.js +157 -0
- package/dist/eval-workflows/artifact-store/node-run-store.d.ts +12 -0
- package/dist/eval-workflows/artifact-store/node-run-store.js +516 -0
- package/dist/eval-workflows/artifact-store/overlay-store.d.ts +12 -0
- package/dist/eval-workflows/artifact-store/overlay-store.js +112 -0
- package/dist/eval-workflows/artifact-store/private-json-file.d.ts +4 -0
- package/dist/eval-workflows/artifact-store/private-json-file.js +50 -0
- package/dist/eval-workflows/assertions/deterministic.d.ts +31 -0
- package/dist/eval-workflows/assertions/deterministic.js +301 -0
- package/dist/eval-workflows/assertions/layers.d.ts +5 -0
- package/dist/eval-workflows/assertions/layers.js +55 -0
- package/dist/eval-workflows/evaluation-defaults.d.ts +4 -0
- package/dist/eval-workflows/evaluation-defaults.js +4 -0
- package/dist/eval-workflows/gold/cli.d.ts +9 -0
- package/dist/eval-workflows/gold/cli.js +61 -0
- package/dist/eval-workflows/gold/human.d.ts +103 -0
- package/dist/eval-workflows/gold/human.js +189 -0
- package/dist/eval-workflows/input-compilation/compile.d.ts +6 -0
- package/dist/eval-workflows/input-compilation/compile.js +1108 -0
- package/dist/eval-workflows/input-compilation/error.d.ts +16 -0
- package/dist/eval-workflows/input-compilation/error.js +14 -0
- package/dist/eval-workflows/input-compilation/index.d.ts +5 -0
- package/dist/eval-workflows/input-compilation/index.js +5 -0
- package/dist/eval-workflows/input-compilation/parse.d.ts +25 -0
- package/dist/eval-workflows/input-compilation/parse.js +546 -0
- package/dist/eval-workflows/input-compilation/registry.d.ts +35 -0
- package/dist/eval-workflows/input-compilation/registry.js +274 -0
- package/dist/eval-workflows/input-compilation/types.d.ts +396 -0
- package/dist/eval-workflows/input-compilation/types.js +4 -0
- package/dist/eval-workflows/inputs/assertion-types.d.ts +6 -0
- package/dist/eval-workflows/inputs/contracts/assertion-kind.d.ts +3 -0
- package/dist/eval-workflows/inputs/contracts/assertion.d.ts +25 -0
- package/dist/eval-workflows/inputs/contracts/config.d.ts +68 -0
- package/dist/eval-workflows/inputs/contracts/index.d.ts +5 -0
- package/dist/eval-workflows/inputs/contracts/sample.d.ts +78 -0
- package/dist/eval-workflows/inputs/contracts/schemas/v1/eval-sample-set.schema.json +620 -0
- package/dist/eval-workflows/inputs/contracts/variant.d.ts +16 -0
- package/dist/eval-workflows/inputs/eval-config.d.ts +19 -0
- package/dist/eval-workflows/inputs/eval-config.js +340 -0
- package/dist/eval-workflows/inputs/eval-samples.d.ts +8 -0
- package/dist/eval-workflows/inputs/eval-samples.js +12 -0
- package/dist/eval-workflows/inputs/load-samples.d.ts +43 -0
- package/dist/eval-workflows/inputs/load-samples.js +290 -0
- package/dist/eval-workflows/inputs/materialize-copy.js +193 -0
- package/dist/eval-workflows/inputs/sample-contract.d.ts +3 -0
- package/dist/eval-workflows/inputs/sample-contract.js +361 -0
- package/dist/eval-workflows/inputs/sample-document.d.ts +10 -0
- package/dist/eval-workflows/inputs/sample-document.js +94 -0
- package/dist/eval-workflows/inputs/sample-locator.d.ts +27 -0
- package/dist/eval-workflows/inputs/sample-locator.js +160 -0
- package/dist/eval-workflows/inputs/schemas/assertion.d.ts +4 -0
- package/dist/eval-workflows/inputs/schemas/assertion.js +23 -0
- package/dist/eval-workflows/inputs/schemas/error.d.ts +8 -0
- package/dist/eval-workflows/inputs/schemas/error.js +36 -0
- package/dist/eval-workflows/inputs/schemas/json-schema.d.ts +1 -0
- package/dist/eval-workflows/inputs/schemas/json-schema.js +27 -0
- package/dist/eval-workflows/inputs/schemas/mock.d.ts +6 -0
- package/dist/eval-workflows/inputs/schemas/mock.js +27 -0
- package/dist/eval-workflows/inputs/schemas/sample-set.d.ts +15 -0
- package/dist/eval-workflows/inputs/schemas/sample-set.js +60 -0
- package/dist/eval-workflows/inputs/skill-loader.d.ts +191 -0
- package/dist/eval-workflows/inputs/skill-loader.js +836 -0
- package/dist/eval-workflows/instruments/contracts/config.d.ts +7 -0
- package/dist/eval-workflows/instruments/contracts/index.d.ts +2 -0
- package/dist/eval-workflows/instruments/contracts/result.d.ts +54 -0
- package/dist/eval-workflows/instruments/judge-trace.d.ts +8 -0
- package/dist/eval-workflows/instruments/judge-trace.js +94 -0
- package/dist/eval-workflows/instruments/judge.d.ts +78 -0
- package/dist/eval-workflows/instruments/judge.js +323 -0
- package/dist/eval-workflows/instruments/prompts/debias-instructions.js +43 -0
- package/dist/eval-workflows/instruments/prompts/judge-prompts.d.ts +30 -0
- package/dist/eval-workflows/instruments/prompts/judge-prompts.js +205 -0
- package/dist/eval-workflows/messages.d.ts +1 -1
- package/dist/eval-workflows/production-host/artifact-graph-persistence.d.ts +11 -0
- package/dist/eval-workflows/production-host/artifact-graph-persistence.js +66 -0
- package/dist/eval-workflows/production-host/environment.d.ts +9 -0
- package/dist/eval-workflows/production-host/environment.js +34 -0
- package/dist/eval-workflows/production-host/holdout.d.ts +8 -0
- package/dist/eval-workflows/production-host/holdout.js +25 -0
- package/dist/eval-workflows/production-host/index.d.ts +9 -0
- package/dist/eval-workflows/production-host/index.js +9 -0
- package/dist/eval-workflows/production-host/measurement-design.d.ts +21 -0
- package/dist/eval-workflows/production-host/measurement-design.js +562 -0
- package/dist/eval-workflows/production-host/node-cli-composition.d.ts +26 -0
- package/dist/eval-workflows/production-host/node-cli-composition.js +474 -0
- package/dist/eval-workflows/production-host/node-cli-evaluation-resolver.d.ts +20 -0
- package/dist/eval-workflows/production-host/node-cli-evaluation-resolver.js +755 -0
- package/dist/eval-workflows/production-host/node-sample-content-resolver.d.ts +43 -0
- package/dist/eval-workflows/production-host/node-sample-content-resolver.js +270 -0
- package/dist/eval-workflows/production-host/orchestration.d.ts +78 -0
- package/dist/eval-workflows/production-host/orchestration.js +274 -0
- package/dist/eval-workflows/production-host/runtime-registry.d.ts +57 -0
- package/dist/eval-workflows/production-host/runtime-registry.js +270 -0
- package/dist/eval-workflows/production-host/safe-http-content-resolver.d.ts +4 -0
- package/dist/eval-workflows/production-host/safe-http-content-resolver.js +219 -0
- package/dist/eval-workflows/production-host/sample-content-resolution.d.ts +44 -0
- package/dist/eval-workflows/production-host/sample-content-resolution.js +242 -0
- package/dist/eval-workflows/production-host/workflow.d.ts +76 -0
- package/dist/eval-workflows/production-host/workflow.js +200 -0
- package/dist/eval-workflows/projections/artifact-graph.d.ts +15 -0
- package/dist/eval-workflows/projections/artifact-graph.js +388 -0
- package/dist/eval-workflows/projections/cli-gate.d.ts +3 -0
- package/dist/eval-workflows/projections/cli-gate.js +45 -0
- package/dist/eval-workflows/projections/cli.d.ts +39 -0
- package/dist/eval-workflows/projections/cli.js +334 -0
- package/dist/eval-workflows/projections/contracts.d.ts +332 -0
- package/dist/eval-workflows/projections/contracts.js +16 -0
- package/dist/eval-workflows/projections/decision.d.ts +4 -0
- package/dist/eval-workflows/projections/decision.js +27 -0
- package/dist/eval-workflows/projections/diagnostic.d.ts +7 -0
- package/dist/eval-workflows/projections/diagnostic.js +160 -0
- package/dist/eval-workflows/projections/evolution.d.ts +13 -0
- package/dist/eval-workflows/projections/evolution.js +144 -0
- package/dist/eval-workflows/projections/gold.d.ts +21 -0
- package/dist/eval-workflows/projections/gold.js +204 -0
- package/dist/eval-workflows/projections/index.d.ts +9 -0
- package/dist/eval-workflows/projections/index.js +9 -0
- package/dist/eval-workflows/projections/managed.d.ts +8 -0
- package/dist/eval-workflows/projections/managed.js +105 -0
- package/dist/eval-workflows/projections/source.d.ts +2 -0
- package/dist/eval-workflows/projections/source.js +48 -0
- package/dist/eval-workflows/resume-admission/admit.d.ts +8 -0
- package/dist/eval-workflows/resume-admission/admit.js +180 -0
- package/dist/eval-workflows/resume-admission/contracts.d.ts +81 -0
- package/dist/eval-workflows/resume-admission/contracts.js +22 -0
- package/dist/eval-workflows/resume-admission/index.d.ts +2 -0
- package/dist/eval-workflows/resume-admission/index.js +2 -0
- package/dist/eval-workflows/runtime-adapter/adapters/anthropic/api.d.ts +31 -0
- package/dist/eval-workflows/runtime-adapter/adapters/anthropic/api.js +286 -0
- package/dist/eval-workflows/runtime-adapter/adapters/anthropic/protocol.d.ts +67 -0
- package/dist/eval-workflows/runtime-adapter/adapters/anthropic/protocol.js +192 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/cli-protocol.d.ts +19 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/cli-protocol.js +322 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/cli.d.ts +48 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/cli.js +607 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/resources.d.ts +151 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/resources.js +664 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk-protocol.d.ts +7 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk-protocol.js +84 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk-runtime.d.ts +40 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk-runtime.js +163 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk.d.ts +29 -0
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk.js +507 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-protocol.d.ts +61 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-protocol.js +36 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-resources.d.ts +13 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-resources.js +16 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli.d.ts +44 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli.js +348 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/content-identity.d.ts +5 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/content-identity.js +3 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/environment.d.ts +4 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/environment.js +2 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/protocol-core.d.ts +21 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/protocol-core.js +232 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/resources.d.ts +108 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/resources.js +331 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-protocol.d.ts +61 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-protocol.js +20 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-runtime.d.ts +33 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-runtime.js +169 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk.d.ts +32 -0
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk.js +375 -0
- package/dist/eval-workflows/runtime-adapter/adapters/custom/command.d.ts +207 -0
- package/dist/eval-workflows/runtime-adapter/adapters/custom/command.js +696 -0
- package/dist/eval-workflows/runtime-adapter/adapters/index.d.ts +8 -0
- package/dist/eval-workflows/runtime-adapter/adapters/index.js +8 -0
- package/dist/eval-workflows/runtime-adapter/adapters/openai/api.d.ts +31 -0
- package/dist/eval-workflows/runtime-adapter/adapters/openai/api.js +292 -0
- package/dist/eval-workflows/runtime-adapter/adapters/openai/protocol.d.ts +67 -0
- package/dist/eval-workflows/runtime-adapter/adapters/openai/protocol.js +236 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/api-http.d.ts +53 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/api-http.js +167 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/api-protocol-core.d.ts +7 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/api-protocol-core.js +96 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/classified-environment.d.ts +21 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/classified-environment.js +43 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/content-identity.d.ts +16 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/content-identity.js +101 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/same-process.d.ts +109 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/same-process.js +322 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/stateless-api-resources.d.ts +80 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/stateless-api-resources.js +230 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-node-contract.d.ts +10 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-node-contract.js +78 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-node.d.ts +3 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-node.js +29 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-parameters.d.ts +31 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-parameters.js +65 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-source-adapter.d.ts +11 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-source-adapter.js +114 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-table.d.ts +259 -0
- package/dist/eval-workflows/runtime-adapter/analysis/agreement-table.js +303 -0
- package/dist/eval-workflows/runtime-adapter/analysis/analysis-support.d.ts +14 -0
- package/dist/eval-workflows/runtime-adapter/analysis/analysis-support.js +44 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-node.d.ts +5 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-node.js +218 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-parameters.d.ts +39 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-parameters.js +49 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer.d.ts +552 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer.js +289 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-node-contract.d.ts +10 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-node-contract.js +89 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-node.d.ts +3 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-node.js +35 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-parameters.d.ts +41 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-parameters.js +111 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-source-adapter.d.ts +9 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-source-adapter.js +28 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-table.d.ts +240 -0
- package/dist/eval-workflows/runtime-adapter/analysis/bootstrap-family-table.js +368 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-node-contract.d.ts +10 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-node-contract.js +80 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-node.d.ts +3 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-node.js +79 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-parameters.d.ts +56 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-parameters.js +92 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-source-adapter.d.ts +13 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-source-adapter.js +60 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-table.d.ts +244 -0
- package/dist/eval-workflows/runtime-adapter/analysis/composite-table.js +199 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-node.d.ts +5 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-node.js +169 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-parameters.d.ts +24 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-parameters.js +44 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-table.d.ts +135 -0
- package/dist/eval-workflows/runtime-adapter/analysis/dimension-table.js +195 -0
- package/dist/eval-workflows/runtime-adapter/analysis/index.d.ts +21 -0
- package/dist/eval-workflows/runtime-adapter/analysis/index.js +21 -0
- package/dist/eval-workflows/runtime-adapter/analysis/judge-aggregation.d.ts +343 -0
- package/dist/eval-workflows/runtime-adapter/analysis/judge-aggregation.js +731 -0
- package/dist/eval-workflows/runtime-adapter/analysis/release-decision-parameters.d.ts +36 -0
- package/dist/eval-workflows/runtime-adapter/analysis/release-decision-parameters.js +114 -0
- package/dist/eval-workflows/runtime-adapter/analysis/release-decision.d.ts +6 -0
- package/dist/eval-workflows/runtime-adapter/analysis/release-decision.js +406 -0
- package/dist/eval-workflows/runtime-adapter/assembly.d.ts +6 -0
- package/dist/eval-workflows/runtime-adapter/assembly.js +787 -0
- package/dist/eval-workflows/runtime-adapter/builtins.d.ts +7 -0
- package/dist/eval-workflows/runtime-adapter/builtins.js +106 -0
- package/dist/eval-workflows/runtime-adapter/composition.d.ts +82 -0
- package/dist/eval-workflows/runtime-adapter/composition.js +824 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/assertion-common.d.ts +17 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/assertion-common.js +72 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/execution-assertions.d.ts +38 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/execution-assertions.js +307 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/index.d.ts +5 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/index.js +5 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-assertions.d.ts +71 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-assertions.js +493 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-judge-invocation.d.ts +49 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-judge-invocation.js +72 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/output-assertions.d.ts +33 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/output-assertions.js +168 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/rubric-judge.d.ts +73 -0
- package/dist/eval-workflows/runtime-adapter/evaluators/rubric-judge.js +431 -0
- package/dist/eval-workflows/runtime-adapter/event-projection.d.ts +34 -0
- package/dist/eval-workflows/runtime-adapter/event-projection.js +175 -0
- package/dist/eval-workflows/runtime-adapter/index.d.ts +11 -0
- package/dist/eval-workflows/runtime-adapter/index.js +11 -0
- package/dist/eval-workflows/runtime-adapter/preflight.d.ts +54 -0
- package/dist/eval-workflows/runtime-adapter/preflight.js +248 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/access.d.ts +14 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/access.js +173 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/index.d.ts +3 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/index.js +3 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/node.d.ts +24 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/node.js +803 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/types.d.ts +105 -0
- package/dist/eval-workflows/runtime-adapter/resource-leases/types.js +25 -0
- package/dist/eval-workflows/runtime-adapter/series-variance.d.ts +6 -0
- package/dist/eval-workflows/runtime-adapter/series-variance.js +94 -0
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.d.ts +57 -0
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.js +66 -0
- package/dist/eval-workflows/runtime-adapter/types.d.ts +159 -0
- package/dist/eval-workflows/runtime-adapter/types.js +12 -0
- package/dist/evidence/graph/contracts.d.ts +93 -0
- package/dist/evidence/graph/doctor.d.ts +22 -0
- package/dist/evidence/graph/doctor.js +572 -0
- package/dist/evidence/graph/schema.d.ts +3 -0
- package/dist/evidence/graph/schema.js +233 -0
- package/dist/evidence/storage/default-dirs.d.ts +34 -0
- package/dist/evidence/storage/default-dirs.js +35 -0
- package/dist/evidence/storage/directories.js +65 -0
- package/dist/evidence/storage/discovery-index.d.ts +88 -0
- package/dist/evidence/storage/discovery-index.js +307 -0
- package/dist/evidence/storage/file-names.d.ts +14 -0
- package/dist/evidence/storage/file-names.js +57 -0
- package/dist/evidence/storage/layout.d.ts +35 -0
- package/dist/evidence/storage/layout.js +50 -0
- package/dist/evidence/storage/report-bundle.d.ts +35 -0
- package/dist/evidence/storage/report-bundle.js +122 -0
- package/dist/evidence/storage/run-id.d.ts +2 -0
- package/dist/evidence/storage/run-id.js +19 -0
- package/dist/executors/anthropic/api.d.ts +3 -2
- package/dist/executors/anthropic/api.js +5 -3
- package/dist/executors/anthropic/claude/cli.d.ts +3 -2
- package/dist/executors/anthropic/claude/cli.js +3 -2
- package/dist/executors/anthropic/claude/protocol.d.ts +1 -1
- package/dist/executors/anthropic/claude/protocol.js +1 -1
- package/dist/executors/anthropic/claude/sdk.d.ts +3 -2
- package/dist/executors/anthropic/claude/sdk.js +7 -2
- package/dist/executors/anthropic/claude/trace.d.ts +1 -1
- package/dist/executors/anthropic/claude/trace.js +4 -4
- package/dist/executors/contracts/index.d.ts +4 -0
- package/dist/executors/contracts/index.js +1 -0
- package/dist/executors/contracts/mock.d.ts +65 -0
- package/dist/executors/contracts/mock.js +1 -0
- package/dist/executors/contracts/ports.d.ts +63 -0
- package/dist/executors/contracts/ports.js +1 -0
- package/dist/executors/contracts/result.d.ts +41 -0
- package/dist/executors/contracts/result.js +1 -0
- package/dist/executors/contracts/runtime.d.ts +40 -0
- package/dist/executors/contracts/runtime.js +1 -0
- package/dist/executors/contracts/trace-source.d.ts +2 -0
- package/dist/executors/contracts/trace-source.js +1 -0
- package/dist/executors/contracts/trace.d.ts +36 -0
- package/dist/executors/contracts/trace.js +1 -0
- package/dist/executors/core/capabilities.d.ts +6 -2
- package/dist/executors/core/runtime-fingerprint.d.ts +2 -1
- package/dist/executors/core/runtime.d.ts +1 -1
- package/dist/executors/core/subprocess.d.ts +1 -0
- package/dist/executors/core/subprocess.js +9 -8
- package/dist/executors/core/trace-source-kind.d.ts +4 -0
- package/dist/executors/core/trace-source-kind.js +13 -0
- package/dist/executors/index.d.ts +1 -1
- package/dist/executors/mock-runtime/mock-hook.cjs +271 -0
- package/dist/executors/mock-runtime/runtime.d.ts +103 -0
- package/dist/executors/mock-runtime/runtime.js +638 -0
- package/dist/executors/openai/api.d.ts +3 -2
- package/dist/executors/openai/api.js +5 -3
- package/dist/executors/openai/codex/cli.d.ts +3 -2
- package/dist/executors/openai/codex/cli.js +3 -1
- package/dist/executors/openai/codex/protocol.d.ts +2 -1
- package/dist/executors/openai/codex/protocol.js +1 -1
- package/dist/executors/openai/codex/sdk.d.ts +3 -2
- package/dist/executors/openai/codex/sdk.js +6 -1
- package/dist/executors/openai/codex/trace.d.ts +1 -1
- package/dist/executors/openai/codex/trace.js +2 -2
- package/dist/executors/preflight/contracts.d.ts +40 -0
- package/dist/executors/preflight/contracts.js +1 -0
- package/dist/executors/preflight/dependencies.d.ts +36 -0
- package/dist/executors/result-validation.d.ts +18 -0
- package/dist/executors/result-validation.js +221 -0
- package/dist/executors/script/index.d.ts +1 -1
- package/dist/executors/script/index.js +2 -2
- package/dist/executors/tool-call-status.d.ts +9 -0
- package/dist/executors/tool-call-status.js +79 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/knowledge-artifacts/authoring/core-evolver.d.ts +65 -0
- package/dist/knowledge-artifacts/authoring/core-evolver.js +277 -0
- package/dist/knowledge-artifacts/authoring/generator.d.ts +100 -0
- package/dist/knowledge-artifacts/authoring/generator.js +1019 -0
- package/dist/knowledge-artifacts/authoring/improvement.d.ts +15 -0
- package/dist/knowledge-artifacts/authoring/improvement.js +52 -0
- package/dist/knowledge-artifacts/contracts.d.ts +18 -0
- package/dist/knowledge-artifacts/contracts.js +1 -0
- package/dist/knowledge-artifacts/doctor/contracts.d.ts +199 -0
- package/dist/knowledge-artifacts/doctor/contracts.js +1 -0
- package/dist/knowledge-artifacts/doctor/endpoint-rule.d.ts +60 -0
- package/dist/knowledge-artifacts/doctor/endpoint-rule.js +382 -0
- package/dist/knowledge-artifacts/doctor/fixer.d.ts +11 -0
- package/dist/knowledge-artifacts/doctor/fixer.js +337 -0
- package/dist/knowledge-artifacts/doctor/health/composer.d.ts +18 -0
- package/dist/knowledge-artifacts/doctor/health/composer.js +407 -0
- package/dist/knowledge-artifacts/doctor/health/dimension-spec.d.ts +52 -0
- package/dist/knowledge-artifacts/doctor/health/merge-prompt.d.ts +22 -0
- package/dist/knowledge-artifacts/doctor/health/prompt-builder.d.ts +21 -0
- package/dist/knowledge-artifacts/doctor/index.d.ts +19 -0
- package/dist/knowledge-artifacts/doctor/index.js +274 -0
- package/dist/knowledge-artifacts/doctor/messages.d.ts +20 -0
- package/dist/knowledge-artifacts/doctor/preflight.d.ts +32 -0
- package/dist/knowledge-artifacts/doctor/preflight.js +31 -0
- package/dist/knowledge-artifacts/doctor/renderer.d.ts +17 -0
- package/dist/knowledge-artifacts/doctor/report-parser.d.ts +3 -0
- package/dist/knowledge-artifacts/doctor/report-parser.js +103 -0
- package/dist/knowledge-artifacts/doctor/report-schema.d.ts +3 -0
- package/dist/knowledge-artifacts/doctor/report-schema.js +3 -0
- package/dist/knowledge-artifacts/doctor/rule-kind.d.ts +2 -0
- package/dist/knowledge-artifacts/doctor/rule-kind.js +3 -0
- package/dist/knowledge-artifacts/doctor/rules.d.ts +32 -0
- package/dist/knowledge-artifacts/doctor/rules.js +290 -0
- package/dist/knowledge-artifacts/governance/contracts.d.ts +158 -0
- package/dist/knowledge-artifacts/governance/contracts.js +1 -0
- package/dist/knowledge-artifacts/governance/evidence.d.ts +24 -0
- package/dist/knowledge-artifacts/governance/evidence.js +109 -0
- package/dist/knowledge-artifacts/governance/index.d.ts +11 -0
- package/dist/knowledge-artifacts/governance/index.js +11 -0
- package/dist/knowledge-artifacts/governance/list-view.d.ts +67 -0
- package/dist/knowledge-artifacts/governance/list-view.js +77 -0
- package/dist/knowledge-artifacts/governance/observe-feedback.d.ts +60 -0
- package/dist/knowledge-artifacts/governance/promote-gate.d.ts +34 -0
- package/dist/knowledge-artifacts/governance/promote-gate.js +32 -0
- package/dist/knowledge-artifacts/governance/source-probe.d.ts +15 -0
- package/dist/knowledge-artifacts/governance/source-probe.js +129 -0
- package/dist/knowledge-artifacts/governance/store.d.ts +141 -0
- package/dist/knowledge-artifacts/governance/store.js +585 -0
- package/dist/knowledge-artifacts/skills/contracts.d.ts +35 -0
- package/dist/knowledge-artifacts/skills/contracts.js +1 -0
- package/dist/knowledge-artifacts/skills/hard-rules.d.ts +7 -0
- package/dist/knowledge-artifacts/sources/content-hash.d.ts +31 -0
- package/dist/knowledge-artifacts/sources/content-hash.js +121 -0
- package/dist/mcp/capture-store.d.ts +25 -0
- package/dist/mcp/capture-store.js +48 -0
- package/dist/mcp/feedback-store.d.ts +78 -0
- package/dist/mcp/feedback-store.js +241 -0
- package/dist/mcp/http.d.ts +26 -0
- package/dist/mcp/http.js +225 -0
- package/dist/mcp/index.d.ts +7 -0
- package/dist/mcp/index.js +7 -0
- package/dist/mcp/mcp-server.d.ts +14 -0
- package/dist/mcp/mcp-server.js +293 -0
- package/dist/mcp/principal.d.ts +25 -0
- package/dist/mcp/principal.js +83 -0
- package/dist/mcp/review-component.d.ts +11 -0
- package/dist/mcp/review-component.js +846 -0
- package/dist/mcp/stdio.d.ts +2 -0
- package/dist/mcp/stdio.js +16 -0
- package/dist/observability/analysis/contracts.d.ts +57 -0
- package/dist/observability/analysis/contracts.js +1 -0
- package/dist/observability/analysis/coverage-analyzer.d.ts +53 -0
- package/dist/observability/analysis/coverage-analyzer.js +306 -0
- package/dist/observability/analysis/gap-analyzer.d.ts +118 -0
- package/dist/observability/analysis/gap-analyzer.js +470 -0
- package/dist/observability/analysis/hedging-classifier.d.ts +34 -0
- package/dist/observability/contracts/experience.d.ts +651 -0
- package/dist/observability/contracts/experience.js +1 -0
- package/dist/observability/contracts/inbox.d.ts +181 -0
- package/dist/observability/contracts/inbox.js +1 -0
- package/dist/observability/contracts/index.d.ts +7 -0
- package/dist/observability/contracts/index.js +1 -0
- package/dist/observability/contracts/problem-patterns.d.ts +50 -0
- package/dist/observability/contracts/problem-patterns.js +1 -0
- package/dist/observability/contracts/review.d.ts +47 -0
- package/dist/observability/contracts/review.js +1 -0
- package/dist/observability/contracts/skill-chain-advisories.d.ts +8 -0
- package/dist/observability/contracts/skill-chain-advisories.js +1 -0
- package/dist/observability/contracts/skill-chain.d.ts +114 -0
- package/dist/observability/contracts/skill-chain.js +1 -0
- package/dist/observability/contracts/trace.d.ts +22 -0
- package/dist/observability/contracts/trace.js +1 -0
- package/dist/observability/conversation/catalog.d.ts +42 -0
- package/dist/observability/conversation/catalog.js +575 -0
- package/dist/observability/conversation/index-process.d.ts +1 -0
- package/dist/observability/conversation/index-process.js +31 -0
- package/dist/observability/conversation/knowledge-debugger.d.ts +7 -0
- package/dist/observability/conversation/knowledge-debugger.js +397 -0
- package/dist/observability/conversation/task-semantic-projection.d.ts +11 -0
- package/dist/observability/conversation/task-window.d.ts +10 -0
- package/dist/observability/conversation/task-window.js +57 -0
- package/dist/observability/conversation/turn-index.d.ts +3 -0
- package/dist/observability/conversation/turn-index.js +255 -0
- package/dist/observability/conversation/view-model.d.ts +5 -0
- package/dist/observability/conversation/view-model.js +100 -0
- package/dist/observability/experience/primitives.d.ts +11 -0
- package/dist/observability/experience/primitives.js +75 -0
- package/dist/observability/experience/report-codec.d.ts +33 -0
- package/dist/observability/experience/report-codec.js +235 -0
- package/dist/observability/experience/report-derivations.d.ts +35 -0
- package/dist/observability/experience/report-derivations.js +283 -0
- package/dist/observability/experience/report-reference-validator.d.ts +18 -0
- package/dist/observability/experience/report-reference-validator.js +596 -0
- package/dist/observability/experience/report-structure.d.ts +9 -0
- package/dist/observability/experience/report-structure.js +93 -0
- package/dist/observability/experience/report-value-guards.d.ts +65 -0
- package/dist/observability/experience/report-value-guards.js +1224 -0
- package/dist/observability/experience/review-checklist.d.ts +43 -0
- package/dist/observability/experience/review-checklist.js +630 -0
- package/dist/observability/experience/reviewer-report.d.ts +17 -0
- package/dist/observability/experience/reviewer-report.js +281 -0
- package/dist/observability/experience/session-story.d.ts +93 -0
- package/dist/observability/experience/session-story.js +1351 -0
- package/dist/observability/experience/text-signals.d.ts +7 -0
- package/dist/observability/experience/text-signals.js +30 -0
- package/dist/observability/experience/timeline.d.ts +29 -0
- package/dist/observability/experience/timeline.js +495 -0
- package/dist/observability/experience.d.ts +13 -43
- package/dist/observability/experience.js +177 -5378
- package/dist/observability/inbox/capture-coverage.d.ts +3 -0
- package/dist/observability/inbox/capture-coverage.js +34 -0
- package/dist/observability/inbox/explicit-capture.d.ts +50 -0
- package/dist/observability/inbox/explicit-capture.js +223 -0
- package/dist/observability/inbox/feedback-projection.d.ts +20 -0
- package/dist/observability/inbox/identity.d.ts +6 -0
- package/dist/observability/inbox/identity.js +44 -0
- package/dist/observability/inbox/index.d.ts +36 -0
- package/dist/observability/inbox/index.js +1339 -0
- package/dist/observability/inbox/paths.d.ts +10 -0
- package/dist/observability/inbox/paths.js +65 -0
- package/dist/observability/inbox/problem-patterns.d.ts +11 -0
- package/dist/observability/inbox/problem-patterns.js +246 -0
- package/dist/observability/inbox/resolved-review.d.ts +36 -0
- package/dist/observability/inbox/review-state.d.ts +25 -0
- package/dist/observability/inbox/review-state.js +352 -0
- package/dist/observability/inbox/source-record-archive.d.ts +14 -0
- package/dist/observability/inbox/source-record-archive.js +328 -0
- package/dist/observability/inbox/view-model.d.ts +44 -0
- package/dist/observability/inbox/view-model.js +186 -0
- package/dist/observability/prompts/document.js +21 -0
- package/dist/observability/skill-health/advisories.d.ts +15 -0
- package/dist/observability/skill-health/analyzer.d.ts +129 -0
- package/dist/observability/skill-health/analyzer.js +328 -0
- package/dist/observability/skill-health/experience-frontmatter.d.ts +36 -0
- package/dist/observability/skill-health/experience-frontmatter.js +99 -0
- package/dist/observability/skill-health/report.d.ts +10 -0
- package/dist/observability/skill-health/report.js +620 -0
- package/dist/observability/skill-health/skill-chain.d.ts +7 -0
- package/dist/observability/skill-health/skill-chain.js +486 -0
- package/dist/observability/soft-standards/llm-extractor.js +1 -1
- package/dist/observability/soft-standards/runtime-evaluator.d.ts +1 -1
- package/dist/observability/soft-standards/skill-standards-store.d.ts +1 -1
- package/dist/observability/soft-standards/skill-standards-store.js +4 -3
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/observability/trace/adapters/codex/conversation-index.d.ts +92 -0
- package/dist/observability/trace/adapters/codex/conversation-index.js +600 -0
- package/dist/observability/trace/adapters/codex/tool-status.d.ts +16 -0
- package/dist/observability/trace/adapters/codex/tool-status.js +114 -0
- package/dist/observability/trace/adapters/codex/trace.d.ts +5 -0
- package/dist/observability/trace/adapters/codex/trace.js +1091 -0
- package/dist/observability/trace/attribution.d.ts +63 -0
- package/dist/observability/trace/attribution.js +397 -0
- package/dist/observability/trace/index.d.ts +26 -0
- package/dist/observability/trace/index.js +20 -0
- package/dist/observability/trace/ingestion.d.ts +9 -0
- package/dist/observability/trace/ingestion.js +80 -0
- package/dist/observability/trace/message-classification.d.ts +6 -0
- package/dist/observability/trace/message-classification.js +34 -0
- package/dist/observability/trace/projection.d.ts +5 -0
- package/dist/observability/trace/projection.js +20 -0
- package/dist/observability/trace/segmentation.d.ts +66 -0
- package/dist/observability/trace/segmentation.js +496 -0
- package/dist/observability/trace/source.d.ts +100 -0
- package/dist/observability/trace/source.js +1233 -0
- package/dist/observability/trace/tool-search.d.ts +8 -0
- package/dist/observability/trace/tool-search.js +160 -0
- package/dist/observability/trace/trace-ir.d.ts +187 -0
- package/dist/observability/trace/trace-ir.js +87 -0
- package/dist/observability/view-models/conversation.d.ts +51 -0
- package/dist/observability/view-models/conversation.js +1 -0
- package/dist/observability/view-models/index.d.ts +3 -0
- package/dist/observability/view-models/index.js +1 -0
- package/dist/observability/view-models/knowledge-debugger.d.ts +84 -0
- package/dist/observability/view-models/knowledge-debugger.js +1 -0
- package/dist/shared/content-hash.d.ts +1 -0
- package/dist/shared/content-hash.js +4 -0
- package/dist/shared/language.d.ts +2 -0
- package/dist/shared/language.js +1 -0
- package/dist/studio/application/index.d.ts +2 -0
- package/dist/studio/application/index.js +2 -0
- package/dist/studio/application/skill-index.d.ts +9 -0
- package/dist/studio/application/skill-index.js +347 -0
- package/dist/studio/application/skill-insights.d.ts +6 -0
- package/dist/studio/application/skill-insights.js +231 -0
- package/dist/studio/core-runs/catalog.d.ts +3 -0
- package/dist/studio/core-runs/catalog.js +15 -0
- package/dist/studio/core-runs/contracts.d.ts +227 -0
- package/dist/studio/core-runs/contracts.js +2 -0
- package/dist/studio/core-runs/index.d.ts +5 -0
- package/dist/studio/core-runs/index.js +5 -0
- package/dist/studio/core-runs/projection.d.ts +4 -0
- package/dist/studio/core-runs/projection.js +307 -0
- package/dist/studio/core-runs/renderer.d.ts +11 -0
- package/dist/studio/core-runs/renderer.js +414 -0
- package/dist/studio/core-runs/route-handler.d.ts +19 -0
- package/dist/studio/core-runs/route-handler.js +122 -0
- package/dist/studio/http/contracts.d.ts +28 -0
- package/dist/studio/http/contracts.js +1 -0
- package/dist/studio/http/errors.d.ts +1 -0
- package/dist/studio/http/errors.js +3 -0
- package/dist/studio/http/report-server.d.ts +4 -0
- package/dist/studio/http/report-server.js +106 -0
- package/dist/studio/http/request-errors.d.ts +8 -0
- package/dist/studio/http/request-errors.js +77 -0
- package/dist/studio/http/request-handler.d.ts +12 -0
- package/dist/studio/http/request-handler.js +137 -0
- package/dist/studio/http/routes/contracts.d.ts +14 -0
- package/dist/studio/http/routes/contracts.js +1 -0
- package/dist/studio/http/routes/conversations.d.ts +8 -0
- package/dist/studio/http/routes/conversations.js +220 -0
- package/dist/studio/http/routes/knowledge.d.ts +14 -0
- package/dist/studio/http/routes/knowledge.js +781 -0
- package/dist/studio/http/routes/observations.d.ts +13 -0
- package/dist/studio/http/routes/observations.js +193 -0
- package/dist/studio/index.d.ts +1 -0
- package/dist/studio/index.js +1 -0
- package/dist/studio/presentation/conversation-renderer.d.ts +17 -0
- package/dist/studio/presentation/conversation-renderer.js +526 -0
- package/dist/studio/presentation/doctor-detail-renderer.d.ts +10 -0
- package/dist/studio/presentation/knowledge-debugger-renderer.d.ts +9 -0
- package/dist/studio/presentation/knowledge-debugger-renderer.js +2393 -0
- package/dist/studio/presentation/layout.d.ts +16 -0
- package/dist/studio/presentation/layout.js +788 -0
- package/dist/studio/presentation/managed-history-renderer.d.ts +5 -0
- package/dist/studio/presentation/managed-history-renderer.js +333 -0
- package/dist/studio/presentation/observation-inbox/client-script.d.ts +2 -0
- package/dist/studio/presentation/observation-inbox/client-script.js +1795 -0
- package/dist/studio/presentation/observation-inbox/experience-workspace-renderer.d.ts +35 -0
- package/dist/studio/presentation/observation-inbox/experience-workspace-renderer.js +1799 -0
- package/dist/studio/presentation/observation-inbox/helpers.d.ts +13 -0
- package/dist/studio/presentation/observation-inbox/metric-renderer.d.ts +50 -0
- package/dist/studio/presentation/observation-inbox/metric-renderer.js +750 -0
- package/dist/studio/presentation/observation-inbox/page-renderer.d.ts +12 -0
- package/dist/studio/presentation/observation-inbox/page-renderer.js +275 -0
- package/dist/studio/presentation/observation-inbox/process-workspace-renderer.d.ts +52 -0
- package/dist/studio/presentation/observation-inbox/process-workspace-renderer.js +380 -0
- package/dist/studio/presentation/observation-inbox/review-renderer.d.ts +28 -0
- package/dist/studio/presentation/observation-inbox/review-renderer.js +323 -0
- package/dist/studio/presentation/observation-inbox/reviewer-report.d.ts +8 -0
- package/dist/studio/presentation/observation-inbox/reviewer-report.js +449 -0
- package/dist/studio/presentation/observation-inbox/signal-renderer.d.ts +20 -0
- package/dist/studio/presentation/observation-inbox/signal-renderer.js +133 -0
- package/dist/studio/presentation/observation-inbox/skill-chain-renderer.d.ts +17 -0
- package/dist/studio/presentation/observation-inbox/skill-chain-renderer.js +738 -0
- package/dist/studio/presentation/observation-inbox/timeline.d.ts +18 -0
- package/dist/studio/presentation/observation-inbox/timeline.js +772 -0
- package/dist/studio/presentation/observation-inbox-renderer.d.ts +4 -0
- package/dist/studio/presentation/observation-inbox-renderer.js +130 -0
- package/dist/studio/presentation/report-shell.d.ts +77 -0
- package/dist/studio/presentation/skill-detail-renderer.d.ts +12 -0
- package/dist/studio/presentation/skill-detail-renderer.js +71 -0
- package/dist/studio/presentation/skill-health-renderer.d.ts +19 -0
- package/dist/studio/presentation/skill-health-renderer.js +315 -0
- package/dist/studio/presentation/skill-list-renderer.d.ts +3 -0
- package/dist/studio/presentation/skill-list-renderer.js +36 -0
- package/dist/studio/presentation/trajectory-evidence.d.ts +8 -0
- package/dist/studio/presentation/trajectory-live.d.ts +43 -0
- package/dist/studio/presentation/trajectory-live.js +261 -0
- package/dist/studio/view-models/index.d.ts +2 -0
- package/dist/studio/view-models/index.js +1 -0
- package/dist/studio/view-models/insight.d.ts +68 -0
- package/dist/studio/view-models/insight.js +1 -0
- package/dist/studio/view-models/skill-index.d.ts +110 -0
- package/dist/studio/view-models/skill-index.js +1 -0
- package/package.json +52 -7
- package/dist/analysis/coverage-analyzer.d.ts +0 -57
- package/dist/analysis/coverage-analyzer.js +0 -324
- package/dist/analysis/failure-clusterer.d.ts +0 -95
- package/dist/analysis/failure-clusterer.js +0 -298
- package/dist/analysis/gap-analyzer.d.ts +0 -120
- package/dist/analysis/gap-analyzer.js +0 -480
- package/dist/analysis/hedging-classifier.d.ts +0 -33
- package/dist/analysis/report-diagnostics.d.ts +0 -40
- package/dist/analysis/report-diagnostics.js +0 -862
- package/dist/analysis/sample-diagnostics.d.ts +0 -84
- package/dist/analysis/sample-diagnostics.js +0 -457
- package/dist/analysis/saturation.d.ts +0 -92
- package/dist/analysis/saturation.js +0 -181
- package/dist/artifact-graph/doctor.d.ts +0 -21
- package/dist/artifact-graph/doctor.js +0 -577
- package/dist/artifact-graph/eval.d.ts +0 -16
- package/dist/artifact-graph/eval.js +0 -471
- package/dist/authoring/evolver.d.ts +0 -263
- package/dist/authoring/evolver.js +0 -1333
- package/dist/authoring/generator.d.ts +0 -99
- package/dist/authoring/generator.js +0 -1018
- package/dist/authoring/sample-fixer.d.ts +0 -54
- package/dist/authoring/sample-fixer.js +0 -297
- package/dist/cli/lib/run-tally.d.ts +0 -17
- package/dist/cli/lib/run-tally.js +0 -30
- package/dist/diagnosis/types.d.ts +0 -5
- package/dist/doctor/endpoint-rule.d.ts +0 -60
- package/dist/doctor/endpoint-rule.js +0 -382
- package/dist/doctor/fixer.d.ts +0 -11
- package/dist/doctor/fixer.js +0 -325
- package/dist/doctor/health/composer.d.ts +0 -18
- package/dist/doctor/health/composer.js +0 -407
- package/dist/doctor/health/dimension-spec.d.ts +0 -52
- package/dist/doctor/health/prompt-builder.d.ts +0 -2
- package/dist/doctor/health/prompt-builder.js +0 -1
- package/dist/doctor/index.d.ts +0 -18
- package/dist/doctor/index.js +0 -273
- package/dist/doctor/messages.d.ts +0 -20
- package/dist/doctor/preflight.d.ts +0 -31
- package/dist/doctor/preflight.js +0 -31
- package/dist/doctor/renderer.d.ts +0 -17
- package/dist/doctor/rules.d.ts +0 -32
- package/dist/doctor/rules.js +0 -290
- package/dist/eval-core/artifact-file-names.d.ts +0 -15
- package/dist/eval-core/artifact-file-names.js +0 -62
- package/dist/eval-core/artifact-index.d.ts +0 -102
- package/dist/eval-core/artifact-index.js +0 -336
- package/dist/eval-core/bootstrap.d.ts +0 -130
- package/dist/eval-core/bootstrap.js +0 -258
- package/dist/eval-core/cache.d.ts +0 -47
- package/dist/eval-core/cache.js +0 -205
- package/dist/eval-core/comparability.d.ts +0 -10
- package/dist/eval-core/comparability.js +0 -302
- package/dist/eval-core/default-dirs.d.ts +0 -34
- package/dist/eval-core/default-dirs.js +0 -36
- package/dist/eval-core/dependency-checker.d.ts +0 -36
- package/dist/eval-core/evaluation-execution.d.ts +0 -66
- package/dist/eval-core/evaluation-execution.js +0 -424
- package/dist/eval-core/evaluation-job.d.ts +0 -57
- package/dist/eval-core/evaluation-job.js +0 -121
- package/dist/eval-core/evaluation-reporting.d.ts +0 -48
- package/dist/eval-core/evaluation-reporting.js +0 -353
- package/dist/eval-core/execution-strategy.d.ts +0 -12
- package/dist/eval-core/execution-strategy.js +0 -174
- package/dist/eval-core/fact-checker.d.ts +0 -37
- package/dist/eval-core/fact-checker.js +0 -128
- package/dist/eval-core/holdout.d.ts +0 -66
- package/dist/eval-core/holdout.js +0 -119
- package/dist/eval-core/judge-independence.d.ts +0 -28
- package/dist/eval-core/judge-independence.js +0 -29
- package/dist/eval-core/layer-gates.d.ts +0 -16
- package/dist/eval-core/layer-gates.js +0 -41
- package/dist/eval-core/measurement-dirs.js +0 -82
- package/dist/eval-core/mock-hook.cjs +0 -271
- package/dist/eval-core/mocks-runtime.d.ts +0 -99
- package/dist/eval-core/mocks-runtime.js +0 -617
- package/dist/eval-core/report-document.d.ts +0 -12
- package/dist/eval-core/report-document.js +0 -1157
- package/dist/eval-core/report-extensions.d.ts +0 -4
- package/dist/eval-core/report-extensions.js +0 -500
- package/dist/eval-core/report-file-migration.d.ts +0 -10
- package/dist/eval-core/report-file-migration.js +0 -95
- package/dist/eval-core/resume-compatibility.d.ts +0 -32
- package/dist/eval-core/resume-compatibility.js +0 -142
- package/dist/eval-core/sample-fingerprint.d.ts +0 -12
- package/dist/eval-core/sample-fingerprint.js +0 -193
- package/dist/eval-core/schema.d.ts +0 -29
- package/dist/eval-core/schema.js +0 -365
- package/dist/eval-core/statistics.d.ts +0 -85
- package/dist/eval-core/statistics.js +0 -209
- package/dist/eval-core/task-planner.d.ts +0 -11
- package/dist/eval-core/task-planner.js +0 -72
- package/dist/eval-core/verdict.d.ts +0 -179
- package/dist/eval-core/verdict.js +0 -669
- package/dist/eval-workflows/batch-evaluation-workflow.d.ts +0 -146
- package/dist/eval-workflows/batch-evaluation-workflow.js +0 -259
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +0 -26
- package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +0 -105
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts +0 -27
- package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +0 -69
- package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +0 -62
- package/dist/eval-workflows/evaluation-pipeline/run-state.js +0 -91
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +0 -21
- package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +0 -52
- package/dist/eval-workflows/evaluation-pipeline.d.ts +0 -99
- package/dist/eval-workflows/evaluation-pipeline.js +0 -188
- package/dist/eval-workflows/evaluation-preparation.d.ts +0 -56
- package/dist/eval-workflows/evaluation-preparation.js +0 -83
- package/dist/eval-workflows/run-evaluation.d.ts +0 -152
- package/dist/eval-workflows/run-evaluation.js +0 -556
- package/dist/grading/assertions.d.ts +0 -38
- package/dist/grading/assertions.js +0 -475
- package/dist/grading/diagnostic.d.ts +0 -46
- package/dist/grading/diagnostic.js +0 -472
- package/dist/grading/gold-cli.d.ts +0 -84
- package/dist/grading/gold-cli.js +0 -246
- package/dist/grading/human-gold.d.ts +0 -103
- package/dist/grading/human-gold.js +0 -189
- package/dist/grading/index.d.ts +0 -60
- package/dist/grading/index.js +0 -150
- package/dist/grading/judge.d.ts +0 -67
- package/dist/grading/judge.js +0 -405
- package/dist/grading/layered-scores.d.ts +0 -26
- package/dist/grading/layered-scores.js +0 -87
- package/dist/inputs/content-hash.d.ts +0 -28
- package/dist/inputs/content-hash.js +0 -106
- package/dist/inputs/eval-config.d.ts +0 -12
- package/dist/inputs/eval-config.js +0 -295
- package/dist/inputs/load-samples.d.ts +0 -39
- package/dist/inputs/load-samples.js +0 -298
- package/dist/inputs/materialize-copy.js +0 -193
- package/dist/inputs/mcp-resolver.d.ts +0 -49
- package/dist/inputs/mcp-resolver.js +0 -309
- package/dist/inputs/sample-document.d.ts +0 -11
- package/dist/inputs/sample-document.js +0 -96
- package/dist/inputs/sample-locator.d.ts +0 -23
- package/dist/inputs/sample-locator.js +0 -195
- package/dist/inputs/skill-loader.d.ts +0 -190
- package/dist/inputs/skill-loader.js +0 -852
- package/dist/inputs/url-fetcher.d.ts +0 -18
- package/dist/inputs/url-fetcher.js +0 -240
- package/dist/managed/evidence.d.ts +0 -24
- package/dist/managed/evidence.js +0 -155
- package/dist/managed/index.d.ts +0 -12
- package/dist/managed/index.js +0 -12
- package/dist/managed/list-view.d.ts +0 -72
- package/dist/managed/list-view.js +0 -78
- package/dist/managed/observe-feedback.d.ts +0 -60
- package/dist/managed/promote-gate.d.ts +0 -40
- package/dist/managed/promote-gate.js +0 -37
- package/dist/managed/source-probe.d.ts +0 -15
- package/dist/managed/source-probe.js +0 -129
- package/dist/managed/store.d.ts +0 -142
- package/dist/managed/store.js +0 -595
- package/dist/managed/version-scores.d.ts +0 -33
- package/dist/managed/version-scores.js +0 -70
- package/dist/observability/codex-conversation-index.d.ts +0 -92
- package/dist/observability/codex-conversation-index.js +0 -600
- package/dist/observability/codex-tool-status.d.ts +0 -16
- package/dist/observability/codex-tool-status.js +0 -114
- package/dist/observability/codex-trace-adapter.d.ts +0 -5
- package/dist/observability/codex-trace-adapter.js +0 -1091
- package/dist/observability/conversation-catalog.d.ts +0 -38
- package/dist/observability/conversation-catalog.js +0 -575
- package/dist/observability/conversation-index-process.js +0 -31
- package/dist/observability/conversation-view-model.d.ts +0 -5
- package/dist/observability/conversation-view-model.js +0 -100
- package/dist/observability/experience-frontmatter.d.ts +0 -36
- package/dist/observability/experience-frontmatter.js +0 -99
- package/dist/observability/feedback-projection.d.ts +0 -20
- package/dist/observability/inbox-view-model.d.ts +0 -43
- package/dist/observability/inbox-view-model.js +0 -185
- package/dist/observability/inbox.d.ts +0 -33
- package/dist/observability/inbox.js +0 -1343
- package/dist/observability/knowledge-debugger.d.ts +0 -4
- package/dist/observability/knowledge-debugger.js +0 -397
- package/dist/observability/problem-patterns.d.ts +0 -11
- package/dist/observability/problem-patterns.js +0 -245
- package/dist/observability/resolved-review.d.ts +0 -36
- package/dist/observability/review-state.d.ts +0 -25
- package/dist/observability/review-state.js +0 -351
- package/dist/observability/skill-chain-advisories.d.ts +0 -15
- package/dist/observability/skill-chain.d.ts +0 -6
- package/dist/observability/skill-chain.js +0 -486
- package/dist/observability/skill-health-analyzer.d.ts +0 -128
- package/dist/observability/skill-health-analyzer.js +0 -328
- package/dist/observability/skill-health-report.d.ts +0 -10
- package/dist/observability/skill-health-report.js +0 -620
- package/dist/observability/source-record-archive.d.ts +0 -13
- package/dist/observability/source-record-archive.js +0 -324
- package/dist/observability/task-semantic-projection.d.ts +0 -11
- package/dist/observability/task-window.d.ts +0 -9
- package/dist/observability/task-window.js +0 -57
- package/dist/observability/text-signals.d.ts +0 -13
- package/dist/observability/text-signals.js +0 -115
- package/dist/observability/trace-adapter.d.ts +0 -27
- package/dist/observability/trace-adapter.js +0 -22
- package/dist/observability/trace-attribution.d.ts +0 -63
- package/dist/observability/trace-attribution.js +0 -397
- package/dist/observability/trace-ingestion.d.ts +0 -9
- package/dist/observability/trace-ingestion.js +0 -80
- package/dist/observability/trace-ir.d.ts +0 -184
- package/dist/observability/trace-ir.js +0 -87
- package/dist/observability/trace-segmenter.d.ts +0 -64
- package/dist/observability/trace-segmenter.js +0 -502
- package/dist/observability/trace-source.d.ts +0 -100
- package/dist/observability/trace-source.js +0 -1232
- package/dist/observability/turn-index.d.ts +0 -3
- package/dist/observability/turn-index.js +0 -251
- package/dist/renderer/conversation-renderer.d.ts +0 -16
- package/dist/renderer/conversation-renderer.js +0 -512
- package/dist/renderer/doctor-detail-renderer.d.ts +0 -9
- package/dist/renderer/html-renderer.d.ts +0 -10
- package/dist/renderer/html-renderer.js +0 -706
- package/dist/renderer/knowledge-debugger-renderer.d.ts +0 -9
- package/dist/renderer/knowledge-debugger-renderer.js +0 -2388
- package/dist/renderer/layout.d.ts +0 -13
- package/dist/renderer/layout.js +0 -788
- package/dist/renderer/managed-history-renderer.d.ts +0 -5
- package/dist/renderer/managed-history-renderer.js +0 -437
- package/dist/renderer/observation-inbox/helpers.d.ts +0 -13
- package/dist/renderer/observation-inbox-renderer.d.ts +0 -4
- package/dist/renderer/observation-inbox-renderer.js +0 -7326
- package/dist/renderer/report-shell.d.ts +0 -77
- package/dist/renderer/skill-detail-renderer.d.ts +0 -12
- package/dist/renderer/skill-detail-renderer.js +0 -3115
- package/dist/renderer/skill-health-renderer.d.ts +0 -19
- package/dist/renderer/skill-health-renderer.js +0 -315
- package/dist/renderer/skill-list-renderer.d.ts +0 -2
- package/dist/renderer/skill-list-renderer.js +0 -394
- package/dist/renderer/summary.d.ts +0 -67
- package/dist/renderer/summary.js +0 -2094
- package/dist/renderer/table.d.ts +0 -2
- package/dist/renderer/table.js +0 -257
- package/dist/renderer/test-view.d.ts +0 -67
- package/dist/renderer/test-view.js +0 -1190
- package/dist/renderer/trajectory-evidence.d.ts +0 -8
- package/dist/renderer/trajectory-live.d.ts +0 -42
- package/dist/renderer/trajectory-live.js +0 -258
- package/dist/renderer/trends.d.ts +0 -5
- package/dist/renderer/trends.js +0 -130
- package/dist/server/indexed-report-store.d.ts +0 -8
- package/dist/server/indexed-report-store.js +0 -126
- package/dist/server/job-store.d.ts +0 -2
- package/dist/server/job-store.js +0 -103
- package/dist/server/report-server.d.ts +0 -39
- package/dist/server/report-server.js +0 -1610
- package/dist/server/report-store.d.ts +0 -51
- package/dist/server/report-store.js +0 -281
- package/dist/server/skill-index.d.ts +0 -25
- package/dist/server/skill-index.js +0 -955
- package/dist/server/skill-insights.d.ts +0 -31
- package/dist/server/skill-insights.js +0 -854
- package/dist/shared/artifact-graph.d.ts +0 -3
- package/dist/shared/artifact-graph.js +0 -224
- package/dist/shared/assertion-types.d.ts +0 -8
- package/dist/shared/diagnosis-schema.d.ts +0 -9
- package/dist/shared/diagnosis-schema.js +0 -181
- package/dist/shared/doctor-report.d.ts +0 -3
- package/dist/shared/doctor-report.js +0 -103
- package/dist/shared/evaluation-job.d.ts +0 -6
- package/dist/shared/evaluation-job.js +0 -217
- package/dist/shared/executor-result.d.ts +0 -17
- package/dist/shared/executor-result.js +0 -221
- package/dist/shared/hard-rules.d.ts +0 -41
- package/dist/shared/llm-prompts/debias-instructions.js +0 -44
- package/dist/shared/llm-prompts/index.js +0 -21
- package/dist/shared/llm-prompts/judge-prompts.d.ts +0 -30
- package/dist/shared/llm-prompts/judge-prompts.js +0 -205
- package/dist/shared/llm-prompts/registry.d.ts +0 -27
- package/dist/shared/llm-prompts/registry.js +0 -70
- package/dist/shared/llm-prompts/skill-health-merge.d.ts +0 -22
- package/dist/shared/llm-prompts/skill-health.d.ts +0 -21
- package/dist/shared/sample-contract.d.ts +0 -4
- package/dist/shared/sample-contract.js +0 -367
- package/dist/shared/tool-call-status.d.ts +0 -8
- package/dist/shared/tool-call-status.js +0 -28
- package/dist/shared/tool-search.d.ts +0 -8
- package/dist/shared/tool-search.js +0 -160
- package/dist/shared/trace-projection.d.ts +0 -5
- package/dist/shared/trace-projection.js +0 -20
- package/dist/shared/trace-source-kind.d.ts +0 -3
- package/dist/shared/trace-source-kind.js +0 -12
- package/dist/types/artifact-graph.d.ts +0 -93
- package/dist/types/dependencies.d.ts +0 -22
- package/dist/types/doctor.d.ts +0 -202
- package/dist/types/doctor.js +0 -6
- package/dist/types/eval.d.ts +0 -413
- package/dist/types/executor.d.ts +0 -191
- package/dist/types/index.d.ts +0 -14
- package/dist/types/index.js +0 -14
- package/dist/types/judge.d.ts +0 -204
- package/dist/types/judge.js +0 -3
- package/dist/types/managed.d.ts +0 -164
- package/dist/types/observability.d.ts +0 -1199
- package/dist/types/observability.js +0 -13
- package/dist/types/report.d.ts +0 -704
- package/dist/types/shared.d.ts +0 -1
- package/dist/types/skill-index.d.ts +0 -206
- package/dist/types/storage.d.ts +0 -20
- package/dist/types/trace.d.ts +0 -2
- /package/dist/{types/diagnosis.d.ts → diagnosis/contracts.d.ts} +0 -0
- /package/dist/{observability/conversation-index-process.d.ts → diagnosis/contracts.js} +0 -0
- /package/dist/diagnosis/{types.js → lifecycle.js} +0 -0
- /package/dist/{types/artifact-graph.js → eval-core/engine/types.js} +0 -0
- /package/dist/{grading/gold-dataset.d.ts → eval-workflows/gold/dataset.d.ts} +0 -0
- /package/dist/{grading/gold-dataset.js → eval-workflows/gold/dataset.js} +0 -0
- /package/dist/{shared → eval-workflows/inputs}/assertion-types.js +0 -0
- /package/dist/{types/dependencies.js → eval-workflows/inputs/contracts/assertion-kind.js} +0 -0
- /package/dist/{types/diagnosis.js → eval-workflows/inputs/contracts/assertion.js} +0 -0
- /package/dist/{types/eval.js → eval-workflows/inputs/contracts/config.js} +0 -0
- /package/dist/{types/executor.js → eval-workflows/inputs/contracts/index.js} +0 -0
- /package/dist/{types/managed.js → eval-workflows/inputs/contracts/sample.js} +0 -0
- /package/dist/{types/report.js → eval-workflows/inputs/contracts/variant.js} +0 -0
- /package/dist/{inputs → eval-workflows/inputs}/materialize-copy.d.ts +0 -0
- /package/dist/{inputs → eval-workflows/inputs}/source-resolver.d.ts +0 -0
- /package/dist/{inputs → eval-workflows/inputs}/source-resolver.js +0 -0
- /package/dist/{types/shared.js → eval-workflows/instruments/contracts/config.js} +0 -0
- /package/dist/{types/skill-index.js → eval-workflows/instruments/contracts/index.js} +0 -0
- /package/dist/{types/storage.js → eval-workflows/instruments/contracts/result.js} +0 -0
- /package/dist/{shared/llm-prompts → eval-workflows/instruments/prompts}/debias-instructions.d.ts +0 -0
- /package/dist/{types/trace.js → evidence/graph/contracts.js} +0 -0
- /package/dist/{eval-core/measurement-dirs.d.ts → evidence/storage/directories.d.ts} +0 -0
- /package/dist/{shared → executors/core}/token-usage.d.ts +0 -0
- /package/dist/{shared → executors/core}/token-usage.js +0 -0
- /package/dist/{shared → executors/core}/tool-identity.d.ts +0 -0
- /package/dist/{shared → executors/core}/tool-identity.js +0 -0
- /package/dist/{eval-core/dependency-checker.js → executors/preflight/dependencies.js} +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/builtin-dimensions.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/builtin-dimensions.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/consensus.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/consensus.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/dimension-registry.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/dimension-registry.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/dimension-spec.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/load-custom-dimensions.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/load-custom-dimensions.js +0 -0
- /package/dist/{shared/llm-prompts/skill-health-merge.js → knowledge-artifacts/doctor/health/merge-prompt.js} +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/parser.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/parser.js +0 -0
- /package/dist/{shared/llm-prompts/skill-health.js → knowledge-artifacts/doctor/health/prompt-builder.js} +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/register.d.ts +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/health/register.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/messages.js +0 -0
- /package/dist/{doctor → knowledge-artifacts/doctor}/renderer.js +0 -0
- /package/dist/{managed → knowledge-artifacts/governance}/list-query.d.ts +0 -0
- /package/dist/{managed → knowledge-artifacts/governance}/list-query.js +0 -0
- /package/dist/{managed → knowledge-artifacts/governance}/observe-feedback.js +0 -0
- /package/dist/{shared → knowledge-artifacts/skills}/hard-rules.js +0 -0
- /package/dist/{analysis → observability/analysis}/hedging-classifier.js +0 -0
- /package/dist/observability/{polling-subscription-hub.d.ts → conversation/polling-subscription-hub.d.ts} +0 -0
- /package/dist/observability/{polling-subscription-hub.js → conversation/polling-subscription-hub.js} +0 -0
- /package/dist/observability/{task-semantic-projection.js → conversation/task-semantic-projection.js} +0 -0
- /package/dist/observability/{feedback-matchers.d.ts → inbox/feedback-matchers.d.ts} +0 -0
- /package/dist/observability/{feedback-matchers.js → inbox/feedback-matchers.js} +0 -0
- /package/dist/observability/{feedback-projection.js → inbox/feedback-projection.js} +0 -0
- /package/dist/observability/{resolved-review.js → inbox/resolved-review.js} +0 -0
- /package/dist/{shared/llm-prompts/index.d.ts → observability/prompts/document.d.ts} +0 -0
- /package/dist/observability/{skill-chain-advisories.js → skill-health/advisories.js} +0 -0
- /package/dist/observability/{codex-exec-command.d.ts → trace/adapters/codex/exec-command.d.ts} +0 -0
- /package/dist/observability/{codex-exec-command.js → trace/adapters/codex/exec-command.js} +0 -0
- /package/dist/observability/{codex-protocol.d.ts → trace/adapters/codex/protocol.d.ts} +0 -0
- /package/dist/observability/{codex-protocol.js → trace/adapters/codex/protocol.js} +0 -0
- /package/dist/observability/{trace-session-index.d.ts → trace/session-index.d.ts} +0 -0
- /package/dist/observability/{trace-session-index.js → trace/session-index.js} +0 -0
- /package/dist/{util/safe-slice.d.ts → shared/json-safe-truncation.d.ts} +0 -0
- /package/dist/{util/safe-slice.js → shared/json-safe-truncation.js} +0 -0
- /package/dist/{renderer → studio/presentation}/doctor-detail-renderer.js +0 -0
- /package/dist/{renderer → studio/presentation}/icons.d.ts +0 -0
- /package/dist/{renderer → studio/presentation}/icons.js +0 -0
- /package/dist/{renderer → studio/presentation}/inline-markdown.d.ts +0 -0
- /package/dist/{renderer → studio/presentation}/inline-markdown.js +0 -0
- /package/dist/{renderer → studio/presentation}/observation-inbox/helpers.js +0 -0
- /package/dist/{renderer → studio/presentation}/observation-inbox/styles.d.ts +0 -0
- /package/dist/{renderer → studio/presentation}/observation-inbox/styles.js +0 -0
- /package/dist/{renderer → studio/presentation}/report-shell.js +0 -0
- /package/dist/{renderer → studio/presentation}/trajectory-evidence.js +0 -0
- /package/dist/{renderer → studio/presentation}/trajectory-routing.d.ts +0 -0
- /package/dist/{renderer → studio/presentation}/trajectory-routing.js +0 -0
|
@@ -26,11 +26,11 @@ omk doctor [target] [flags]
|
|
|
26
26
|
- `--executor` `option`:执行器名。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。指定为测试 fixture 路径可在测试里跑。
|
|
27
27
|
- `--fix` `boolean`:交互式修复:根据 doctor 报告问题,用 LLM agent 修复 skill。
|
|
28
28
|
- `--gate` `boolean`:静默模式,只在 fail 时输出 stderr 摘要,exit code 标识结果。
|
|
29
|
-
- `--global` `boolean`:写全局 ~/.oh-my-knowledge/
|
|
29
|
+
- `--global` `boolean`:写全局 ~/.oh-my-knowledge/doctor,而非项目 .omk/doctor
|
|
30
30
|
- `--json` `boolean`:JSON 输出到 stdout,适合 CI / 外部脚本消费。
|
|
31
31
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
32
32
|
- `--model` `option`:LLM model 名。Codex 自动读取本机配置;也可用 OMK_MODEL 设置环境偏好。
|
|
33
|
-
- `--output-dir` `option`:报告输出目录,默认项目级 .omk/
|
|
33
|
+
- `--output-dir` `option`:报告输出目录,默认项目级 .omk/doctor(--global 写全局)。
|
|
34
34
|
- `--repeat` `option`:健康度体检重复采样次数(self-consistency)。默认 2:并行跑 2 遍、finding 取并集并用 LLM 聚类归并同根因、标注支持度 k/N,压低单次采样方差。设 1 = 单次快速体检(不采样、不归并,最省)。
|
|
35
35
|
- `--static-only` `boolean`:只跑静态检测(不调 LLM、不读 samples.json):skill 可读性 / frontmatter 合法性 / 正文引用的脚本·CLI·文件·env 是否存在。CI 无 LLM 凭证或断网时用。
|
|
36
36
|
- `--timeout` `option`:单次 LLM 会话超时秒数,默认 600(10 分钟)。
|
|
@@ -86,7 +86,7 @@ omk eval [flags]
|
|
|
86
86
|
- `--dry-run` `boolean`:只 plan 不实跑
|
|
87
87
|
- `--effort` `option`:被测 LLM 扩展思考预算 low/medium/high/xhigh/max(默认 low;跨 effort 报告不严格可比)。
|
|
88
88
|
- `--executor` `option`:执行器:claude / claude-sdk / codex / codex-sdk / anthropic-api / openai-api / 自定义命令。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
|
|
89
|
-
- `--global` `boolean`:报告写全局 ~/.oh-my-knowledge/
|
|
89
|
+
- `--global` `boolean`:报告写全局 ~/.oh-my-knowledge/eval,而非项目 .omk/eval
|
|
90
90
|
- `--gold-dir` `option`:gold dataset 目录
|
|
91
91
|
- `--holdout-ratio` `option`:留出比例 0-1(如 0.3);切出 holdout 子集,对比 train/holdout 综合分检测过拟合
|
|
92
92
|
- `--judge-models` `option`:评委配置,格式 executor:model[,...],例 claude:haiku 或 codex:<model>(≥ 2 个 = ensemble)。默认跟随所选执行器;Codex 沿用被测模型。
|
|
@@ -97,24 +97,24 @@ omk eval [flags]
|
|
|
97
97
|
- `--model` `option`:被测模型
|
|
98
98
|
- `--no-cache` `boolean`:跳过 executor cache
|
|
99
99
|
- `--no-debias-length` `boolean`:关 length-debias(默认开)
|
|
100
|
-
- `--no-diagnostic` `boolean
|
|
100
|
+
- `--no-diagnostic` `boolean`:关闭基于 Core 失败、缺失、排除与稳定 reason code 的诊断投影。
|
|
101
101
|
- `--no-evidence` `boolean`:不把本次评测写成证据追加进受管记录(默认会为已 install 的 skill 自动写)。
|
|
102
102
|
- `--no-gate` `boolean`:关 verdict gate
|
|
103
103
|
- `--no-judge` `boolean`:跳过 LLM judge
|
|
104
104
|
- `--no-serve` `boolean`:不启 report server
|
|
105
105
|
- `--no-strict-baseline` `boolean`:关闭 baseline 隔离
|
|
106
|
-
- `--output-dir` `option`:报告输出目录(默认项目级 .omk/
|
|
106
|
+
- `--output-dir` `option`:报告输出目录(默认项目级 .omk/eval)
|
|
107
107
|
- `--repeat` `option`:每个 sample 重复跑 N 次
|
|
108
108
|
- `--report-only` `boolean`:生成报告并打印 verdict,但始终 exit 0(不参与 CI gate)。
|
|
109
|
-
- `--resume` `option
|
|
109
|
+
- `--resume` `option`:复用经过完整契约校验的 Core runId;拒绝时失败关闭
|
|
110
110
|
- `--retry` `option`:失败 sample 重试次数
|
|
111
|
-
- `--samples` `option
|
|
111
|
+
- `--samples` `option`:用例路径。自动发现项目级或单 treatment 目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。
|
|
112
112
|
- `--skill-dir` `option`:skill 目录,默认 skills
|
|
113
113
|
- `--skip-connectivity` `boolean`:跳 LLM 连通性预检
|
|
114
114
|
- `--skip-doctor` `boolean`:escape hatch:跳 doctor 健康检查门禁(默认强制启用)。沙箱 mock 提供依赖时绕开 doctor 物理路径误报;garbage-in 风险自负。
|
|
115
115
|
- `--strict-baseline` `boolean`:强制 baseline 隔离(default true)
|
|
116
116
|
- `--threshold` `option`:verdict 阈值,默认 3.5
|
|
117
|
-
- `--timeout` `option`:单用例超时秒,默认
|
|
117
|
+
- `--timeout` `option`:单用例超时秒,默认 120
|
|
118
118
|
- `--treatment` `option`:treatment variant 列表,逗号分隔(仅 artifact 身份)
|
|
119
119
|
- `--treatment-cwd` `option`:treatment 的 runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)
|
|
120
120
|
- `--trivial-diff` `option`:可忽略 diff 容差,0 表示不启用容差
|
|
@@ -150,26 +150,29 @@ omk eval gold [flags]
|
|
|
150
150
|
|
|
151
151
|
## omk eval gold compare
|
|
152
152
|
|
|
153
|
-
|
|
153
|
+
把一组 Core run 观测跟 gold dataset 对比,计算 bootstrap CI 后的 agreement。
|
|
154
154
|
|
|
155
155
|
**用法:**
|
|
156
156
|
|
|
157
157
|
```bash
|
|
158
|
-
omk eval gold compare <
|
|
158
|
+
omk eval gold compare <runId> [flags]
|
|
159
159
|
```
|
|
160
160
|
|
|
161
161
|
**参数:**
|
|
162
162
|
|
|
163
|
-
- `
|
|
163
|
+
- `runId`(必填):Core run ID。
|
|
164
164
|
|
|
165
165
|
**Flags:**
|
|
166
166
|
|
|
167
167
|
- `--bootstrap-samples` `option`:bootstrap 重采样次数,默认 1000
|
|
168
|
+
- `--evaluator` `option`:显式选择 Core evaluator ID。
|
|
168
169
|
- `--gold-dir` `option`:gold dataset 目录,必填
|
|
169
170
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
170
|
-
- `--
|
|
171
|
+
- `--metric` `option`:显式选择 Core metric ID。
|
|
172
|
+
- `--reports-dir` `option`:只读取指定 Core 报告目录(默认当前项目 + 全局)
|
|
171
173
|
- `--seed` `option`:bootstrap seed,可复现
|
|
172
|
-
- `--
|
|
174
|
+
- `--target` `option`:显式选择 Core target ID。
|
|
175
|
+
- `--trial-index` `option`:显式选择 trial index。
|
|
173
176
|
|
|
174
177
|
## omk eval gold init
|
|
175
178
|
|
|
@@ -221,32 +224,22 @@ omk evolve <skillPath> [flags]
|
|
|
221
224
|
|
|
222
225
|
**Flags:**
|
|
223
226
|
|
|
224
|
-
- `--auto-fix-samples` `boolean`:每轮先修 skill,再修 sample,随后一起评估候选结果
|
|
225
227
|
- `--concurrency` `option` (默认 `1`):评测并发数,默认 1
|
|
226
228
|
- `--edit-budget` `option` (默认 `0.2`):单轮最多改动的 skill 行占比(默认 0.2)。超预算的候选评测前直接判拒,省 eval 成本
|
|
227
229
|
- `--effort` `option`:reasoning effort: low/medium/high/xhigh/max
|
|
228
230
|
- `--executor` `option`:执行器名。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
|
|
229
|
-
- `--holdout-ratio` `option` (默认 `0`):留出验收集比例(0..1,默认 0=关)。> 0 时按 holdout 分接受候选、weak-sample 只取训练集,防 train-on-test
|
|
230
231
|
- `--improve-mode` `agent|rewrite` (默认 `agent`):改写策略(默认:agent)
|
|
231
232
|
- `--improve-model` `option`:负责重写 skill 的 LLM,默认沿用被测模型
|
|
232
233
|
- `--judge-models` `option`:评委 model(单评委约束),格式 executor:model。默认跟随所选执行器;Codex 沿用被测模型。
|
|
233
234
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
234
235
|
- `--model` `option`:被评测的 LLM。Codex 自动读取本机配置;无用例时也用作自动生成用例的出题模型。
|
|
235
|
-
- `--no-diagnostic` `boolean`:关 LLM diagnostic 调用
|
|
236
236
|
- `--no-edit-budget` `boolean`:关掉 edit budget 约束(允许任意大小的单轮改动)
|
|
237
237
|
- `--no-reject-memory` `boolean`:关掉 rejected-edit 记忆(不把被拒改法回灌下一轮 prompt)
|
|
238
|
-
- `--no-significance-gate` `boolean`:关掉显著性接受门,退回「候选分高一点点就收」的点估计判定(默认门开:只收统计显著的提升)
|
|
239
|
-
- `--reuse-latest-eval` `boolean`:复用可比的最新 eval 报告作为 round-0
|
|
240
238
|
- `--rounds` `option` (默认 `5`):最大迭代轮数,默认 5
|
|
241
|
-
- `--sample-fix-max-attempts` `option` (默认 `2`):每条 sample 自动修复最多尝试次数(默认:2)
|
|
242
239
|
- `--samples` `option` (默认 `eval-samples.json`):用例文件路径,默认 eval-samples.json
|
|
243
|
-
- `--significance-alpha` `option` (默认 `0.05`):显著性门的 diff CI 显著性水平(默认 0.05 = 95% CI)
|
|
244
|
-
- `--skip-connectivity` `boolean`:跳过 LLM 连通性预检
|
|
245
240
|
- `--skip-doctor` `boolean`:跳过 doctor 门禁(escape hatch,自负 garbage-in 风险)
|
|
246
|
-
- `--snapshot-only` `boolean`:只产候选、不写回 source:胜出版本留在 evolve
|
|
247
|
-
- `--stop-on-assertions-pass` `boolean`:普通用例断言全过时提前停止
|
|
241
|
+
- `--snapshot-only` `boolean`:只产候选、不写回 source:胜出版本留在 evolve/,再由你人工选择。受管 skill 默认会写回 source 并记 Core 证据。
|
|
248
242
|
- `--target` `option`:目标 composite 分数,达到即停。不传则跑满 rounds
|
|
249
|
-
- `--test-ratio` `option` (默认 `0`):锁定 test 集比例(0..1,默认 0=关),需配 --holdout-ratio。全程不参与选择,收尾读一次给无偏泛化分
|
|
250
243
|
- `--timeout` `option` (默认 `600`):单用例超时秒,默认 600
|
|
251
244
|
|
|
252
245
|
**示例:**
|
|
@@ -279,7 +272,9 @@ omk init [targetDir] [flags]
|
|
|
279
272
|
|
|
280
273
|
**Flags:**
|
|
281
274
|
|
|
275
|
+
- `--force` `boolean`:允许覆盖目标目录中已有的 omk 脚手架文件
|
|
282
276
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
277
|
+
- `--samples` `3|20` (默认 `3`):官方起步用例数量:3 条用于快速跑通,20 条用于达到注册样本量下限
|
|
283
278
|
|
|
284
279
|
**示例:**
|
|
285
280
|
|
|
@@ -295,6 +290,12 @@ omk init
|
|
|
295
290
|
omk init my-project
|
|
296
291
|
```
|
|
297
292
|
|
|
293
|
+
> 使用达到注册样本量下限的 20 条官方用例初始化
|
|
294
|
+
|
|
295
|
+
```bash
|
|
296
|
+
omk init my-project --samples 20
|
|
297
|
+
```
|
|
298
|
+
|
|
298
299
|
## omk install
|
|
299
300
|
|
|
300
301
|
安装 omk 官方 Agent Skill,或登记并分发用户自己的 skill(内置 id omk-agent-skill,本地路径,或 git:<ref>:<name> 取当前仓库某个 ref 的 skill)。默认写入本机已检测 agent 目标;安装用户 skill 时同时登记一条受管记录。
|
|
@@ -370,7 +371,7 @@ omk list [flags]
|
|
|
370
371
|
|
|
371
372
|
**Flags:**
|
|
372
373
|
|
|
373
|
-
- `--global` `boolean`:看全局受管目录(~/.oh-my-knowledge/managed)而非项目 .omk/managed
|
|
374
|
+
- `--global` `boolean`:看全局受管目录(~/.oh-my-knowledge/governance/managed)而非项目 .omk/governance/managed
|
|
374
375
|
- `--json` `boolean`:输出 JSON(含完整可比性 marker),供脚本消费
|
|
375
376
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
376
377
|
|
|
@@ -412,11 +413,11 @@ omk observe [sessionsDir] [flags]
|
|
|
412
413
|
|
|
413
414
|
- `--feedback` `boolean`:把生产健康观测反哺已纳管的同名 skill(--no-feedback 关闭)
|
|
414
415
|
- `--from` `option`:起始时间 ISO,优先级高于 --last
|
|
415
|
-
- `--global` `boolean`:写全局 ~/.oh-my-knowledge/observe
|
|
416
|
+
- `--global` `boolean`:写全局 ~/.oh-my-knowledge/observe/health,而非项目 .omk/observe/health
|
|
416
417
|
- `--kb` `option`:知识库 root,启用 KB-aware 分析
|
|
417
418
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
418
419
|
- `--last` `option`:时间窗(7d / 24h / 30m)
|
|
419
|
-
- `--output-dir` `option`:健康报告输出目录,默认项目级 .omk/observe
|
|
420
|
+
- `--output-dir` `option`:健康报告输出目录,默认项目级 .omk/observe/health(--global 写全局)
|
|
420
421
|
- `--skills` `option`:只看指定 skill,逗号分隔
|
|
421
422
|
- `--to` `option`:结束时间 ISO
|
|
422
423
|
|
|
@@ -443,9 +444,9 @@ omk observe inbox [flags]
|
|
|
443
444
|
- `--by-skill` `boolean`:按 skill 聚合输出
|
|
444
445
|
- `--executor` `option`:LLM 增强复盘使用的执行器。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
|
|
445
446
|
- `--explore` `option`:抽样 N 条 medium/low 长尾(replaces limit)
|
|
446
|
-
- `--global` `boolean`:直接读取全局 ~/.oh-my-knowledge/observe
|
|
447
|
+
- `--global` `boolean`:直接读取全局 ~/.oh-my-knowledge/observe/inbox(跳过项目级与兜底)。
|
|
447
448
|
- `--include-noise` `boolean`:explore 时也包含 noise 桶
|
|
448
|
-
- `--input-dir` `option`:inbox 数据目录,默认 .omk/observe
|
|
449
|
+
- `--input-dir` `option`:inbox 数据目录,默认 .omk/observe/inbox(项目级,相对于 cwd);目录不存在时兜底读 ~/.oh-my-knowledge/observe/inbox。
|
|
449
450
|
- `--json` `boolean`:JSON 格式输出
|
|
450
451
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
451
452
|
- `--limit` `option`:限制条数,默认 20
|
|
@@ -470,10 +471,10 @@ omk observe ingest <traceDir> [flags]
|
|
|
470
471
|
|
|
471
472
|
**Flags:**
|
|
472
473
|
|
|
473
|
-
- `--global` `boolean`:写入全局 ~/.oh-my-knowledge/observe
|
|
474
|
+
- `--global` `boolean`:写入全局 ~/.oh-my-knowledge/observe/inbox,而非项目 .omk/observe/inbox。
|
|
474
475
|
- `--json` `boolean`:把完整 observation inbox 报告输出到 stdout;默认只输出摘要。
|
|
475
476
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
476
|
-
- `--output-dir` `option`:输出目录,默认 .omk/observe
|
|
477
|
+
- `--output-dir` `option`:输出目录,默认 .omk/observe/inbox(项目级,相对于 cwd;--global 写全局)。
|
|
477
478
|
|
|
478
479
|
## omk observe show
|
|
479
480
|
|
|
@@ -491,7 +492,7 @@ omk observe show <inboxId> [flags]
|
|
|
491
492
|
|
|
492
493
|
**Flags:**
|
|
493
494
|
|
|
494
|
-
- `--global` `boolean`:直接读取全局 ~/.oh-my-knowledge/observe
|
|
495
|
+
- `--global` `boolean`:直接读取全局 ~/.oh-my-knowledge/observe/inbox(跳过项目级与兜底)。
|
|
495
496
|
- `--input-dir` `option`:inbox 数据目录
|
|
496
497
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
497
498
|
|
|
@@ -514,7 +515,7 @@ omk promote <name> [flags]
|
|
|
514
515
|
- `--accept-cautious` `boolean`:把 CAUTIOUS 也算可接受(默认仅 PROGRESS)
|
|
515
516
|
- `--actor` `option`:决定的 actor(默认取 git config user.name)
|
|
516
517
|
- `--force` `boolean`:越过可越门拦截强制 promote,记为人工 override 决定(无当前证据或源 hash 已变时仍拒)
|
|
517
|
-
- `--global` `boolean`:操作全局受管目录而非项目 .omk/managed
|
|
518
|
+
- `--global` `boolean`:操作全局受管目录而非项目 .omk/governance/managed
|
|
518
519
|
- `--json` `boolean`:输出 JSON(版本化信封)供脚本消费
|
|
519
520
|
- `--kind` `option` (默认 `skill`):artifact 类型(当前仅 skill)
|
|
520
521
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
@@ -557,7 +558,7 @@ omk rollback <name> [flags]
|
|
|
557
558
|
**Flags:**
|
|
558
559
|
|
|
559
560
|
- `--actor` `option`:决定的 actor(默认取 git config user.name)
|
|
560
|
-
- `--global` `boolean`:操作全局受管目录而非项目 .omk/managed
|
|
561
|
+
- `--global` `boolean`:操作全局受管目录而非项目 .omk/governance/managed
|
|
561
562
|
- `--json` `boolean`:输出 JSON(版本化信封)供脚本消费
|
|
562
563
|
- `--kind` `option` (默认 `skill`):artifact 类型(当前仅 skill)
|
|
563
564
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
@@ -579,7 +580,7 @@ omk rollback review --reason "线上发现回归"
|
|
|
579
580
|
|
|
580
581
|
## omk sample
|
|
581
582
|
|
|
582
|
-
为指定 skill
|
|
583
|
+
为指定 skill 生成评测用例,支持 batch、single 与 from-traces 模式。
|
|
583
584
|
|
|
584
585
|
**用法:**
|
|
585
586
|
|
|
@@ -589,25 +590,22 @@ omk sample [skillPath] [flags]
|
|
|
589
590
|
|
|
590
591
|
**参数:**
|
|
591
592
|
|
|
592
|
-
- `skillPath`(可选):skill 文件路径或 SKILL.md 路径。batch 模式不需要;single
|
|
593
|
+
- `skillPath`(可选):skill 文件路径或 SKILL.md 路径。batch 模式不需要;single 模式必填。
|
|
593
594
|
|
|
594
595
|
**Flags:**
|
|
595
596
|
|
|
596
|
-
- `--append` `boolean`:在已有用例文件上追加新生成的用例(撞 sample_id 自动加后缀去重,保留原 json/yaml 格式)。仅单 skill 模式,不支持 --batch / --from-traces
|
|
597
|
+
- `--append` `boolean`:在已有用例文件上追加新生成的用例(撞 sample_id 自动加后缀去重,保留原 json/yaml 格式)。仅单 skill 模式,不支持 --batch / --from-traces。不传则已有文件时报错保护。常配 --focus 补特定场景。
|
|
597
598
|
- `--batch` `boolean`:批量模式:扫 --skill-dir 下所有缺 samples 的 skill,逐个生成。
|
|
598
599
|
- `--count` `option`:生成用例条数。不传由 LLM 按 skill 类型自动决定。
|
|
599
600
|
- `--executor` `option`:执行器名。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
|
|
600
|
-
- `--fix` `boolean`:fix 模式:基于最近评测报告自动修复 sample_design 类型失败。
|
|
601
601
|
- `--focus` `option`:生成焦点(自然语言提示)。控制 LLM 偏向哪类用例。
|
|
602
602
|
- `--from-traces` `boolean`:from-traces 模式:从 observe inbox 的失败信号回流生成评测用例草稿(provenance: production-trace),落草稿待人工 review。
|
|
603
603
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
604
604
|
- `--model` `option`:生成 LLM model 名。Codex 自动读取本机配置;也可用 OMK_MODEL 设置环境偏好。
|
|
605
605
|
- `--no-mock` `boolean`:不生成 mocks。执行器不支持工具拦截时会自动启用,避免产生必然失败的 mock_hit。
|
|
606
|
-
- `--observations-dir` `option`:observe inbox 目录(from-traces 模式用),默认项目 .omk/observe
|
|
607
|
-
- `--reports-dir` `option`:报告目录(fix 模式用),默认 ~/.oh-my-knowledge/reports。
|
|
606
|
+
- `--observations-dir` `option`:observe inbox 目录(from-traces 模式用),默认项目 .omk/observe/inbox。
|
|
608
607
|
- `--skill` `option`:仅从指定 skill 的 observe inbox 信号生成草稿(仅 from-traces 模式用)。
|
|
609
608
|
- `--skill-dir` `option` (默认 `skills`):skill 根目录,默认 skills。batch 模式扫此目录。
|
|
610
|
-
- `--treatment` `option`:指定 treatment 名(fix 模式用),默认推断自 skill 路径。
|
|
611
609
|
|
|
612
610
|
**示例:**
|
|
613
611
|
|
|
@@ -623,12 +621,6 @@ omk sample skills/my-skill/SKILL.md
|
|
|
623
621
|
omk sample --batch --skill-dir skills
|
|
624
622
|
```
|
|
625
623
|
|
|
626
|
-
> 根据最近评测报告自动修复 sample_design 类型失败
|
|
627
|
-
|
|
628
|
-
```bash
|
|
629
|
-
omk sample skills/my-skill/SKILL.md --fix
|
|
630
|
-
```
|
|
631
|
-
|
|
632
624
|
> 从 observe inbox 的失败信号回流生成评测用例草稿
|
|
633
625
|
|
|
634
626
|
```bash
|
|
@@ -647,16 +639,16 @@ omk studio [flags]
|
|
|
647
639
|
|
|
648
640
|
**Flags:**
|
|
649
641
|
|
|
650
|
-
- `--analyses-dir` `option`:观测健康报告目录(可选,默认项目级 .omk/observe
|
|
642
|
+
- `--analyses-dir` `option`:观测健康报告目录(可选,默认项目级 .omk/observe/health,空则全局兜底)
|
|
651
643
|
- `--dev` `boolean`:dev 模式:子进程启动 + 热更新
|
|
652
|
-
- `--doctors-dir` `option`:体检报告目录(可选,默认项目级 .omk/
|
|
653
|
-
- `--global` `boolean`:只看全局
|
|
644
|
+
- `--doctors-dir` `option`:体检报告目录(可选,默认项目级 .omk/doctor,空则全局兜底)
|
|
645
|
+
- `--global` `boolean`:只看全局 eval / observe/health / doctor / observe/inbox 目录(~/.oh-my-knowledge/),而非机器级聚合 / 项目优先;governance/managed 不受影响
|
|
654
646
|
- `--host` `option`:监听 host,默认 localhost。改为 0.0.0.0 暴露给局域网
|
|
655
647
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
656
648
|
- `--no-open` `boolean`:不自动打开浏览器
|
|
657
|
-
- `--observations-dir` `option`:观测收件箱数据目录(可选,默认 .omk/observe
|
|
649
|
+
- `--observations-dir` `option`:观测收件箱数据目录(可选,默认 .omk/observe/inbox)
|
|
658
650
|
- `--port` `option` (默认 `7799`):监听端口,默认 7799。传 0 让 OS 分配
|
|
659
|
-
- `--reports-dir` `option
|
|
651
|
+
- `--reports-dir` `option`:只看指定 Core 报告目录(可选;默认聚合当前项目 + 全局)
|
|
660
652
|
|
|
661
653
|
**示例:**
|
|
662
654
|
|
|
@@ -690,6 +682,6 @@ omk studio --port 8080 --no-open
|
|
|
690
682
|
| `provenance` | 否 | 用例来源(`omk sample` 自动打) |
|
|
691
683
|
| `mocks` | 否 | 工具调用 mock 返回(sandbox 评测) |
|
|
692
684
|
| `environment` | 否 | 题设环境声明(仅注入 prompt,不物化) |
|
|
693
|
-
| `tripwire` | 否 |
|
|
685
|
+
| `tripwire` | 否 | 标记为「故意诱错」用例并作为 Core Sample annotation 保留;不改变评分或 Decision |
|
|
694
686
|
|
|
695
687
|
完整 schema 见 [docs/specs/sample-design-spec.md](https://github.com/lizhiyao/oh-my-knowledge/blob/main/docs/specs/sample-design-spec.md)。
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { readFileSync, rmSync } from 'node:fs';
|
|
2
|
+
import { dirname, resolve } from 'node:path';
|
|
3
3
|
import { Args, Flags } from '@oclif/core';
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
|
|
5
5
|
import { BaseCommand } from '../oclif/base-command.js';
|
|
@@ -8,14 +8,13 @@ import { CliExit } from '../lib/cli-exit.js';
|
|
|
8
8
|
import { tCli } from '../lib/i18n.js';
|
|
9
9
|
import { makeDoctorProgress } from '../lib/progress.js';
|
|
10
10
|
import { resolveCliExecutor, resolveRuntimeSelection } from '../lib/runtime-defaults.js';
|
|
11
|
-
import { DEFAULT_DOCTORS_DIR } from '../../
|
|
12
|
-
import { indexDoctorWrite, removeDoctorCard } from '../../
|
|
13
|
-
import { doctorReportFileStem
|
|
14
|
-
import {
|
|
15
|
-
import { projectDoctorsDir, globalDoctorsDir } from '../../
|
|
16
|
-
import { persistDoctorGraphSidecars, removeDoctorGraphSidecars } from '../../
|
|
17
|
-
import { parseDoctorReport } from '../../
|
|
18
|
-
import { writeJsonFileAtomic } from '../../shared/atomic-json.js';
|
|
11
|
+
import { DEFAULT_DOCTORS_DIR } from '../../evidence/storage/default-dirs.js';
|
|
12
|
+
import { indexDoctorWrite, removeDoctorCard } from '../../evidence/storage/discovery-index.js';
|
|
13
|
+
import { doctorReportFileStem } from '../../evidence/storage/file-names.js';
|
|
14
|
+
import { listMeasurementReportPaths, measurementRecordIdFromReportPath, writeMeasurementReportBundle, } from '../../evidence/storage/report-bundle.js';
|
|
15
|
+
import { projectDoctorsDir, globalDoctorsDir } from '../../evidence/storage/directories.js';
|
|
16
|
+
import { persistDoctorGraphSidecars, removeDoctorGraphSidecars } from '../../evidence/graph/doctor.js';
|
|
17
|
+
import { parseDoctorReport } from '../../knowledge-artifacts/doctor/report-parser.js';
|
|
19
18
|
export default class Doctor extends BaseCommand {
|
|
20
19
|
static description = bilingual({
|
|
21
20
|
zh: '体检 omk 工作目录:先跑静态规则,再对 skill 做多维度 LLM 健康度审计(默认 --repeat 2 采样 + 共识归并)。',
|
|
@@ -97,14 +96,14 @@ export default class Doctor extends BaseCommand {
|
|
|
97
96
|
}),
|
|
98
97
|
'output-dir': Flags.string({
|
|
99
98
|
description: bilingual({
|
|
100
|
-
zh: '报告输出目录,默认项目级 .omk/
|
|
101
|
-
en: 'Report output dir, default project-level .omk/
|
|
99
|
+
zh: '报告输出目录,默认项目级 .omk/doctor(--global 写全局)。',
|
|
100
|
+
en: 'Report output dir, default project-level .omk/doctor (--global for global).',
|
|
102
101
|
}),
|
|
103
102
|
}),
|
|
104
103
|
global: Flags.boolean({
|
|
105
104
|
description: bilingual({
|
|
106
|
-
zh: '写全局 ~/.oh-my-knowledge/
|
|
107
|
-
en: 'Write to global ~/.oh-my-knowledge/
|
|
105
|
+
zh: '写全局 ~/.oh-my-knowledge/doctor,而非项目 .omk/doctor',
|
|
106
|
+
en: 'Write to global ~/.oh-my-knowledge/doctor instead of project .omk/doctor',
|
|
108
107
|
}),
|
|
109
108
|
}),
|
|
110
109
|
dimensions: Flags.string({
|
|
@@ -179,18 +178,18 @@ export default class Doctor extends BaseCommand {
|
|
|
179
178
|
const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
|
|
180
179
|
const cwd = process.cwd();
|
|
181
180
|
// 副作用 import: 注册 7 内置维度 spec + skill_health composer rule。
|
|
182
|
-
await import('../../doctor/health/register.js');
|
|
181
|
+
await import('../../knowledge-artifacts/doctor/health/register.js');
|
|
183
182
|
if (flags.dimensions) {
|
|
184
|
-
const { loadAndRegisterCustomDimensions } = await import('../../doctor/health/load-custom-dimensions.js');
|
|
183
|
+
const { loadAndRegisterCustomDimensions } = await import('../../knowledge-artifacts/doctor/health/load-custom-dimensions.js');
|
|
185
184
|
const count = loadAndRegisterCustomDimensions(resolve(flags.dimensions));
|
|
186
185
|
if (count > 0) {
|
|
187
186
|
process.stderr.write(lang === 'zh' ? `已加载 ${count} 个自定义维度\n` : `Loaded ${count} custom dimension(s)\n`);
|
|
188
187
|
}
|
|
189
188
|
}
|
|
190
|
-
const { runDoctor } = await import('../../doctor/index.js');
|
|
191
|
-
const { renderDoctorReportText, renderDoctorReportJson, renderDoctorActionPlanText } = await import('../../doctor/renderer.js');
|
|
192
|
-
const { getRegisteredRules } = await import('../../doctor/rules.js');
|
|
193
|
-
const { isComposerRule } = await import('../../
|
|
189
|
+
const { runDoctor } = await import('../../knowledge-artifacts/doctor/index.js');
|
|
190
|
+
const { renderDoctorReportText, renderDoctorReportJson, renderDoctorActionPlanText } = await import('../../knowledge-artifacts/doctor/renderer.js');
|
|
191
|
+
const { getRegisteredRules } = await import('../../knowledge-artifacts/doctor/rules.js');
|
|
192
|
+
const { isComposerRule } = await import('../../knowledge-artifacts/doctor/rule-kind.js');
|
|
194
193
|
// 默认:静态规则 + 在线检查(LLM health composer + endpoint 自定义维度 external=true)。
|
|
195
194
|
// --static-only:只跑静态检测(纯静态内置 rule),但排除 samples_contract_aligned
|
|
196
195
|
// (那条要 samples.json,与离线解耦) → 且不加载 samples,依赖检查只扫 skill 正文。
|
|
@@ -260,7 +259,7 @@ export default class Doctor extends BaseCommand {
|
|
|
260
259
|
process.stderr.write(lang === 'zh' ? '✅ 没有需要修复的问题\n' : '✅ Nothing to fix\n');
|
|
261
260
|
throw new CliExit(0);
|
|
262
261
|
}
|
|
263
|
-
const { runDoctorFix } = await import('../../doctor/fixer.js');
|
|
262
|
+
const { runDoctorFix } = await import('../../knowledge-artifacts/doctor/fixer.js');
|
|
264
263
|
const changed = await runDoctorFix({ report: existing, executorName, model, timeoutMs, effort });
|
|
265
264
|
throw new CliExit(changed ? 0 : (existing.outcome === 'failed' ? 1 : 0));
|
|
266
265
|
}
|
|
@@ -273,8 +272,6 @@ export default class Doctor extends BaseCommand {
|
|
|
273
272
|
const DOCTOR_HISTORY_MAX_PER_SKILL = 50;
|
|
274
273
|
function persistDoctorReport(report, outputDir, lang = 'zh') {
|
|
275
274
|
const dir = outputDir ?? DEFAULT_DOCTORS_DIR;
|
|
276
|
-
mkdirSync(dir, { recursive: true });
|
|
277
|
-
migrateLegacyReportFiles(dir, 'doctor');
|
|
278
275
|
for (const skill of report.skills) {
|
|
279
276
|
const counts = {
|
|
280
277
|
pass: 0,
|
|
@@ -306,11 +303,17 @@ function persistDoctorReport(report, outputDir, lang = 'zh') {
|
|
|
306
303
|
outcome,
|
|
307
304
|
};
|
|
308
305
|
const cardId = doctorReportFileStem(skill.skillName, report.id);
|
|
309
|
-
const filePath = reportFilePath(dir, cardId);
|
|
310
306
|
const parsed = parseDoctorReport(perSkill);
|
|
311
307
|
if (!parsed)
|
|
312
308
|
throw new Error('invalid doctor report');
|
|
313
|
-
|
|
309
|
+
const { reportPath: filePath } = writeMeasurementReportBundle({
|
|
310
|
+
rootDir: dir,
|
|
311
|
+
measurementDomain: 'doctor',
|
|
312
|
+
recordId: cardId,
|
|
313
|
+
reportId: report.id,
|
|
314
|
+
createdAt: report.timestamp,
|
|
315
|
+
report: parsed,
|
|
316
|
+
});
|
|
314
317
|
// 产物发现索引:per-skill 报告落项目本地后,best-effort 追加全局轻卡片,让 studio 跨项目聚合。
|
|
315
318
|
indexDoctorWrite({
|
|
316
319
|
id: cardId, path: filePath, skillName: skill.skillName, reportId: report.id, timestamp: report.timestamp,
|
|
@@ -343,36 +346,36 @@ export function pruneDoctorHistory(dir, skillName, maxKeep) {
|
|
|
343
346
|
if (!Number.isSafeInteger(maxKeep) || maxKeep < 0) {
|
|
344
347
|
throw new TypeError('maxKeep must be a non-negative safe integer');
|
|
345
348
|
}
|
|
346
|
-
migrateLegacyReportFiles(dir, 'doctor');
|
|
347
349
|
const candidates = [];
|
|
348
|
-
for (const
|
|
349
|
-
if (!isReportFileName(file))
|
|
350
|
-
continue;
|
|
350
|
+
for (const path of listMeasurementReportPaths(dir, 'doctor')) {
|
|
351
351
|
try {
|
|
352
|
-
const data = parseDoctorReport(JSON.parse(readFileSync(
|
|
352
|
+
const data = parseDoctorReport(JSON.parse(readFileSync(path, 'utf-8')));
|
|
353
353
|
if (!data || data.skills.length !== 1)
|
|
354
354
|
continue;
|
|
355
355
|
if (data.skills[0].skillName !== skillName)
|
|
356
356
|
continue;
|
|
357
357
|
const expectedStem = doctorReportFileStem(skillName, data.id);
|
|
358
|
-
if (
|
|
358
|
+
if (measurementRecordIdFromReportPath(path) !== expectedStem)
|
|
359
359
|
continue;
|
|
360
|
-
candidates.push({
|
|
360
|
+
candidates.push({
|
|
361
|
+
path,
|
|
362
|
+
graphStem: expectedStem,
|
|
363
|
+
timestamp: data.timestamp,
|
|
364
|
+
});
|
|
361
365
|
}
|
|
362
366
|
catch { /* skip corrupt / unrelated json */ }
|
|
363
367
|
}
|
|
364
368
|
if (candidates.length <= maxKeep)
|
|
365
369
|
return;
|
|
366
370
|
candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
367
|
-
for (const {
|
|
371
|
+
for (const { path, graphStem } of candidates.slice(maxKeep)) {
|
|
368
372
|
try {
|
|
369
|
-
|
|
373
|
+
rmSync(dirname(path), { recursive: true, force: true });
|
|
370
374
|
}
|
|
371
375
|
catch { /* ignore */ }
|
|
372
376
|
// 连带删卡片:否则被 prune 掉的报告会经 listDoctorCards 合并在本项目 studio「复活」(正文已删、卡片还在)。
|
|
373
377
|
// 卡片 id = 文件 stem(`{name}-{id}`),与 indexDoctorWrite 写入口径一致。
|
|
374
|
-
|
|
375
|
-
removeDoctorCard(doctorStem);
|
|
378
|
+
removeDoctorCard(graphStem);
|
|
376
379
|
removeDoctorGraphSidecars(dir, graphStem);
|
|
377
380
|
}
|
|
378
381
|
}
|
|
@@ -2,12 +2,15 @@ import { BaseCommand } from '../../../oclif/base-command.js';
|
|
|
2
2
|
export default class EvalGoldCompare extends BaseCommand {
|
|
3
3
|
static description: string;
|
|
4
4
|
static args: {
|
|
5
|
-
|
|
5
|
+
runId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
6
6
|
};
|
|
7
7
|
static flags: {
|
|
8
8
|
lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
9
9
|
'gold-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
10
|
-
|
|
10
|
+
target: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
11
|
+
evaluator: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
+
metric: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
|
+
'trial-index': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
11
14
|
'reports-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
15
|
'bootstrap-samples': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
13
16
|
seed: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
@@ -4,16 +4,15 @@ import { BaseCommand } from '../../../oclif/base-command.js';
|
|
|
4
4
|
import { LANG_FLAG, bilingual } from '../../../oclif/i18n.js';
|
|
5
5
|
import { integerStringParser } from '../../../oclif/parsers.js';
|
|
6
6
|
import { CliExit } from '../../../lib/cli-exit.js';
|
|
7
|
-
import { projectReportsDir, globalReportsDir } from '../../../../
|
|
8
|
-
import { requireEvaluationReport } from '../../../lib/shared.js';
|
|
7
|
+
import { projectReportsDir, globalReportsDir } from '../../../../evidence/storage/directories.js';
|
|
9
8
|
export default class EvalGoldCompare extends BaseCommand {
|
|
10
9
|
static description = bilingual({
|
|
11
|
-
zh: '
|
|
12
|
-
en: 'Compare
|
|
10
|
+
zh: '把一组 Core run 观测跟 gold dataset 对比,计算 bootstrap CI 后的 agreement。',
|
|
11
|
+
en: 'Compare Core run observations against a gold dataset with bootstrap-CI agreement.',
|
|
13
12
|
});
|
|
14
13
|
static args = {
|
|
15
|
-
|
|
16
|
-
description: bilingual({ zh: '
|
|
14
|
+
runId: Args.string({
|
|
15
|
+
description: bilingual({ zh: 'Core run ID。', en: 'Core run ID.' }),
|
|
17
16
|
required: true,
|
|
18
17
|
}),
|
|
19
18
|
};
|
|
@@ -22,16 +21,26 @@ export default class EvalGoldCompare extends BaseCommand {
|
|
|
22
21
|
'gold-dir': Flags.string({
|
|
23
22
|
description: bilingual({ zh: 'gold dataset 目录,必填', en: 'Gold dataset dir (required)' }),
|
|
24
23
|
}),
|
|
25
|
-
|
|
24
|
+
target: Flags.string({
|
|
26
25
|
description: bilingual({
|
|
27
|
-
zh: '
|
|
28
|
-
en: '
|
|
26
|
+
zh: '显式选择 Core target ID。',
|
|
27
|
+
en: 'Explicit Core target ID.',
|
|
29
28
|
}),
|
|
30
29
|
}),
|
|
30
|
+
evaluator: Flags.string({
|
|
31
|
+
description: bilingual({ zh: '显式选择 Core evaluator ID。', en: 'Explicit Core evaluator ID.' }),
|
|
32
|
+
}),
|
|
33
|
+
metric: Flags.string({
|
|
34
|
+
description: bilingual({ zh: '显式选择 Core metric ID。', en: 'Explicit Core metric ID.' }),
|
|
35
|
+
}),
|
|
36
|
+
'trial-index': Flags.string({
|
|
37
|
+
description: bilingual({ zh: '显式选择 trial index。', en: 'Explicit trial index.' }),
|
|
38
|
+
parse: integerStringParser('--trial-index', { min: 0 }),
|
|
39
|
+
}),
|
|
31
40
|
'reports-dir': Flags.string({
|
|
32
41
|
description: bilingual({
|
|
33
|
-
zh: '
|
|
34
|
-
en: '
|
|
42
|
+
zh: '只读取指定 Core 报告目录(默认当前项目 + 全局)',
|
|
43
|
+
en: 'Read only this Core reports dir (default: current project + global)',
|
|
35
44
|
}),
|
|
36
45
|
}),
|
|
37
46
|
'bootstrap-samples': Flags.string({
|
|
@@ -50,9 +59,9 @@ export default class EvalGoldCompare extends BaseCommand {
|
|
|
50
59
|
const { args, flags } = await this.parse(EvalGoldCompare);
|
|
51
60
|
const lang = this.lang;
|
|
52
61
|
await this.runWithCliExit(async () => {
|
|
53
|
-
const
|
|
54
|
-
if (!
|
|
55
|
-
console.error('Usage: omk eval gold compare <
|
|
62
|
+
const runId = args.runId;
|
|
63
|
+
if (!runId) {
|
|
64
|
+
console.error('Usage: omk eval gold compare <runId> --gold-dir <dir> --target <id> --evaluator <id> --metric <id>');
|
|
56
65
|
throw new CliExit(1);
|
|
57
66
|
}
|
|
58
67
|
const goldDir = flags['gold-dir'];
|
|
@@ -60,9 +69,9 @@ export default class EvalGoldCompare extends BaseCommand {
|
|
|
60
69
|
console.error('--gold-dir is required');
|
|
61
70
|
throw new CliExit(1);
|
|
62
71
|
}
|
|
63
|
-
const { loadGoldDataset } = await import('../../../../
|
|
64
|
-
const {
|
|
65
|
-
const {
|
|
72
|
+
const { loadGoldDataset } = await import('../../../../eval-workflows/gold/dataset.js');
|
|
73
|
+
const { createNodeCoreContentStore, createNodeCoreRunArtifactStore, createOverlayCoreRunArtifactStore, } = await import('../../../../eval-workflows/artifact-store/index.js');
|
|
74
|
+
const { compareGoldToCoreRun } = await import('../../../../eval-workflows/projections/gold.js');
|
|
66
75
|
const { dataset, issues } = loadGoldDataset(goldDir);
|
|
67
76
|
if (!dataset) {
|
|
68
77
|
console.error('Cannot load gold dataset:');
|
|
@@ -72,22 +81,42 @@ export default class EvalGoldCompare extends BaseCommand {
|
|
|
72
81
|
}
|
|
73
82
|
for (const i of issues)
|
|
74
83
|
console.error(`warn: ${i.message}`);
|
|
75
|
-
|
|
76
|
-
|
|
84
|
+
const storeOf = (directory) => createNodeCoreRunArtifactStore(directory, {
|
|
85
|
+
contentResolver: createNodeCoreContentStore(resolve(directory, 'content')),
|
|
86
|
+
});
|
|
77
87
|
const store = flags['reports-dir']
|
|
78
|
-
?
|
|
79
|
-
:
|
|
80
|
-
const
|
|
88
|
+
? storeOf(resolve(flags['reports-dir']))
|
|
89
|
+
: createOverlayCoreRunArtifactStore(storeOf(projectReportsDir()), [storeOf(globalReportsDir())]);
|
|
90
|
+
const source = await store.get(runId);
|
|
91
|
+
if (source === undefined) {
|
|
92
|
+
console.error(lang === 'zh'
|
|
93
|
+
? `找不到 Core run「${runId}」;旧 evaluation report 不再支持 gold compare。`
|
|
94
|
+
: `Core run "${runId}" was not found; legacy evaluation reports are no longer supported.`);
|
|
95
|
+
throw new CliExit(1);
|
|
96
|
+
}
|
|
97
|
+
if (!flags.target || !flags.evaluator || !flags.metric) {
|
|
98
|
+
console.error(lang === 'zh'
|
|
99
|
+
? '--target、--evaluator、--metric 均为必填;Core 不会隐式合并多个观测。'
|
|
100
|
+
: '--target, --evaluator, and --metric are required; Core never pools observations implicitly.');
|
|
101
|
+
throw new CliExit(1);
|
|
102
|
+
}
|
|
81
103
|
const samples = Math.max(100, Number(flags['bootstrap-samples'] ?? 1000) || 1000);
|
|
82
104
|
const seedVal = flags.seed != null ? Number(flags.seed) : undefined;
|
|
83
|
-
const result =
|
|
84
|
-
|
|
105
|
+
const result = compareGoldToCoreRun({
|
|
106
|
+
source,
|
|
85
107
|
gold: dataset,
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
108
|
+
selector: {
|
|
109
|
+
targetId: flags.target,
|
|
110
|
+
evaluatorId: flags.evaluator,
|
|
111
|
+
metricId: flags.metric,
|
|
112
|
+
...(flags['trial-index'] === undefined ? {} : {
|
|
113
|
+
trialIndex: Number(flags['trial-index']),
|
|
114
|
+
}),
|
|
115
|
+
},
|
|
116
|
+
bootstrapSamples: samples,
|
|
117
|
+
...(Number.isFinite(seedVal) ? { bootstrapSeed: seedVal } : {}),
|
|
89
118
|
});
|
|
90
|
-
console.log(
|
|
119
|
+
console.log(JSON.stringify(result, null, 2));
|
|
91
120
|
});
|
|
92
121
|
}
|
|
93
122
|
}
|
|
@@ -27,7 +27,7 @@ export default class EvalGoldInit extends BaseCommand {
|
|
|
27
27
|
const { flags } = await this.parse(EvalGoldInit);
|
|
28
28
|
const lang = this.lang;
|
|
29
29
|
await this.runWithCliExit(async () => {
|
|
30
|
-
const { initGoldDataset } = await import('../../../../
|
|
30
|
+
const { initGoldDataset } = await import('../../../../eval-workflows/gold/cli.js');
|
|
31
31
|
try {
|
|
32
32
|
const written = initGoldDataset(flags.out, {
|
|
33
33
|
annotator: flags.annotator,
|