oh-my-knowledge 1.0.0-beta.2 → 1.0.0-beta.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +1 -1
  2. package/README.zh.md +2 -2
  3. package/dist/assets/agent-skills/omk/references/commands.md +17 -17
  4. package/dist/cli/commands/eval/index.js +17 -17
  5. package/dist/eval-core/contracts/comparability.d.ts +10 -10
  6. package/dist/eval-core/contracts/comparability.js +42 -42
  7. package/dist/eval-core/contracts/schemas/v2/comparability-assessment.schema.json +452 -0
  8. package/dist/eval-core/contracts/schemas/v2/series-analysis-bundle.schema.json +1299 -0
  9. package/dist/eval-core/contracts/series.d.ts +8 -8
  10. package/dist/eval-core/contracts/series.js +2 -2
  11. package/dist/eval-core/index.d.ts +1 -1
  12. package/dist/eval-core/schemas.js +2 -0
  13. package/dist/eval-runtime/adapters/executor-fn.js +4 -30
  14. package/dist/eval-runtime/adapters/invoke-contract.d.ts +57 -0
  15. package/dist/eval-runtime/adapters/invoke-contract.js +58 -0
  16. package/dist/eval-runtime/adapters/json-executor.d.ts +43 -0
  17. package/dist/eval-runtime/adapters/json-executor.js +142 -0
  18. package/dist/eval-runtime/advanced.d.ts +32 -0
  19. package/dist/eval-runtime/advanced.js +17 -0
  20. package/dist/eval-runtime/conformance/executor.d.ts +20 -9
  21. package/dist/eval-runtime/conformance/executor.js +166 -34
  22. package/dist/eval-runtime/contracts.d.ts +4 -0
  23. package/dist/eval-runtime/contracts.js +2 -0
  24. package/dist/eval-runtime/evaluate.d.ts +219 -0
  25. package/dist/eval-runtime/evaluate.js +649 -0
  26. package/dist/eval-runtime/index.d.ts +3 -25
  27. package/dist/eval-runtime/index.js +2 -13
  28. package/dist/eval-runtime/judges/rubric-contracts.d.ts +57 -0
  29. package/dist/eval-runtime/judges/rubric-contracts.js +74 -0
  30. package/dist/eval-runtime/judges/rubric-judge.d.ts +9 -53
  31. package/dist/eval-runtime/judges/rubric-judge.js +5 -73
  32. package/dist/eval-runtime/judges/rubric-kit.d.ts +51 -0
  33. package/dist/eval-runtime/judges/rubric-kit.js +139 -0
  34. package/dist/eval-runtime/runner.d.ts +34 -0
  35. package/dist/eval-runtime/runner.js +93 -0
  36. package/dist/eval-workflows/instruments/judge.d.ts +2 -2
  37. package/dist/eval-workflows/instruments/judge.js +2 -2
  38. package/dist/eval-workflows/runtime-adapter/adapters/claude/resources.d.ts +1 -1
  39. package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-protocol.d.ts +1 -1
  40. package/dist/eval-workflows/runtime-adapter/adapters/codex/resources.d.ts +1 -1
  41. package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-protocol.d.ts +1 -1
  42. package/dist/eval-workflows/runtime-adapter/adapters/shared/stateless-api-resources.d.ts +1 -1
  43. package/dist/index.d.ts +2 -4
  44. package/dist/index.js +2 -4
  45. package/dist/studio/core-runs/renderer.js +1 -1
  46. package/package.json +9 -1
package/README.md CHANGED
@@ -228,7 +228,7 @@ The full docs are published at **[oh-my-knowledge.pages.dev](https://oh-my-knowl
228
228
  - **[CLI reference](docs/reference/cli.md)** — all top-level commands with bash examples and flag tables
229
229
  - **[Migrate to the 1.0 preview](docs/guides/v1-preview-migration.md)** — install channel, storage reset, sample protocol, CLI automation, and embedded API changes since 0.54
230
230
  - **[Evaluation Core cutover](docs/guides/eval-core-cutover.md)** — `BREAKING-SCHEMA` storage, resume, Studio, Gold, managed-evidence, and evolve migration
231
- - **[Embed OMK in a service](docs/guides/eval-runtime.md)** — lightweight `eval-runtime` assembly for Node.js and FaaS hosts
231
+ - **[Embed OMK in a service](docs/guides/eval-runtime.md)** — one `evaluate()` API for Node.js and FaaS hosts
232
232
  - **[Storage layout v2](docs/specs/storage-layout-spec.md)** — project/global domains, compatibility boundary, and Git policy
233
233
  - **[Executors](docs/reference/executors.md)** & **[artifact layout](docs/reference/artifact-layout.md)** — built-in / custom executors; how `variant` resolves to an artifact + runtime context
234
234
  - **[How-to guides](docs/guides/agent-eval.md)** — [evaluate an agent](docs/guides/agent-eval.md) (project runtime context) and [use non-Claude models](docs/guides/non-claude-models.md) (GLM / Qwen / DeepSeek / Moonshot / Ollama)
package/README.zh.md CHANGED
@@ -157,7 +157,7 @@ RAG 专项评测请看 RAGAS(独立 niche,跟 omk 互补)。完整对比
157
157
 
158
158
  | 特性 | 说明 |
159
159
  |------|------|
160
- | **Core 发布决定** | 六种结论 + 稳定 reason code + exit code 路由;Studio 投影同一份经过认证的 Decision |
160
+ | **Core 发布决定** | 六种判定 + 稳定 reason code + exit code 路由;Studio 投影同一份经过认证的 Decision |
161
161
  | **五层 evidence graph** | Assertion / LLM / Judge / Dimension / Composite 保持独立,coverage、成本、状态与 lineage 与分数正交 |
162
162
  | **多执行器** | 支持 Claude CLI / Claude SDK / Codex CLI / Codex SDK / DeepSeek Harness / OpenAI / Anthropic API 及自定义命令 |
163
163
  | **30+ 种断言** | 包含子串、正则、JSON Schema、ROUGE/BLEU/Levenshtein 相似度、Agent 工具调用、语义相似度、自定义函数等 |
@@ -228,7 +228,7 @@ omk-mcp
228
228
  - **[CLI 参考](docs/zh/reference/cli.md)** —— 顶层命令的 bash 示例和 flag 表
229
229
  - **[迁移到 1.0 预览版](docs/zh/guides/v1-preview-migration.md)** —— 从 `0.54` 升级时的安装渠道、存储重建、用例协议、CLI 自动化与嵌入式 API 变化
230
230
  - **[Evaluation Core 生产切换](docs/zh/guides/eval-core-cutover.md)** —— `BREAKING-SCHEMA` 存储、resume、Studio、Gold、受管证据与 evolve 迁移
231
- - **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS 宿主的轻量 `eval-runtime` 装配
231
+ - **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS 宿主的单一 `evaluate()` API
232
232
  - **[存储布局 v2](docs/zh/specs/storage-layout-spec.md)** —— 项目/全局领域、迁移兼容与 Git 策略
233
233
  - **[执行器](docs/zh/reference/executors.md)** & **[知识载体布局](docs/zh/reference/artifact-layout.md)** —— 内置 / 自定义执行器;variant 如何解析为 artifact + runtime context
234
234
  - **[操作指南](docs/zh/guides/agent-eval.md)** —— [评测 agent](docs/zh/guides/agent-eval.md)(项目 runtime context)与[使用非 Claude 模型](docs/zh/guides/non-claude-models.md)(GLM / 通义 / DeepSeek / Moonshot / Ollama)
@@ -63,7 +63,7 @@ omk doctor --json --gate
63
63
 
64
64
  ## omk eval
65
65
 
66
- 跑评测:对一个 control vs 多个 treatment skill 做对照试验,产 verdict 报告。
66
+ 跑评测:用一个对照组与多个实验组 skill 做对照试验,生成判定报告。
67
67
 
68
68
  **用法:**
69
69
 
@@ -73,16 +73,16 @@ omk eval [flags]
73
73
 
74
74
  **Flags:**
75
75
 
76
- - `--batch` `boolean`:batch 模式:baseline vs 每个 skill
76
+ - `--batch` `boolean`:batch 模式:baseline 作为对照组,逐个 skill 作为实验组
77
77
  - `--bootstrap` `boolean`:加 bootstrap CI
78
78
  - `--bootstrap-samples` `option`:bootstrap 重采样次数,默认 1000
79
- - `--budget-per-sample-ms` `option`:单 sample 时长上限 ms(必须 > 0,不传则无上限)
80
- - `--budget-per-sample-usd` `option`:单 sample 预算上限 USD(必须 > 0,不传则无上限)
79
+ - `--budget-per-sample-ms` `option`:单用例时长上限 ms(必须 > 0,不传则无上限)
80
+ - `--budget-per-sample-usd` `option`:单用例预算上限 USD(必须 > 0,不传则无上限)
81
81
  - `--budget-usd` `option`:总预算上限 USD(必须 > 0,不传则无上限)
82
82
  - `--concurrency` `option`:并发数,默认 1
83
83
  - `--config` `option`:eval.yaml 路径
84
- - `--control` `option`:control variant 表达式(仅 artifact 身份)
85
- - `--control-cwd` `option`:control runtime context 目录
84
+ - `--control` `option`:对照组(control)的 variant 表达式(仅 artifact 身份)
85
+ - `--control-cwd` `option`:对照组(control)的 runtime context 目录
86
86
  - `--dry-run` `boolean`:只 plan 不实跑
87
87
  - `--effort` `option`:被测 LLM 扩展思考预算 low/medium/high/xhigh/max(默认 low;跨 effort 报告不严格可比)。
88
88
  - `--executor` `option`:执行器:claude / claude-sdk / codex / codex-sdk / anthropic-api / openai-api / 自定义命令。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
@@ -90,7 +90,7 @@ omk eval [flags]
90
90
  - `--gold-dir` `option`:gold dataset 目录
91
91
  - `--holdout-ratio` `option`:留出比例 0-1(如 0.3);切出 holdout 子集,对比 train/holdout 综合分检测过拟合
92
92
  - `--judge-models` `option`:评委配置,格式 executor:model[,...],例 claude:haiku 或 codex:<model>(≥ 2 个 = ensemble)。默认跟随所选执行器;Codex 沿用被测模型。
93
- - `--judge-repeat` `option`:每个 dim 评 N 次
93
+ - `--judge-repeat` `option`:每个维度由评委评价 N 次
94
94
  - `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
95
95
  - `--layered-stats` `boolean`:输出分层统计
96
96
  - `--mcp-config` `option`:MCP 配置文件路径
@@ -99,30 +99,30 @@ omk eval [flags]
99
99
  - `--no-debias-length` `boolean`:关 length-debias(默认开)
100
100
  - `--no-diagnostic` `boolean`:关闭基于 Core 失败、缺失、排除与稳定 reason code 的诊断投影。
101
101
  - `--no-evidence` `boolean`:不把本次评测写成证据追加进受管记录(默认会为已 install 的 skill 自动写)。
102
- - `--no-gate` `boolean`:关 verdict gate
103
- - `--no-judge` `boolean`:跳过 LLM judge
102
+ - `--no-gate` `boolean`:关闭判定门禁
103
+ - `--no-judge` `boolean`:跳过 LLM 评委
104
104
  - `--no-serve` `boolean`:不启 report server
105
105
  - `--no-strict-baseline` `boolean`:关闭 baseline 隔离
106
106
  - `--output-dir` `option`:报告输出目录(默认项目级 .omk/eval)
107
- - `--repeat` `option`:每个 sample 重复跑 N 次
108
- - `--report-only` `boolean`:生成报告并打印 verdict,但始终 exit 0(不参与 CI gate)。
107
+ - `--repeat` `option`:每个用例重复运行 N 次
108
+ - `--report-only` `boolean`:生成报告并打印判定,但始终 exit 0(不参与 CI gate)。
109
109
  - `--resume` `option`:复用经过完整契约校验的 Core runId;拒绝时失败关闭
110
- - `--retry` `option`:失败 sample 重试次数
111
- - `--samples` `option`:用例路径。自动发现项目级或单 treatment 目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。
110
+ - `--retry` `option`:单用例失败重试次数
111
+ - `--samples` `option`:用例路径。自动发现项目级或单个实验组(treatment)目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。
112
112
  - `--skill-dir` `option`:skill 目录,默认 skills
113
113
  - `--skip-connectivity` `boolean`:跳 LLM 连通性预检
114
114
  - `--skip-doctor` `boolean`:escape hatch:跳 doctor 健康检查门禁(默认强制启用)。沙箱 mock 提供依赖时绕开 doctor 物理路径误报;garbage-in 风险自负。
115
115
  - `--strict-baseline` `boolean`:强制 baseline 隔离(default true)
116
- - `--threshold` `option`:verdict 阈值,默认 3.5
116
+ - `--threshold` `option`:判定阈值,默认 3.5
117
117
  - `--timeout` `option`:单用例超时秒,默认 120
118
- - `--treatment` `option`:treatment variant 列表,逗号分隔(仅 artifact 身份)
119
- - `--treatment-cwd` `option`:treatment runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)
118
+ - `--treatment` `option`:实验组(treatment)的 variant 列表,逗号分隔(仅 artifact 身份)
119
+ - `--treatment-cwd` `option`:实验组(treatment)的 runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)
120
120
  - `--trivial-diff` `option`:可忽略 diff 容差,0 表示不启用容差
121
121
  - `--verbose` `boolean`:详细日志
122
122
 
123
123
  **示例:**
124
124
 
125
- > 最简对照:baseline vs my-skill
125
+ > 最简对照:baseline 作为对照组,my-skill 作为实验组
126
126
 
127
127
  ```bash
128
128
  omk eval --control baseline --treatment my-skill
@@ -174,13 +174,13 @@ async function runEval(_args, flags, lang) {
174
174
  }
175
175
  export default class Eval extends BaseCommand {
176
176
  static description = bilingual({
177
- zh: '跑评测:对一个 control vs 多个 treatment skill 做对照试验,产 verdict 报告。',
177
+ zh: '跑评测:用一个对照组与多个实验组 skill 做对照试验,生成判定报告。',
178
178
  en: 'Run evaluation: control vs treatment(s) comparison, produce verdict report.',
179
179
  });
180
180
  static examples = [
181
181
  {
182
182
  description: bilingual({
183
- zh: '最简对照:baseline vs my-skill',
183
+ zh: '最简对照:baseline 作为对照组,my-skill 作为实验组',
184
184
  en: 'Minimal A/B: baseline vs my-skill',
185
185
  }),
186
186
  command: '<%= config.bin %> eval --control baseline --treatment my-skill',
@@ -197,23 +197,23 @@ export default class Eval extends BaseCommand {
197
197
  lang: LANG_FLAG,
198
198
  // ── 实验角色 ──
199
199
  control: Flags.string({
200
- description: bilingual({ zh: 'control variant 表达式(仅 artifact 身份)', en: 'Control variant expr (artifact identity only)' }),
200
+ description: bilingual({ zh: '对照组(control)的 variant 表达式(仅 artifact 身份)', en: 'Control variant expr (artifact identity only)' }),
201
201
  }),
202
202
  treatment: Flags.string({
203
203
  description: bilingual({
204
- zh: 'treatment variant 列表,逗号分隔(仅 artifact 身份)',
204
+ zh: '实验组(treatment)的 variant 列表,逗号分隔(仅 artifact 身份)',
205
205
  en: 'Treatment variants, comma-separated (artifact identity only)',
206
206
  }),
207
207
  }),
208
208
  'control-cwd': Flags.string({
209
209
  description: bilingual({
210
- zh: 'control runtime context 目录',
210
+ zh: '对照组(control)的 runtime context 目录',
211
211
  en: 'Runtime context dir for control',
212
212
  }),
213
213
  }),
214
214
  'treatment-cwd': Flags.string({
215
215
  description: bilingual({
216
- zh: 'treatment runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)',
216
+ zh: '实验组(treatment)的 runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)',
217
217
  en: 'Runtime context dirs for treatments, comma-separated, index-aligned with --treatment (blank = none)',
218
218
  }),
219
219
  }),
@@ -222,7 +222,7 @@ export default class Eval extends BaseCommand {
222
222
  }),
223
223
  samples: Flags.string({
224
224
  description: bilingual({
225
- zh: '用例路径。自动发现项目级或单 treatment 目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。',
225
+ zh: '用例路径。自动发现项目级或单个实验组(treatment)目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。',
226
226
  en: 'Samples path. Auto-discovers eval-samples.json / eval-samples.yaml at project scope or for a single directory-skill treatment; an explicit path may be a JSON / YAML file or split directory.',
227
227
  }),
228
228
  }),
@@ -256,7 +256,7 @@ export default class Eval extends BaseCommand {
256
256
  }),
257
257
  // ── 评测 toggle ──
258
258
  'no-judge': Flags.boolean({
259
- description: bilingual({ zh: '跳过 LLM judge', en: 'Skip LLM judge' }),
259
+ description: bilingual({ zh: '跳过 LLM 评委', en: 'Skip LLM judge' }),
260
260
  }),
261
261
  'no-cache': Flags.boolean({
262
262
  description: bilingual({ zh: '跳过 executor cache', en: 'Skip executor cache' }),
@@ -277,7 +277,7 @@ export default class Eval extends BaseCommand {
277
277
  }),
278
278
  batch: Flags.boolean({
279
279
  description: bilingual({
280
- zh: 'batch 模式:baseline vs 每个 skill',
280
+ zh: 'batch 模式:baseline 作为对照组,逐个 skill 作为实验组',
281
281
  en: 'Batch mode: baseline vs each skill',
282
282
  }),
283
283
  }),
@@ -300,7 +300,7 @@ export default class Eval extends BaseCommand {
300
300
  description: bilingual({ zh: '详细日志', en: 'Verbose logging' }),
301
301
  }),
302
302
  retry: Flags.string({
303
- description: bilingual({ zh: '失败 sample 重试次数', en: 'Per-sample retry count' }),
303
+ description: bilingual({ zh: '单用例失败重试次数', en: 'Per-sample retry count' }),
304
304
  parse: integerStringParser('--retry', { min: 0 }),
305
305
  }),
306
306
  resume: Flags.string({
@@ -333,7 +333,7 @@ export default class Eval extends BaseCommand {
333
333
  }),
334
334
  // ── eval-runner extra ──
335
335
  repeat: Flags.string({
336
- description: bilingual({ zh: '每个 sample 重复跑 N 次', en: 'Repeat each sample N times' }),
336
+ description: bilingual({ zh: '每个用例重复运行 N 次', en: 'Repeat each sample N times' }),
337
337
  parse: integerStringParser('--repeat', { min: 1 }),
338
338
  }),
339
339
  'holdout-ratio': Flags.string({
@@ -344,7 +344,7 @@ export default class Eval extends BaseCommand {
344
344
  parse: numberStringParser('--holdout-ratio', { min: 0, max: 1 }),
345
345
  }),
346
346
  'judge-repeat': Flags.string({
347
- description: bilingual({ zh: '每个 dim 评 N 次', en: 'Judge each dim N times' }),
347
+ description: bilingual({ zh: '每个维度由评委评价 N 次', en: 'Judge each dim N times' }),
348
348
  parse: integerStringParser('--judge-repeat', { min: 1 }),
349
349
  }),
350
350
  bootstrap: Flags.boolean({
@@ -365,16 +365,16 @@ export default class Eval extends BaseCommand {
365
365
  parse: numberStringParser('--budget-usd', { minExclusive: 0 }),
366
366
  }),
367
367
  'budget-per-sample-usd': Flags.string({
368
- description: bilingual({ zh: ' sample 预算上限 USD(必须 > 0,不传则无上限)', en: 'Per-sample budget cap USD (must be > 0; omit for no cap)' }),
368
+ description: bilingual({ zh: '单用例预算上限 USD(必须 > 0,不传则无上限)', en: 'Per-sample budget cap USD (must be > 0; omit for no cap)' }),
369
369
  parse: numberStringParser('--budget-per-sample-usd', { minExclusive: 0 }),
370
370
  }),
371
371
  'budget-per-sample-ms': Flags.string({
372
- description: bilingual({ zh: ' sample 时长上限 ms(必须 > 0,不传则无上限)', en: 'Per-sample time cap ms (must be > 0; omit for no cap)' }),
372
+ description: bilingual({ zh: '单用例时长上限 ms(必须 > 0,不传则无上限)', en: 'Per-sample time cap ms (must be > 0; omit for no cap)' }),
373
373
  parse: integerStringParser('--budget-per-sample-ms', { min: 1 }),
374
374
  }),
375
375
  threshold: Flags.string({
376
376
  description: bilingual({
377
- zh: `verdict 阈值,默认 ${DEFAULT_GATE_THRESHOLD}`,
377
+ zh: `判定阈值,默认 ${DEFAULT_GATE_THRESHOLD}`,
378
378
  en: `Verdict threshold, default ${DEFAULT_GATE_THRESHOLD}`,
379
379
  }),
380
380
  parse: numberStringParser('--threshold'),
@@ -385,12 +385,12 @@ export default class Eval extends BaseCommand {
385
385
  }),
386
386
  'report-only': Flags.boolean({
387
387
  description: bilingual({
388
- zh: '生成报告并打印 verdict,但始终 exit 0(不参与 CI gate)。',
388
+ zh: '生成报告并打印判定,但始终 exit 0(不参与 CI gate)。',
389
389
  en: 'Produce the report and print verdict, but always exit 0 (no CI gate).',
390
390
  }),
391
391
  }),
392
392
  'no-gate': Flags.boolean({
393
- description: bilingual({ zh: '关 verdict gate', en: 'Disable verdict gate' }),
393
+ description: bilingual({ zh: '关闭判定门禁', en: 'Disable verdict gate' }),
394
394
  }),
395
395
  'no-evidence': Flags.boolean({
396
396
  description: bilingual({
@@ -7,7 +7,7 @@ import { type Sha256Digest } from './json.js';
7
7
  import { type SealedRunPlan } from '../internal/sealed-run-plan.js';
8
8
  export type ComparabilityRunPlan = SealedRunPlan;
9
9
  export declare const COMPARABILITY_POLICY_SCHEMA_VERSION: "omk.comparability-policy/v1";
10
- export declare const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION: "omk.comparability-assessment/v1";
10
+ export declare const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION: "omk.comparability-assessment/v2";
11
11
  export declare const ComparisonScopeSchema: z.ZodEnum<{
12
12
  evaluation: "evaluation";
13
13
  analysis: "analysis";
@@ -123,7 +123,7 @@ export declare const ComparabilityArtifactIdentitySchema: z.ZodObject<{
123
123
  }>;
124
124
  artifactDigest: z.ZodString;
125
125
  }, z.core.$strict>;
126
- export declare const ComparabilityCandidateIdentitySchema: z.ZodObject<{
126
+ export declare const ComparabilityRunIdentitySchema: z.ZodObject<{
127
127
  runContractDigest: z.ZodString;
128
128
  planDigests: z.ZodObject<{
129
129
  datasetRevisionDigest: z.ZodString;
@@ -222,7 +222,7 @@ export declare const ComparabilityCandidateIdentitySchema: z.ZodObject<{
222
222
  }>;
223
223
  verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
224
224
  }, z.core.$strict>>;
225
- candidateDigest: z.ZodString;
225
+ runIdentityDigest: z.ZodString;
226
226
  }, z.core.$strict>;
227
227
  export declare const COMPARABILITY_REASON_CODES: readonly ["comparability-identity-declared-subject-change", "comparability-design-subject-mapping-invalid", "comparability-design-undeclared-subject-change", "comparability-design-evaluation-input-mismatch", "comparability-design-evaluation-instrument-mismatch", "comparability-design-sampling-mismatch", "comparability-design-randomization-mismatch", "comparability-design-analysis-mismatch", "comparability-design-comparison-mismatch", "comparability-design-decision-mismatch", "comparability-design-schema-mismatch", "comparability-design-projection-mismatch", "comparability-evidence-source-absent", "comparability-evidence-verification-indeterminate", "comparability-evidence-assurance-unverified", "comparability-evidence-source-untrusted", "comparability-evidence-runtime-identity-opaque"];
228
228
  export declare const ComparabilityReasonCodeSchema: z.ZodEnum<{
@@ -282,7 +282,7 @@ export declare const ComparabilityReasonSchema: z.ZodObject<{
282
282
  }, z.core.$strict>;
283
283
  export type ComparabilityReasonCode = z.infer<typeof ComparabilityReasonCodeSchema>;
284
284
  export declare const ComparabilityAssessmentSchema: z.ZodObject<{
285
- schemaVersion: z.ZodLiteral<"omk.comparability-assessment/v1">;
285
+ schemaVersion: z.ZodLiteral<"omk.comparability-assessment/v2">;
286
286
  policyDigest: z.ZodString;
287
287
  designMode: z.ZodLiteral<"exact-measurement-design">;
288
288
  comparisonScope: z.ZodEnum<{
@@ -389,7 +389,7 @@ export declare const ComparabilityAssessmentSchema: z.ZodObject<{
389
389
  }>;
390
390
  verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
391
391
  }, z.core.$strict>>;
392
- candidateDigest: z.ZodString;
392
+ runIdentityDigest: z.ZodString;
393
393
  }, z.core.$strict>;
394
394
  right: z.ZodObject<{
395
395
  runContractDigest: z.ZodString;
@@ -490,7 +490,7 @@ export declare const ComparabilityAssessmentSchema: z.ZodObject<{
490
490
  }>;
491
491
  verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
492
492
  }, z.core.$strict>>;
493
- candidateDigest: z.ZodString;
493
+ runIdentityDigest: z.ZodString;
494
494
  }, z.core.$strict>;
495
495
  designStatus: z.ZodEnum<{
496
496
  incompatible: "incompatible";
@@ -550,10 +550,10 @@ export type ComparabilitySubject = z.infer<typeof ComparabilitySubjectSchema>;
550
550
  export type ComparabilityPolicy = z.infer<typeof ComparabilityPolicySchema>;
551
551
  export type ComparabilitySourceVerificationFact = z.infer<typeof ComparabilitySourceVerificationFactSchema>;
552
552
  export type RuntimeQualificationFact = z.infer<typeof RuntimeQualificationFactSchema>;
553
- export type ComparabilityCandidateIdentity = z.infer<typeof ComparabilityCandidateIdentitySchema>;
553
+ export type ComparabilityRunIdentity = z.infer<typeof ComparabilityRunIdentitySchema>;
554
554
  export type ComparabilityReason = z.infer<typeof ComparabilityReasonSchema>;
555
555
  export type ComparabilityAssessment = z.infer<typeof ComparabilityAssessmentSchema>;
556
- export type ComparabilityValidationErrorCode = 'COMPARABILITY_POLICY_DIGEST_MISMATCH' | 'COMPARABILITY_CANDIDATE_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_RECOMPUTATION_MISMATCH' | 'COMPARABILITY_SOURCE_PREFIX_INVALID' | 'COMPARABILITY_DECISION_SOURCE_PLAN_MISMATCH' | 'COMPARABILITY_VERIFICATION_CONTEXT_INVALID';
556
+ export type ComparabilityValidationErrorCode = 'COMPARABILITY_POLICY_DIGEST_MISMATCH' | 'COMPARABILITY_RUN_IDENTITY_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_RECOMPUTATION_MISMATCH' | 'COMPARABILITY_SOURCE_PREFIX_INVALID' | 'COMPARABILITY_DECISION_SOURCE_PLAN_MISMATCH' | 'COMPARABILITY_VERIFICATION_CONTEXT_INVALID';
557
557
  export declare class ComparabilityValidationError extends TypeError {
558
558
  readonly code: ComparabilityValidationErrorCode;
559
559
  constructor(code: ComparabilityValidationErrorCode, message: string);
@@ -583,8 +583,8 @@ export interface ComparabilityVerificationContext {
583
583
  export interface ComparabilityAssessmentPlanVerification {
584
584
  readonly assessmentComputationStatus: 'verified';
585
585
  readonly policyDigest: Sha256Digest;
586
- readonly leftCandidateDigest: Sha256Digest;
587
- readonly rightCandidateDigest: Sha256Digest;
586
+ readonly leftRunIdentityDigest: Sha256Digest;
587
+ readonly rightRunIdentityDigest: Sha256Digest;
588
588
  }
589
589
  export interface ComparabilityAssessmentSource {
590
590
  readonly assessment: ComparabilityAssessment;
@@ -12,7 +12,7 @@ import { canonicalizeJson, deepFreezeCanonicalJson, digestCanonicalJson, parseWi
12
12
  import { ANALYSIS_PLAN_SCHEMA_VERSION, DECISION_PLAN_SCHEMA_VERSION, EVALUATION_PLAN_SCHEMA_VERSION, EXECUTION_PLAN_SCHEMA_VERSION, PlanDigestsSchema, RUN_PLAN_SCHEMA_VERSION, } from './plans.js';
13
13
  import { assertSealedRunPlan, } from '../internal/sealed-run-plan.js';
14
14
  export const COMPARABILITY_POLICY_SCHEMA_VERSION = 'omk.comparability-policy/v1';
15
- export const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION = 'omk.comparability-assessment/v1';
15
+ export const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION = 'omk.comparability-assessment/v2';
16
16
  export const ComparisonScopeSchema = z.enum(['evaluation', 'analysis', 'decision']);
17
17
  export const ComparabilityStageSchema = z.enum([
18
18
  'execution',
@@ -224,41 +224,41 @@ function compareRuntimeQualifications(left, right) {
224
224
  function runtimeQualificationKey(fact) {
225
225
  return `${fact.stage}\u0000${fact.runtimeKind}\u0000${fact.referenceId}`;
226
226
  }
227
- export const ComparabilityCandidateIdentitySchema = z.object({
227
+ export const ComparabilityRunIdentitySchema = z.object({
228
228
  runContractDigest: Sha256DigestSchema,
229
229
  planDigests: PlanDigestsSchema,
230
230
  randomizationDesignDigest: Sha256DigestSchema,
231
231
  artifacts: z.array(ComparabilityArtifactIdentitySchema),
232
232
  sourceVerification: z.array(ComparabilitySourceVerificationFactSchema),
233
233
  runtimeQualification: z.array(RuntimeQualificationFactSchema),
234
- candidateDigest: Sha256DigestSchema,
235
- }).strict().superRefine((candidate, context) => {
236
- if (!isCanonicalArray(candidate.artifacts, compareArtifacts)
237
- || new Set(candidate.artifacts.map((artifact) => artifact.stage)).size
238
- !== candidate.artifacts.length) {
234
+ runIdentityDigest: Sha256DigestSchema,
235
+ }).strict().superRefine((runIdentity, context) => {
236
+ if (!isCanonicalArray(runIdentity.artifacts, compareArtifacts)
237
+ || new Set(runIdentity.artifacts.map((artifact) => artifact.stage)).size
238
+ !== runIdentity.artifacts.length) {
239
239
  context.addIssue({ code: 'custom', path: ['artifacts'], message: 'Artifacts must be canonical.' });
240
240
  }
241
- if (candidate.runContractDigest !== candidate.planDigests.runContractDigest
242
- || candidate.randomizationDesignDigest
243
- !== candidate.planDigests.randomizationDesignDigest) {
241
+ if (runIdentity.runContractDigest !== runIdentity.planDigests.runContractDigest
242
+ || runIdentity.randomizationDesignDigest
243
+ !== runIdentity.planDigests.randomizationDesignDigest) {
244
244
  context.addIssue({
245
245
  code: 'custom',
246
246
  path: ['planDigests'],
247
- message: 'Candidate summary digests must equal their Plan digest entries.',
247
+ message: 'Run identity digests must equal their Plan digest entries.',
248
248
  });
249
249
  }
250
- if (!isCanonicalArray(candidate.sourceVerification, compareSourceFacts)
251
- || new Set(candidate.sourceVerification.map(sourceFactKey)).size
252
- !== candidate.sourceVerification.length) {
250
+ if (!isCanonicalArray(runIdentity.sourceVerification, compareSourceFacts)
251
+ || new Set(runIdentity.sourceVerification.map(sourceFactKey)).size
252
+ !== runIdentity.sourceVerification.length) {
253
253
  context.addIssue({
254
254
  code: 'custom',
255
255
  path: ['sourceVerification'],
256
256
  message: 'Source verification facts must be unique and canonical.',
257
257
  });
258
258
  }
259
- if (!isCanonicalArray(candidate.runtimeQualification, compareRuntimeQualifications)
260
- || new Set(candidate.runtimeQualification.map(runtimeQualificationKey)).size
261
- !== candidate.runtimeQualification.length) {
259
+ if (!isCanonicalArray(runIdentity.runtimeQualification, compareRuntimeQualifications)
260
+ || new Set(runIdentity.runtimeQualification.map(runtimeQualificationKey)).size
261
+ !== runIdentity.runtimeQualification.length) {
262
262
  context.addIssue({
263
263
  code: 'custom',
264
264
  path: ['runtimeQualification'],
@@ -318,8 +318,8 @@ export const ComparabilityAssessmentSchema = z.object({
318
318
  policyDigest: Sha256DigestSchema,
319
319
  designMode: z.literal('exact-measurement-design'),
320
320
  comparisonScope: ComparisonScopeSchema,
321
- left: ComparabilityCandidateIdentitySchema,
322
- right: ComparabilityCandidateIdentitySchema,
321
+ left: ComparabilityRunIdentitySchema,
322
+ right: ComparabilityRunIdentitySchema,
323
323
  designStatus: z.enum(['compatible', 'incompatible']),
324
324
  evidenceQualificationStatus: z.enum(['verified', 'conditional', 'rejected']),
325
325
  comparabilityStatus: z.enum(['compatible', 'conditional', 'incompatible']),
@@ -398,13 +398,13 @@ export function parseComparabilityPolicyDocument(value) {
398
398
  }
399
399
  return policy;
400
400
  }
401
- function computeCandidateDigest(candidate) {
402
- return digestCanonicalJson(candidate);
401
+ function computeRunIdentityDigest(runIdentity) {
402
+ return digestCanonicalJson(runIdentity);
403
403
  }
404
- function assertCandidateDigest(candidate) {
405
- const { candidateDigest, ...payload } = candidate;
406
- if (computeCandidateDigest(payload) !== candidateDigest) {
407
- throw new ComparabilityValidationError('COMPARABILITY_CANDIDATE_DIGEST_MISMATCH', 'Comparability candidate digest does not match its canonical payload.');
404
+ function assertRunIdentityDigest(runIdentity) {
405
+ const { runIdentityDigest, ...payload } = runIdentity;
406
+ if (computeRunIdentityDigest(payload) !== runIdentityDigest) {
407
+ throw new ComparabilityValidationError('COMPARABILITY_RUN_IDENTITY_DIGEST_MISMATCH', 'Comparability Run identity digest does not match its canonical payload.');
408
408
  }
409
409
  }
410
410
  function computeAssessmentDigest(assessment) {
@@ -412,8 +412,8 @@ function computeAssessmentDigest(assessment) {
412
412
  }
413
413
  export function parseComparabilityAssessmentDocument(value) {
414
414
  const assessment = parseWireDocument(ComparabilityAssessmentSchema, value);
415
- assertCandidateDigest(assessment.left);
416
- assertCandidateDigest(assessment.right);
415
+ assertRunIdentityDigest(assessment.left);
416
+ assertRunIdentityDigest(assessment.right);
417
417
  const { assessmentDigest, ...payload } = assessment;
418
418
  if (computeAssessmentDigest(payload) !== assessmentDigest) {
419
419
  throw new ComparabilityValidationError('COMPARABILITY_ASSESSMENT_DIGEST_MISMATCH', 'ComparabilityAssessment digest does not match its canonical payload.');
@@ -1081,7 +1081,7 @@ function requiredStages(scope) {
1081
1081
  return ['execution', 'evaluation', 'analysis'];
1082
1082
  return ['execution', 'evaluation', 'analysis', 'decision'];
1083
1083
  }
1084
- function candidateIdentity(plan, scope, source, attestations) {
1084
+ function comparabilityRunIdentity(plan, scope, source, attestations) {
1085
1085
  const artifacts = requiredStages(scope)
1086
1086
  .filter((stage) => source[stage] !== undefined)
1087
1087
  .map((stage) => ({ stage, artifactDigest: sourceDigest(source, stage) }))
@@ -1096,18 +1096,18 @@ function candidateIdentity(plan, scope, source, attestations) {
1096
1096
  };
1097
1097
  return {
1098
1098
  ...payload,
1099
- candidateDigest: computeCandidateDigest(payload),
1099
+ runIdentityDigest: computeRunIdentityDigest(payload),
1100
1100
  };
1101
1101
  }
1102
- function evidenceReasons(scope, leftCandidate, rightCandidate, leftProjection, rightProjection) {
1102
+ function evidenceReasons(scope, leftRunIdentity, rightRunIdentity, leftProjection, rightProjection) {
1103
1103
  const codes = new Set();
1104
1104
  const required = requiredStages(scope);
1105
- const leftStages = new Set(leftCandidate.artifacts.map((artifact) => artifact.stage));
1106
- const rightStages = new Set(rightCandidate.artifacts.map((artifact) => artifact.stage));
1105
+ const leftStages = new Set(leftRunIdentity.artifacts.map((artifact) => artifact.stage));
1106
+ const rightStages = new Set(rightRunIdentity.artifacts.map((artifact) => artifact.stage));
1107
1107
  if (required.some((stage) => !leftStages.has(stage) || !rightStages.has(stage))) {
1108
1108
  addReason(codes, 'comparability-evidence-source-absent');
1109
1109
  }
1110
- const facts = [...leftCandidate.sourceVerification, ...rightCandidate.sourceVerification];
1110
+ const facts = [...leftRunIdentity.sourceVerification, ...rightRunIdentity.sourceVerification];
1111
1111
  if (facts.some((fact) => fact.verificationFactKind === 'verification-axis'
1112
1112
  && fact.verificationStatus === 'indeterminate')) {
1113
1113
  addReason(codes, 'comparability-evidence-verification-indeterminate');
@@ -1117,17 +1117,17 @@ function evidenceReasons(scope, leftCandidate, rightCandidate, leftProjection, r
1117
1117
  addReason(codes, 'comparability-evidence-source-untrusted');
1118
1118
  }
1119
1119
  const runtimes = [
1120
- ...leftCandidate.runtimeQualification,
1121
- ...rightCandidate.runtimeQualification,
1120
+ ...leftRunIdentity.runtimeQualification,
1121
+ ...rightRunIdentity.runtimeQualification,
1122
1122
  ];
1123
1123
  if (trustFacts.some((fact) => fact.trust === 'declared' || fact.trust === 'unknown')
1124
1124
  || runtimes.some((runtime) => runtime.effectiveAssuranceLevel !== 'verified')) {
1125
1125
  addReason(codes, 'comparability-evidence-assurance-unverified');
1126
1126
  }
1127
- const opaqueInvariantRuntime = (candidate, projection) => candidate.runtimeQualification.some((runtime) => (runtime.fingerprintBasis === 'opaque'
1127
+ const opaqueInvariantRuntime = (runIdentity, projection) => runIdentity.runtimeQualification.some((runtime) => (runtime.fingerprintBasis === 'opaque'
1128
1128
  && !(runtime.runtimeKind === 'executor' && projection.mappedTargetIds.has(runtime.referenceId))));
1129
- if (opaqueInvariantRuntime(leftCandidate, leftProjection)
1130
- || opaqueInvariantRuntime(rightCandidate, rightProjection)) {
1129
+ if (opaqueInvariantRuntime(leftRunIdentity, leftProjection)
1130
+ || opaqueInvariantRuntime(rightRunIdentity, rightProjection)) {
1131
1131
  addReason(codes, 'comparability-evidence-runtime-identity-opaque');
1132
1132
  }
1133
1133
  return codes;
@@ -1145,8 +1145,8 @@ function makeAssessmentSource(assessment) {
1145
1145
  planVerification: {
1146
1146
  assessmentComputationStatus: 'verified',
1147
1147
  policyDigest: assessment.policyDigest,
1148
- leftCandidateDigest: assessment.left.candidateDigest,
1149
- rightCandidateDigest: assessment.right.candidateDigest,
1148
+ leftRunIdentityDigest: assessment.left.runIdentityDigest,
1149
+ rightRunIdentityDigest: assessment.right.runIdentityDigest,
1150
1150
  },
1151
1151
  });
1152
1152
  comparabilityAssessmentSources.add(source);
@@ -1160,8 +1160,8 @@ export function assessComparability(policyInput, leftPlan, rightPlan, leftSource
1160
1160
  const normalizedLeftSource = validateSourcePrefix(leftPlan, policy.comparisonScope, leftSource);
1161
1161
  const normalizedRightSource = validateSourcePrefix(rightPlan, policy.comparisonScope, rightSource);
1162
1162
  const design = designReasons(policy, leftPlan, rightPlan);
1163
- const left = candidateIdentity(leftPlan, policy.comparisonScope, normalizedLeftSource, attestations);
1164
- const right = candidateIdentity(rightPlan, policy.comparisonScope, normalizedRightSource, attestations);
1163
+ const left = comparabilityRunIdentity(leftPlan, policy.comparisonScope, normalizedLeftSource, attestations);
1164
+ const right = comparabilityRunIdentity(rightPlan, policy.comparisonScope, normalizedRightSource, attestations);
1165
1165
  const codes = new Set(design.codes);
1166
1166
  for (const code of evidenceReasons(policy.comparisonScope, left, right, design.left, design.right))
1167
1167
  codes.add(code);