oh-my-knowledge 1.0.0-beta.2 → 1.0.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +2 -2
- package/dist/assets/agent-skills/omk/references/commands.md +17 -17
- package/dist/cli/commands/eval/index.js +17 -17
- package/dist/eval-core/contracts/comparability.d.ts +10 -10
- package/dist/eval-core/contracts/comparability.js +42 -42
- package/dist/eval-core/contracts/schemas/v2/comparability-assessment.schema.json +452 -0
- package/dist/eval-core/contracts/schemas/v2/series-analysis-bundle.schema.json +1299 -0
- package/dist/eval-core/contracts/series.d.ts +8 -8
- package/dist/eval-core/contracts/series.js +2 -2
- package/dist/eval-core/index.d.ts +1 -1
- package/dist/eval-core/schemas.js +2 -0
- package/dist/eval-runtime/adapters/executor-fn.js +4 -30
- package/dist/eval-runtime/adapters/invoke-contract.d.ts +57 -0
- package/dist/eval-runtime/adapters/invoke-contract.js +58 -0
- package/dist/eval-runtime/adapters/json-executor.d.ts +43 -0
- package/dist/eval-runtime/adapters/json-executor.js +142 -0
- package/dist/eval-runtime/advanced.d.ts +32 -0
- package/dist/eval-runtime/advanced.js +17 -0
- package/dist/eval-runtime/conformance/executor.d.ts +20 -9
- package/dist/eval-runtime/conformance/executor.js +166 -34
- package/dist/eval-runtime/contracts.d.ts +4 -0
- package/dist/eval-runtime/contracts.js +2 -0
- package/dist/eval-runtime/evaluate.d.ts +219 -0
- package/dist/eval-runtime/evaluate.js +649 -0
- package/dist/eval-runtime/index.d.ts +3 -25
- package/dist/eval-runtime/index.js +2 -13
- package/dist/eval-runtime/judges/rubric-contracts.d.ts +57 -0
- package/dist/eval-runtime/judges/rubric-contracts.js +74 -0
- package/dist/eval-runtime/judges/rubric-judge.d.ts +9 -53
- package/dist/eval-runtime/judges/rubric-judge.js +5 -73
- package/dist/eval-runtime/judges/rubric-kit.d.ts +51 -0
- package/dist/eval-runtime/judges/rubric-kit.js +139 -0
- package/dist/eval-runtime/runner.d.ts +34 -0
- package/dist/eval-runtime/runner.js +93 -0
- package/dist/eval-workflows/instruments/judge.d.ts +2 -2
- package/dist/eval-workflows/instruments/judge.js +2 -2
- package/dist/eval-workflows/runtime-adapter/adapters/claude/resources.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli-protocol.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/codex/resources.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk-protocol.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/shared/stateless-api-resources.d.ts +1 -1
- package/dist/index.d.ts +2 -4
- package/dist/index.js +2 -4
- package/dist/studio/core-runs/renderer.js +1 -1
- package/package.json +9 -1
package/README.md
CHANGED
|
@@ -228,7 +228,7 @@ The full docs are published at **[oh-my-knowledge.pages.dev](https://oh-my-knowl
|
|
|
228
228
|
- **[CLI reference](docs/reference/cli.md)** — all top-level commands with bash examples and flag tables
|
|
229
229
|
- **[Migrate to the 1.0 preview](docs/guides/v1-preview-migration.md)** — install channel, storage reset, sample protocol, CLI automation, and embedded API changes since 0.54
|
|
230
230
|
- **[Evaluation Core cutover](docs/guides/eval-core-cutover.md)** — `BREAKING-SCHEMA` storage, resume, Studio, Gold, managed-evidence, and evolve migration
|
|
231
|
-
- **[Embed OMK in a service](docs/guides/eval-runtime.md)** —
|
|
231
|
+
- **[Embed OMK in a service](docs/guides/eval-runtime.md)** — one `evaluate()` API for Node.js and FaaS hosts
|
|
232
232
|
- **[Storage layout v2](docs/specs/storage-layout-spec.md)** — project/global domains, compatibility boundary, and Git policy
|
|
233
233
|
- **[Executors](docs/reference/executors.md)** & **[artifact layout](docs/reference/artifact-layout.md)** — built-in / custom executors; how `variant` resolves to an artifact + runtime context
|
|
234
234
|
- **[How-to guides](docs/guides/agent-eval.md)** — [evaluate an agent](docs/guides/agent-eval.md) (project runtime context) and [use non-Claude models](docs/guides/non-claude-models.md) (GLM / Qwen / DeepSeek / Moonshot / Ollama)
|
package/README.zh.md
CHANGED
|
@@ -157,7 +157,7 @@ RAG 专项评测请看 RAGAS(独立 niche,跟 omk 互补)。完整对比
|
|
|
157
157
|
|
|
158
158
|
| 特性 | 说明 |
|
|
159
159
|
|------|------|
|
|
160
|
-
| **Core 发布决定** |
|
|
160
|
+
| **Core 发布决定** | 六种判定 + 稳定 reason code + exit code 路由;Studio 投影同一份经过认证的 Decision |
|
|
161
161
|
| **五层 evidence graph** | Assertion / LLM / Judge / Dimension / Composite 保持独立,coverage、成本、状态与 lineage 与分数正交 |
|
|
162
162
|
| **多执行器** | 支持 Claude CLI / Claude SDK / Codex CLI / Codex SDK / DeepSeek Harness / OpenAI / Anthropic API 及自定义命令 |
|
|
163
163
|
| **30+ 种断言** | 包含子串、正则、JSON Schema、ROUGE/BLEU/Levenshtein 相似度、Agent 工具调用、语义相似度、自定义函数等 |
|
|
@@ -228,7 +228,7 @@ omk-mcp
|
|
|
228
228
|
- **[CLI 参考](docs/zh/reference/cli.md)** —— 顶层命令的 bash 示例和 flag 表
|
|
229
229
|
- **[迁移到 1.0 预览版](docs/zh/guides/v1-preview-migration.md)** —— 从 `0.54` 升级时的安装渠道、存储重建、用例协议、CLI 自动化与嵌入式 API 变化
|
|
230
230
|
- **[Evaluation Core 生产切换](docs/zh/guides/eval-core-cutover.md)** —— `BREAKING-SCHEMA` 存储、resume、Studio、Gold、受管证据与 evolve 迁移
|
|
231
|
-
- **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS
|
|
231
|
+
- **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS 宿主的单一 `evaluate()` API
|
|
232
232
|
- **[存储布局 v2](docs/zh/specs/storage-layout-spec.md)** —— 项目/全局领域、迁移兼容与 Git 策略
|
|
233
233
|
- **[执行器](docs/zh/reference/executors.md)** & **[知识载体布局](docs/zh/reference/artifact-layout.md)** —— 内置 / 自定义执行器;variant 如何解析为 artifact + runtime context
|
|
234
234
|
- **[操作指南](docs/zh/guides/agent-eval.md)** —— [评测 agent](docs/zh/guides/agent-eval.md)(项目 runtime context)与[使用非 Claude 模型](docs/zh/guides/non-claude-models.md)(GLM / 通义 / DeepSeek / Moonshot / Ollama)
|
|
@@ -63,7 +63,7 @@ omk doctor --json --gate
|
|
|
63
63
|
|
|
64
64
|
## omk eval
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
跑评测:用一个对照组与多个实验组 skill 做对照试验,生成判定报告。
|
|
67
67
|
|
|
68
68
|
**用法:**
|
|
69
69
|
|
|
@@ -73,16 +73,16 @@ omk eval [flags]
|
|
|
73
73
|
|
|
74
74
|
**Flags:**
|
|
75
75
|
|
|
76
|
-
- `--batch` `boolean`:batch
|
|
76
|
+
- `--batch` `boolean`:batch 模式:baseline 作为对照组,逐个 skill 作为实验组
|
|
77
77
|
- `--bootstrap` `boolean`:加 bootstrap CI
|
|
78
78
|
- `--bootstrap-samples` `option`:bootstrap 重采样次数,默认 1000
|
|
79
|
-
- `--budget-per-sample-ms` `option
|
|
80
|
-
- `--budget-per-sample-usd` `option
|
|
79
|
+
- `--budget-per-sample-ms` `option`:单用例时长上限 ms(必须 > 0,不传则无上限)
|
|
80
|
+
- `--budget-per-sample-usd` `option`:单用例预算上限 USD(必须 > 0,不传则无上限)
|
|
81
81
|
- `--budget-usd` `option`:总预算上限 USD(必须 > 0,不传则无上限)
|
|
82
82
|
- `--concurrency` `option`:并发数,默认 1
|
|
83
83
|
- `--config` `option`:eval.yaml 路径
|
|
84
|
-
- `--control` `option
|
|
85
|
-
- `--control-cwd` `option
|
|
84
|
+
- `--control` `option`:对照组(control)的 variant 表达式(仅 artifact 身份)
|
|
85
|
+
- `--control-cwd` `option`:对照组(control)的 runtime context 目录
|
|
86
86
|
- `--dry-run` `boolean`:只 plan 不实跑
|
|
87
87
|
- `--effort` `option`:被测 LLM 扩展思考预算 low/medium/high/xhigh/max(默认 low;跨 effort 报告不严格可比)。
|
|
88
88
|
- `--executor` `option`:执行器:claude / claude-sdk / codex / codex-sdk / anthropic-api / openai-api / 自定义命令。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。
|
|
@@ -90,7 +90,7 @@ omk eval [flags]
|
|
|
90
90
|
- `--gold-dir` `option`:gold dataset 目录
|
|
91
91
|
- `--holdout-ratio` `option`:留出比例 0-1(如 0.3);切出 holdout 子集,对比 train/holdout 综合分检测过拟合
|
|
92
92
|
- `--judge-models` `option`:评委配置,格式 executor:model[,...],例 claude:haiku 或 codex:<model>(≥ 2 个 = ensemble)。默认跟随所选执行器;Codex 沿用被测模型。
|
|
93
|
-
- `--judge-repeat` `option
|
|
93
|
+
- `--judge-repeat` `option`:每个维度由评委评价 N 次
|
|
94
94
|
- `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
|
|
95
95
|
- `--layered-stats` `boolean`:输出分层统计
|
|
96
96
|
- `--mcp-config` `option`:MCP 配置文件路径
|
|
@@ -99,30 +99,30 @@ omk eval [flags]
|
|
|
99
99
|
- `--no-debias-length` `boolean`:关 length-debias(默认开)
|
|
100
100
|
- `--no-diagnostic` `boolean`:关闭基于 Core 失败、缺失、排除与稳定 reason code 的诊断投影。
|
|
101
101
|
- `--no-evidence` `boolean`:不把本次评测写成证据追加进受管记录(默认会为已 install 的 skill 自动写)。
|
|
102
|
-
- `--no-gate` `boolean
|
|
103
|
-
- `--no-judge` `boolean`:跳过 LLM
|
|
102
|
+
- `--no-gate` `boolean`:关闭判定门禁
|
|
103
|
+
- `--no-judge` `boolean`:跳过 LLM 评委
|
|
104
104
|
- `--no-serve` `boolean`:不启 report server
|
|
105
105
|
- `--no-strict-baseline` `boolean`:关闭 baseline 隔离
|
|
106
106
|
- `--output-dir` `option`:报告输出目录(默认项目级 .omk/eval)
|
|
107
|
-
- `--repeat` `option
|
|
108
|
-
- `--report-only` `boolean
|
|
107
|
+
- `--repeat` `option`:每个用例重复运行 N 次
|
|
108
|
+
- `--report-only` `boolean`:生成报告并打印判定,但始终 exit 0(不参与 CI gate)。
|
|
109
109
|
- `--resume` `option`:复用经过完整契约校验的 Core runId;拒绝时失败关闭
|
|
110
|
-
- `--retry` `option
|
|
111
|
-
- `--samples` `option
|
|
110
|
+
- `--retry` `option`:单用例失败重试次数
|
|
111
|
+
- `--samples` `option`:用例路径。自动发现项目级或单个实验组(treatment)目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。
|
|
112
112
|
- `--skill-dir` `option`:skill 目录,默认 skills
|
|
113
113
|
- `--skip-connectivity` `boolean`:跳 LLM 连通性预检
|
|
114
114
|
- `--skip-doctor` `boolean`:escape hatch:跳 doctor 健康检查门禁(默认强制启用)。沙箱 mock 提供依赖时绕开 doctor 物理路径误报;garbage-in 风险自负。
|
|
115
115
|
- `--strict-baseline` `boolean`:强制 baseline 隔离(default true)
|
|
116
|
-
- `--threshold` `option
|
|
116
|
+
- `--threshold` `option`:判定阈值,默认 3.5
|
|
117
117
|
- `--timeout` `option`:单用例超时秒,默认 120
|
|
118
|
-
- `--treatment` `option
|
|
119
|
-
- `--treatment-cwd` `option
|
|
118
|
+
- `--treatment` `option`:实验组(treatment)的 variant 列表,逗号分隔(仅 artifact 身份)
|
|
119
|
+
- `--treatment-cwd` `option`:实验组(treatment)的 runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)
|
|
120
120
|
- `--trivial-diff` `option`:可忽略 diff 容差,0 表示不启用容差
|
|
121
121
|
- `--verbose` `boolean`:详细日志
|
|
122
122
|
|
|
123
123
|
**示例:**
|
|
124
124
|
|
|
125
|
-
>
|
|
125
|
+
> 最简对照:baseline 作为对照组,my-skill 作为实验组
|
|
126
126
|
|
|
127
127
|
```bash
|
|
128
128
|
omk eval --control baseline --treatment my-skill
|
|
@@ -174,13 +174,13 @@ async function runEval(_args, flags, lang) {
|
|
|
174
174
|
}
|
|
175
175
|
export default class Eval extends BaseCommand {
|
|
176
176
|
static description = bilingual({
|
|
177
|
-
zh: '
|
|
177
|
+
zh: '跑评测:用一个对照组与多个实验组 skill 做对照试验,生成判定报告。',
|
|
178
178
|
en: 'Run evaluation: control vs treatment(s) comparison, produce verdict report.',
|
|
179
179
|
});
|
|
180
180
|
static examples = [
|
|
181
181
|
{
|
|
182
182
|
description: bilingual({
|
|
183
|
-
zh: '
|
|
183
|
+
zh: '最简对照:baseline 作为对照组,my-skill 作为实验组',
|
|
184
184
|
en: 'Minimal A/B: baseline vs my-skill',
|
|
185
185
|
}),
|
|
186
186
|
command: '<%= config.bin %> eval --control baseline --treatment my-skill',
|
|
@@ -197,23 +197,23 @@ export default class Eval extends BaseCommand {
|
|
|
197
197
|
lang: LANG_FLAG,
|
|
198
198
|
// ── 实验角色 ──
|
|
199
199
|
control: Flags.string({
|
|
200
|
-
description: bilingual({ zh: 'control variant 表达式(仅 artifact 身份)', en: 'Control variant expr (artifact identity only)' }),
|
|
200
|
+
description: bilingual({ zh: '对照组(control)的 variant 表达式(仅 artifact 身份)', en: 'Control variant expr (artifact identity only)' }),
|
|
201
201
|
}),
|
|
202
202
|
treatment: Flags.string({
|
|
203
203
|
description: bilingual({
|
|
204
|
-
zh: 'treatment variant 列表,逗号分隔(仅 artifact 身份)',
|
|
204
|
+
zh: '实验组(treatment)的 variant 列表,逗号分隔(仅 artifact 身份)',
|
|
205
205
|
en: 'Treatment variants, comma-separated (artifact identity only)',
|
|
206
206
|
}),
|
|
207
207
|
}),
|
|
208
208
|
'control-cwd': Flags.string({
|
|
209
209
|
description: bilingual({
|
|
210
|
-
zh: 'control
|
|
210
|
+
zh: '对照组(control)的 runtime context 目录',
|
|
211
211
|
en: 'Runtime context dir for control',
|
|
212
212
|
}),
|
|
213
213
|
}),
|
|
214
214
|
'treatment-cwd': Flags.string({
|
|
215
215
|
description: bilingual({
|
|
216
|
-
zh: 'treatment
|
|
216
|
+
zh: '实验组(treatment)的 runtime context 目录列表,逗号分隔、与 --treatment 按序对齐(空位 = 无 cwd)',
|
|
217
217
|
en: 'Runtime context dirs for treatments, comma-separated, index-aligned with --treatment (blank = none)',
|
|
218
218
|
}),
|
|
219
219
|
}),
|
|
@@ -222,7 +222,7 @@ export default class Eval extends BaseCommand {
|
|
|
222
222
|
}),
|
|
223
223
|
samples: Flags.string({
|
|
224
224
|
description: bilingual({
|
|
225
|
-
zh: '
|
|
225
|
+
zh: '用例路径。自动发现项目级或单个实验组(treatment)目录 skill 下的 eval-samples.json / eval-samples.yaml;显式路径可为 JSON / YAML 文件或分片目录。',
|
|
226
226
|
en: 'Samples path. Auto-discovers eval-samples.json / eval-samples.yaml at project scope or for a single directory-skill treatment; an explicit path may be a JSON / YAML file or split directory.',
|
|
227
227
|
}),
|
|
228
228
|
}),
|
|
@@ -256,7 +256,7 @@ export default class Eval extends BaseCommand {
|
|
|
256
256
|
}),
|
|
257
257
|
// ── 评测 toggle ──
|
|
258
258
|
'no-judge': Flags.boolean({
|
|
259
|
-
description: bilingual({ zh: '跳过 LLM
|
|
259
|
+
description: bilingual({ zh: '跳过 LLM 评委', en: 'Skip LLM judge' }),
|
|
260
260
|
}),
|
|
261
261
|
'no-cache': Flags.boolean({
|
|
262
262
|
description: bilingual({ zh: '跳过 executor cache', en: 'Skip executor cache' }),
|
|
@@ -277,7 +277,7 @@ export default class Eval extends BaseCommand {
|
|
|
277
277
|
}),
|
|
278
278
|
batch: Flags.boolean({
|
|
279
279
|
description: bilingual({
|
|
280
|
-
zh: 'batch
|
|
280
|
+
zh: 'batch 模式:baseline 作为对照组,逐个 skill 作为实验组',
|
|
281
281
|
en: 'Batch mode: baseline vs each skill',
|
|
282
282
|
}),
|
|
283
283
|
}),
|
|
@@ -300,7 +300,7 @@ export default class Eval extends BaseCommand {
|
|
|
300
300
|
description: bilingual({ zh: '详细日志', en: 'Verbose logging' }),
|
|
301
301
|
}),
|
|
302
302
|
retry: Flags.string({
|
|
303
|
-
description: bilingual({ zh: '
|
|
303
|
+
description: bilingual({ zh: '单用例失败重试次数', en: 'Per-sample retry count' }),
|
|
304
304
|
parse: integerStringParser('--retry', { min: 0 }),
|
|
305
305
|
}),
|
|
306
306
|
resume: Flags.string({
|
|
@@ -333,7 +333,7 @@ export default class Eval extends BaseCommand {
|
|
|
333
333
|
}),
|
|
334
334
|
// ── eval-runner extra ──
|
|
335
335
|
repeat: Flags.string({
|
|
336
|
-
description: bilingual({ zh: '
|
|
336
|
+
description: bilingual({ zh: '每个用例重复运行 N 次', en: 'Repeat each sample N times' }),
|
|
337
337
|
parse: integerStringParser('--repeat', { min: 1 }),
|
|
338
338
|
}),
|
|
339
339
|
'holdout-ratio': Flags.string({
|
|
@@ -344,7 +344,7 @@ export default class Eval extends BaseCommand {
|
|
|
344
344
|
parse: numberStringParser('--holdout-ratio', { min: 0, max: 1 }),
|
|
345
345
|
}),
|
|
346
346
|
'judge-repeat': Flags.string({
|
|
347
|
-
description: bilingual({ zh: '
|
|
347
|
+
description: bilingual({ zh: '每个维度由评委评价 N 次', en: 'Judge each dim N times' }),
|
|
348
348
|
parse: integerStringParser('--judge-repeat', { min: 1 }),
|
|
349
349
|
}),
|
|
350
350
|
bootstrap: Flags.boolean({
|
|
@@ -365,16 +365,16 @@ export default class Eval extends BaseCommand {
|
|
|
365
365
|
parse: numberStringParser('--budget-usd', { minExclusive: 0 }),
|
|
366
366
|
}),
|
|
367
367
|
'budget-per-sample-usd': Flags.string({
|
|
368
|
-
description: bilingual({ zh: '
|
|
368
|
+
description: bilingual({ zh: '单用例预算上限 USD(必须 > 0,不传则无上限)', en: 'Per-sample budget cap USD (must be > 0; omit for no cap)' }),
|
|
369
369
|
parse: numberStringParser('--budget-per-sample-usd', { minExclusive: 0 }),
|
|
370
370
|
}),
|
|
371
371
|
'budget-per-sample-ms': Flags.string({
|
|
372
|
-
description: bilingual({ zh: '
|
|
372
|
+
description: bilingual({ zh: '单用例时长上限 ms(必须 > 0,不传则无上限)', en: 'Per-sample time cap ms (must be > 0; omit for no cap)' }),
|
|
373
373
|
parse: integerStringParser('--budget-per-sample-ms', { min: 1 }),
|
|
374
374
|
}),
|
|
375
375
|
threshold: Flags.string({
|
|
376
376
|
description: bilingual({
|
|
377
|
-
zh:
|
|
377
|
+
zh: `判定阈值,默认 ${DEFAULT_GATE_THRESHOLD}`,
|
|
378
378
|
en: `Verdict threshold, default ${DEFAULT_GATE_THRESHOLD}`,
|
|
379
379
|
}),
|
|
380
380
|
parse: numberStringParser('--threshold'),
|
|
@@ -385,12 +385,12 @@ export default class Eval extends BaseCommand {
|
|
|
385
385
|
}),
|
|
386
386
|
'report-only': Flags.boolean({
|
|
387
387
|
description: bilingual({
|
|
388
|
-
zh: '
|
|
388
|
+
zh: '生成报告并打印判定,但始终 exit 0(不参与 CI gate)。',
|
|
389
389
|
en: 'Produce the report and print verdict, but always exit 0 (no CI gate).',
|
|
390
390
|
}),
|
|
391
391
|
}),
|
|
392
392
|
'no-gate': Flags.boolean({
|
|
393
|
-
description: bilingual({ zh: '
|
|
393
|
+
description: bilingual({ zh: '关闭判定门禁', en: 'Disable verdict gate' }),
|
|
394
394
|
}),
|
|
395
395
|
'no-evidence': Flags.boolean({
|
|
396
396
|
description: bilingual({
|
|
@@ -7,7 +7,7 @@ import { type Sha256Digest } from './json.js';
|
|
|
7
7
|
import { type SealedRunPlan } from '../internal/sealed-run-plan.js';
|
|
8
8
|
export type ComparabilityRunPlan = SealedRunPlan;
|
|
9
9
|
export declare const COMPARABILITY_POLICY_SCHEMA_VERSION: "omk.comparability-policy/v1";
|
|
10
|
-
export declare const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION: "omk.comparability-assessment/
|
|
10
|
+
export declare const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION: "omk.comparability-assessment/v2";
|
|
11
11
|
export declare const ComparisonScopeSchema: z.ZodEnum<{
|
|
12
12
|
evaluation: "evaluation";
|
|
13
13
|
analysis: "analysis";
|
|
@@ -123,7 +123,7 @@ export declare const ComparabilityArtifactIdentitySchema: z.ZodObject<{
|
|
|
123
123
|
}>;
|
|
124
124
|
artifactDigest: z.ZodString;
|
|
125
125
|
}, z.core.$strict>;
|
|
126
|
-
export declare const
|
|
126
|
+
export declare const ComparabilityRunIdentitySchema: z.ZodObject<{
|
|
127
127
|
runContractDigest: z.ZodString;
|
|
128
128
|
planDigests: z.ZodObject<{
|
|
129
129
|
datasetRevisionDigest: z.ZodString;
|
|
@@ -222,7 +222,7 @@ export declare const ComparabilityCandidateIdentitySchema: z.ZodObject<{
|
|
|
222
222
|
}>;
|
|
223
223
|
verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
|
|
224
224
|
}, z.core.$strict>>;
|
|
225
|
-
|
|
225
|
+
runIdentityDigest: z.ZodString;
|
|
226
226
|
}, z.core.$strict>;
|
|
227
227
|
export declare const COMPARABILITY_REASON_CODES: readonly ["comparability-identity-declared-subject-change", "comparability-design-subject-mapping-invalid", "comparability-design-undeclared-subject-change", "comparability-design-evaluation-input-mismatch", "comparability-design-evaluation-instrument-mismatch", "comparability-design-sampling-mismatch", "comparability-design-randomization-mismatch", "comparability-design-analysis-mismatch", "comparability-design-comparison-mismatch", "comparability-design-decision-mismatch", "comparability-design-schema-mismatch", "comparability-design-projection-mismatch", "comparability-evidence-source-absent", "comparability-evidence-verification-indeterminate", "comparability-evidence-assurance-unverified", "comparability-evidence-source-untrusted", "comparability-evidence-runtime-identity-opaque"];
|
|
228
228
|
export declare const ComparabilityReasonCodeSchema: z.ZodEnum<{
|
|
@@ -282,7 +282,7 @@ export declare const ComparabilityReasonSchema: z.ZodObject<{
|
|
|
282
282
|
}, z.core.$strict>;
|
|
283
283
|
export type ComparabilityReasonCode = z.infer<typeof ComparabilityReasonCodeSchema>;
|
|
284
284
|
export declare const ComparabilityAssessmentSchema: z.ZodObject<{
|
|
285
|
-
schemaVersion: z.ZodLiteral<"omk.comparability-assessment/
|
|
285
|
+
schemaVersion: z.ZodLiteral<"omk.comparability-assessment/v2">;
|
|
286
286
|
policyDigest: z.ZodString;
|
|
287
287
|
designMode: z.ZodLiteral<"exact-measurement-design">;
|
|
288
288
|
comparisonScope: z.ZodEnum<{
|
|
@@ -389,7 +389,7 @@ export declare const ComparabilityAssessmentSchema: z.ZodObject<{
|
|
|
389
389
|
}>;
|
|
390
390
|
verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
|
|
391
391
|
}, z.core.$strict>>;
|
|
392
|
-
|
|
392
|
+
runIdentityDigest: z.ZodString;
|
|
393
393
|
}, z.core.$strict>;
|
|
394
394
|
right: z.ZodObject<{
|
|
395
395
|
runContractDigest: z.ZodString;
|
|
@@ -490,7 +490,7 @@ export declare const ComparabilityAssessmentSchema: z.ZodObject<{
|
|
|
490
490
|
}>;
|
|
491
491
|
verifiedByAttestationDigest: z.ZodOptional<z.ZodString>;
|
|
492
492
|
}, z.core.$strict>>;
|
|
493
|
-
|
|
493
|
+
runIdentityDigest: z.ZodString;
|
|
494
494
|
}, z.core.$strict>;
|
|
495
495
|
designStatus: z.ZodEnum<{
|
|
496
496
|
incompatible: "incompatible";
|
|
@@ -550,10 +550,10 @@ export type ComparabilitySubject = z.infer<typeof ComparabilitySubjectSchema>;
|
|
|
550
550
|
export type ComparabilityPolicy = z.infer<typeof ComparabilityPolicySchema>;
|
|
551
551
|
export type ComparabilitySourceVerificationFact = z.infer<typeof ComparabilitySourceVerificationFactSchema>;
|
|
552
552
|
export type RuntimeQualificationFact = z.infer<typeof RuntimeQualificationFactSchema>;
|
|
553
|
-
export type
|
|
553
|
+
export type ComparabilityRunIdentity = z.infer<typeof ComparabilityRunIdentitySchema>;
|
|
554
554
|
export type ComparabilityReason = z.infer<typeof ComparabilityReasonSchema>;
|
|
555
555
|
export type ComparabilityAssessment = z.infer<typeof ComparabilityAssessmentSchema>;
|
|
556
|
-
export type ComparabilityValidationErrorCode = 'COMPARABILITY_POLICY_DIGEST_MISMATCH' | '
|
|
556
|
+
export type ComparabilityValidationErrorCode = 'COMPARABILITY_POLICY_DIGEST_MISMATCH' | 'COMPARABILITY_RUN_IDENTITY_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_DIGEST_MISMATCH' | 'COMPARABILITY_ASSESSMENT_RECOMPUTATION_MISMATCH' | 'COMPARABILITY_SOURCE_PREFIX_INVALID' | 'COMPARABILITY_DECISION_SOURCE_PLAN_MISMATCH' | 'COMPARABILITY_VERIFICATION_CONTEXT_INVALID';
|
|
557
557
|
export declare class ComparabilityValidationError extends TypeError {
|
|
558
558
|
readonly code: ComparabilityValidationErrorCode;
|
|
559
559
|
constructor(code: ComparabilityValidationErrorCode, message: string);
|
|
@@ -583,8 +583,8 @@ export interface ComparabilityVerificationContext {
|
|
|
583
583
|
export interface ComparabilityAssessmentPlanVerification {
|
|
584
584
|
readonly assessmentComputationStatus: 'verified';
|
|
585
585
|
readonly policyDigest: Sha256Digest;
|
|
586
|
-
readonly
|
|
587
|
-
readonly
|
|
586
|
+
readonly leftRunIdentityDigest: Sha256Digest;
|
|
587
|
+
readonly rightRunIdentityDigest: Sha256Digest;
|
|
588
588
|
}
|
|
589
589
|
export interface ComparabilityAssessmentSource {
|
|
590
590
|
readonly assessment: ComparabilityAssessment;
|
|
@@ -12,7 +12,7 @@ import { canonicalizeJson, deepFreezeCanonicalJson, digestCanonicalJson, parseWi
|
|
|
12
12
|
import { ANALYSIS_PLAN_SCHEMA_VERSION, DECISION_PLAN_SCHEMA_VERSION, EVALUATION_PLAN_SCHEMA_VERSION, EXECUTION_PLAN_SCHEMA_VERSION, PlanDigestsSchema, RUN_PLAN_SCHEMA_VERSION, } from './plans.js';
|
|
13
13
|
import { assertSealedRunPlan, } from '../internal/sealed-run-plan.js';
|
|
14
14
|
export const COMPARABILITY_POLICY_SCHEMA_VERSION = 'omk.comparability-policy/v1';
|
|
15
|
-
export const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION = 'omk.comparability-assessment/
|
|
15
|
+
export const COMPARABILITY_ASSESSMENT_SCHEMA_VERSION = 'omk.comparability-assessment/v2';
|
|
16
16
|
export const ComparisonScopeSchema = z.enum(['evaluation', 'analysis', 'decision']);
|
|
17
17
|
export const ComparabilityStageSchema = z.enum([
|
|
18
18
|
'execution',
|
|
@@ -224,41 +224,41 @@ function compareRuntimeQualifications(left, right) {
|
|
|
224
224
|
function runtimeQualificationKey(fact) {
|
|
225
225
|
return `${fact.stage}\u0000${fact.runtimeKind}\u0000${fact.referenceId}`;
|
|
226
226
|
}
|
|
227
|
-
export const
|
|
227
|
+
export const ComparabilityRunIdentitySchema = z.object({
|
|
228
228
|
runContractDigest: Sha256DigestSchema,
|
|
229
229
|
planDigests: PlanDigestsSchema,
|
|
230
230
|
randomizationDesignDigest: Sha256DigestSchema,
|
|
231
231
|
artifacts: z.array(ComparabilityArtifactIdentitySchema),
|
|
232
232
|
sourceVerification: z.array(ComparabilitySourceVerificationFactSchema),
|
|
233
233
|
runtimeQualification: z.array(RuntimeQualificationFactSchema),
|
|
234
|
-
|
|
235
|
-
}).strict().superRefine((
|
|
236
|
-
if (!isCanonicalArray(
|
|
237
|
-
|| new Set(
|
|
238
|
-
!==
|
|
234
|
+
runIdentityDigest: Sha256DigestSchema,
|
|
235
|
+
}).strict().superRefine((runIdentity, context) => {
|
|
236
|
+
if (!isCanonicalArray(runIdentity.artifacts, compareArtifacts)
|
|
237
|
+
|| new Set(runIdentity.artifacts.map((artifact) => artifact.stage)).size
|
|
238
|
+
!== runIdentity.artifacts.length) {
|
|
239
239
|
context.addIssue({ code: 'custom', path: ['artifacts'], message: 'Artifacts must be canonical.' });
|
|
240
240
|
}
|
|
241
|
-
if (
|
|
242
|
-
||
|
|
243
|
-
!==
|
|
241
|
+
if (runIdentity.runContractDigest !== runIdentity.planDigests.runContractDigest
|
|
242
|
+
|| runIdentity.randomizationDesignDigest
|
|
243
|
+
!== runIdentity.planDigests.randomizationDesignDigest) {
|
|
244
244
|
context.addIssue({
|
|
245
245
|
code: 'custom',
|
|
246
246
|
path: ['planDigests'],
|
|
247
|
-
message: '
|
|
247
|
+
message: 'Run identity digests must equal their Plan digest entries.',
|
|
248
248
|
});
|
|
249
249
|
}
|
|
250
|
-
if (!isCanonicalArray(
|
|
251
|
-
|| new Set(
|
|
252
|
-
!==
|
|
250
|
+
if (!isCanonicalArray(runIdentity.sourceVerification, compareSourceFacts)
|
|
251
|
+
|| new Set(runIdentity.sourceVerification.map(sourceFactKey)).size
|
|
252
|
+
!== runIdentity.sourceVerification.length) {
|
|
253
253
|
context.addIssue({
|
|
254
254
|
code: 'custom',
|
|
255
255
|
path: ['sourceVerification'],
|
|
256
256
|
message: 'Source verification facts must be unique and canonical.',
|
|
257
257
|
});
|
|
258
258
|
}
|
|
259
|
-
if (!isCanonicalArray(
|
|
260
|
-
|| new Set(
|
|
261
|
-
!==
|
|
259
|
+
if (!isCanonicalArray(runIdentity.runtimeQualification, compareRuntimeQualifications)
|
|
260
|
+
|| new Set(runIdentity.runtimeQualification.map(runtimeQualificationKey)).size
|
|
261
|
+
!== runIdentity.runtimeQualification.length) {
|
|
262
262
|
context.addIssue({
|
|
263
263
|
code: 'custom',
|
|
264
264
|
path: ['runtimeQualification'],
|
|
@@ -318,8 +318,8 @@ export const ComparabilityAssessmentSchema = z.object({
|
|
|
318
318
|
policyDigest: Sha256DigestSchema,
|
|
319
319
|
designMode: z.literal('exact-measurement-design'),
|
|
320
320
|
comparisonScope: ComparisonScopeSchema,
|
|
321
|
-
left:
|
|
322
|
-
right:
|
|
321
|
+
left: ComparabilityRunIdentitySchema,
|
|
322
|
+
right: ComparabilityRunIdentitySchema,
|
|
323
323
|
designStatus: z.enum(['compatible', 'incompatible']),
|
|
324
324
|
evidenceQualificationStatus: z.enum(['verified', 'conditional', 'rejected']),
|
|
325
325
|
comparabilityStatus: z.enum(['compatible', 'conditional', 'incompatible']),
|
|
@@ -398,13 +398,13 @@ export function parseComparabilityPolicyDocument(value) {
|
|
|
398
398
|
}
|
|
399
399
|
return policy;
|
|
400
400
|
}
|
|
401
|
-
function
|
|
402
|
-
return digestCanonicalJson(
|
|
401
|
+
function computeRunIdentityDigest(runIdentity) {
|
|
402
|
+
return digestCanonicalJson(runIdentity);
|
|
403
403
|
}
|
|
404
|
-
function
|
|
405
|
-
const {
|
|
406
|
-
if (
|
|
407
|
-
throw new ComparabilityValidationError('
|
|
404
|
+
function assertRunIdentityDigest(runIdentity) {
|
|
405
|
+
const { runIdentityDigest, ...payload } = runIdentity;
|
|
406
|
+
if (computeRunIdentityDigest(payload) !== runIdentityDigest) {
|
|
407
|
+
throw new ComparabilityValidationError('COMPARABILITY_RUN_IDENTITY_DIGEST_MISMATCH', 'Comparability Run identity digest does not match its canonical payload.');
|
|
408
408
|
}
|
|
409
409
|
}
|
|
410
410
|
function computeAssessmentDigest(assessment) {
|
|
@@ -412,8 +412,8 @@ function computeAssessmentDigest(assessment) {
|
|
|
412
412
|
}
|
|
413
413
|
export function parseComparabilityAssessmentDocument(value) {
|
|
414
414
|
const assessment = parseWireDocument(ComparabilityAssessmentSchema, value);
|
|
415
|
-
|
|
416
|
-
|
|
415
|
+
assertRunIdentityDigest(assessment.left);
|
|
416
|
+
assertRunIdentityDigest(assessment.right);
|
|
417
417
|
const { assessmentDigest, ...payload } = assessment;
|
|
418
418
|
if (computeAssessmentDigest(payload) !== assessmentDigest) {
|
|
419
419
|
throw new ComparabilityValidationError('COMPARABILITY_ASSESSMENT_DIGEST_MISMATCH', 'ComparabilityAssessment digest does not match its canonical payload.');
|
|
@@ -1081,7 +1081,7 @@ function requiredStages(scope) {
|
|
|
1081
1081
|
return ['execution', 'evaluation', 'analysis'];
|
|
1082
1082
|
return ['execution', 'evaluation', 'analysis', 'decision'];
|
|
1083
1083
|
}
|
|
1084
|
-
function
|
|
1084
|
+
function comparabilityRunIdentity(plan, scope, source, attestations) {
|
|
1085
1085
|
const artifacts = requiredStages(scope)
|
|
1086
1086
|
.filter((stage) => source[stage] !== undefined)
|
|
1087
1087
|
.map((stage) => ({ stage, artifactDigest: sourceDigest(source, stage) }))
|
|
@@ -1096,18 +1096,18 @@ function candidateIdentity(plan, scope, source, attestations) {
|
|
|
1096
1096
|
};
|
|
1097
1097
|
return {
|
|
1098
1098
|
...payload,
|
|
1099
|
-
|
|
1099
|
+
runIdentityDigest: computeRunIdentityDigest(payload),
|
|
1100
1100
|
};
|
|
1101
1101
|
}
|
|
1102
|
-
function evidenceReasons(scope,
|
|
1102
|
+
function evidenceReasons(scope, leftRunIdentity, rightRunIdentity, leftProjection, rightProjection) {
|
|
1103
1103
|
const codes = new Set();
|
|
1104
1104
|
const required = requiredStages(scope);
|
|
1105
|
-
const leftStages = new Set(
|
|
1106
|
-
const rightStages = new Set(
|
|
1105
|
+
const leftStages = new Set(leftRunIdentity.artifacts.map((artifact) => artifact.stage));
|
|
1106
|
+
const rightStages = new Set(rightRunIdentity.artifacts.map((artifact) => artifact.stage));
|
|
1107
1107
|
if (required.some((stage) => !leftStages.has(stage) || !rightStages.has(stage))) {
|
|
1108
1108
|
addReason(codes, 'comparability-evidence-source-absent');
|
|
1109
1109
|
}
|
|
1110
|
-
const facts = [...
|
|
1110
|
+
const facts = [...leftRunIdentity.sourceVerification, ...rightRunIdentity.sourceVerification];
|
|
1111
1111
|
if (facts.some((fact) => fact.verificationFactKind === 'verification-axis'
|
|
1112
1112
|
&& fact.verificationStatus === 'indeterminate')) {
|
|
1113
1113
|
addReason(codes, 'comparability-evidence-verification-indeterminate');
|
|
@@ -1117,17 +1117,17 @@ function evidenceReasons(scope, leftCandidate, rightCandidate, leftProjection, r
|
|
|
1117
1117
|
addReason(codes, 'comparability-evidence-source-untrusted');
|
|
1118
1118
|
}
|
|
1119
1119
|
const runtimes = [
|
|
1120
|
-
...
|
|
1121
|
-
...
|
|
1120
|
+
...leftRunIdentity.runtimeQualification,
|
|
1121
|
+
...rightRunIdentity.runtimeQualification,
|
|
1122
1122
|
];
|
|
1123
1123
|
if (trustFacts.some((fact) => fact.trust === 'declared' || fact.trust === 'unknown')
|
|
1124
1124
|
|| runtimes.some((runtime) => runtime.effectiveAssuranceLevel !== 'verified')) {
|
|
1125
1125
|
addReason(codes, 'comparability-evidence-assurance-unverified');
|
|
1126
1126
|
}
|
|
1127
|
-
const opaqueInvariantRuntime = (
|
|
1127
|
+
const opaqueInvariantRuntime = (runIdentity, projection) => runIdentity.runtimeQualification.some((runtime) => (runtime.fingerprintBasis === 'opaque'
|
|
1128
1128
|
&& !(runtime.runtimeKind === 'executor' && projection.mappedTargetIds.has(runtime.referenceId))));
|
|
1129
|
-
if (opaqueInvariantRuntime(
|
|
1130
|
-
|| opaqueInvariantRuntime(
|
|
1129
|
+
if (opaqueInvariantRuntime(leftRunIdentity, leftProjection)
|
|
1130
|
+
|| opaqueInvariantRuntime(rightRunIdentity, rightProjection)) {
|
|
1131
1131
|
addReason(codes, 'comparability-evidence-runtime-identity-opaque');
|
|
1132
1132
|
}
|
|
1133
1133
|
return codes;
|
|
@@ -1145,8 +1145,8 @@ function makeAssessmentSource(assessment) {
|
|
|
1145
1145
|
planVerification: {
|
|
1146
1146
|
assessmentComputationStatus: 'verified',
|
|
1147
1147
|
policyDigest: assessment.policyDigest,
|
|
1148
|
-
|
|
1149
|
-
|
|
1148
|
+
leftRunIdentityDigest: assessment.left.runIdentityDigest,
|
|
1149
|
+
rightRunIdentityDigest: assessment.right.runIdentityDigest,
|
|
1150
1150
|
},
|
|
1151
1151
|
});
|
|
1152
1152
|
comparabilityAssessmentSources.add(source);
|
|
@@ -1160,8 +1160,8 @@ export function assessComparability(policyInput, leftPlan, rightPlan, leftSource
|
|
|
1160
1160
|
const normalizedLeftSource = validateSourcePrefix(leftPlan, policy.comparisonScope, leftSource);
|
|
1161
1161
|
const normalizedRightSource = validateSourcePrefix(rightPlan, policy.comparisonScope, rightSource);
|
|
1162
1162
|
const design = designReasons(policy, leftPlan, rightPlan);
|
|
1163
|
-
const left =
|
|
1164
|
-
const right =
|
|
1163
|
+
const left = comparabilityRunIdentity(leftPlan, policy.comparisonScope, normalizedLeftSource, attestations);
|
|
1164
|
+
const right = comparabilityRunIdentity(rightPlan, policy.comparisonScope, normalizedRightSource, attestations);
|
|
1165
1165
|
const codes = new Set(design.codes);
|
|
1166
1166
|
for (const code of evidenceReasons(policy.comparisonScope, left, right, design.left, design.right))
|
|
1167
1167
|
codes.add(code);
|