oh-my-knowledge 1.0.0-beta.0 → 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/README.zh.md +1 -0
- package/dist/dsh-plugin/core-adapter.js +1 -1
- package/dist/eval-core/analysis/runtime.js +30 -3
- package/dist/eval-core/analysis/types.d.ts +4 -0
- package/dist/eval-core/analysis/types.js +2 -0
- package/dist/eval-core/contracts/analysis-bundle.js +25 -1
- package/dist/eval-core/contracts/artifacts.d.ts +65 -4
- package/dist/eval-core/contracts/artifacts.js +6 -4
- package/dist/eval-core/contracts/json-schema.d.ts +2 -0
- package/dist/eval-core/contracts/json-schema.js +11 -4
- package/dist/eval-core/contracts/schemas/v1/analysis-bundle.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/analysis-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/budget-summary.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/comparability-assessment.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/comparability-policy.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/decision-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-bundle.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-definition.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-event.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-report.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-definition.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/evaluation-series-report.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/execution-bundle.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/execution-facts.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/execution-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/executor-capabilities.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/measurement-policy.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/run-plan.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v1/series-analysis-bundle.schema.json +1 -1
- package/dist/eval-core/contracts/schemas/v2/analysis-bundle.schema.json +1022 -0
- package/dist/eval-core/contracts/schemas/v2/evaluation-report.schema.json +1041 -0
- package/dist/eval-core/schema-url.d.ts +2 -2
- package/dist/eval-core/schema-url.js +3 -3
- package/dist/eval-core/schemas.d.ts +1 -0
- package/dist/eval-core/schemas.js +6 -0
- package/dist/eval-runtime/adapters/executor-fn.d.ts +21 -0
- package/dist/eval-runtime/adapters/executor-fn.js +100 -0
- package/dist/{eval-workflows/runtime-adapter/adapters/shared → eval-runtime/adapters}/same-process.d.ts +27 -24
- package/dist/{eval-workflows/runtime-adapter/adapters/shared → eval-runtime/adapters}/same-process.js +1 -1
- package/dist/eval-runtime/builders/exact-match.d.ts +31 -0
- package/dist/eval-runtime/builders/exact-match.js +48 -0
- package/dist/eval-runtime/builders/paired-comparison.d.ts +42 -0
- package/dist/eval-runtime/builders/paired-comparison.js +123 -0
- package/dist/eval-runtime/builders/policy.d.ts +11 -0
- package/dist/eval-runtime/builders/policy.js +57 -0
- package/dist/eval-runtime/clock.d.ts +3 -0
- package/dist/eval-runtime/clock.js +25 -0
- package/dist/eval-runtime/conformance/executor.d.ts +35 -0
- package/dist/eval-runtime/conformance/executor.js +99 -0
- package/dist/eval-runtime/engine.d.ts +3 -0
- package/dist/eval-runtime/engine.js +3 -0
- package/dist/eval-runtime/evaluators/exact-match.d.ts +16 -0
- package/dist/eval-runtime/evaluators/exact-match.js +70 -0
- package/dist/eval-runtime/identity.d.ts +56 -0
- package/dist/eval-runtime/identity.js +88 -0
- package/dist/eval-runtime/index.d.ts +25 -0
- package/dist/eval-runtime/index.js +13 -0
- package/dist/eval-runtime/judges/invocation.d.ts +43 -0
- package/dist/eval-runtime/judges/invocation.js +72 -0
- package/dist/eval-runtime/judges/rubric-judge.d.ts +127 -0
- package/dist/eval-runtime/judges/rubric-judge.js +540 -0
- package/dist/eval-runtime/judges/rubric-prompt.d.ts +5 -0
- package/dist/eval-runtime/judges/rubric-prompt.js +59 -0
- package/dist/eval-runtime/judges/trace-summary.d.ts +8 -0
- package/dist/eval-runtime/judges/trace-summary.js +98 -0
- package/dist/eval-runtime/runtime.d.ts +35 -0
- package/dist/eval-runtime/runtime.js +111 -0
- package/dist/eval-runtime/traces/source-neutral.d.ts +57 -0
- package/dist/eval-runtime/traces/source-neutral.js +216 -0
- package/dist/eval-workflows/artifact-store/contracts.d.ts +4 -4
- package/dist/eval-workflows/instruments/judge-trace.d.ts +1 -8
- package/dist/eval-workflows/instruments/judge-trace.js +1 -94
- package/dist/eval-workflows/instruments/prompts/debias-instructions.d.ts +1 -2
- package/dist/eval-workflows/instruments/prompts/debias-instructions.js +1 -16
- package/dist/eval-workflows/instruments/prompts/judge-prompts.d.ts +11 -8
- package/dist/eval-workflows/instruments/prompts/judge-prompts.js +4 -54
- package/dist/eval-workflows/production-host/node-cli-evaluation-resolver.js +41 -3
- package/dist/eval-workflows/production-host/runtime-registry.d.ts +1 -2
- package/dist/eval-workflows/production-host/runtime-registry.js +2 -24
- package/dist/eval-workflows/runtime-adapter/adapters/anthropic/api.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/claude/cli.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/claude/sdk.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/codex/cli.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/codex/sdk.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/custom/command.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/index.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/index.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/openai/api.js +1 -1
- package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.d.ts +15 -0
- package/dist/eval-workflows/runtime-adapter/adapters/shared/omk-resource-same-process.js +7 -0
- package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-node.js +6 -1
- package/dist/eval-workflows/runtime-adapter/builtins.js +1 -1
- package/dist/eval-workflows/runtime-adapter/evaluators/execution-assertions.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-assertions.d.ts +2 -2
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-assertions.js +1 -1
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-judge-invocation.d.ts +2 -43
- package/dist/eval-workflows/runtime-adapter/evaluators/llm-judge-invocation.js +1 -72
- package/dist/eval-workflows/runtime-adapter/evaluators/output-assertions.d.ts +1 -1
- package/dist/eval-workflows/runtime-adapter/evaluators/rubric-judge.d.ts +4 -71
- package/dist/eval-workflows/runtime-adapter/evaluators/rubric-judge.js +19 -404
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.d.ts +1 -57
- package/dist/eval-workflows/runtime-adapter/source-neutral-trace.js +1 -66
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -228,6 +228,7 @@ The full docs are published at **[oh-my-knowledge.pages.dev](https://oh-my-knowl
|
|
|
228
228
|
- **[CLI reference](docs/reference/cli.md)** — all top-level commands with bash examples and flag tables
|
|
229
229
|
- **[Migrate to the 1.0 preview](docs/guides/v1-preview-migration.md)** — install channel, storage reset, sample protocol, CLI automation, and embedded API changes since 0.54
|
|
230
230
|
- **[Evaluation Core cutover](docs/guides/eval-core-cutover.md)** — `BREAKING-SCHEMA` storage, resume, Studio, Gold, managed-evidence, and evolve migration
|
|
231
|
+
- **[Embed OMK in a service](docs/guides/eval-runtime.md)** — lightweight `eval-runtime` assembly for Node.js and FaaS hosts
|
|
231
232
|
- **[Storage layout v2](docs/specs/storage-layout-spec.md)** — project/global domains, compatibility boundary, and Git policy
|
|
232
233
|
- **[Executors](docs/reference/executors.md)** & **[artifact layout](docs/reference/artifact-layout.md)** — built-in / custom executors; how `variant` resolves to an artifact + runtime context
|
|
233
234
|
- **[How-to guides](docs/guides/agent-eval.md)** — [evaluate an agent](docs/guides/agent-eval.md) (project runtime context) and [use non-Claude models](docs/guides/non-claude-models.md) (GLM / Qwen / DeepSeek / Moonshot / Ollama)
|
package/README.zh.md
CHANGED
|
@@ -228,6 +228,7 @@ omk-mcp
|
|
|
228
228
|
- **[CLI 参考](docs/zh/reference/cli.md)** —— 顶层命令的 bash 示例和 flag 表
|
|
229
229
|
- **[迁移到 1.0 预览版](docs/zh/guides/v1-preview-migration.md)** —— 从 `0.54` 升级时的安装渠道、存储重建、用例协议、CLI 自动化与嵌入式 API 变化
|
|
230
230
|
- **[Evaluation Core 生产切换](docs/zh/guides/eval-core-cutover.md)** —— `BREAKING-SCHEMA` 存储、resume、Studio、Gold、受管证据与 evolve 迁移
|
|
231
|
+
- **[在服务中嵌入 OMK](docs/zh/guides/eval-runtime.md)** —— 面向 Node.js 与 FaaS 宿主的轻量 `eval-runtime` 装配
|
|
231
232
|
- **[存储布局 v2](docs/zh/specs/storage-layout-spec.md)** —— 项目/全局领域、迁移兼容与 Git 策略
|
|
232
233
|
- **[执行器](docs/zh/reference/executors.md)** & **[知识载体布局](docs/zh/reference/artifact-layout.md)** —— 内置 / 自定义执行器;variant 如何解析为 artifact + runtime context
|
|
233
234
|
- **[操作指南](docs/zh/guides/agent-eval.md)** —— [评测 agent](docs/zh/guides/agent-eval.md)(项目 runtime context)与[使用非 Claude 模型](docs/zh/guides/non-claude-models.md)(GLM / 通义 / DeepSeek / Moonshot / Ollama)
|
|
@@ -3,7 +3,7 @@ import { JsonValueSchema, RuntimeIdentitySchema, canonicalizeJson, deepFreezeCan
|
|
|
3
3
|
import { ExecutionPortFailure, } from '../eval-core/execution/index.js';
|
|
4
4
|
import { createDshHostRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
|
|
5
5
|
import { captureClaudeCliRunState, captureClaudeCliTarget, disposeClaudeCliTrial, openClaudeCliTrial, } from '../eval-workflows/runtime-adapter/adapters/claude/resources.js';
|
|
6
|
-
import { createSameProcessExecutorAdapter } from '../eval-
|
|
6
|
+
import { createSameProcessExecutorAdapter } from '../eval-runtime/adapters/same-process.js';
|
|
7
7
|
import { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, dshHostCoreExecutorCapabilities, parseDshHostCoreResult, } from './core-protocol.js';
|
|
8
8
|
import { supportsDshTraceEventType } from './trace-adapter.js';
|
|
9
9
|
export { DSH_HOST_CORE_ADAPTER_IMPLEMENTATION_VERSION, createDshHostCoreSchemaValidators, } from './core-protocol.js';
|
|
@@ -260,20 +260,28 @@ function rowsForInputs(inputs) {
|
|
|
260
260
|
}
|
|
261
261
|
return [...byId.values()].sort((left, right) => compareStrings(left.rowId, right.rowId));
|
|
262
262
|
}
|
|
263
|
-
function observationCoverage(rows, includedRowIds, comparableRowIds) {
|
|
263
|
+
function observationCoverage(rows, includedRowIds, comparableRowIds, notApplicableRowIds = []) {
|
|
264
264
|
const included = new Set(includedRowIds);
|
|
265
265
|
const comparable = new Set(comparableRowIds);
|
|
266
|
+
const notApplicable = new Set(notApplicableRowIds);
|
|
266
267
|
const known = new Set(rows.map((row) => row.rowId));
|
|
267
268
|
if (included.size !== includedRowIds.length
|
|
268
269
|
|| comparable.size !== comparableRowIds.length
|
|
270
|
+
|| notApplicable.size !== notApplicableRowIds.length
|
|
269
271
|
|| [...included].some((rowId) => !known.has(rowId))
|
|
270
|
-
|| [...comparable].some((rowId) => !included.has(rowId))
|
|
272
|
+
|| [...comparable].some((rowId) => !included.has(rowId))
|
|
273
|
+
|| [...notApplicable].some((rowId) => !known.has(rowId) || included.has(rowId))) {
|
|
271
274
|
throw new TypeError('Analysis implementation returned invalid row membership.');
|
|
272
275
|
}
|
|
276
|
+
if (rows.some((row) => (notApplicable.has(row.rowId)
|
|
277
|
+
&& (row.rowStatus !== 'missing' || row.censored)))) {
|
|
278
|
+
throw new TypeError('Analysis implementation marked an ineligible row as not applicable.');
|
|
279
|
+
}
|
|
273
280
|
const coverage = {
|
|
274
281
|
planned: rows.length,
|
|
275
282
|
observed: 0,
|
|
276
283
|
missing: 0,
|
|
284
|
+
notApplicable: notApplicable.size,
|
|
277
285
|
invalid: 0,
|
|
278
286
|
evaluationFailed: 0,
|
|
279
287
|
sourceUnavailable: 0,
|
|
@@ -284,6 +292,8 @@ function observationCoverage(rows, includedRowIds, comparableRowIds) {
|
|
|
284
292
|
comparable: comparable.size,
|
|
285
293
|
};
|
|
286
294
|
for (const row of rows) {
|
|
295
|
+
if (notApplicable.has(row.rowId))
|
|
296
|
+
continue;
|
|
287
297
|
if (row.rowStatus === 'observed')
|
|
288
298
|
coverage.observed += 1;
|
|
289
299
|
else if (row.rowStatus === 'missing')
|
|
@@ -319,6 +329,18 @@ function exclusionFacts(rows, includedRowIds) {
|
|
|
319
329
|
}))
|
|
320
330
|
.sort((left, right) => compareStrings(left.rowId, right.rowId));
|
|
321
331
|
}
|
|
332
|
+
function notApplicableFacts(rows, notApplicableRowIds) {
|
|
333
|
+
const notApplicable = new Set(notApplicableRowIds);
|
|
334
|
+
return rows
|
|
335
|
+
.filter((row) => notApplicable.has(row.rowId))
|
|
336
|
+
.map((row) => {
|
|
337
|
+
if (row.rowStatus === 'observed') {
|
|
338
|
+
throw new TypeError('Observed Analysis rows cannot be structurally not applicable.');
|
|
339
|
+
}
|
|
340
|
+
return { rowId: row.rowId, reasonCode: row.reasonCode };
|
|
341
|
+
})
|
|
342
|
+
.sort((left, right) => compareStrings(left.rowId, right.rowId));
|
|
343
|
+
}
|
|
322
344
|
function safeError(error) {
|
|
323
345
|
if (error instanceof AnalysisPortFailure) {
|
|
324
346
|
const parsed = EvaluationErrorSchema.safeParse(error.evaluationError);
|
|
@@ -583,6 +605,7 @@ async function runAnalysis(plan, ports, options, prepared, stream) {
|
|
|
583
605
|
inputReferences: binding.node.inputs,
|
|
584
606
|
coverage: emptyCoverage(rows),
|
|
585
607
|
exclusions: exclusionFacts(rows, []),
|
|
608
|
+
notApplicableRows: [],
|
|
586
609
|
assumptionChecks: [],
|
|
587
610
|
analysisMode: plan.analysis.analysisGraph.analysisMode,
|
|
588
611
|
derivedAt: ports.clock.timestamp(),
|
|
@@ -735,9 +758,10 @@ async function runAnalysis(plan, ports, options, prepared, stream) {
|
|
|
735
758
|
.map((row) => row.rowId);
|
|
736
759
|
const includedRowIds = output.includedRowIds ?? observedRowIds;
|
|
737
760
|
const comparableRowIds = output.comparableRowIds ?? includedRowIds;
|
|
761
|
+
const notApplicableRowIds = output.notApplicableRowIds ?? [];
|
|
738
762
|
let coverage;
|
|
739
763
|
try {
|
|
740
|
-
coverage = observationCoverage(rows, includedRowIds, comparableRowIds);
|
|
764
|
+
coverage = observationCoverage(rows, includedRowIds, comparableRowIds, notApplicableRowIds);
|
|
741
765
|
}
|
|
742
766
|
catch (error) {
|
|
743
767
|
const evaluationError = safeError(error);
|
|
@@ -784,6 +808,7 @@ async function runAnalysis(plan, ports, options, prepared, stream) {
|
|
|
784
808
|
...base,
|
|
785
809
|
coverage,
|
|
786
810
|
exclusions: exclusionFacts(rows, includedRowIds),
|
|
811
|
+
notApplicableRows: notApplicableFacts(rows, notApplicableRowIds),
|
|
787
812
|
assumptionChecks: checks,
|
|
788
813
|
}, runtimeDependencies, {
|
|
789
814
|
analysisStatus: 'inconclusive',
|
|
@@ -832,6 +857,7 @@ async function runAnalysis(plan, ports, options, prepared, stream) {
|
|
|
832
857
|
...base,
|
|
833
858
|
coverage,
|
|
834
859
|
exclusions: exclusionFacts(rows, includedRowIds),
|
|
860
|
+
notApplicableRows: notApplicableFacts(rows, notApplicableRowIds),
|
|
835
861
|
assumptionChecks: checks,
|
|
836
862
|
}, runtimeDependencies, {
|
|
837
863
|
analysisStatus: 'completed',
|
|
@@ -852,6 +878,7 @@ async function runAnalysis(plan, ports, options, prepared, stream) {
|
|
|
852
878
|
...base,
|
|
853
879
|
coverage,
|
|
854
880
|
exclusions: exclusionFacts(rows, includedRowIds),
|
|
881
|
+
notApplicableRows: notApplicableFacts(rows, notApplicableRowIds),
|
|
855
882
|
assumptionChecks: checks,
|
|
856
883
|
}, runtimeDependencies, {
|
|
857
884
|
analysisStatus: 'inconclusive',
|
|
@@ -80,12 +80,14 @@ export type AnalysisNodeExecutionResult = {
|
|
|
80
80
|
value: JsonValue;
|
|
81
81
|
includedRowIds?: readonly Sha256Digest[];
|
|
82
82
|
comparableRowIds?: readonly Sha256Digest[];
|
|
83
|
+
notApplicableRowIds?: readonly Sha256Digest[];
|
|
83
84
|
assumptionChecks?: readonly Omit<AssumptionCheck, 'nodeId'>[];
|
|
84
85
|
} | {
|
|
85
86
|
analysisStatus: 'inconclusive';
|
|
86
87
|
reasonCodes: readonly string[];
|
|
87
88
|
includedRowIds?: readonly Sha256Digest[];
|
|
88
89
|
comparableRowIds?: readonly Sha256Digest[];
|
|
90
|
+
notApplicableRowIds?: readonly Sha256Digest[];
|
|
89
91
|
assumptionChecks?: readonly Omit<AssumptionCheck, 'nodeId'>[];
|
|
90
92
|
};
|
|
91
93
|
export declare const AnalysisNodeExecutionResultSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -101,6 +103,7 @@ export declare const AnalysisNodeExecutionResultSchema: z.ZodDiscriminatedUnion<
|
|
|
101
103
|
value: z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>;
|
|
102
104
|
includedRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
103
105
|
comparableRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
106
|
+
notApplicableRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
104
107
|
assumptionChecks: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
105
108
|
details: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
106
109
|
reasonCode: z.ZodOptional<z.ZodString>;
|
|
@@ -116,6 +119,7 @@ export declare const AnalysisNodeExecutionResultSchema: z.ZodDiscriminatedUnion<
|
|
|
116
119
|
reasonCodes: z.ZodArray<z.ZodString>;
|
|
117
120
|
includedRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
118
121
|
comparableRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
122
|
+
notApplicableRowIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
119
123
|
assumptionChecks: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
120
124
|
details: z.ZodOptional<z.ZodType<JsonValue, unknown, z.core.$ZodTypeInternals<JsonValue, unknown>>>;
|
|
121
125
|
reasonCode: z.ZodOptional<z.ZodString>;
|
|
@@ -26,6 +26,7 @@ export const AnalysisNodeExecutionResultSchema = z.discriminatedUnion('analysisS
|
|
|
26
26
|
value: JsonValueSchema,
|
|
27
27
|
includedRowIds: z.array(Sha256DigestSchema).optional(),
|
|
28
28
|
comparableRowIds: z.array(Sha256DigestSchema).optional(),
|
|
29
|
+
notApplicableRowIds: z.array(Sha256DigestSchema).optional(),
|
|
29
30
|
assumptionChecks: z.array(PortAssumptionCheckSchema).optional(),
|
|
30
31
|
}).strict(),
|
|
31
32
|
z.object({
|
|
@@ -33,6 +34,7 @@ export const AnalysisNodeExecutionResultSchema = z.discriminatedUnion('analysisS
|
|
|
33
34
|
reasonCodes: z.array(IdentifierSchema).min(1),
|
|
34
35
|
includedRowIds: z.array(Sha256DigestSchema).optional(),
|
|
35
36
|
comparableRowIds: z.array(Sha256DigestSchema).optional(),
|
|
37
|
+
notApplicableRowIds: z.array(Sha256DigestSchema).optional(),
|
|
36
38
|
assumptionChecks: z.array(PortAssumptionCheckSchema).optional(),
|
|
37
39
|
}).strict(),
|
|
38
40
|
]);
|
|
@@ -24,6 +24,7 @@ function compareStrings(left, right) {
|
|
|
24
24
|
function assertObservationCoverage(coverage) {
|
|
25
25
|
if (coverage.planned !== coverage.observed
|
|
26
26
|
+ coverage.missing
|
|
27
|
+
+ coverage.notApplicable
|
|
27
28
|
+ coverage.invalid
|
|
28
29
|
+ coverage.evaluationFailed
|
|
29
30
|
+ coverage.sourceUnavailable
|
|
@@ -58,6 +59,20 @@ function assertRecords(bundle) {
|
|
|
58
59
|
if (record.exclusions.length !== record.coverage.excluded) {
|
|
59
60
|
throw new AnalysisBundleValidationError('ANALYSIS_BUNDLE_COVERAGE_INVALID', 'Analysis exclusions must account for every excluded observation row.');
|
|
60
61
|
}
|
|
62
|
+
if (record.notApplicableRows.length !== record.coverage.notApplicable) {
|
|
63
|
+
throw new AnalysisBundleValidationError('ANALYSIS_BUNDLE_COVERAGE_INVALID', 'Analysis not-applicable rows must account for structural coverage.');
|
|
64
|
+
}
|
|
65
|
+
const exclusionById = new Map(record.exclusions.map((entry) => [entry.rowId, entry]));
|
|
66
|
+
for (let index = 0; index < record.notApplicableRows.length; index += 1) {
|
|
67
|
+
const fact = record.notApplicableRows[index];
|
|
68
|
+
const previousFact = record.notApplicableRows[index - 1];
|
|
69
|
+
if (previousFact !== undefined && compareStrings(previousFact.rowId, fact.rowId) >= 0) {
|
|
70
|
+
throw new AnalysisBundleValidationError('ANALYSIS_BUNDLE_COVERAGE_INVALID', 'Analysis not-applicable rows must use unique canonical row order.');
|
|
71
|
+
}
|
|
72
|
+
if (exclusionById.get(fact.rowId)?.reasonCode !== fact.reasonCode) {
|
|
73
|
+
throw new AnalysisBundleValidationError('ANALYSIS_BUNDLE_COVERAGE_INVALID', 'Every not-applicable row must be retained as a matching exclusion fact.');
|
|
74
|
+
}
|
|
75
|
+
}
|
|
61
76
|
for (let exclusionIndex = 0; exclusionIndex < record.exclusions.length; exclusionIndex += 1) {
|
|
62
77
|
const exclusion = record.exclusions[exclusionIndex];
|
|
63
78
|
const previousExclusion = record.exclusions[exclusionIndex - 1];
|
|
@@ -282,10 +297,12 @@ function expectedRows(plan, execution, evaluation, node) {
|
|
|
282
297
|
return rows.sort((left, right) => compareStrings(left.rowId, right.rowId));
|
|
283
298
|
}
|
|
284
299
|
function assertSourceCoverage(record, rows) {
|
|
300
|
+
const notApplicableIds = new Set(record.notApplicableRows.map((entry) => entry.rowId));
|
|
285
301
|
const expected = {
|
|
286
302
|
planned: rows.length,
|
|
287
303
|
observed: rows.filter((row) => row.rowStatus === 'observed').length,
|
|
288
|
-
missing: rows.filter((row) => row.rowStatus === 'missing').length,
|
|
304
|
+
missing: rows.filter((row) => (row.rowStatus === 'missing' && !notApplicableIds.has(row.rowId))).length,
|
|
305
|
+
notApplicable: record.notApplicableRows.length,
|
|
289
306
|
invalid: rows.filter((row) => row.rowStatus === 'invalid').length,
|
|
290
307
|
evaluationFailed: rows.filter((row) => row.rowStatus === 'evaluation-failed').length,
|
|
291
308
|
sourceUnavailable: rows.filter((row) => row.rowStatus === 'source-unavailable').length,
|
|
@@ -296,6 +313,7 @@ function assertSourceCoverage(record, rows) {
|
|
|
296
313
|
planned: record.coverage.planned,
|
|
297
314
|
observed: record.coverage.observed,
|
|
298
315
|
missing: record.coverage.missing,
|
|
316
|
+
notApplicable: record.coverage.notApplicable,
|
|
299
317
|
invalid: record.coverage.invalid,
|
|
300
318
|
evaluationFailed: record.coverage.evaluationFailed,
|
|
301
319
|
sourceUnavailable: record.coverage.sourceUnavailable,
|
|
@@ -308,6 +326,12 @@ function assertSourceCoverage(record, rows) {
|
|
|
308
326
|
const byId = new Map(rows.map((row) => [row.rowId, row]));
|
|
309
327
|
const exclusionById = new Map(record.exclusions.map((entry) => [entry.rowId, entry]));
|
|
310
328
|
if (record.exclusions.some((entry) => !byId.has(entry.rowId))
|
|
329
|
+
|| record.notApplicableRows.some((entry) => {
|
|
330
|
+
const row = byId.get(entry.rowId);
|
|
331
|
+
return row?.rowStatus !== 'missing'
|
|
332
|
+
|| row.censored
|
|
333
|
+
|| row.reasonCode !== entry.reasonCode;
|
|
334
|
+
})
|
|
311
335
|
|| rows.some((row) => row.rowStatus !== 'observed'
|
|
312
336
|
&& (exclusionById.get(row.rowId)?.reasonCode !== row.reasonCode))) {
|
|
313
337
|
throw new AnalysisBundleValidationError('ANALYSIS_BUNDLE_SOURCE_MISMATCH', 'Analysis exclusions do not match source observation identities and reasons.');
|
|
@@ -2,8 +2,8 @@ import { z } from 'zod';
|
|
|
2
2
|
export declare const EVALUATION_EVENT_SCHEMA_VERSION: "omk.evaluation-event/v1";
|
|
3
3
|
export declare const EXECUTION_BUNDLE_SCHEMA_VERSION: "omk.execution-bundle/v1";
|
|
4
4
|
export declare const EVALUATION_BUNDLE_SCHEMA_VERSION: "omk.evaluation-bundle/v1";
|
|
5
|
-
export declare const ANALYSIS_BUNDLE_SCHEMA_VERSION: "omk.analysis-bundle/
|
|
6
|
-
export declare const EVALUATION_REPORT_SCHEMA_VERSION: "omk.evaluation-report/
|
|
5
|
+
export declare const ANALYSIS_BUNDLE_SCHEMA_VERSION: "omk.analysis-bundle/v2";
|
|
6
|
+
export declare const EVALUATION_REPORT_SCHEMA_VERSION: "omk.evaluation-report/v2";
|
|
7
7
|
export declare const TimingRecordSchema: z.ZodObject<{
|
|
8
8
|
startedAt: z.ZodString;
|
|
9
9
|
completedAt: z.ZodOptional<z.ZodString>;
|
|
@@ -7259,6 +7259,7 @@ export declare const AnalysisObservationCoverageSchema: z.ZodObject<{
|
|
|
7259
7259
|
planned: z.ZodNumber;
|
|
7260
7260
|
observed: z.ZodNumber;
|
|
7261
7261
|
missing: z.ZodNumber;
|
|
7262
|
+
notApplicable: z.ZodNumber;
|
|
7262
7263
|
invalid: z.ZodNumber;
|
|
7263
7264
|
evaluationFailed: z.ZodNumber;
|
|
7264
7265
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7355,6 +7356,7 @@ export declare const CompletedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7355
7356
|
planned: z.ZodNumber;
|
|
7356
7357
|
observed: z.ZodNumber;
|
|
7357
7358
|
missing: z.ZodNumber;
|
|
7359
|
+
notApplicable: z.ZodNumber;
|
|
7358
7360
|
invalid: z.ZodNumber;
|
|
7359
7361
|
evaluationFailed: z.ZodNumber;
|
|
7360
7362
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7368,6 +7370,10 @@ export declare const CompletedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7368
7370
|
rowId: z.ZodString;
|
|
7369
7371
|
reasonCode: z.ZodString;
|
|
7370
7372
|
}, z.core.$strict>>;
|
|
7373
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7374
|
+
rowId: z.ZodString;
|
|
7375
|
+
reasonCode: z.ZodString;
|
|
7376
|
+
}, z.core.$strict>>;
|
|
7371
7377
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7372
7378
|
assumptionId: z.ZodString;
|
|
7373
7379
|
nodeId: z.ZodString;
|
|
@@ -7469,6 +7475,7 @@ export declare const InconclusiveAnalysisRecordSchema: z.ZodObject<{
|
|
|
7469
7475
|
planned: z.ZodNumber;
|
|
7470
7476
|
observed: z.ZodNumber;
|
|
7471
7477
|
missing: z.ZodNumber;
|
|
7478
|
+
notApplicable: z.ZodNumber;
|
|
7472
7479
|
invalid: z.ZodNumber;
|
|
7473
7480
|
evaluationFailed: z.ZodNumber;
|
|
7474
7481
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7482,6 +7489,10 @@ export declare const InconclusiveAnalysisRecordSchema: z.ZodObject<{
|
|
|
7482
7489
|
rowId: z.ZodString;
|
|
7483
7490
|
reasonCode: z.ZodString;
|
|
7484
7491
|
}, z.core.$strict>>;
|
|
7492
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7493
|
+
rowId: z.ZodString;
|
|
7494
|
+
reasonCode: z.ZodString;
|
|
7495
|
+
}, z.core.$strict>>;
|
|
7485
7496
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7486
7497
|
assumptionId: z.ZodString;
|
|
7487
7498
|
nodeId: z.ZodString;
|
|
@@ -7575,6 +7586,7 @@ export declare const FailedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7575
7586
|
planned: z.ZodNumber;
|
|
7576
7587
|
observed: z.ZodNumber;
|
|
7577
7588
|
missing: z.ZodNumber;
|
|
7589
|
+
notApplicable: z.ZodNumber;
|
|
7578
7590
|
invalid: z.ZodNumber;
|
|
7579
7591
|
evaluationFailed: z.ZodNumber;
|
|
7580
7592
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7588,6 +7600,10 @@ export declare const FailedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7588
7600
|
rowId: z.ZodString;
|
|
7589
7601
|
reasonCode: z.ZodString;
|
|
7590
7602
|
}, z.core.$strict>>;
|
|
7603
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7604
|
+
rowId: z.ZodString;
|
|
7605
|
+
reasonCode: z.ZodString;
|
|
7606
|
+
}, z.core.$strict>>;
|
|
7591
7607
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7592
7608
|
assumptionId: z.ZodString;
|
|
7593
7609
|
nodeId: z.ZodString;
|
|
@@ -7681,6 +7697,7 @@ export declare const NotEvaluatedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7681
7697
|
planned: z.ZodNumber;
|
|
7682
7698
|
observed: z.ZodNumber;
|
|
7683
7699
|
missing: z.ZodNumber;
|
|
7700
|
+
notApplicable: z.ZodNumber;
|
|
7684
7701
|
invalid: z.ZodNumber;
|
|
7685
7702
|
evaluationFailed: z.ZodNumber;
|
|
7686
7703
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7694,6 +7711,10 @@ export declare const NotEvaluatedAnalysisRecordSchema: z.ZodObject<{
|
|
|
7694
7711
|
rowId: z.ZodString;
|
|
7695
7712
|
reasonCode: z.ZodString;
|
|
7696
7713
|
}, z.core.$strict>>;
|
|
7714
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7715
|
+
rowId: z.ZodString;
|
|
7716
|
+
reasonCode: z.ZodString;
|
|
7717
|
+
}, z.core.$strict>>;
|
|
7697
7718
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7698
7719
|
assumptionId: z.ZodString;
|
|
7699
7720
|
nodeId: z.ZodString;
|
|
@@ -7787,6 +7808,7 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
7787
7808
|
planned: z.ZodNumber;
|
|
7788
7809
|
observed: z.ZodNumber;
|
|
7789
7810
|
missing: z.ZodNumber;
|
|
7811
|
+
notApplicable: z.ZodNumber;
|
|
7790
7812
|
invalid: z.ZodNumber;
|
|
7791
7813
|
evaluationFailed: z.ZodNumber;
|
|
7792
7814
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7800,6 +7822,10 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
7800
7822
|
rowId: z.ZodString;
|
|
7801
7823
|
reasonCode: z.ZodString;
|
|
7802
7824
|
}, z.core.$strict>>;
|
|
7825
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7826
|
+
rowId: z.ZodString;
|
|
7827
|
+
reasonCode: z.ZodString;
|
|
7828
|
+
}, z.core.$strict>>;
|
|
7803
7829
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7804
7830
|
assumptionId: z.ZodString;
|
|
7805
7831
|
nodeId: z.ZodString;
|
|
@@ -7900,6 +7926,7 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
7900
7926
|
planned: z.ZodNumber;
|
|
7901
7927
|
observed: z.ZodNumber;
|
|
7902
7928
|
missing: z.ZodNumber;
|
|
7929
|
+
notApplicable: z.ZodNumber;
|
|
7903
7930
|
invalid: z.ZodNumber;
|
|
7904
7931
|
evaluationFailed: z.ZodNumber;
|
|
7905
7932
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -7913,6 +7940,10 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
7913
7940
|
rowId: z.ZodString;
|
|
7914
7941
|
reasonCode: z.ZodString;
|
|
7915
7942
|
}, z.core.$strict>>;
|
|
7943
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
7944
|
+
rowId: z.ZodString;
|
|
7945
|
+
reasonCode: z.ZodString;
|
|
7946
|
+
}, z.core.$strict>>;
|
|
7916
7947
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
7917
7948
|
assumptionId: z.ZodString;
|
|
7918
7949
|
nodeId: z.ZodString;
|
|
@@ -8005,6 +8036,7 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
8005
8036
|
planned: z.ZodNumber;
|
|
8006
8037
|
observed: z.ZodNumber;
|
|
8007
8038
|
missing: z.ZodNumber;
|
|
8039
|
+
notApplicable: z.ZodNumber;
|
|
8008
8040
|
invalid: z.ZodNumber;
|
|
8009
8041
|
evaluationFailed: z.ZodNumber;
|
|
8010
8042
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8018,6 +8050,10 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
8018
8050
|
rowId: z.ZodString;
|
|
8019
8051
|
reasonCode: z.ZodString;
|
|
8020
8052
|
}, z.core.$strict>>;
|
|
8053
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8054
|
+
rowId: z.ZodString;
|
|
8055
|
+
reasonCode: z.ZodString;
|
|
8056
|
+
}, z.core.$strict>>;
|
|
8021
8057
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8022
8058
|
assumptionId: z.ZodString;
|
|
8023
8059
|
nodeId: z.ZodString;
|
|
@@ -8110,6 +8146,7 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
8110
8146
|
planned: z.ZodNumber;
|
|
8111
8147
|
observed: z.ZodNumber;
|
|
8112
8148
|
missing: z.ZodNumber;
|
|
8149
|
+
notApplicable: z.ZodNumber;
|
|
8113
8150
|
invalid: z.ZodNumber;
|
|
8114
8151
|
evaluationFailed: z.ZodNumber;
|
|
8115
8152
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8123,6 +8160,10 @@ export declare const AnalysisRecordSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
8123
8160
|
rowId: z.ZodString;
|
|
8124
8161
|
reasonCode: z.ZodString;
|
|
8125
8162
|
}, z.core.$strict>>;
|
|
8163
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8164
|
+
rowId: z.ZodString;
|
|
8165
|
+
reasonCode: z.ZodString;
|
|
8166
|
+
}, z.core.$strict>>;
|
|
8126
8167
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8127
8168
|
assumptionId: z.ZodString;
|
|
8128
8169
|
nodeId: z.ZodString;
|
|
@@ -8160,7 +8201,7 @@ export declare const AnalysisCoverageSchema: z.ZodObject<{
|
|
|
8160
8201
|
notStarted: z.ZodNumber;
|
|
8161
8202
|
}, z.core.$strict>;
|
|
8162
8203
|
export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
8163
|
-
schemaVersion: z.ZodLiteral<"omk.analysis-bundle/
|
|
8204
|
+
schemaVersion: z.ZodLiteral<"omk.analysis-bundle/v2">;
|
|
8164
8205
|
bundleId: z.ZodString;
|
|
8165
8206
|
runContractDigest: z.ZodString;
|
|
8166
8207
|
evaluationBundleDigest: z.ZodString;
|
|
@@ -8244,6 +8285,7 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8244
8285
|
planned: z.ZodNumber;
|
|
8245
8286
|
observed: z.ZodNumber;
|
|
8246
8287
|
missing: z.ZodNumber;
|
|
8288
|
+
notApplicable: z.ZodNumber;
|
|
8247
8289
|
invalid: z.ZodNumber;
|
|
8248
8290
|
evaluationFailed: z.ZodNumber;
|
|
8249
8291
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8257,6 +8299,10 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8257
8299
|
rowId: z.ZodString;
|
|
8258
8300
|
reasonCode: z.ZodString;
|
|
8259
8301
|
}, z.core.$strict>>;
|
|
8302
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8303
|
+
rowId: z.ZodString;
|
|
8304
|
+
reasonCode: z.ZodString;
|
|
8305
|
+
}, z.core.$strict>>;
|
|
8260
8306
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8261
8307
|
assumptionId: z.ZodString;
|
|
8262
8308
|
nodeId: z.ZodString;
|
|
@@ -8357,6 +8403,7 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8357
8403
|
planned: z.ZodNumber;
|
|
8358
8404
|
observed: z.ZodNumber;
|
|
8359
8405
|
missing: z.ZodNumber;
|
|
8406
|
+
notApplicable: z.ZodNumber;
|
|
8360
8407
|
invalid: z.ZodNumber;
|
|
8361
8408
|
evaluationFailed: z.ZodNumber;
|
|
8362
8409
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8370,6 +8417,10 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8370
8417
|
rowId: z.ZodString;
|
|
8371
8418
|
reasonCode: z.ZodString;
|
|
8372
8419
|
}, z.core.$strict>>;
|
|
8420
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8421
|
+
rowId: z.ZodString;
|
|
8422
|
+
reasonCode: z.ZodString;
|
|
8423
|
+
}, z.core.$strict>>;
|
|
8373
8424
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8374
8425
|
assumptionId: z.ZodString;
|
|
8375
8426
|
nodeId: z.ZodString;
|
|
@@ -8462,6 +8513,7 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8462
8513
|
planned: z.ZodNumber;
|
|
8463
8514
|
observed: z.ZodNumber;
|
|
8464
8515
|
missing: z.ZodNumber;
|
|
8516
|
+
notApplicable: z.ZodNumber;
|
|
8465
8517
|
invalid: z.ZodNumber;
|
|
8466
8518
|
evaluationFailed: z.ZodNumber;
|
|
8467
8519
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8475,6 +8527,10 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8475
8527
|
rowId: z.ZodString;
|
|
8476
8528
|
reasonCode: z.ZodString;
|
|
8477
8529
|
}, z.core.$strict>>;
|
|
8530
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8531
|
+
rowId: z.ZodString;
|
|
8532
|
+
reasonCode: z.ZodString;
|
|
8533
|
+
}, z.core.$strict>>;
|
|
8478
8534
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8479
8535
|
assumptionId: z.ZodString;
|
|
8480
8536
|
nodeId: z.ZodString;
|
|
@@ -8567,6 +8623,7 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8567
8623
|
planned: z.ZodNumber;
|
|
8568
8624
|
observed: z.ZodNumber;
|
|
8569
8625
|
missing: z.ZodNumber;
|
|
8626
|
+
notApplicable: z.ZodNumber;
|
|
8570
8627
|
invalid: z.ZodNumber;
|
|
8571
8628
|
evaluationFailed: z.ZodNumber;
|
|
8572
8629
|
sourceUnavailable: z.ZodNumber;
|
|
@@ -8580,6 +8637,10 @@ export declare const AnalysisBundleSchema: z.ZodObject<{
|
|
|
8580
8637
|
rowId: z.ZodString;
|
|
8581
8638
|
reasonCode: z.ZodString;
|
|
8582
8639
|
}, z.core.$strict>>;
|
|
8640
|
+
notApplicableRows: z.ZodArray<z.ZodObject<{
|
|
8641
|
+
rowId: z.ZodString;
|
|
8642
|
+
reasonCode: z.ZodString;
|
|
8643
|
+
}, z.core.$strict>>;
|
|
8583
8644
|
assumptionChecks: z.ZodArray<z.ZodObject<{
|
|
8584
8645
|
assumptionId: z.ZodString;
|
|
8585
8646
|
nodeId: z.ZodString;
|
|
@@ -8801,7 +8862,7 @@ export declare const DecisionResultSchema: z.ZodDiscriminatedUnion<[z.ZodObject<
|
|
|
8801
8862
|
decisionDigest: z.ZodString;
|
|
8802
8863
|
}, z.core.$strict>], "decisionStatus">;
|
|
8803
8864
|
export declare const EvaluationReportSchema: z.ZodObject<{
|
|
8804
|
-
schemaVersion: z.ZodLiteral<"omk.evaluation-report/
|
|
8865
|
+
schemaVersion: z.ZodLiteral<"omk.evaluation-report/v2">;
|
|
8805
8866
|
reportId: z.ZodString;
|
|
8806
8867
|
runContractDigest: z.ZodString;
|
|
8807
8868
|
status: z.ZodObject<{
|
|
@@ -5,8 +5,8 @@ import { BudgetSummarySchema } from './budget.js';
|
|
|
5
5
|
export const EVALUATION_EVENT_SCHEMA_VERSION = 'omk.evaluation-event/v1';
|
|
6
6
|
export const EXECUTION_BUNDLE_SCHEMA_VERSION = 'omk.execution-bundle/v1';
|
|
7
7
|
export const EVALUATION_BUNDLE_SCHEMA_VERSION = 'omk.evaluation-bundle/v1';
|
|
8
|
-
export const ANALYSIS_BUNDLE_SCHEMA_VERSION = 'omk.analysis-bundle/
|
|
9
|
-
export const EVALUATION_REPORT_SCHEMA_VERSION = 'omk.evaluation-report/
|
|
8
|
+
export const ANALYSIS_BUNDLE_SCHEMA_VERSION = 'omk.analysis-bundle/v2';
|
|
9
|
+
export const EVALUATION_REPORT_SCHEMA_VERSION = 'omk.evaluation-report/v2';
|
|
10
10
|
export const TimingRecordSchema = z.object({
|
|
11
11
|
startedAt: TimestampSchema,
|
|
12
12
|
completedAt: TimestampSchema.optional(),
|
|
@@ -280,6 +280,7 @@ export const AnalysisObservationCoverageSchema = z.object({
|
|
|
280
280
|
planned: z.number().int().nonnegative(),
|
|
281
281
|
observed: z.number().int().nonnegative(),
|
|
282
282
|
missing: z.number().int().nonnegative(),
|
|
283
|
+
notApplicable: z.number().int().nonnegative(),
|
|
283
284
|
invalid: z.number().int().nonnegative(),
|
|
284
285
|
evaluationFailed: z.number().int().nonnegative(),
|
|
285
286
|
sourceUnavailable: z.number().int().nonnegative(),
|
|
@@ -332,6 +333,7 @@ const AnalysisRecordBaseSchema = z.object({
|
|
|
332
333
|
])).min(1),
|
|
333
334
|
coverage: AnalysisObservationCoverageSchema,
|
|
334
335
|
exclusions: z.array(AnalysisExclusionSchema),
|
|
336
|
+
notApplicableRows: z.array(AnalysisExclusionSchema),
|
|
335
337
|
assumptionChecks: z.array(AssumptionCheckSchema),
|
|
336
338
|
runtimeDependencies: z.array(AnalysisRuntimeDependencySchema),
|
|
337
339
|
analysisMode: z.enum(['preregistered', 'exploratory']),
|
|
@@ -394,7 +396,7 @@ export const AnalysisBundleSchema = z.object({
|
|
|
394
396
|
bundleDigest: Sha256DigestSchema,
|
|
395
397
|
extensions: ExtensionsSchema.optional(),
|
|
396
398
|
}).strict().meta({
|
|
397
|
-
title: 'OMK Analysis Bundle
|
|
399
|
+
title: 'OMK Analysis Bundle v2',
|
|
398
400
|
});
|
|
399
401
|
export const EvaluationStatusSchema = z.object({
|
|
400
402
|
runStatus: z.enum(['completed', 'cancelled', 'budget-exhausted', 'failed']),
|
|
@@ -448,7 +450,7 @@ export const EvaluationReportSchema = z.object({
|
|
|
448
450
|
reportDigest: Sha256DigestSchema,
|
|
449
451
|
extensions: ExtensionsSchema.optional(),
|
|
450
452
|
}).strict().meta({
|
|
451
|
-
title: 'OMK Evaluation Report
|
|
453
|
+
title: 'OMK Evaluation Report v2',
|
|
452
454
|
});
|
|
453
455
|
export const EvaluationEventSchema = z.object({
|
|
454
456
|
schemaVersion: z.literal(EVALUATION_EVENT_SCHEMA_VERSION),
|
|
@@ -7,6 +7,8 @@ export interface WireSchemaCatalogEntry {
|
|
|
7
7
|
schema: z.ZodType;
|
|
8
8
|
includedInRunContract?: false;
|
|
9
9
|
}
|
|
10
|
+
export type WireSchemaCatalogVersion = `v${number}`;
|
|
11
|
+
export declare function wireSchemaCatalogVersion(entry: Pick<WireSchemaCatalogEntry, 'schemaVersion'>): WireSchemaCatalogVersion;
|
|
10
12
|
export declare const WIRE_SCHEMA_CATALOG: readonly WireSchemaCatalogEntry[];
|
|
11
13
|
export declare function generateWireJsonSchemas(): Readonly<Record<string, JsonValue>>;
|
|
12
14
|
export declare function generateWireSchemaIdentities(): SchemaIdentity[];
|
|
@@ -8,6 +8,13 @@ import { COMPARABILITY_ASSESSMENT_SCHEMA_VERSION, COMPARABILITY_POLICY_SCHEMA_VE
|
|
|
8
8
|
import { EVALUATION_SERIES_DEFINITION_SCHEMA_VERSION, EVALUATION_SERIES_PLAN_SCHEMA_VERSION, EVALUATION_SERIES_REPORT_SCHEMA_VERSION, EvaluationSeriesDefinitionSchema, EvaluationSeriesPlanSchema, EvaluationSeriesReportSchema, SERIES_ANALYSIS_BUNDLE_SCHEMA_VERSION, SeriesAnalysisBundleSchema, } from './series.js';
|
|
9
9
|
import { EXECUTOR_CAPABILITIES_SCHEMA_VERSION, ExecutorCapabilitiesSchema, } from './runtime-capabilities.js';
|
|
10
10
|
import { EXECUTION_FACTS_SCHEMA_VERSION, ExecutionFactsSchema, } from './execution-facts.js';
|
|
11
|
+
export function wireSchemaCatalogVersion(entry) {
|
|
12
|
+
const version = entry.schemaVersion.match(/\/(v[1-9]\d*)$/)?.[1];
|
|
13
|
+
if (version === undefined) {
|
|
14
|
+
throw new Error(`Wire schema version is not explicitly versioned: ${entry.schemaVersion}`);
|
|
15
|
+
}
|
|
16
|
+
return version;
|
|
17
|
+
}
|
|
11
18
|
export const WIRE_SCHEMA_CATALOG = [
|
|
12
19
|
{
|
|
13
20
|
fileName: 'execution-facts.schema.json',
|
|
@@ -121,9 +128,9 @@ export const WIRE_SCHEMA_CATALOG = [
|
|
|
121
128
|
includedInRunContract: false,
|
|
122
129
|
},
|
|
123
130
|
];
|
|
124
|
-
//
|
|
125
|
-
//
|
|
126
|
-
const SCHEMA_BASE_URI = 'https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
131
|
+
// Keep the public identity aligned with the canonical catalog location so schema tooling can
|
|
132
|
+
// resolve the identifier instead of treating a stale repository path as an opaque URI.
|
|
133
|
+
const SCHEMA_BASE_URI = 'https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core';
|
|
127
134
|
function assertNoEmptySchemaNode(value, path = '$') {
|
|
128
135
|
if (Array.isArray(value)) {
|
|
129
136
|
value.forEach((entry, index) => assertNoEmptySchemaNode(entry, `${path}[${index}]`));
|
|
@@ -145,7 +152,7 @@ export function generateWireJsonSchemas() {
|
|
|
145
152
|
cycles: 'ref',
|
|
146
153
|
reused: 'ref',
|
|
147
154
|
});
|
|
148
|
-
const schemaUri = `${SCHEMA_BASE_URI}/${entry.fileName}`;
|
|
155
|
+
const schemaUri = `${SCHEMA_BASE_URI}/${wireSchemaCatalogVersion(entry)}/${entry.fileName}`;
|
|
149
156
|
const schema = { ...generated, $id: schemaUri };
|
|
150
157
|
assertNoEmptySchemaNode(schema);
|
|
151
158
|
return [entry.fileName, schema];
|
|
@@ -933,7 +933,7 @@
|
|
|
933
933
|
"type": "string"
|
|
934
934
|
}
|
|
935
935
|
},
|
|
936
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
936
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/analysis-bundle.schema.json",
|
|
937
937
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
938
938
|
"additionalProperties": false,
|
|
939
939
|
"properties": {
|
|
@@ -830,7 +830,7 @@
|
|
|
830
830
|
"type": "array"
|
|
831
831
|
}
|
|
832
832
|
},
|
|
833
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
833
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/analysis-plan.schema.json",
|
|
834
834
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
835
835
|
"additionalProperties": false,
|
|
836
836
|
"properties": {
|
|
@@ -400,7 +400,7 @@
|
|
|
400
400
|
"type": "number"
|
|
401
401
|
}
|
|
402
402
|
},
|
|
403
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
403
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/budget-summary.schema.json",
|
|
404
404
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
405
405
|
"additionalProperties": false,
|
|
406
406
|
"properties": {
|
|
@@ -397,7 +397,7 @@
|
|
|
397
397
|
"type": "string"
|
|
398
398
|
}
|
|
399
399
|
},
|
|
400
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
400
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/comparability-assessment.schema.json",
|
|
401
401
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
402
402
|
"additionalProperties": false,
|
|
403
403
|
"properties": {
|
|
@@ -53,7 +53,7 @@
|
|
|
53
53
|
"type": "string"
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
56
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/comparability-policy.schema.json",
|
|
57
57
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
58
58
|
"additionalProperties": false,
|
|
59
59
|
"properties": {
|
|
@@ -351,7 +351,7 @@
|
|
|
351
351
|
"type": "object"
|
|
352
352
|
}
|
|
353
353
|
},
|
|
354
|
-
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/
|
|
354
|
+
"$id": "https://raw.githubusercontent.com/lizhiyao/oh-my-knowledge/main/schemas/eval-core/v1/decision-plan.schema.json",
|
|
355
355
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
356
356
|
"additionalProperties": false,
|
|
357
357
|
"properties": {
|