oh-my-knowledge 0.53.0 → 1.0.0-beta.0

package/dist/eval-workflows/runtime-adapter/analysis/agreement-table.js

@@ -0,0 +1,303 @@
+import { z } from 'zod';
+import { drawBootstrapMetric, summarizeBootstrapMetric, } from '../../analysis/bootstrap.js';
+import { computeKrippendorffAlpha, computePearson, computeWeightedKappa, } from '../../gold/human.js';
+import { IdentifierSchema, Sha256DigestSchema, canonicalizeJson, schemaIdentityKey, } from '../../../eval-core/contracts/index.js';
+import { analysisJsonSchema, analysisSchemaIdentity, createAnalysisSchemaValidator, } from './analysis-support.js';
+import { AgreementParametersSchema, parseAgreementParameters, } from './agreement-parameters.js';
+export const AGREEMENT_TABLE_SCHEMA_VERSION = 'omk.agreement-table/v1';
+export const AGREEMENT_STATISTIC_DECIMALS = 4;
+const CountSchema = z.number().int().nonnegative().safe();
+const PositiveCountSchema = z.number().int().positive().safe();
+const RatingSchema = z.number().finite();
+const RoundedStatisticSchema = z.number().finite().refine((value) => Math.round(value * 10 ** AGREEMENT_STATISTIC_DECIMALS)
+    / 10 ** AGREEMENT_STATISTIC_DECIMALS === value, { message: 'Agreement statistics must use four-decimal precision.' });
+const ObservedGoldRatingSchema = z.object({
+    ratingStatus: z.literal('observed'),
+    score: RatingSchema,
+}).strict();
+const UnavailableGoldRatingSchema = z.object({
+    ratingStatus: z.literal('unavailable'),
+    reasonCode: z.literal('gold-rating-unavailable'),
+}).strict();
+const GoldRatingSchema = z.discriminatedUnion('ratingStatus', [
+    ObservedGoldRatingSchema,
+    UnavailableGoldRatingSchema,
+]);
+const JudgeCoverageSchema = z.object({
+    plannedGroups: CountSchema,
+    observedGroups: CountSchema,
+    missingGroups: CountSchema,
+}).strict();
+const ObservedJudgeRatingSchema = z.object({
+    ratingStatus: z.literal('observed'),
+    score: RatingSchema,
+    sourceGroupIds: z.array(Sha256DigestSchema).min(1),
+    coverage: JudgeCoverageSchema,
+}).strict();
+const MissingJudgeRatingSchema = z.object({
+    ratingStatus: z.literal('missing'),
+    reasonCode: z.literal('dimension-unobserved'),
+    sourceGroupIds: z.array(Sha256DigestSchema).min(1),
+    coverage: JudgeCoverageSchema,
+}).strict();
+const UnavailableJudgeRatingSchema = z.object({
+    ratingStatus: z.literal('unavailable'),
+    reasonCode: z.literal('dimension-group-unavailable'),
+    sourceGroupIds: z.tuple([]),
+    coverage: z.object({
+        plannedGroups: z.literal(0),
+        observedGroups: z.literal(0),
+        missingGroups: z.literal(0),
+    }).strict(),
+}).strict();
+const JudgeRatingSchema = z.discriminatedUnion('ratingStatus', [
+    ObservedJudgeRatingSchema,
+    MissingJudgeRatingSchema,
+    UnavailableJudgeRatingSchema,
+]);
+export const AgreementPairSchema = z.object({
+    sampleId: IdentifierSchema,
+    gold: GoldRatingSchema,
+    judge: JudgeRatingSchema,
+}).strict();
+const AgreementCoverageSchema = z.object({
+    plannedPairs: PositiveCountSchema,
+    comparablePairs: CountSchema,
+    goldObservedPairs: CountSchema,
+    goldUnavailablePairs: CountSchema,
+    judgeObservedPairs: CountSchema,
+    judgeMissingPairs: CountSchema,
+    judgeUnavailablePairs: CountSchema,
+}).strict();
+const ObservedStatisticSchema = z.object({
+    statisticStatus: z.literal('observed'),
+    value: RoundedStatisticSchema,
+}).strict();
+const MissingStatisticSchema = z.object({
+    statisticStatus: z.literal('missing'),
+    reasonCode: z.enum([
+        'agreement-insufficient-pairs',
+        'agreement-zero-expected-disagreement',
+        'agreement-statistic-undefined',
+    ]),
+}).strict();
+const AgreementStatisticSchema = z.discriminatedUnion('statisticStatus', [
+    ObservedStatisticSchema,
+    MissingStatisticSchema,
+]);
+const DrawCoverageSchema = z.object({
+    plannedDraws: PositiveCountSchema,
+    observedDraws: CountSchema,
+    missingDraws: CountSchema,
+}).strict();
+const ObservedAlphaIntervalSchema = z.object({
+    intervalStatus: z.literal('observed'),
+    lower: RoundedStatisticSchema,
+    upper: RoundedStatisticSchema,
+    estimate: RoundedStatisticSchema,
+    samples: PositiveCountSchema,
+    confidenceLevel: z.number().finite().gt(0).lt(1),
+    drawCoverage: DrawCoverageSchema,
+}).strict();
+const MissingAlphaIntervalSchema = z.object({
+    intervalStatus: z.literal('missing'),
+    reasonCode: z.enum([
+        'agreement-point-unobserved',
+        'agreement-no-valid-bootstrap-draws',
+    ]),
+    drawCoverage: DrawCoverageSchema,
+}).strict();
+const AlphaIntervalSchema = z.discriminatedUnion('intervalStatus', [
+    ObservedAlphaIntervalSchema,
+    MissingAlphaIntervalSchema,
+]);
+const AgreementStatisticsSchema = z.object({
+    krippendorffAlpha: AgreementStatisticSchema,
+    alphaInterval: AlphaIntervalSchema,
+    weightedKappa: AgreementStatisticSchema,
+    pearson: AgreementStatisticSchema,
+}).strict();
+const AgreementTableValueSchema = z.object({
+    schemaVersion: z.literal(AGREEMENT_TABLE_SCHEMA_VERSION),
+    configuration: AgreementParametersSchema,
+    pairs: z.array(AgreementPairSchema).min(1),
+    coverage: AgreementCoverageSchema,
+    statistics: AgreementStatisticsSchema,
+}).strict();
+function legacyRound4(value) {
+    return Math.round(value * 10_000) / 10_000;
+}
+function parsePairs(parameters, rawPairs) {
+    const pairs = rawPairs.map((pair) => AgreementPairSchema.parse(pair));
+    const order = new Map(parameters.sampleIds.map((sampleId, index) => [sampleId, index]));
+    const pairIds = pairs.map((pair) => pair.sampleId);
+    if (new Set(pairIds).size !== pairIds.length
+        || canonicalizeJson([...pairIds].sort())
+            !== canonicalizeJson([...parameters.sampleIds].sort())) {
+        throw new TypeError('Agreement pairs must map every sealed sample exactly once.');
+    }
+    const sourceGroupIds = pairs.flatMap((pair) => pair.judge.sourceGroupIds);
+    if (new Set(sourceGroupIds).size !== sourceGroupIds.length) {
+        throw new TypeError('Agreement judge source group identities must be globally unique.');
+    }
+    for (const pair of pairs) {
+        for (const rating of [pair.gold, pair.judge]) {
+            if (rating.ratingStatus === 'observed'
+                && (rating.score < parameters.gold.scale.min
+                    || rating.score > parameters.gold.scale.max)) {
+                throw new TypeError('Agreement ratings must stay inside the sealed scale.');
+            }
+        }
+        const coverage = pair.judge.coverage;
+        if (coverage.observedGroups + coverage.missingGroups !== coverage.plannedGroups
+            || pair.judge.sourceGroupIds.length !== coverage.plannedGroups
+            || (pair.judge.ratingStatus === 'observed' && coverage.observedGroups === 0)
+            || (pair.judge.ratingStatus === 'missing'
+                && (coverage.plannedGroups === 0 || coverage.observedGroups !== 0))
+            || (pair.judge.ratingStatus === 'unavailable' && coverage.plannedGroups !== 0)) {
+            throw new TypeError('Agreement judge coverage does not match its rating status.');
+        }
+    }
+    return pairs.sort((left, right) => ((order.get(left.sampleId) ?? -1) - (order.get(right.sampleId) ?? -1)));
+}
+export function agreementCoverage(pairs) {
+    return {
+        plannedPairs: pairs.length,
+        comparablePairs: pairs.filter((pair) => (pair.gold.ratingStatus === 'observed' && pair.judge.ratingStatus === 'observed')).length,
+        goldObservedPairs: pairs.filter((pair) => pair.gold.ratingStatus === 'observed').length,
+        goldUnavailablePairs: pairs.filter((pair) => pair.gold.ratingStatus === 'unavailable').length,
+        judgeObservedPairs: pairs.filter((pair) => pair.judge.ratingStatus === 'observed').length,
+        judgeMissingPairs: pairs.filter((pair) => pair.judge.ratingStatus === 'missing').length,
+        judgeUnavailablePairs: pairs.filter((pair) => pair.judge.ratingStatus === 'unavailable').length,
+    };
+}
+function observedOrMissing(value, insufficient, undefinedReason) {
+    if (insufficient) {
+        return { statisticStatus: 'missing', reasonCode: 'agreement-insufficient-pairs' };
+    }
+    return Number.isFinite(value)
+        ? { statisticStatus: 'observed', value: legacyRound4(value) }
+        : { statisticStatus: 'missing', reasonCode: undefinedReason };
+}
+export function agreementStatistics(parameters, pairs) {
+    const ratings = pairs.flatMap((pair) => (pair.gold.ratingStatus === 'observed' && pair.judge.ratingStatus === 'observed'
+        ? [{ unitId: pair.sampleId, coderA: pair.gold.score, coderB: pair.judge.score }]
+        : []));
+    const insufficient = ratings.length < 2;
+    const rawAlpha = insufficient ? Number.NaN : computeKrippendorffAlpha(ratings);
+    const krippendorffAlpha = observedOrMissing(rawAlpha, insufficient, 'agreement-zero-expected-disagreement');
+    const weightedKappa = observedOrMissing(insufficient ? Number.NaN : computeWeightedKappa(ratings, parameters.gold.scale), insufficient, 'agreement-statistic-undefined');
+    const pearson = observedOrMissing(insufficient ? Number.NaN : computePearson(ratings), insufficient, 'agreement-statistic-undefined');
+    const emptyDrawCoverage = {
+        plannedDraws: parameters.resamples,
+        observedDraws: 0,
+        missingDraws: parameters.resamples,
+    };
+    if (krippendorffAlpha.statisticStatus !== 'observed') {
+        return {
+            krippendorffAlpha,
+            alphaInterval: {
+                intervalStatus: 'missing',
+                reasonCode: 'agreement-point-unobserved',
+                drawCoverage: emptyDrawCoverage,
+            },
+            weightedKappa,
+            pearson,
+        };
+    }
+    const indices = ratings.map((_, index) => index);
+    const distribution = drawBootstrapMetric(indices, (resampled) => computeKrippendorffAlpha(resampled.map((index) => ratings[index])), parameters.resamples, parameters.seed);
+    const finiteDraws = distribution.draws.filter(Number.isFinite);
+    const drawCoverage = {
+        plannedDraws: parameters.resamples,
+        observedDraws: finiteDraws.length,
+        missingDraws: parameters.resamples - finiteDraws.length,
+    };
+    const alphaInterval = finiteDraws.length === 0
+        ? {
+            intervalStatus: 'missing',
+            reasonCode: 'agreement-no-valid-bootstrap-draws',
+            drawCoverage,
+        }
+        : (() => {
+            const interval = summarizeBootstrapMetric(rawAlpha, finiteDraws, parameters.alpha, finiteDraws.length);
+            return {
+                intervalStatus: 'observed',
+                lower: interval.low,
+                upper: interval.high,
+                estimate: interval.estimate,
+                samples: interval.samples,
+                confidenceLevel: 1 - parameters.alpha,
+                drawCoverage,
+            };
+        })();
+    return { krippendorffAlpha, alphaInterval, weightedKappa, pearson };
+}
+export function buildAgreementTable(rawParameters, rawPairs) {
+    const parameters = parseAgreementParameters(rawParameters);
+    const pairs = parsePairs(parameters, rawPairs);
+    return {
+        schemaVersion: AGREEMENT_TABLE_SCHEMA_VERSION,
+        configuration: parameters,
+        pairs,
+        coverage: agreementCoverage(pairs),
+        statistics: agreementStatistics(parameters, pairs),
+    };
+}
+function validateAgreementTable(value, context) {
+    let expected;
+    try {
+        expected = buildAgreementTable(value.configuration, value.pairs);
+    }
+    catch (error) {
+        context.addIssue({
+            code: 'custom',
+            path: ['pairs'],
+            message: error instanceof Error ? error.message : 'Agreement pairs are invalid.',
+        });
+        return;
+    }
+    if (canonicalizeJson(value) !== canonicalizeJson(expected)) {
+        context.addIssue({
+            code: 'custom',
+            path: [],
+            message: 'Agreement table is not canonically ordered or statistically recomputable.',
+        });
+    }
+}
+const AgreementTableEnvelopeSchema = z.object({
+    resultType: z.literal('table'),
+    value: AgreementTableValueSchema,
+}).strict().superRefine((envelope, context) => {
+    validateAgreementTable(envelope.value, context);
+});
+export const AGREEMENT_TABLE_SCHEMA = analysisSchemaIdentity(AGREEMENT_TABLE_SCHEMA_VERSION, 'urn:omk:analysis-result:agreement-table:v1', analysisJsonSchema(AgreementTableEnvelopeSchema, [
+    'pairs map every sealed sample exactly once and preserve gold and Dimension availability separately',
+    'judge group coverage, globally unique source lineage, pair coverage, and sample ordering are recomputable',
+    'all observed ratings stay within the sealed external-annotation scale',
+    'Krippendorff alpha uses interval distance squared and requires at least two comparable rating pairs',
+    'quadratic-weighted kappa and Pearson are auxiliary statistics over the identical comparable pair set',
+    'bootstrap resamples complete pair indices using the sealed Mulberry32 stream',
+    'undefined alpha draws are excluded from interval quantiles and retained as missing draw coverage',
+    'zero finite draws, zero expected disagreement, and insufficient pairs produce structured missing rather than NaN or zero',
+    'finite point estimates and percentile bounds use the legacy four-decimal persistence rounding',
+    'the complete table is statistically recomputable during live and transported validation',
+]));
+export function parseAgreementTableEnvelope(value) {
+    return AgreementTableEnvelopeSchema.parse(value);
+}
+function validateSealedConfiguration(value, context) {
+    const parsed = parseAgreementTableEnvelope(value);
+    if (context?.validationKind !== 'analysis-output') {
+        throw new TypeError('Agreement Analysis output validation requires sealed node parameters.');
+    }
+    const sealed = parseAgreementParameters(context.parameters);
+    if (canonicalizeJson(parsed.value.configuration) !== canonicalizeJson(sealed)) {
+        throw new TypeError('Agreement table configuration does not match sealed node parameters.');
+    }
+    return parsed;
+}
+export function createAgreementTableSchemaValidators() {
+    const validator = createAnalysisSchemaValidator(AGREEMENT_TABLE_SCHEMA, validateSealedConfiguration);
+    return new Map([[schemaIdentityKey(validator.schema), validator]]);
+}

package/dist/eval-workflows/runtime-adapter/analysis/analysis-support.d.ts

@@ -0,0 +1,14 @@
+import { z } from 'zod';
+import { type CoreSchemaValidationContext, type CoreSchemaValidator, type JsonValue, type RuntimeIdentity, type SchemaIdentity } from '../../../eval-core/contracts/index.js';
+import type { AnalysisNodeExecutionContext, AnalysisNodeExecutionResult, AnalysisNodeImplementation } from '../../../eval-core/analysis/index.js';
+export declare function compareStrings(left: string, right: string): number;
+export declare function round(value: number, decimals: number): number;
+export declare function analysisJsonSchema(schema: z.ZodType, invariants: readonly string[]): JsonValue;
+export declare function analysisSchemaIdentity(schemaVersion: string, schemaUri: string, schema: JsonValue): SchemaIdentity;
+export declare function createAnalysisSchemaValidator(schema: SchemaIdentity, parse: (value: unknown, context?: Readonly<CoreSchemaValidationContext>) => JsonValue): CoreSchemaValidator;
+export declare function createStatelessAnalysisImplementation(input: Readonly<{
+    identity: RuntimeIdentity;
+    outputSchema: SchemaIdentity;
+    parseParameters(parameters: unknown): void;
+    execute(context: AnalysisNodeExecutionContext): AnalysisNodeExecutionResult | Promise<AnalysisNodeExecutionResult>;
+}>): AnalysisNodeImplementation;

package/dist/eval-workflows/runtime-adapter/analysis/analysis-support.js

@@ -0,0 +1,44 @@
+import { z } from 'zod';
+import { deepFreezeCanonicalJson, digestCanonicalJson, } from '../../../eval-core/contracts/index.js';
+export function compareStrings(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+export function round(value, decimals) {
+    return Number(value.toFixed(decimals));
+}
+export function analysisJsonSchema(schema, invariants) {
+    const generated = z.toJSONSchema(schema, {
+        target: 'draft-2020-12',
+        unrepresentable: 'throw',
+        cycles: 'ref',
+        reused: 'ref',
+    });
+    return { ...generated, 'x-omk-invariants': [...invariants] };
+}
+export function analysisSchemaIdentity(schemaVersion, schemaUri, schema) {
+    return deepFreezeCanonicalJson({
+        schemaVersion,
+        schemaUri,
+        schemaDigest: digestCanonicalJson(schema),
+    });
+}
+export function createAnalysisSchemaValidator(schema, parse) {
+    return Object.freeze({ schema, parse });
+}
+export function createStatelessAnalysisImplementation(input) {
+    return Object.freeze({
+        identity: input.identity,
+        outputSchema: input.outputSchema,
+        async openRun() {
+            return {
+                async execute(context) {
+                    if (context.signal.aborted)
+                        throw context.signal.reason;
+                    input.parseParameters(context.node.parameters);
+                    return input.execute(context);
+                },
+                dispose() { },
+            };
+        },
+    });
+}

package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-node.d.ts

@@ -0,0 +1,5 @@
+import { type RuntimeIdentity } from '../../../eval-core/contracts/index.js';
+import type { AnalysisNodeImplementation } from '../../../eval-core/analysis/index.js';
+export declare const ASSERTION_LAYER_ANALYSIS_IMPLEMENTATION_ID: "omk.assertion-layer-table/v1";
+export declare const ASSERTION_LAYER_ANALYSIS_IDENTITY: RuntimeIdentity;
+export declare function createAssertionLayerAnalysisNodes(): ReadonlyMap<string, AnalysisNodeImplementation>;

package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-node.js

@@ -0,0 +1,218 @@
+import { RuntimeIdentitySchema, canonicalizeJson, deepFreezeCanonicalJson, digestCanonicalJson, } from '../../../eval-core/contracts/index.js';
+import { ASSERTION_LAYER_PARAMETERS_SCHEMA, parseAssertionLayerParameters, } from './assertion-layer-parameters.js';
+import { ASSERTION_LAYER_SCORE_DECIMALS, ASSERTION_LAYER_SCORE_MAX, ASSERTION_LAYER_SCORE_MIN, ASSERTION_LAYER_TABLE_SCHEMA, ASSERTION_LAYER_TABLE_SCHEMA_VERSION, ASSERTION_NOT_APPLICABLE_REASON, assertionLayerAggregate, assertionLayerCoverage, assertionLayerGroupId, compareAssertionLayerGroups, parseAssertionLayerTableValue, } from './assertion-layer.js';
+import { compareStrings, createStatelessAnalysisImplementation, } from './analysis-support.js';
+export const ASSERTION_LAYER_ANALYSIS_IMPLEMENTATION_ID = 'omk.assertion-layer-table/v1';
+const ALGORITHM_VERSION = 'omk.assertion-layer-aggregation/v1';
+const ASSERTION_LAYER_CAPABILITIES = {
+    capabilityKind: 'analysis-node',
+    analysisNodeKinds: ['reducer'],
+    inputDomains: [{
+            inputKind: 'metric-observations',
+            valueTypes: ['boolean'],
+            missingPolicyIds: ['exclude/v1'],
+        }],
+    outputSchema: ASSERTION_LAYER_TABLE_SCHEMA,
+    parameterSchema: ASSERTION_LAYER_PARAMETERS_SCHEMA,
+    inputCardinalities: {
+        metricObservations: { min: 1 },
+        analysisResults: { min: 0, max: 0 },
+        comparisons: { min: 0, max: 0 },
+    },
+    schemas: [],
+};
+export const ASSERTION_LAYER_ANALYSIS_IDENTITY = deepFreezeCanonicalJson(RuntimeIdentitySchema.parse({
+    implementationId: ASSERTION_LAYER_ANALYSIS_IMPLEMENTATION_ID,
+    version: '1.0.0',
+    fingerprint: digestCanonicalJson({
+        implementationId: ASSERTION_LAYER_ANALYSIS_IMPLEMENTATION_ID,
+        algorithmVersion: ALGORITHM_VERSION,
+        scoreMapping: {
+            min: ASSERTION_LAYER_SCORE_MIN,
+            max: ASSERTION_LAYER_SCORE_MAX,
+            decimals: ASSERTION_LAYER_SCORE_DECIMALS,
+        },
+        missingPolicyId: 'exclude/v1',
+        metricContract: { scope: 'sample', direction: 'higher-is-better' },
+        structuralNotApplicableReason: ASSERTION_NOT_APPLICABLE_REASON,
+        samplingUnitLineage: 'preserved-from-analysis-metric-rows',
+        criterionDesign: 'sealed-explicit-parameters',
+        outputSchema: ASSERTION_LAYER_TABLE_SCHEMA,
+        parameterSchema: ASSERTION_LAYER_PARAMETERS_SCHEMA,
+        declaredCapabilities: ASSERTION_LAYER_CAPABILITIES,
+    }),
+    fingerprintBasis: 'self-reported',
+    assuranceLevel: 'declared',
+    capabilities: ASSERTION_LAYER_CAPABILITIES,
+    implementationManifest: { coverageKind: 'fingerprint-complete' },
+}));
+function metricInputs(context) {
+    const inputs = context.inputs.filter((input) => (input.inputKind === 'metric-observations'));
+    if (inputs.length === 0 || inputs.length !== context.inputs.length) {
+        throw new TypeError('Assertion-layer Analysis requires one or more Metric inputs only.');
+    }
+    for (const input of inputs) {
+        if (input.referenceId !== input.metric.metricId
+            || input.metric.valueType !== 'boolean'
+            || input.metric.scope !== 'sample'
+            || input.metric.direction !== 'higher-is-better'
+            || input.metric.missingPolicyId !== 'exclude/v1') {
+            throw new TypeError('Assertion-layer Analysis requires reference-aligned, sample-scoped, '
+                + 'higher-is-better, exclude/v1 Boolean Metrics.');
+        }
+    }
+    return inputs;
+}
+function baseUnitKey(row) {
+    return canonicalizeJson([row.targetId, row.sampleId, row.trialIndex, row.trialId]);
+}
+function entryFromRow(row, criterion) {
+    const base = {
+        ...criterion,
+        rowId: row.rowId,
+        evaluatorId: row.evaluatorId,
+        censored: row.censored,
+    };
+    if (row.rowStatus === 'observed') {
+        if (typeof row.value !== 'boolean') {
+            throw new TypeError('Observed assertion reading must be Boolean.');
+        }
+        return { ...base, applicability: 'applicable', rowStatus: 'observed', value: row.value };
+    }
+    if (row.reasonCode === ASSERTION_NOT_APPLICABLE_REASON) {
+        if (row.rowStatus !== 'missing' || row.censored) {
+            throw new TypeError('criterion-not-applicable must be a non-censored missing row.');
+        }
+        return {
+            ...base,
+            censored: false,
+            applicability: 'not-applicable',
+            rowStatus: 'missing',
+            reasonCode: ASSERTION_NOT_APPLICABLE_REASON,
+        };
+    }
+    return {
+        ...base,
+        applicability: 'applicable',
+        rowStatus: row.rowStatus,
+        reasonCode: row.reasonCode,
+    };
+}
+function validateInputDesign(inputs, parameters) {
+    const criteriaByMetric = new Map(parameters.criteria.map((criterion) => [
+        criterion.metricId,
+        criterion,
+    ]));
+    const inputMetricIds = inputs.map((input) => input.metric.metricId);
+    if (new Set(inputMetricIds).size !== inputMetricIds.length
+        || canonicalizeJson([...inputMetricIds].sort(compareStrings))
+            !== canonicalizeJson(parameters.criteria.map((criterion) => criterion.metricId))) {
+        throw new TypeError('Assertion-layer parameters must map every input Metric exactly once and no others.');
+    }
+    return criteriaByMetric;
+}
+function buildAssertionLayerTable(inputs, parameters, signal) {
+    const criteriaByMetric = validateInputDesign(inputs, parameters);
+    const groupedRows = new Map();
+    for (const input of inputs) {
+        for (const row of input.rows) {
+            if (signal.aborted)
+                throw signal.reason;
+            if (row.metricId !== input.metric.metricId || row.valueType !== 'boolean') {
+                throw new TypeError('Assertion-layer row identity or value type disagrees with its Metric.');
+            }
+            const key = baseUnitKey(row);
+            const members = groupedRows.get(key) ?? [];
+            members.push(row);
+            groupedRows.set(key, members);
+        }
+    }
+    const groups = [...groupedRows.entries()]
+        .sort(([left], [right]) => compareStrings(left, right))
+        .map(([, rows]) => {
+        const orderedRows = [...rows].sort((left, right) => (compareStrings(left.metricId, right.metricId)));
+        const first = orderedRows[0];
+        if (orderedRows.length !== parameters.criteria.length
+            || new Set(orderedRows.map((row) => row.metricId)).size !== orderedRows.length) {
+            throw new TypeError('Every assertion measurement unit must contain exactly one row per Metric.');
+        }
+        if (new Set(orderedRows.map((row) => canonicalizeJson(row.samplingUnitIds))).size !== 1) {
+            throw new TypeError('Assertion rows disagree on sealed sampling-unit lineage.');
+        }
+        const entries = orderedRows.map((row) => {
+            const criterion = criteriaByMetric.get(row.metricId);
+            if (criterion === undefined) {
+                throw new TypeError(`Assertion Metric "${row.metricId}" is not mapped by parameters.`);
+            }
+            return entryFromRow(row, criterion);
+        });
+        const fact = entries.filter((entry) => entry.layerDisposition === 'fact');
+        const behavior = entries.filter((entry) => entry.layerDisposition === 'behavior');
+        const excluded = entries.filter((entry) => (entry.layerDisposition === 'excluded-mixed-layer'));
+        const withoutGroupId = {
+            targetId: first.targetId,
+            sampleId: first.sampleId,
+            trialIndex: first.trialIndex,
+            trialId: first.trialId,
+            samplingUnitIds: first.samplingUnitIds,
+            entries,
+            layers: {
+                fact: assertionLayerAggregate(fact),
+                behavior: assertionLayerAggregate(behavior),
+            },
+            excludedMixedLayer: { coverage: assertionLayerCoverage(excluded) },
+        };
+        return { groupId: assertionLayerGroupId(withoutGroupId), ...withoutGroupId };
+    });
+    groups.sort(compareAssertionLayerGroups);
+    return parseAssertionLayerTableValue({
+        schemaVersion: ASSERTION_LAYER_TABLE_SCHEMA_VERSION,
+        groups,
+    });
+}
+export function createAssertionLayerAnalysisNodes() {
+    const implementation = createStatelessAnalysisImplementation({
+        identity: ASSERTION_LAYER_ANALYSIS_IDENTITY,
+        outputSchema: ASSERTION_LAYER_TABLE_SCHEMA,
+        parseParameters: (parameters) => { parseAssertionLayerParameters(parameters); },
+        execute(context) {
+            const parameters = parseAssertionLayerParameters(context.node.parameters);
+            const inputs = metricInputs(context);
+            validateInputDesign(inputs, parameters);
+            if (inputs.every((input) => input.rows.length === 0)) {
+                return {
+                    analysisStatus: 'inconclusive',
+                    reasonCodes: ['assertion-layer-no-planned-rows'],
+                    includedRowIds: [],
+                    comparableRowIds: [],
+                    assumptionChecks: [{
+                            assumptionId: 'assertion-layer-contract',
+                            checkStatus: 'failed',
+                            reasonCode: 'assertion-layer-no-planned-rows',
+                        }],
+                };
+            }
+            const table = buildAssertionLayerTable(inputs, parameters, context.signal);
+            const dispositionByMetric = new Map(parameters.criteria.map((criterion) => [
+                criterion.metricId,
+                criterion.layerDisposition,
+            ]));
+            const includedRowIds = inputs.flatMap((input) => input.rows.flatMap((row) => (row.rowStatus === 'observed'
+                && dispositionByMetric.get(row.metricId) !== 'excluded-mixed-layer'
+                ? [row.rowId]
+                : []))).sort(compareStrings);
+            return {
+                analysisStatus: 'completed',
+                resultType: 'table',
+                value: table,
+                includedRowIds,
+                comparableRowIds: includedRowIds,
+                assumptionChecks: [{
+                        assumptionId: 'assertion-layer-contract',
+                        checkStatus: 'passed',
+                    }],
+            };
+        },
+    });
+    return new Map([[ASSERTION_LAYER_ANALYSIS_IMPLEMENTATION_ID, implementation]]);
+}

package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-parameters.d.ts

@@ -0,0 +1,39 @@
+import { z } from 'zod';
+import { type CoreSchemaValidator } from '../../../eval-core/contracts/index.js';
+export declare const AssertionLayerDispositionSchema: z.ZodEnum<{
+    behavior: "behavior";
+    fact: "fact";
+    "excluded-mixed-layer": "excluded-mixed-layer";
+}>;
+export declare const AssertionLayerCriterionParameterSchema: z.ZodObject<{
+    criterionId: z.ZodString;
+    metricId: z.ZodString;
+    layerDisposition: z.ZodEnum<{
+        behavior: "behavior";
+        fact: "fact";
+        "excluded-mixed-layer": "excluded-mixed-layer";
+    }>;
+    weight: z.ZodNumber;
+}, z.core.$strict>;
+declare const AssertionLayerParametersSchema: z.ZodObject<{
+    criteria: z.ZodArray<z.ZodObject<{
+        criterionId: z.ZodString;
+        metricId: z.ZodString;
+        layerDisposition: z.ZodEnum<{
+            behavior: "behavior";
+            fact: "fact";
+            "excluded-mixed-layer": "excluded-mixed-layer";
+        }>;
+        weight: z.ZodNumber;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type AssertionLayerParameters = z.infer<typeof AssertionLayerParametersSchema>;
+export type AssertionLayerCriterionParameter = z.infer<typeof AssertionLayerCriterionParameterSchema>;
+export declare function parseAssertionLayerParameters(value: unknown): AssertionLayerParameters;
+export declare const ASSERTION_LAYER_PARAMETERS_SCHEMA: {
+    schemaVersion: string;
+    schemaUri: string;
+    schemaDigest: string;
+};
+export declare function createAssertionLayerParameterSchemaValidators(): ReadonlyMap<string, CoreSchemaValidator>;
+export {};

package/dist/eval-workflows/runtime-adapter/analysis/assertion-layer-parameters.js

@@ -0,0 +1,49 @@
+import { z } from 'zod';
+import { IdentifierSchema, schemaIdentityKey, } from '../../../eval-core/contracts/index.js';
+import { analysisJsonSchema, analysisSchemaIdentity, compareStrings, createAnalysisSchemaValidator, } from './analysis-support.js';
+const PARAMETERS_SCHEMA_VERSION = 'omk.parameters.assertion-layer/v1';
+export const AssertionLayerDispositionSchema = z.enum([
+    'fact',
+    'behavior',
+    'excluded-mixed-layer',
+]);
+export const AssertionLayerCriterionParameterSchema = z.object({
+    criterionId: IdentifierSchema,
+    metricId: IdentifierSchema,
+    layerDisposition: AssertionLayerDispositionSchema,
+    weight: z.number().finite().positive(),
+}).strict();
+const AssertionLayerParametersSchema = z.object({
+    criteria: z.array(AssertionLayerCriterionParameterSchema).min(1),
+}).strict().superRefine((parameters, context) => {
+    for (const field of ['criterionId', 'metricId']) {
+        const values = parameters.criteria.map((criterion) => criterion[field]);
+        if (new Set(values).size !== values.length) {
+            context.addIssue({
+                code: 'custom',
+                path: ['criteria'],
+                message: `Assertion-layer ${field} values must be unique.`,
+            });
+        }
+    }
+});
+function compareCriteria(left, right) {
+    return compareStrings(left.metricId, right.metricId)
+        || compareStrings(left.criterionId, right.criterionId);
+}
+export function parseAssertionLayerParameters(value) {
+    const parsed = AssertionLayerParametersSchema.parse(value);
+    return {
+        criteria: [...parsed.criteria].sort(compareCriteria),
+    };
+}
+export const ASSERTION_LAYER_PARAMETERS_SCHEMA = analysisSchemaIdentity(PARAMETERS_SCHEMA_VERSION, 'urn:omk:parameters:assertion-layer:v1', analysisJsonSchema(AssertionLayerParametersSchema, [
+    'criteria are normalized by metricId and criterionId before plan sealing',
+    'criterionId and metricId are independently unique',
+    'layer disposition is explicit and never inferred from evaluator or assertion strings',
+    'weights are finite and strictly positive',
+]));
+export function createAssertionLayerParameterSchemaValidators() {
+    const validator = createAnalysisSchemaValidator(ASSERTION_LAYER_PARAMETERS_SCHEMA, (value) => parseAssertionLayerParameters(value));
+    return new Map([[schemaIdentityKey(validator.schema), validator]]);
+}