openmerit 0.1.4 → 0.1.6-preview.0

package/dist/protocol/src/schemas.js

@@ -0,0 +1,280 @@
+import { Type } from "typebox";
+import { Check, Errors } from "typebox/value";
+export const OPENMERIT_SCHEMA_DIALECT = "https://json-schema.org/draft/2020-12/schema";
+export const OPENMERIT_RESULT_SCHEMA_VERSION = "0.2";
+const strictObject = (properties, options = {}) => Type.Object(properties, { additionalProperties: false, ...options });
+const metricIds = [
+    "task_quality", "task_success", "consistency", "instruction_following",
+    "tool_use_performance", "structured_output_reliability", "hallucination_rate",
+    "reasoning_efficiency", "input_token_consumption", "output_token_consumption",
+    "total_task_cost", "cost_per_successful_task", "time_to_first_useful_output",
+    "end_to_end_task_latency", "throughput", "retry_rate", "agent_steps",
+    "recovery_ability", "context_handling", "retrieval_use_quality",
+    "long_horizon_performance", "human_intervention_rate", "preference_fit",
+];
+const evidenceSources = [
+    "harness_trace", "evaluation_artifact", "observability_record", "external_source", "user_feedback",
+];
+const stringEnum = (values) => Type.Union(values.map((value) => Type.Literal(value)));
+const cadenceSchema = () => strictObject({
+    afterCompletedTasks: Type.Optional(Type.Integer({ minimum: 1 })),
+    afterElapsedSeconds: Type.Optional(Type.Integer({ minimum: 1 })),
+});
+export const EvidenceReferenceSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    source: stringEnum(evidenceSources),
+    uri: Type.String({ minLength: 1 }),
+    mediaType: Type.Optional(Type.String({ minLength: 1 })),
+    digest: Type.Optional(Type.String({ minLength: 1 })),
+});
+export const MetricObjectiveSchema = strictObject({
+    metricId: stringEnum(metricIds),
+    required: Type.Boolean(),
+    minimumSamples: Type.Integer({ minimum: 1 }),
+    tolerance: Type.Number({ minimum: 0 }),
+    samplingPlan: strictObject({
+        strategy: stringEnum([
+            "single_run", "repeated_same_input", "representative_task_set",
+            "repeated_representative_task_set", "load_test",
+        ]),
+        representativeCaseCount: Type.Integer({ minimum: 1 }),
+        repetitionsPerCase: Type.Integer({ minimum: 1 }),
+        aggregation: stringEnum(["single_value", "mean", "rate", "distribution", "percentile"]),
+        rationale: Type.String({ minLength: 1 }),
+    }),
+    constraint: Type.Optional(strictObject({
+        operator: stringEnum(["at_least", "at_most"]),
+        value: Type.Number(),
+    })),
+});
+export const ApplicationLlmTargetSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    kind: Type.Literal("application_llm_call"),
+    name: Type.String({ minLength: 1 }),
+    applicationId: Type.String({ minLength: 1 }),
+    callSites: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    routeKey: Type.String({ minLength: 1 }),
+    confirmedAt: Type.String({ minLength: 1 }),
+});
+export const TaskProfileSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    revision: Type.Integer({ minimum: 1 }),
+    name: Type.String({ minLength: 1 }),
+    goal: Type.String({ minLength: 1 }),
+    applicationTarget: ApplicationLlmTargetSchema,
+    objectives: Type.Array(MetricObjectiveSchema, { minItems: 1 }),
+    createdAt: Type.String({ minLength: 1 }),
+    confirmedAt: Type.Optional(Type.String({ minLength: 1 })),
+});
+export const ObservabilityCoverageSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    status: stringEnum(["ready", "incomplete"]),
+    requiredMetricIds: Type.Array(stringEnum(metricIds)),
+    coveredMetricIds: Type.Array(stringEnum(metricIds)),
+    missingMetricIds: Type.Array(stringEnum(metricIds)),
+    checkedAt: Type.String({ minLength: 1 }),
+});
+const ConfirmedTaskProfileSchema = strictObject({
+    ...TaskProfileSchema.properties,
+    confirmedAt: Type.String({ minLength: 1 }),
+});
+export const SupervisedGraduationPolicySchema = strictObject({
+    mode: Type.Literal("supervised_graduation"),
+    evaluationBudget: strictObject({
+        currency: Type.String({ minLength: 1 }),
+        maximumSpend: Type.Number({ minimum: 0 }),
+        maximumCandidateCount: Type.Optional(Type.Integer({ minimum: 1 })),
+        maximumRunsPerCandidate: Type.Optional(Type.Integer({ minimum: 1 })),
+        expiresAt: Type.Optional(Type.String({ minLength: 1 })),
+    }),
+    automaticSwapsEnabled: Type.Boolean(),
+    confirmationRequiredUntilVerifiedSwaps: Type.Integer({ minimum: 0 }),
+    requirePostSwapVerification: Type.Literal(true),
+    rollbackOnRegression: Type.Boolean(),
+    checkPolicy: strictObject({
+        baselineAssessment: cadenceSchema(),
+        frontierReassessment: strictObject({
+            afterCompletedTasks: Type.Optional(Type.Integer({ minimum: 1 })),
+            afterElapsedSeconds: Type.Optional(Type.Integer({ minimum: 1 })),
+            onModelCatalogChange: Type.Boolean(),
+        }),
+        regressionCheck: strictObject({
+            afterCompletedTasks: Type.Optional(Type.Integer({ minimum: 1 })),
+            afterElapsedSeconds: Type.Optional(Type.Integer({ minimum: 1 })),
+            thresholds: Type.Array(strictObject({
+                metricId: stringEnum(metricIds),
+                relativeChangeAtLeast: Type.Number({ exclusiveMinimum: 0 }),
+            })),
+        }),
+        postSwapVerification: cadenceSchema(),
+        cooldownSeconds: Type.Integer({ minimum: 0 }),
+        execution: strictObject({
+            mode: stringEnum(["active_session_only", "persistent"]),
+            infrastructureChangesAllowed: Type.Boolean(),
+        }),
+    }),
+});
+export const MetricRecordSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    metricId: stringEnum(metricIds),
+    state: stringEnum(["measured", "not_applicable", "not_configured", "insufficient_evidence"]),
+    scope: stringEnum(["single_run", "sample_window"]),
+    direction: stringEnum(["maximize", "minimize"]),
+    method: stringEnum(["observed", "derived", "evaluator", "human", "external"]),
+    aggregation: stringEnum(["single_value", "sum", "mean", "rate", "distribution", "percentile"]),
+    value: Type.Optional(Type.Number()),
+    unit: Type.String({ minLength: 1 }),
+    sampleCount: Type.Integer({ minimum: 0 }),
+    interval: Type.Optional(strictObject({
+        lower: Type.Number(),
+        upper: Type.Number(),
+        confidenceLevel: Type.Optional(Type.Number({ minimum: 0, maximum: 1 })),
+    })),
+    window: Type.Optional(strictObject({
+        startedAt: Type.String({ minLength: 1 }),
+        endedAt: Type.String({ minLength: 1 }),
+    })),
+    evidence: Type.Array(EvidenceReferenceSchema),
+    note: Type.Optional(Type.String()),
+});
+export const CandidateDescriptorSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    candidateId: Type.String({ minLength: 1 }),
+    modelId: Type.String({ minLength: 1 }),
+    modelVersion: Type.Optional(Type.String({ minLength: 1 })),
+    reasons: Type.Array(Type.String()),
+});
+export const PreliminaryModelLeadSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    modelId: Type.String({ minLength: 1 }),
+    modelVersion: Type.Optional(Type.String({ minLength: 1 })),
+    estimatedPrice: strictObject({
+        currency: Type.String({ minLength: 1 }),
+        inputPerMillionTokens: Type.Number({ minimum: 0 }),
+        outputPerMillionTokens: Type.Number({ minimum: 0 }),
+        sourceUri: Type.String({ minLength: 1 }),
+        checkedAt: Type.String({ minLength: 1 }),
+    }),
+    reputation: strictObject({
+        summary: Type.String({ minLength: 1 }),
+        sourceUri: Type.String({ minLength: 1 }),
+        checkedAt: Type.String({ minLength: 1 }),
+    }),
+});
+export const CandidateAssessmentSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    targetId: Type.String({ minLength: 1 }),
+    taskProfileId: Type.String({ minLength: 1 }),
+    taskProfileRevision: Type.Integer({ minimum: 1 }),
+    candidateId: Type.String({ minLength: 1 }),
+    modelId: Type.String({ minLength: 1 }),
+    modelVersion: Type.Optional(Type.String({ minLength: 1 })),
+    productRevision: Type.String({ minLength: 1 }),
+    harnessId: Type.String({ minLength: 1 }),
+    startedAt: Type.String({ minLength: 1 }),
+    completedAt: Type.String({ minLength: 1 }),
+    metrics: Type.Array(MetricRecordSchema),
+});
+export const FrontierSnapshotSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    targetId: Type.String({ minLength: 1 }),
+    taskProfileId: Type.String({ minLength: 1 }),
+    taskProfileRevision: Type.Integer({ minimum: 1 }),
+    productRevision: Type.String({ minLength: 1 }),
+    createdAt: Type.String({ minLength: 1 }),
+    frontierCandidateIds: Type.Array(Type.String()),
+    dominatedCandidateIds: Type.Array(Type.String()),
+    ineligibleCandidateIds: Type.Array(Type.String()),
+    unresolvedCandidateIds: Type.Array(Type.String()),
+    comparisons: Type.Array(strictObject({
+        candidateId: Type.String({ minLength: 1 }),
+        otherCandidateId: Type.String({ minLength: 1 }),
+        state: stringEnum(["dominates", "does_not_dominate", "unresolved"]),
+        reasons: Type.Array(Type.String()),
+    })),
+    assessmentIds: Type.Array(Type.String()),
+    calculationEvidence: Type.Array(EvidenceReferenceSchema, { minItems: 1 }),
+});
+export const FrontierRecommendationSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    frontierSnapshotId: Type.String({ minLength: 1 }),
+    selectedCandidateId: Type.String({ minLength: 1 }),
+    currentCandidateId: Type.String({ minLength: 1 }),
+    reasons: Type.Array(Type.String()),
+    preferenceEvidence: Type.Array(EvidenceReferenceSchema),
+});
+export const INTENT_OUTPUT_SCHEMAS = {
+    establish_evals: strictObject({
+        taskProfile: ConfirmedTaskProfileSchema,
+        incumbent: CandidateDescriptorSchema,
+        policy: SupervisedGraduationPolicySchema,
+        observability: ObservabilityCoverageSchema,
+        preliminaryModelLeads: Type.Optional(Type.Array(PreliminaryModelLeadSchema)),
+    }),
+    instrument_observability: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        coveredMetricIds: Type.Array(stringEnum(metricIds)),
+        missingMetricIds: Type.Array(stringEnum(metricIds)),
+    }),
+    run_assessment: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        baselineEvidenceSufficient: Type.Boolean(),
+        metrics: Type.Array(MetricRecordSchema),
+    }),
+    discover_candidates: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        candidates: Type.Array(CandidateDescriptorSchema),
+    }),
+    run_challenger_trials: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        assessments: Type.Array(CandidateAssessmentSchema),
+        experimentManifest: EvidenceReferenceSchema,
+    }),
+    calculate_frontier: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        assessments: Type.Array(CandidateAssessmentSchema),
+        frontierSnapshot: FrontierSnapshotSchema,
+        recommendation: Type.Optional(FrontierRecommendationSchema),
+    }),
+    investigate_regression: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        regressionDetected: Type.Boolean(),
+        affectedMetricIds: Type.Array(stringEnum(metricIds)),
+        likelyCause: Type.Optional(Type.String()),
+    }),
+    apply_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        appliedCandidateId: Type.String({ minLength: 1 }),
+        previousCandidateId: Type.Optional(Type.String({ minLength: 1 })),
+    }),
+    verify_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        verified: Type.Boolean(),
+        regressionDetected: Type.Boolean(),
+    }),
+    rollback_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
+        restoredCandidateId: Type.String({ minLength: 1 }),
+    }),
+};
+export function intentOutputSchemaFor(kind) {
+    return INTENT_OUTPUT_SCHEMAS[kind];
+}
+export function validateIntentOutputSchema(kind, output) {
+    const schema = intentOutputSchemaFor(kind);
+    if (Check(schema, output))
+        return [];
+    return [...Errors(schema, output)].map((error) => `${error.instancePath || "/"} ${error.message}`);
+}
+export function completionToolSchemaFor(kind) {
+    return strictObject({
+        intentId: Type.String({ minLength: 1 }),
+        status: stringEnum(["succeeded", "failed", "cancelled"]),
+        summary: Type.String({ minLength: 1 }),
+        evidence: Type.Array(EvidenceReferenceSchema),
+        outputs: Type.Optional(intentOutputSchemaFor(kind)),
+        errorCode: Type.Optional(Type.String({ minLength: 1 })),
+        errorMessage: Type.Optional(Type.String({ minLength: 1 })),
+        retryable: Type.Optional(Type.Boolean()),
+    });
+}