openmerit 0.1.5 → 0.1.6-preview.0

package/dist/protocol/src/schemas.d.ts

@@ -14,21 +14,53 @@ export declare const MetricObjectiveSchema: Type.TObject<{
     required: Type.TBoolean;
     minimumSamples: Type.TInteger;
     tolerance: Type.TNumber;
+    samplingPlan: Type.TObject<{
+        strategy: Type.TUnion<Type.TLiteral<string>[]>;
+        representativeCaseCount: Type.TInteger;
+        repetitionsPerCase: Type.TInteger;
+        aggregation: Type.TUnion<Type.TLiteral<string>[]>;
+        rationale: Type.TString;
+    }>;
     constraint: Type.TOptional<Type.TObject<{
         operator: Type.TUnion<Type.TLiteral<string>[]>;
         value: Type.TNumber;
     }>>;
 }>;
+export declare const ApplicationLlmTargetSchema: Type.TObject<{
+    id: Type.TString;
+    kind: Type.TLiteral<"application_llm_call">;
+    name: Type.TString;
+    applicationId: Type.TString;
+    callSites: Type.TArray<Type.TString>;
+    routeKey: Type.TString;
+    confirmedAt: Type.TString;
+}>;
 export declare const TaskProfileSchema: Type.TObject<{
     id: Type.TString;
     revision: Type.TInteger;
     name: Type.TString;
     goal: Type.TString;
+    applicationTarget: Type.TObject<{
+        id: Type.TString;
+        kind: Type.TLiteral<"application_llm_call">;
+        name: Type.TString;
+        applicationId: Type.TString;
+        callSites: Type.TArray<Type.TString>;
+        routeKey: Type.TString;
+        confirmedAt: Type.TString;
+    }>;
     objectives: Type.TArray<Type.TObject<{
         metricId: Type.TUnion<Type.TLiteral<string>[]>;
         required: Type.TBoolean;
         minimumSamples: Type.TInteger;
         tolerance: Type.TNumber;
+        samplingPlan: Type.TObject<{
+            strategy: Type.TUnion<Type.TLiteral<string>[]>;
+            representativeCaseCount: Type.TInteger;
+            repetitionsPerCase: Type.TInteger;
+            aggregation: Type.TUnion<Type.TLiteral<string>[]>;
+            rationale: Type.TString;
+        }>;
         constraint: Type.TOptional<Type.TObject<{
             operator: Type.TUnion<Type.TLiteral<string>[]>;
             value: Type.TNumber;
@@ -38,6 +70,7 @@ export declare const TaskProfileSchema: Type.TObject<{
     confirmedAt: Type.TOptional<Type.TString>;
 }>;
 export declare const ObservabilityCoverageSchema: Type.TObject<{
+    targetId: Type.TString;
     status: Type.TUnion<Type.TLiteral<string>[]>;
     requiredMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
     coveredMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
@@ -87,11 +120,13 @@ export declare const SupervisedGraduationPolicySchema: Type.TObject<{
     }>;
 }>;
 export declare const MetricRecordSchema: Type.TObject<{
+    targetId: Type.TString;
     metricId: Type.TUnion<Type.TLiteral<string>[]>;
     state: Type.TUnion<Type.TLiteral<string>[]>;
     scope: Type.TUnion<Type.TLiteral<string>[]>;
     direction: Type.TUnion<Type.TLiteral<string>[]>;
     method: Type.TUnion<Type.TLiteral<string>[]>;
+    aggregation: Type.TUnion<Type.TLiteral<string>[]>;
     value: Type.TOptional<Type.TNumber>;
     unit: Type.TString;
     sampleCount: Type.TInteger;
@@ -114,13 +149,32 @@ export declare const MetricRecordSchema: Type.TObject<{
     note: Type.TOptional<Type.TString>;
 }>;
 export declare const CandidateDescriptorSchema: Type.TObject<{
+    targetId: Type.TString;
     candidateId: Type.TString;
     modelId: Type.TString;
     modelVersion: Type.TOptional<Type.TString>;
     reasons: Type.TArray<Type.TString>;
 }>;
+export declare const PreliminaryModelLeadSchema: Type.TObject<{
+    targetId: Type.TString;
+    modelId: Type.TString;
+    modelVersion: Type.TOptional<Type.TString>;
+    estimatedPrice: Type.TObject<{
+        currency: Type.TString;
+        inputPerMillionTokens: Type.TNumber;
+        outputPerMillionTokens: Type.TNumber;
+        sourceUri: Type.TString;
+        checkedAt: Type.TString;
+    }>;
+    reputation: Type.TObject<{
+        summary: Type.TString;
+        sourceUri: Type.TString;
+        checkedAt: Type.TString;
+    }>;
+}>;
 export declare const CandidateAssessmentSchema: Type.TObject<{
     id: Type.TString;
+    targetId: Type.TString;
     taskProfileId: Type.TString;
     taskProfileRevision: Type.TInteger;
     candidateId: Type.TString;
@@ -131,11 +185,13 @@ export declare const CandidateAssessmentSchema: Type.TObject<{
     startedAt: Type.TString;
     completedAt: Type.TString;
     metrics: Type.TArray<Type.TObject<{
+        targetId: Type.TString;
         metricId: Type.TUnion<Type.TLiteral<string>[]>;
         state: Type.TUnion<Type.TLiteral<string>[]>;
         scope: Type.TUnion<Type.TLiteral<string>[]>;
         direction: Type.TUnion<Type.TLiteral<string>[]>;
         method: Type.TUnion<Type.TLiteral<string>[]>;
+        aggregation: Type.TUnion<Type.TLiteral<string>[]>;
         value: Type.TOptional<Type.TNumber>;
         unit: Type.TString;
         sampleCount: Type.TInteger;
@@ -160,6 +216,7 @@ export declare const CandidateAssessmentSchema: Type.TObject<{
 }>;
 export declare const FrontierSnapshotSchema: Type.TObject<{
     id: Type.TString;
+    targetId: Type.TString;
     taskProfileId: Type.TString;
     taskProfileRevision: Type.TInteger;
     productRevision: Type.TString;
@@ -184,6 +241,7 @@ export declare const FrontierSnapshotSchema: Type.TObject<{
     }>>;
 }>;
 export declare const FrontierRecommendationSchema: Type.TObject<{
+    targetId: Type.TString;
     frontierSnapshotId: Type.TString;
     selectedCandidateId: Type.TString;
     currentCandidateId: Type.TString;
@@ -204,11 +262,27 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             revision: Type.TInteger;
             name: Type.TString;
             goal: Type.TString;
+            applicationTarget: Type.TObject<{
+                id: Type.TString;
+                kind: Type.TLiteral<"application_llm_call">;
+                name: Type.TString;
+                applicationId: Type.TString;
+                callSites: Type.TArray<Type.TString>;
+                routeKey: Type.TString;
+                confirmedAt: Type.TString;
+            }>;
             objectives: Type.TArray<Type.TObject<{
                 metricId: Type.TUnion<Type.TLiteral<string>[]>;
                 required: Type.TBoolean;
                 minimumSamples: Type.TInteger;
                 tolerance: Type.TNumber;
+                samplingPlan: Type.TObject<{
+                    strategy: Type.TUnion<Type.TLiteral<string>[]>;
+                    representativeCaseCount: Type.TInteger;
+                    repetitionsPerCase: Type.TInteger;
+                    aggregation: Type.TUnion<Type.TLiteral<string>[]>;
+                    rationale: Type.TString;
+                }>;
                 constraint: Type.TOptional<Type.TObject<{
                     operator: Type.TUnion<Type.TLiteral<string>[]>;
                     value: Type.TNumber;
@@ -216,6 +290,13 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             }>>;
             createdAt: Type.TString;
         }>;
+        incumbent: Type.TObject<{
+            targetId: Type.TString;
+            candidateId: Type.TString;
+            modelId: Type.TString;
+            modelVersion: Type.TOptional<Type.TString>;
+            reasons: Type.TArray<Type.TString>;
+        }>;
         policy: Type.TObject<{
             mode: Type.TLiteral<"supervised_graduation">;
             evaluationBudget: Type.TObject<{
@@ -259,25 +340,47 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             }>;
         }>;
         observability: Type.TObject<{
+            targetId: Type.TString;
             status: Type.TUnion<Type.TLiteral<string>[]>;
             requiredMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
             coveredMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
             missingMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
             checkedAt: Type.TString;
         }>;
+        preliminaryModelLeads: Type.TOptional<Type.TArray<Type.TObject<{
+            targetId: Type.TString;
+            modelId: Type.TString;
+            modelVersion: Type.TOptional<Type.TString>;
+            estimatedPrice: Type.TObject<{
+                currency: Type.TString;
+                inputPerMillionTokens: Type.TNumber;
+                outputPerMillionTokens: Type.TNumber;
+                sourceUri: Type.TString;
+                checkedAt: Type.TString;
+            }>;
+            reputation: Type.TObject<{
+                summary: Type.TString;
+                sourceUri: Type.TString;
+                checkedAt: Type.TString;
+            }>;
+        }>>>;
     }>;
     readonly instrument_observability: Type.TObject<{
+        targetId: Type.TString;
         coveredMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
         missingMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
     }>;
     readonly run_assessment: Type.TObject<{
+        targetId: Type.TString;
         baselineEvidenceSufficient: Type.TBoolean;
-        metrics: Type.TOptional<Type.TArray<Type.TObject<{
+        metrics: Type.TArray<Type.TObject<{
+            targetId: Type.TString;
             metricId: Type.TUnion<Type.TLiteral<string>[]>;
             state: Type.TUnion<Type.TLiteral<string>[]>;
             scope: Type.TUnion<Type.TLiteral<string>[]>;
             direction: Type.TUnion<Type.TLiteral<string>[]>;
             method: Type.TUnion<Type.TLiteral<string>[]>;
+            aggregation: Type.TUnion<Type.TLiteral<string>[]>;
             value: Type.TOptional<Type.TNumber>;
             unit: Type.TString;
             sampleCount: Type.TInteger;
@@ -298,10 +401,12 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
                 digest: Type.TOptional<Type.TString>;
             }>>;
             note: Type.TOptional<Type.TString>;
-        }>>>;
+        }>>;
     }>;
     readonly discover_candidates: Type.TObject<{
+        targetId: Type.TString;
         candidates: Type.TArray<Type.TObject<{
+            targetId: Type.TString;
             candidateId: Type.TString;
             modelId: Type.TString;
             modelVersion: Type.TOptional<Type.TString>;
@@ -309,8 +414,10 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
         }>>;
     }>;
     readonly run_challenger_trials: Type.TObject<{
+        targetId: Type.TString;
         assessments: Type.TArray<Type.TObject<{
             id: Type.TString;
+            targetId: Type.TString;
             taskProfileId: Type.TString;
             taskProfileRevision: Type.TInteger;
             candidateId: Type.TString;
@@ -321,11 +428,13 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             startedAt: Type.TString;
             completedAt: Type.TString;
             metrics: Type.TArray<Type.TObject<{
+                targetId: Type.TString;
                 metricId: Type.TUnion<Type.TLiteral<string>[]>;
                 state: Type.TUnion<Type.TLiteral<string>[]>;
                 scope: Type.TUnion<Type.TLiteral<string>[]>;
                 direction: Type.TUnion<Type.TLiteral<string>[]>;
                 method: Type.TUnion<Type.TLiteral<string>[]>;
+                aggregation: Type.TUnion<Type.TLiteral<string>[]>;
                 value: Type.TOptional<Type.TNumber>;
                 unit: Type.TString;
                 sampleCount: Type.TInteger;
@@ -348,10 +457,19 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
                 note: Type.TOptional<Type.TString>;
             }>>;
         }>>;
+        experimentManifest: Type.TObject<{
+            id: Type.TString;
+            source: Type.TUnion<Type.TLiteral<string>[]>;
+            uri: Type.TString;
+            mediaType: Type.TOptional<Type.TString>;
+            digest: Type.TOptional<Type.TString>;
+        }>;
     }>;
     readonly calculate_frontier: Type.TObject<{
+        targetId: Type.TString;
         assessments: Type.TArray<Type.TObject<{
             id: Type.TString;
+            targetId: Type.TString;
             taskProfileId: Type.TString;
             taskProfileRevision: Type.TInteger;
             candidateId: Type.TString;
@@ -362,11 +480,13 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             startedAt: Type.TString;
             completedAt: Type.TString;
             metrics: Type.TArray<Type.TObject<{
+                targetId: Type.TString;
                 metricId: Type.TUnion<Type.TLiteral<string>[]>;
                 state: Type.TUnion<Type.TLiteral<string>[]>;
                 scope: Type.TUnion<Type.TLiteral<string>[]>;
                 direction: Type.TUnion<Type.TLiteral<string>[]>;
                 method: Type.TUnion<Type.TLiteral<string>[]>;
+                aggregation: Type.TUnion<Type.TLiteral<string>[]>;
                 value: Type.TOptional<Type.TNumber>;
                 unit: Type.TString;
                 sampleCount: Type.TInteger;
@@ -391,6 +511,7 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
         }>>;
         frontierSnapshot: Type.TObject<{
             id: Type.TString;
+            targetId: Type.TString;
             taskProfileId: Type.TString;
             taskProfileRevision: Type.TInteger;
             productRevision: Type.TString;
@@ -415,6 +536,7 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
             }>>;
         }>;
         recommendation: Type.TOptional<Type.TObject<{
+            targetId: Type.TString;
             frontierSnapshotId: Type.TString;
             selectedCandidateId: Type.TString;
             currentCandidateId: Type.TString;
@@ -429,19 +551,23 @@ export declare const INTENT_OUTPUT_SCHEMAS: {
         }>>;
     }>;
     readonly investigate_regression: Type.TObject<{
+        targetId: Type.TString;
         regressionDetected: Type.TBoolean;
         affectedMetricIds: Type.TArray<Type.TUnion<Type.TLiteral<string>[]>>;
         likelyCause: Type.TOptional<Type.TString>;
     }>;
     readonly apply_model_swap: Type.TObject<{
+        targetId: Type.TString;
         appliedCandidateId: Type.TString;
         previousCandidateId: Type.TOptional<Type.TString>;
     }>;
     readonly verify_model_swap: Type.TObject<{
+        targetId: Type.TString;
         verified: Type.TBoolean;
         regressionDetected: Type.TBoolean;
     }>;
     readonly rollback_model_swap: Type.TObject<{
+        targetId: Type.TString;
         restoredCandidateId: Type.TString;
     }>;
 };

package/dist/protocol/src/schemas.js

@@ -32,21 +32,42 @@ export const MetricObjectiveSchema = strictObject({
     required: Type.Boolean(),
     minimumSamples: Type.Integer({ minimum: 1 }),
     tolerance: Type.Number({ minimum: 0 }),
+    samplingPlan: strictObject({
+        strategy: stringEnum([
+            "single_run", "repeated_same_input", "representative_task_set",
+            "repeated_representative_task_set", "load_test",
+        ]),
+        representativeCaseCount: Type.Integer({ minimum: 1 }),
+        repetitionsPerCase: Type.Integer({ minimum: 1 }),
+        aggregation: stringEnum(["single_value", "mean", "rate", "distribution", "percentile"]),
+        rationale: Type.String({ minLength: 1 }),
+    }),
     constraint: Type.Optional(strictObject({
         operator: stringEnum(["at_least", "at_most"]),
         value: Type.Number(),
     })),
 });
+export const ApplicationLlmTargetSchema = strictObject({
+    id: Type.String({ minLength: 1 }),
+    kind: Type.Literal("application_llm_call"),
+    name: Type.String({ minLength: 1 }),
+    applicationId: Type.String({ minLength: 1 }),
+    callSites: Type.Array(Type.String({ minLength: 1 }), { minItems: 1 }),
+    routeKey: Type.String({ minLength: 1 }),
+    confirmedAt: Type.String({ minLength: 1 }),
+});
 export const TaskProfileSchema = strictObject({
     id: Type.String({ minLength: 1 }),
     revision: Type.Integer({ minimum: 1 }),
     name: Type.String({ minLength: 1 }),
     goal: Type.String({ minLength: 1 }),
+    applicationTarget: ApplicationLlmTargetSchema,
     objectives: Type.Array(MetricObjectiveSchema, { minItems: 1 }),
     createdAt: Type.String({ minLength: 1 }),
     confirmedAt: Type.Optional(Type.String({ minLength: 1 })),
 });
 export const ObservabilityCoverageSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
     status: stringEnum(["ready", "incomplete"]),
     requiredMetricIds: Type.Array(stringEnum(metricIds)),
     coveredMetricIds: Type.Array(stringEnum(metricIds)),
@@ -94,11 +115,13 @@ export const SupervisedGraduationPolicySchema = strictObject({
     }),
 });
 export const MetricRecordSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
     metricId: stringEnum(metricIds),
     state: stringEnum(["measured", "not_applicable", "not_configured", "insufficient_evidence"]),
     scope: stringEnum(["single_run", "sample_window"]),
     direction: stringEnum(["maximize", "minimize"]),
     method: stringEnum(["observed", "derived", "evaluator", "human", "external"]),
+    aggregation: stringEnum(["single_value", "sum", "mean", "rate", "distribution", "percentile"]),
     value: Type.Optional(Type.Number()),
     unit: Type.String({ minLength: 1 }),
     sampleCount: Type.Integer({ minimum: 0 }),
@@ -115,13 +138,32 @@ export const MetricRecordSchema = strictObject({
     note: Type.Optional(Type.String()),
 });
 export const CandidateDescriptorSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
     candidateId: Type.String({ minLength: 1 }),
     modelId: Type.String({ minLength: 1 }),
     modelVersion: Type.Optional(Type.String({ minLength: 1 })),
     reasons: Type.Array(Type.String()),
 });
+export const PreliminaryModelLeadSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
+    modelId: Type.String({ minLength: 1 }),
+    modelVersion: Type.Optional(Type.String({ minLength: 1 })),
+    estimatedPrice: strictObject({
+        currency: Type.String({ minLength: 1 }),
+        inputPerMillionTokens: Type.Number({ minimum: 0 }),
+        outputPerMillionTokens: Type.Number({ minimum: 0 }),
+        sourceUri: Type.String({ minLength: 1 }),
+        checkedAt: Type.String({ minLength: 1 }),
+    }),
+    reputation: strictObject({
+        summary: Type.String({ minLength: 1 }),
+        sourceUri: Type.String({ minLength: 1 }),
+        checkedAt: Type.String({ minLength: 1 }),
+    }),
+});
 export const CandidateAssessmentSchema = strictObject({
     id: Type.String({ minLength: 1 }),
+    targetId: Type.String({ minLength: 1 }),
     taskProfileId: Type.String({ minLength: 1 }),
     taskProfileRevision: Type.Integer({ minimum: 1 }),
     candidateId: Type.String({ minLength: 1 }),
@@ -135,6 +177,7 @@ export const CandidateAssessmentSchema = strictObject({
 });
 export const FrontierSnapshotSchema = strictObject({
     id: Type.String({ minLength: 1 }),
+    targetId: Type.String({ minLength: 1 }),
     taskProfileId: Type.String({ minLength: 1 }),
     taskProfileRevision: Type.Integer({ minimum: 1 }),
     productRevision: Type.String({ minLength: 1 }),
@@ -153,6 +196,7 @@ export const FrontierSnapshotSchema = strictObject({
     calculationEvidence: Type.Array(EvidenceReferenceSchema, { minItems: 1 }),
 });
 export const FrontierRecommendationSchema = strictObject({
+    targetId: Type.String({ minLength: 1 }),
     frontierSnapshotId: Type.String({ minLength: 1 }),
     selectedCandidateId: Type.String({ minLength: 1 }),
     currentCandidateId: Type.String({ minLength: 1 }),
@@ -162,42 +206,54 @@ export const FrontierRecommendationSchema = strictObject({
 export const INTENT_OUTPUT_SCHEMAS = {
     establish_evals: strictObject({
         taskProfile: ConfirmedTaskProfileSchema,
+        incumbent: CandidateDescriptorSchema,
         policy: SupervisedGraduationPolicySchema,
         observability: ObservabilityCoverageSchema,
+        preliminaryModelLeads: Type.Optional(Type.Array(PreliminaryModelLeadSchema)),
     }),
     instrument_observability: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         coveredMetricIds: Type.Array(stringEnum(metricIds)),
         missingMetricIds: Type.Array(stringEnum(metricIds)),
     }),
     run_assessment: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         baselineEvidenceSufficient: Type.Boolean(),
-        metrics: Type.Optional(Type.Array(MetricRecordSchema)),
+        metrics: Type.Array(MetricRecordSchema),
     }),
     discover_candidates: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         candidates: Type.Array(CandidateDescriptorSchema),
     }),
     run_challenger_trials: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         assessments: Type.Array(CandidateAssessmentSchema),
+        experimentManifest: EvidenceReferenceSchema,
     }),
     calculate_frontier: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         assessments: Type.Array(CandidateAssessmentSchema),
         frontierSnapshot: FrontierSnapshotSchema,
         recommendation: Type.Optional(FrontierRecommendationSchema),
     }),
     investigate_regression: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         regressionDetected: Type.Boolean(),
         affectedMetricIds: Type.Array(stringEnum(metricIds)),
         likelyCause: Type.Optional(Type.String()),
     }),
     apply_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         appliedCandidateId: Type.String({ minLength: 1 }),
         previousCandidateId: Type.Optional(Type.String({ minLength: 1 })),
     }),
     verify_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         verified: Type.Boolean(),
         regressionDetected: Type.Boolean(),
     }),
     rollback_model_swap: strictObject({
+        targetId: Type.String({ minLength: 1 }),
         restoredCandidateId: Type.String({ minLength: 1 }),
     }),
 };