openmerit 0.1.5 → 0.1.6-preview.0

package/dist/core/src/index.js

@@ -1,14 +1,14 @@
 import { METRIC_CATALOG, OPENMERIT_SCHEMA_DIALECT, PROTOCOL_VERSION, REQUIRED_CAPABILITIES, validateIntentOutputSchema, } from "../../protocol/src/index.js";
 export { OPENMERIT_DIRECTORY, ProjectStore, } from "./store.js";
 const outcomes = {
-    establish_evals: "Infer the current task profile and metric objectives, ask the user to confirm requirements, evaluation budget, check cadence, thresholds, scheduling mode, and automation permissions, establish applicable evals and observability, verify them, and return the confirmed taskProfile and policy. Do not change model configuration.",
-    instrument_observability: "Establish and verify the missing observability required by the confirmed task profile.",
-    run_assessment: "Assess whether current baseline evidence satisfies every required metric and sample threshold. Return measured, missing, and insufficient-evidence metrics explicitly.",
-    discover_candidates: "Discover credible model candidates for the confirmed task profile using current capability, reputation, price, availability, and benchmark signals without exceeding the approved evaluation budget.",
-    run_challenger_trials: "Run comparable controlled challenger trials under the approved budget and return candidate assessments with durable evidence.",
+    establish_evals: "Identify the real application LLM call or shared application route under evaluation and its exact incumbent application model. Infer the metrics required for that application's task from its outputs, tools, failure modes, and representative work. Before running setup, tell the user which metrics will be collected, why each matters, whether it uses repeated identical inputs, representative task instances, both, or load testing, the case and repetition counts, aggregation, any true performance threshold, and estimated evaluation cost; ask the user to confirm or edit that plan together with the target, budget, check cadence, scheduling mode, and automation permissions. Never turn a sample-count requirement into a metric-value constraint. Rate, distribution, reliability, percentile, variance, and average claims require multiple runs. Build runnable task-specific evaluation cases and application-call collectors for every confirmed metric. Execute a representative smoke run through the same application authentication and runtime path, run the grader, and inspect actual outputs. Preserve the commands and resulting artifacts as setup evidence. Mark any metric missing if its collector or grader cannot produce a value; do not declare coverage ready based only on code existing. Do not count a setup smoke run as a production baseline sample. Return the confirmed taskProfile, incumbent, and policy. During setup, research a preliminary model shortlist with current sourced price estimates and reputation; report an empty list if no credible leads are available. These are research priors only: do not treat them as measured task results or change model configuration. The coding harness model is not an evaluation target.",
+    instrument_observability: "Establish the missing observability required by the confirmed task profile. Exercise each collector and task-specific grader through a representative application run with the same authentication and runtime path, inspect actual metric outputs, and preserve verification artifacts. Report a metric covered only if that path produced its value; otherwise keep it missing. Do not count a setup smoke run as a production baseline sample.",
+    run_assessment: "Assess existing application-target baseline metric windows against every required metric and sample threshold. Do not synthesize samples, count setup smoke runs as production data, or wait for new data during this check. If a required window is absent or immature, promptly return baselineEvidenceSufficient false with measured, missing, and insufficient-evidence metrics explicitly and cite the records inspected.",
+    discover_candidates: "Refresh the preliminary model leads collected at setup and discover other credible candidates for the confirmed task profile using current capability, reputation, price, availability, and benchmark signals without exceeding the approved evaluation budget. Preliminary leads are research priors, not measured task evidence.",
+    run_challenger_trials: "Run a balanced, reproducible comparison of the frozen incumbent and challenger set under the approved budget. Use the same evaluation cases, application revision, prompt, tools, grader, and run count for every usable model; use a recorded seed to shuffle execution order. Attribute every run to the exact application model, preserve per-run quality, cost, token, and latency evidence, and return candidate assessments plus one durable experiment manifest. Never substitute catalogue prices for measured application-call cost.",
     calculate_frontier: "Calculate the Pareto frontier using the OpenMerit conformance definition, explain every candidate classification, and return a FrontierSnapshot with calculation evidence.",
     investigate_regression: "Investigate the observed regression and return its likely cause, affected metrics, and supporting evidence.",
-    apply_model_swap: "Apply the approved model change in the harness or product target and return proof of the resulting configuration. Do not broaden the authorized change.",
+    apply_model_swap: "Apply the approved model change only to the bound application LLM target and return proof of the resulting application configuration. Never change the coding harness model and do not broaden the authorized change.",
     verify_model_swap: "Verify the applied model against the approved post-swap evaluation window and report any material regression.",
     rollback_model_swap: "Restore the previously verified model configuration and validate the rollback.",
 };
@@ -17,31 +17,47 @@ export function createOpenMeritIntent(kind, options = {}) {
         protocolVersion: PROTOCOL_VERSION,
         id: options.id ?? `intent-${crypto.randomUUID()}`,
         kind,
+        ...(options.targetId ? { targetId: options.targetId } : {}),
         taskProfileId: options.taskProfileId ?? "current-project",
         authorizationPolicyId: options.authorizationPolicyId ?? "observe-and-evaluate-only",
         requestedOutcome: outcomes[kind],
         constraints: {
             modelMutationAllowed: kind === "apply_model_swap" || kind === "rollback_model_swap",
+            harnessModelMutationAllowed: false,
+            mutationTarget: "application_llm_call",
             requireVerifiedArtifacts: true,
             distinguishSingleRunFromMultiRunClaims: true,
             ...(options.policy ? { evaluationBudget: options.policy.evaluationBudget } : {}),
+            ...(options.policy ? {
+                evaluationSpend: options.evaluationSpend ?? 0,
+                remainingEvaluationBudget: Math.max(0, options.policy.evaluationBudget.maximumSpend - (options.evaluationSpend ?? 0)),
+            } : {}),
             ...(options.policy ? { checkPolicy: options.policy.checkPolicy } : {}),
+            ...(options.additionalConstraints ?? {}),
         },
         requiredEvidence: kind === "establish_evals"
             ? [
                 { id: "task-profile", description: "The inferred task profile and the user constraints it represents." },
+                { id: "application-target", description: "The user-confirmed application LLM call or shared route, including stable call-site and route identifiers." },
+                { id: "incumbent-model", description: "The exact application model configured before challenger trials." },
                 { id: "evaluation-artifacts", description: "Runnable evaluation artifacts and verification results." },
                 { id: "observability-coverage", description: "The observable metrics and explicit coverage gaps." },
+                { id: "preliminary-model-research", description: "Sources for preliminary model pricing and reputation, or an explanation that no credible leads were found." },
             ]
-            : kind === "calculate_frontier"
+            : kind === "run_challenger_trials"
                 ? [
-                    { id: "frontier-snapshot", description: "A complete harness-calculated FrontierSnapshot." },
-                    { id: "frontier-calculation", description: "Durable evidence of the frontier calculation." },
+                    { id: "experiment-manifest", description: "A durable manifest containing the seed, frozen cases, application revision, exact model IDs, and per-run evidence." },
+                    { id: "assessment-results", description: "Comparable candidate assessments tied to the active task profile." },
                 ]
-                : [
-                    { id: "assessment-results", description: "Assessment results tied to the active task profile." },
-                    { id: "metric-coverage", description: "Observed, derived, and insufficient-evidence metric states." },
-                ],
+                : kind === "calculate_frontier"
+                    ? [
+                        { id: "frontier-snapshot", description: "A complete harness-calculated FrontierSnapshot." },
+                        { id: "frontier-calculation", description: "Durable evidence of the frontier calculation." },
+                    ]
+                    : [
+                        { id: "assessment-results", description: "Assessment results tied to the active task profile." },
+                        { id: "metric-coverage", description: "Observed, derived, and insufficient-evidence metric states." },
+                    ],
         requestedAt: options.requestedAt ?? new Date().toISOString(),
     };
 }
@@ -126,13 +142,16 @@ const knownMetricIds = new Set(METRIC_CATALOG.map((metric) => metric.id));
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function validateSetupOutput(output) {
-    if (!isRecord(output) || !isRecord(output.taskProfile) || !isRecord(output.policy)) {
-        return ["setup output requires taskProfile and policy"];
+function validateSetupOutput(output, expectedIncumbentModelId) {
+    if (!isRecord(output) || !isRecord(output.taskProfile) || !isRecord(output.incumbent) || !isRecord(output.policy)) {
+        return ["setup output requires taskProfile, incumbent, and policy"];
     }
     const profile = output.taskProfile;
     const policy = output.policy;
     const errors = [];
+    if (expectedIncumbentModelId && output.incumbent.modelId !== expectedIncumbentModelId) {
+        errors.push(`incumbent model ${String(output.incumbent.modelId)} does not match the application source model ${expectedIncumbentModelId}`);
+    }
     if (typeof profile.id !== "string" || typeof profile.name !== "string" ||
         typeof profile.goal !== "string" || !Number.isInteger(profile.revision) ||
         typeof profile.confirmedAt !== "string" || !Array.isArray(profile.objectives) ||
@@ -148,6 +167,49 @@ function validateSetupOutput(output) {
                 errors.push("taskProfile contains an invalid metric objective");
                 break;
             }
+            const samplingPlan = objective.samplingPlan;
+            if (!isRecord(samplingPlan) || typeof samplingPlan.strategy !== "string" ||
+                !Number.isInteger(samplingPlan.representativeCaseCount) || samplingPlan.representativeCaseCount < 1 ||
+                !Number.isInteger(samplingPlan.repetitionsPerCase) || samplingPlan.repetitionsPerCase < 1 ||
+                typeof samplingPlan.aggregation !== "string" || typeof samplingPlan.rationale !== "string" ||
+                !samplingPlan.rationale.trim()) {
+                errors.push(`${objective.metricId} requires a complete sampling plan`);
+                continue;
+            }
+            const plannedSamples = samplingPlan.representativeCaseCount *
+                samplingPlan.repetitionsPerCase;
+            if (objective.minimumSamples < plannedSamples) {
+                errors.push(`${objective.metricId} minimumSamples must cover its sampling plan`);
+            }
+            const aggregateClaim = ["mean", "rate", "distribution", "percentile"].includes(samplingPlan.aggregation);
+            const metricDefinition = METRIC_CATALOG.find((metric) => metric.id === objective.metricId);
+            if ((aggregateClaim || metricDefinition?.requiresRepeatedRuns) && plannedSamples < 2) {
+                errors.push(`${objective.metricId} requires multiple planned runs for its ${String(samplingPlan.aggregation)} claim`);
+            }
+            if ((aggregateClaim || metricDefinition?.requiresRepeatedRuns) && samplingPlan.strategy === "single_run") {
+                errors.push(`${objective.metricId} cannot use single_run for a repeated-run claim`);
+            }
+            if (isRecord(objective.constraint) &&
+                ["task_success", "structured_output_reliability", "hallucination_rate", "retry_rate", "recovery_ability", "human_intervention_rate"].includes(objective.metricId) &&
+                (typeof objective.constraint.value !== "number" || objective.constraint.value < 0 || objective.constraint.value > 1)) {
+                errors.push(`${objective.metricId} ratio constraint must be between 0 and 1`);
+            }
+        }
+    }
+    if (!isRecord(profile.applicationTarget) || profile.applicationTarget.kind !== "application_llm_call" ||
+        typeof profile.applicationTarget.id !== "string" || !profile.applicationTarget.id.trim() ||
+        typeof profile.applicationTarget.applicationId !== "string" || !profile.applicationTarget.applicationId.trim() ||
+        typeof profile.applicationTarget.routeKey !== "string" || !profile.applicationTarget.routeKey.trim() ||
+        typeof profile.applicationTarget.confirmedAt !== "string" ||
+        !Array.isArray(profile.applicationTarget.callSites) || profile.applicationTarget.callSites.length === 0 ||
+        profile.applicationTarget.callSites.some((callSite) => typeof callSite !== "string" || !callSite.trim())) {
+        errors.push("taskProfile requires a user-confirmed application LLM target with stable call sites and route key");
+    }
+    if (isRecord(profile.applicationTarget)) {
+        errors.push(...validateTargetId(output.incumbent.targetId, profile.applicationTarget.id, "incumbent"));
+        if (typeof output.incumbent.candidateId !== "string" || !output.incumbent.candidateId.trim() ||
+            typeof output.incumbent.modelId !== "string" || !output.incumbent.modelId.trim()) {
+            errors.push("incumbent requires stable candidateId and exact modelId");
         }
     }
     const budget = policy.evaluationBudget;
@@ -158,20 +220,71 @@ function validateSetupOutput(output) {
         policy.requirePostSwapVerification !== true || typeof policy.rollbackOnRegression !== "boolean") {
         errors.push("policy must be a valid supervised-graduation policy with an evaluation budget");
     }
+    const checkPolicy = policy.checkPolicy;
+    if (!isRecord(checkPolicy) || !isRecord(checkPolicy.baselineAssessment) ||
+        (typeof checkPolicy.baselineAssessment.afterCompletedTasks !== "number" &&
+            typeof checkPolicy.baselineAssessment.afterElapsedSeconds !== "number")) {
+        errors.push("policy requires a baseline assessment cadence");
+    }
+    if (!isRecord(checkPolicy) || !isRecord(checkPolicy.postSwapVerification) ||
+        (typeof checkPolicy.postSwapVerification.afterCompletedTasks !== "number" &&
+            typeof checkPolicy.postSwapVerification.afterElapsedSeconds !== "number")) {
+        errors.push("policy requires a post-swap verification cadence");
+    }
     if (!isRecord(output.observability)) {
         errors.push("setup output requires an observability coverage report");
     }
     else if (Array.isArray(profile.objectives)) {
         errors.push(...validateObservabilityCoverage(profile, output.observability, false));
     }
+    if (Array.isArray(output.preliminaryModelLeads) && isRecord(profile.applicationTarget)) {
+        const seen = new Set();
+        for (const lead of output.preliminaryModelLeads) {
+            if (!isRecord(lead))
+                continue;
+            errors.push(...validateTargetId(lead.targetId, profile.applicationTarget.id, `preliminary lead ${String(lead.modelId)}`));
+            const identity = `${String(lead.modelId)}@${String(lead.modelVersion ?? "")}`;
+            if (seen.has(identity))
+                errors.push(`duplicate preliminary model lead ${identity}`);
+            seen.add(identity);
+        }
+    }
+    return errors;
+}
+function validateMetricAggregation(metric, objective) {
+    const errors = [];
+    if (metric.metricId === "total_task_cost" && metric.state === "measured" &&
+        metric.scope === "sample_window" && metric.aggregation !== "sum") {
+        errors.push("total_task_cost sample windows must use sum aggregation because the value is used for evaluation-budget accounting");
+    }
+    if (objective?.samplingPlan && metric.state === "measured" &&
+        metric.aggregation !== objective.samplingPlan.aggregation &&
+        !(objective.metricId === "total_task_cost" && metric.aggregation === "sum")) {
+        errors.push(`${objective.metricId} aggregation ${metric.aggregation} does not match its sampling plan ${objective.samplingPlan.aggregation}`);
+    }
     return errors;
 }
+function validateMetricWindow(metric, startedAt, completedAt) {
+    if (!metric.window)
+        return [];
+    const start = Date.parse(metric.window.startedAt);
+    const end = Date.parse(metric.window.endedAt);
+    const intentStart = Date.parse(startedAt);
+    const intentEnd = Date.parse(completedAt);
+    if (![start, end, intentStart, intentEnd].every(Number.isFinite) || end < start || start < intentStart || end > intentEnd) {
+        return [`${metric.metricId} window must be valid and contained within the intent execution window`];
+    }
+    return [];
+}
 export function validateObservabilityCoverage(profile, coverage, requireReady) {
     const required = new Set(profile.objectives.filter((item) => item.required).map((item) => item.metricId));
     const declaredRequired = new Set(coverage.requiredMetricIds);
     const covered = new Set(coverage.coveredMetricIds);
     const missing = new Set(coverage.missingMetricIds);
     const errors = [];
+    if (coverage.targetId !== profile.applicationTarget.id) {
+        errors.push("observability target does not match the confirmed application LLM target");
+    }
     for (const metricId of required) {
         if (!declaredRequired.has(metricId))
             errors.push(`observability omits required metric ${metricId}`);
@@ -191,6 +304,15 @@ export function validateObservabilityCoverage(profile, coverage, requireReady) {
     }
     return errors;
 }
+function activeTarget(config) {
+    return config.taskProfiles.find((item) => item.id === config.activeTaskProfileId)?.applicationTarget;
+}
+function validateTargetId(actual, expected, subject) {
+    return actual === expected ? [] : [`${subject} target does not match the confirmed application LLM target`];
+}
+function validateMetricTargets(metrics, targetId, subject) {
+    return metrics.flatMap((metric) => validateTargetId(metric.targetId, targetId, `${subject} metric ${metric.metricId}`));
+}
 export function interpretIntentResult(intent, result, cycle, config) {
     const errors = [];
     if (result.protocolVersion !== PROTOCOL_VERSION)
@@ -208,6 +330,35 @@ export function interpretIntentResult(intent, result, cycle, config) {
     }
     const output = result.outputs;
     errors.push(...validateIntentOutputSchema(intent.kind, output).map((error) => `result outputs ${error}`));
+    if (isRecord(output) && (intent.kind === "run_assessment" || intent.kind === "run_challenger_trials")) {
+        const profile = config.taskProfiles.find((item) => item.id === config.activeTaskProfileId);
+        const objectives = new Map((profile?.objectives ?? []).map((objective) => [objective.metricId, objective]));
+        const metricGroups = intent.kind === "run_assessment"
+            ? [{ metrics: output.metrics, startedAt: undefined, completedAt: undefined }]
+            : (Array.isArray(output.assessments) ? output.assessments.map((assessment) => isRecord(assessment)
+                ? { metrics: assessment.metrics, startedAt: assessment.startedAt, completedAt: assessment.completedAt }
+                : { metrics: undefined, startedAt: undefined, completedAt: undefined }) : []);
+        for (const group of metricGroups) {
+            const metrics = group.metrics;
+            if (!Array.isArray(metrics))
+                continue;
+            for (const metric of metrics) {
+                if (!isRecord(metric))
+                    continue;
+                const typedMetric = metric;
+                errors.push(...validateMetricAggregation(typedMetric, objectives.get(typedMetric.metricId)));
+                if (group.startedAt && group.completedAt)
+                    errors.push(...validateMetricWindow(typedMetric, String(group.startedAt), String(group.completedAt)));
+            }
+        }
+    }
+    const target = activeTarget(config);
+    if (intent.kind !== "establish_evals") {
+        if (!target)
+            errors.push("no confirmed application LLM target exists");
+        else
+            errors.push(...validateTargetId(intent.targetId, target.id, "intent"));
+    }
     if (errors.length) {
         return { valid: false, errors, nextStage: completedStage(intent.kind, false) };
     }
@@ -216,20 +367,27 @@ export function interpretIntentResult(intent, result, cycle, config) {
     let cyclePatch;
     switch (intent.kind) {
         case "establish_evals": {
-            errors.push(...validateSetupOutput(output));
+            errors.push(...validateSetupOutput(output, typeof intent.constraints.expectedIncumbentModelId === "string"
+                ? intent.constraints.expectedIncumbentModelId : undefined));
             if (!errors.length && isRecord(output)) {
                 const taskProfile = output.taskProfile;
                 const policy = output.policy;
                 const observability = output.observability;
+                const incumbent = output.incumbent;
+                const preliminaryModelLeads = (output.preliminaryModelLeads ?? []);
                 nextConfig = {
                     schemaVersion: 1,
                     activeTaskProfileId: taskProfile.id,
                     taskProfiles: [...config.taskProfiles.filter((item) => item.id !== taskProfile.id), taskProfile],
                     policy,
+                    preliminaryModelLeads,
                 };
                 cyclePatch = {
+                    targetId: taskProfile.applicationTarget.id,
                     taskProfileId: taskProfile.id,
+                    activeCandidateId: incumbent.candidateId,
                     policy,
+                    baselineEvidenceSufficient: false,
                     observabilityReady: observability.status === "ready",
                 };
                 if (observability.status !== "ready")
@@ -246,7 +404,9 @@ export function interpretIntentResult(intent, result, cycle, config) {
                 if (!profile)
                     errors.push("no confirmed task profile exists for observability verification");
                 else {
+                    errors.push(...validateTargetId(output.targetId, profile.applicationTarget.id, "observability result"));
                     const coverage = {
+                        targetId: profile.applicationTarget.id,
                         status: output.missingMetricIds.length ? "incomplete" : "ready",
                         requiredMetricIds: profile.objectives.filter((item) => item.required).map((item) => item.metricId),
                         coveredMetricIds: output.coveredMetricIds,
@@ -259,10 +419,18 @@ export function interpretIntentResult(intent, result, cycle, config) {
             }
             break;
         case "run_assessment":
-            if (!isRecord(output) || typeof output.baselineEvidenceSufficient !== "boolean") {
+            if (!isRecord(output) || typeof output.baselineEvidenceSufficient !== "boolean" || !Array.isArray(output.metrics)) {
                 errors.push("assessment result requires baselineEvidenceSufficient");
             }
             else {
+                if (target) {
+                    errors.push(...validateTargetId(output.targetId, target.id, "baseline assessment"));
+                    errors.push(...validateMetricTargets(output.metrics, target.id, "baseline assessment"));
+                    const profile = config.taskProfiles.find((item) => item.id === config.activeTaskProfileId);
+                    if (output.baselineEvidenceSufficient) {
+                        errors.push(...baselineReadinessErrors(profile, output.metrics));
+                    }
+                }
                 cyclePatch = { baselineEvidenceSufficient: output.baselineEvidenceSufficient };
                 if (!output.baselineEvidenceSufficient)
                     nextStage = "collecting_baseline";
@@ -271,10 +439,61 @@ export function interpretIntentResult(intent, result, cycle, config) {
         case "discover_candidates":
             if (!isRecord(output) || !Array.isArray(output.candidates))
                 errors.push("candidate discovery requires candidates");
+            else if (target) {
+                errors.push(...validateTargetId(output.targetId, target.id, "candidate discovery"));
+                const candidateIds = new Set();
+                for (const candidate of output.candidates) {
+                    if (isRecord(candidate)) {
+                        errors.push(...validateTargetId(candidate.targetId, target.id, `candidate ${String(candidate.candidateId)}`));
+                        if (typeof candidate.candidateId === "string") {
+                            if (candidateIds.has(candidate.candidateId))
+                                errors.push(`duplicate candidate ${candidate.candidateId}`);
+                            candidateIds.add(candidate.candidateId);
+                        }
+                    }
+                }
+                if (cycle.activeCandidateId && !candidateIds.has(cycle.activeCandidateId)) {
+                    errors.push("candidate discovery must include the incumbent application model");
+                }
+                const maximum = config.policy?.evaluationBudget.maximumCandidateCount;
+                if (maximum !== undefined && output.candidates.length > maximum) {
+                    errors.push(`candidate discovery exceeds maximumCandidateCount ${maximum}`);
+                }
+            }
             break;
         case "run_challenger_trials":
-            if (!isRecord(output) || !Array.isArray(output.assessments))
-                errors.push("challenger trials require assessments");
+            if (!isRecord(output) || !Array.isArray(output.assessments) || !isRecord(output.experimentManifest)) {
+                errors.push("challenger trials require assessments and an experimentManifest");
+            }
+            else {
+                if (target)
+                    errors.push(...validateTargetId(output.targetId, target.id, "challenger trials"));
+                const expectedCandidateIds = Array.isArray(intent.constraints.candidates)
+                    ? new Set(intent.constraints.candidates.flatMap((candidate) => isRecord(candidate) && typeof candidate.candidateId === "string" ? [candidate.candidateId] : []))
+                    : undefined;
+                const assessedCandidateIds = new Set();
+                const productRevisions = new Set();
+                for (const assessment of output.assessments) {
+                    assessedCandidateIds.add(assessment.candidateId);
+                    productRevisions.add(assessment.productRevision);
+                    if (target) {
+                        errors.push(...validateTargetId(assessment.targetId, target.id, `challenger assessment ${assessment.id}`));
+                        errors.push(...validateMetricTargets(assessment.metrics, target.id, `challenger assessment ${assessment.id}`));
+                    }
+                    const costs = assessment.metrics.filter((metric) => metric.metricId === "total_task_cost" && metric.state === "measured" &&
+                        metric.value !== undefined && metric.value >= 0 && metric.evidence.length > 0);
+                    if (costs.length !== 1) {
+                        errors.push(`challenger assessment ${assessment.id} requires exactly one measured total_task_cost with evidence`);
+                    }
+                }
+                if (expectedCandidateIds &&
+                    (expectedCandidateIds.size !== assessedCandidateIds.size ||
+                        [...expectedCandidateIds].some((candidateId) => !assessedCandidateIds.has(candidateId)))) {
+                    errors.push("challenger assessments must exactly cover the frozen candidate set");
+                }
+                if (productRevisions.size !== 1)
+                    errors.push("challenger assessments must use one application product revision");
+            }
             break;
         case "calculate_frontier": {
             if (!isRecord(output) || !Array.isArray(output.assessments) || !isRecord(output.frontierSnapshot)) {
@@ -286,6 +505,16 @@ export function interpretIntentResult(intent, result, cycle, config) {
                 errors.push("no confirmed task profile exists for frontier verification");
                 break;
             }
+            errors.push(...validateTargetId(output.targetId, profile.applicationTarget.id, "frontier result"));
+            const expectedAssessmentIds = Array.isArray(intent.constraints.assessments)
+                ? new Set(intent.constraints.assessments.flatMap((assessment) => isRecord(assessment) && typeof assessment.id === "string" ? [assessment.id] : []))
+                : undefined;
+            const suppliedAssessmentIds = new Set(output.assessments.map((assessment) => assessment.id));
+            if (expectedAssessmentIds &&
+                (expectedAssessmentIds.size !== suppliedAssessmentIds.size ||
+                    [...expectedAssessmentIds].some((assessmentId) => !suppliedAssessmentIds.has(assessmentId)))) {
+                errors.push("frontier calculation must use exactly the accepted challenger assessments");
+            }
             const verification = verifyFrontierSnapshot(profile, output.assessments, output.frontierSnapshot);
             errors.push(...verification.errors);
             if (output.recommendation !== undefined) {
@@ -295,6 +524,12 @@ export function interpretIntentResult(intent, result, cycle, config) {
                 else if (!output.frontierSnapshot.frontierCandidateIds.includes(output.recommendation.selectedCandidateId)) {
                     errors.push("recommended candidate is not on the verified frontier");
                 }
+                else if (output.recommendation.targetId !== profile.applicationTarget.id) {
+                    errors.push("frontier recommendation target does not match the confirmed application LLM target");
+                }
+                else if (cycle.activeCandidateId && output.recommendation.currentCandidateId !== cycle.activeCandidateId) {
+                    errors.push("frontier recommendation current candidate does not match the incumbent application model");
+                }
                 else {
                     cyclePatch = { proposedCandidateId: output.recommendation.selectedCandidateId };
                 }
@@ -305,11 +540,16 @@ export function interpretIntentResult(intent, result, cycle, config) {
             if (!isRecord(output) || typeof output.regressionDetected !== "boolean" || !Array.isArray(output.affectedMetricIds)) {
                 errors.push("regression result requires regressionDetected and affectedMetricIds");
             }
+            else if (target)
+                errors.push(...validateTargetId(output.targetId, target.id, "regression investigation"));
             break;
         case "apply_model_swap":
             if (!isRecord(output) || typeof output.appliedCandidateId !== "string") {
                 errors.push("swap result requires appliedCandidateId");
             }
+            else if (target && output.targetId !== target.id) {
+                errors.push("swap target does not match the confirmed application LLM target");
+            }
             else if (cycle.proposedCandidateId && output.appliedCandidateId !== cycle.proposedCandidateId) {
                 errors.push("applied candidate does not match the authorized proposal");
             }
@@ -324,6 +564,9 @@ export function interpretIntentResult(intent, result, cycle, config) {
             if (!isRecord(output) || typeof output.verified !== "boolean" || typeof output.regressionDetected !== "boolean") {
                 errors.push("swap verification requires verified and regressionDetected");
             }
+            else if (target && output.targetId !== target.id) {
+                errors.push("swap verification target does not match the confirmed application LLM target");
+            }
             else if (!output.verified || output.regressionDetected) {
                 nextStage = "verification_failed";
             }
@@ -335,6 +578,9 @@ export function interpretIntentResult(intent, result, cycle, config) {
             if (!isRecord(output) || typeof output.restoredCandidateId !== "string") {
                 errors.push("rollback result requires restoredCandidateId");
             }
+            else if (target && output.targetId !== target.id) {
+                errors.push("rollback target does not match the confirmed application LLM target");
+            }
             else if (cycle.previousCandidateId && output.restoredCandidateId !== cycle.previousCandidateId) {
                 errors.push("rollback did not restore the previous verified candidate");
             }
@@ -359,7 +605,7 @@ export function nextImprovementAction(state) {
                 : { action: "issue_intent", intentKind: "instrument_observability", reason: "Required metric instrumentation is incomplete." };
         case "collecting_baseline":
             return state.baselineEvidenceSufficient
-                ? { action: "issue_intent", intentKind: "run_assessment", reason: "Baseline evidence has reached the approved sufficiency threshold." }
+                ? { action: "issue_intent", intentKind: "discover_candidates", reason: "The baseline is ready for candidate discovery." }
                 : { action: "observe", reason: "Continue collecting baseline production evidence." };
         case "baseline_ready":
             return { action: "issue_intent", intentKind: "discover_candidates", reason: "The baseline is ready for budgeted candidate discovery." };
@@ -411,22 +657,71 @@ function bounds(metric) {
 }
 function readinessErrors(profile, assessment) {
     const errors = [];
+    if (assessment.targetId !== profile.applicationTarget.id) {
+        errors.push("assessment target does not match the confirmed application LLM target");
+    }
     for (const objective of profile.objectives) {
         if (!objective.required)
             continue;
         const metric = metricFor(assessment, objective);
         if (!metric)
             errors.push(`${objective.metricId}: missing`);
-        else if (metric.state !== "measured")
-            errors.push(`${objective.metricId}: ${metric.state}`);
-        else if (metric.value === undefined)
-            errors.push(`${objective.metricId}: no numeric value`);
-        else if (metric.sampleCount < objective.minimumSamples) {
-            errors.push(`${objective.metricId}: ${metric.sampleCount}/${objective.minimumSamples} samples`);
+        else {
+            if (metric.state !== "measured")
+                errors.push(`${objective.metricId}: ${metric.state}`);
+            else if (metric.value === undefined)
+                errors.push(`${objective.metricId}: no numeric value`);
+            errors.push(...validateMetricAggregation(metric, objective));
+            if (metric.sampleCount < objective.minimumSamples) {
+                errors.push(`${objective.metricId}: ${metric.sampleCount}/${objective.minimumSamples} samples`);
+            }
         }
     }
     return errors;
 }
+function baselineReadinessErrors(profile, metrics) {
+    if (!profile)
+        return ["confirmed task profile is missing"];
+    const errors = [];
+    if (!profile.objectives.some((objective) => objective.required)) {
+        errors.push("task profile has no required metric objectives");
+    }
+    for (const objective of profile.objectives) {
+        if (!objective.required)
+            continue;
+        const metric = metrics.find((item) => item.metricId === objective.metricId &&
+            item.targetId === profile.applicationTarget.id);
+        if (!metric)
+            errors.push(`${objective.metricId}: missing`);
+        else {
+            if (metric.state !== "measured")
+                errors.push(`${objective.metricId}: ${metric.state}`);
+            else if (metric.value === undefined || !Number.isFinite(metric.value))
+                errors.push(`${objective.metricId}: no finite numeric value`);
+            errors.push(...validateMetricAggregation(metric, objective));
+            if (metric.sampleCount < objective.minimumSamples) {
+                errors.push(`${objective.metricId}: ${metric.sampleCount}/${objective.minimumSamples} samples`);
+            }
+            else if (!metric.evidence.length)
+                errors.push(`${objective.metricId}: no evidence reference`);
+        }
+    }
+    return errors;
+}
+function baselineAssessmentErrors(state, profile, metrics) {
+    return [
+        ...(state.cycle.observabilityReady ? [] : ["required evaluation and observability setup is not ready"]),
+        ...baselineReadinessErrors(profile, metrics),
+    ];
+}
+function updatedBaselineWindows(previous, signal) {
+    if (signal.type !== "metric_window_available")
+        return previous ?? [];
+    const windows = new Map((previous ?? []).map((metric) => [metric.metricId, metric]));
+    for (const metric of signal.metrics)
+        windows.set(metric.metricId, metric);
+    return [...windows.values()];
+}
 function feasibility(profile, assessment) {
     if (readinessErrors(profile, assessment).length)
         return "unresolved";
@@ -525,6 +820,9 @@ export function verifyFrontierSnapshot(profile, assessments, snapshot) {
     if (snapshot.taskProfileId !== profile.id || snapshot.taskProfileRevision !== profile.revision) {
         errors.push("snapshot task profile does not match");
     }
+    if (snapshot.targetId !== profile.applicationTarget.id) {
+        errors.push("snapshot target does not match the confirmed application LLM target");
+    }
     if (snapshot.calculationEvidence.length === 0)
         errors.push("calculation evidence is required");
     if (!sameMembers(snapshot.assessmentIds, assessments.map((assessment) => assessment.id))) {
@@ -534,6 +832,11 @@ export function verifyFrontierSnapshot(profile, assessments, snapshot) {
     const expectedIneligible = [];
     const expectedUnresolved = new Set();
     for (const assessment of assessments) {
+        if (assessment.targetId !== profile.applicationTarget.id) {
+            errors.push(`${assessment.candidateId}: application target mismatch`);
+            continue;
+        }
+        errors.push(...validateMetricTargets(assessment.metrics, profile.applicationTarget.id, assessment.candidateId));
         if (assessment.taskProfileId !== profile.id || assessment.taskProfileRevision !== profile.revision) {
             errors.push(`${assessment.candidateId}: task profile revision mismatch`);
             continue;
@@ -611,6 +914,14 @@ function earliest(values) {
         .sort((left, right) => Date.parse(left) - Date.parse(right))[0];
 }
 export function buildAutomationPlan(state, descriptor) {
+    if (state.automation.paused) {
+        return {
+            mode: state.cycle.policy?.checkPolicy.execution.mode ?? "active_session_only",
+            requiredSignals: [],
+            scheduling: "active_session_fallback",
+            gaps: ["Proactive OpenMerit checks are paused."],
+        };
+    }
     const policy = state.cycle.policy?.checkPolicy;
     if (!policy) {
         return {
@@ -640,7 +951,7 @@ export function buildAutomationPlan(state, descriptor) {
         required.add("verification_window_completed");
     }
     const nextWakeupAt = state.cycle.stage === "collecting_baseline"
-        ? nextAt(state.automation.lastAssessmentAt ?? state.updatedAt, policy.baselineAssessment.afterElapsedSeconds)
+        ? nextAt(state.automation.lastAssessmentAt ?? state.automation.baselineStartedAt ?? state.lastResult?.completedAt ?? state.updatedAt, policy.baselineAssessment.afterElapsedSeconds)
         : ["monitoring", "swap_verified"].includes(state.cycle.stage)
             ? earliest([
                 nextAt(state.automation.lastReassessmentAt ?? state.updatedAt, policy.frontierReassessment.afterElapsedSeconds),
@@ -701,6 +1012,8 @@ function regressionMetric(signal, previous, policy) {
     return undefined;
 }
 function dueDecision(state, signal, now) {
+    if (state.automation.paused)
+        return { action: "observe", reason: "Proactive OpenMerit checks are paused." };
     if (state.activeIntent)
         return { action: "observe", reason: "Harness work is already active." };
     if (state.automation.pendingIntentKind) {
@@ -719,7 +1032,7 @@ function dueDecision(state, signal, now) {
         .sort((left, right) => Date.parse(right) - Date.parse(left))[0];
     const cooldownSatisfied = !cooldownAnchor ||
         Date.parse(now) - Date.parse(cooldownAnchor) >= policy.checkPolicy.cooldownSeconds * 1000;
-    if (state.cycle.stage === "collecting_baseline" && cadenceDue(policy.checkPolicy.baselineAssessment, completed, state.automation.lastAssessmentTaskCount, state.automation.lastAssessmentAt ?? state.updatedAt, now)) {
+    if (state.cycle.stage === "collecting_baseline" && cadenceDue(policy.checkPolicy.baselineAssessment, completed, state.automation.lastAssessmentTaskCount, state.automation.lastAssessmentAt ?? state.automation.baselineStartedAt ?? state.lastResult?.completedAt ?? state.updatedAt, now)) {
         return { action: "issue_intent", intentKind: "run_assessment", reason: "The user-confirmed baseline assessment cadence is due." };
     }
     if (["monitoring", "swap_verified"].includes(state.cycle.stage) && cooldownSatisfied) {
@@ -749,14 +1062,17 @@ function auditSignalData(signal) {
     switch (signal.type) {
         case "product_task_completed":
             return {
+                targetId: signal.targetId,
                 modelId: signal.modelId,
-                contextTokens: signal.contextTokens ?? null,
+                ...(signal.telemetry ? { telemetry: signal.telemetry } : {}),
                 evidenceIds: (signal.evidence ?? []).map((item) => item.id),
             };
         case "metric_window_available":
         case "verification_window_completed":
             return {
+                targetId: signal.targetId,
                 metrics: (signal.metrics ?? []).map((metric) => ({
+                    targetId: metric.targetId,
                     metricId: metric.metricId,
                     state: metric.state,
                     scope: metric.scope,
@@ -771,9 +1087,9 @@ function auditSignalData(signal) {
                 })),
             };
         case "model_catalog_changed":
-            return { fingerprint: signal.fingerprint };
+            return { targetId: signal.targetId, fingerprint: signal.fingerprint };
         case "scheduled_tick":
-            return {};
+            return { targetId: signal.targetId };
     }
 }
 /** Harness-neutral durable coordinator. Adapters transport jobs; this class owns lifecycle state. */
@@ -784,7 +1100,7 @@ export class OpenMeritCoordinator {
         this.store = store;
         this.adapter = adapter;
     }
-    async issue(kind) {
+    async issue(kind, options = {}) {
         const missing = missingHarnessCapabilities(this.adapter.descriptor, kind);
         if (missing.length)
             throw new Error(`Harness lacks required capabilities: ${missing.join(", ")}`);
@@ -792,16 +1108,80 @@ export class OpenMeritCoordinator {
         const existing = await this.store.readState();
         if (existing?.activeIntent)
             throw new Error(`OpenMerit intent ${existing.activeIntent.id} is already active.`);
+        if (kind === "run_challenger_trials" && config.policy) {
+            const spent = existing?.automation.evaluationSpend ?? 0;
+            const limit = config.policy.evaluationBudget.maximumSpend;
+            if (spent >= limit) {
+                throw new Error(`Evaluation budget exhausted: ${spent.toFixed(6)} ${config.policy.evaluationBudget.currency} spent of ${limit.toFixed(6)}.`);
+            }
+        }
+        const targetId = activeTarget(config)?.id;
+        const confirmedProfile = config.taskProfiles.find((item) => item.id === config.activeTaskProfileId);
+        if (kind !== "establish_evals" && !targetId) {
+            throw new Error(`OpenMerit cannot issue ${kind} without a confirmed application LLM target.`);
+        }
         const intent = createOpenMeritIntent(kind, {
             taskProfileId: config.activeTaskProfileId,
             policy: config.policy,
+            evaluationSpend: existing?.automation.evaluationSpend,
+            targetId,
+            additionalConstraints: {
+                ...options.additionalConstraints,
+                ...(confirmedProfile ? { taskProfile: confirmedProfile } : {}),
+                ...(kind === "discover_candidates"
+                    ? {
+                        preliminaryModelLeads: (config.preliminaryModelLeads ?? []),
+                        incumbentCandidate: (existing?.candidates?.find((candidate) => candidate.candidateId === existing.cycle.activeCandidateId) ?? existing?.candidates?.[0]),
+                        candidateSelection: {
+                            totalCandidateCount: Math.min(3, config.policy?.evaluationBudget.maximumCandidateCount ?? 3),
+                            includeIncumbent: true,
+                            requireAvailabilityProbe: true,
+                            requireExactModelIds: true,
+                            requirePriceAndCapabilitySources: true,
+                        },
+                    }
+                    : {}),
+                ...(kind === "run_challenger_trials"
+                    ? { candidates: (existing?.candidates ?? []) }
+                    : {}),
+                ...(kind === "calculate_frontier"
+                    ? {
+                        assessments: (existing?.assessments ?? []),
+                        ...(existing?.experimentManifest
+                            ? { experimentManifest: existing.experimentManifest }
+                            : {}),
+                    }
+                    : {}),
+            },
         });
         const now = new Date().toISOString();
         const state = existing
             ? {
                 ...existing,
-                cycle: { ...existing.cycle, stage: startedStage(kind) },
+                cycle: kind === "establish_evals"
+                    ? {
+                        ...existing.cycle,
+                        id: `cycle-${crypto.randomUUID()}`,
+                        targetId: undefined,
+                        taskProfileId: undefined,
+                        activeCandidateId: undefined,
+                        previousCandidateId: undefined,
+                        proposedCandidateId: undefined,
+                        stage: startedStage(kind),
+                        verifiedSwapCount: 0,
+                        baselineEvidenceSufficient: false,
+                        observabilityReady: false,
+                        policy: undefined,
+                    }
+                    : { ...existing.cycle, stage: startedStage(kind) },
                 activeIntent: intent,
+                ...(kind === "establish_evals" ? {
+                    candidates: undefined,
+                    assessments: undefined,
+                    experimentManifest: undefined,
+                    observability: undefined,
+                    lastResult: undefined,
+                } : {}),
                 automation: {
                     ...existing.automation,
                     ...(existing.automation.pendingIntentKind === kind
@@ -889,6 +1269,16 @@ export class OpenMeritCoordinator {
             signal: "scheduled_tick",
             reason: "Wake the harness so OpenMerit can evaluate the user-confirmed check policy.",
         });
+        if (!receipt.accepted) {
+            return {
+                plan: {
+                    ...plan,
+                    scheduling: "external_scheduler_required",
+                    gaps: [...plan.gaps, `Harness wakeup provisioning failed: ${receipt.reason ?? "unknown reason"}`],
+                },
+                receipt,
+            };
+        }
         return { plan, receipt };
     }
     async acceptResult(result) {
@@ -903,6 +1293,17 @@ export class OpenMeritCoordinator {
         if (interpretation.config)
             await this.store.writeConfig(interpretation.config);
         const now = new Date().toISOString();
+        const evaluationSpend = result.status === "succeeded" && state.activeIntent.kind === "run_challenger_trials"
+            ? result.outputs.assessments.reduce((sum, assessment) => sum + assessment.metrics
+                .filter((metric) => metric.metricId === "total_task_cost" && metric.state === "measured")
+                .reduce((metricSum, metric) => metricSum + (metric.value ?? 0), 0), state.automation.evaluationSpend ?? 0)
+            : state.automation.evaluationSpend;
+        if (config.policy && evaluationSpend !== undefined &&
+            evaluationSpend > config.policy.evaluationBudget.maximumSpend) {
+            throw new Error(`OpenMerit rejected challenger results: evaluation spend ${evaluationSpend.toFixed(6)} ` +
+                `${config.policy.evaluationBudget.currency} exceeds the approved limit ` +
+                `${config.policy.evaluationBudget.maximumSpend.toFixed(6)}.`);
+        }
         let observability = state.observability;
         if (result.status === "succeeded" && state.activeIntent.kind === "establish_evals") {
             observability = result.outputs.observability;
@@ -913,6 +1314,7 @@ export class OpenMeritCoordinator {
                 const output = result.outputs;
                 const requiredMetricIds = profile.objectives.filter((item) => item.required).map((item) => item.metricId);
                 observability = {
+                    targetId: profile.applicationTarget.id,
                     status: "ready",
                     requiredMetricIds,
                     coveredMetricIds: output.coveredMetricIds,
@@ -930,9 +1332,31 @@ export class OpenMeritCoordinator {
             },
             activeIntent: undefined,
             lastResult: result,
+            ...(result.status === "succeeded" && state.activeIntent.kind === "establish_evals"
+                ? { candidates: [result.outputs.incumbent] }
+                : {}),
+            ...(result.status === "succeeded" && state.activeIntent.kind === "discover_candidates"
+                ? { candidates: result.outputs.candidates }
+                : {}),
+            ...(result.status === "succeeded" && state.activeIntent.kind === "run_challenger_trials"
+                ? {
+                    assessments: result.outputs.assessments,
+                    experimentManifest: result.outputs.experimentManifest,
+                }
+                : {}),
             ...(observability ? { observability } : {}),
             automation: {
                 ...state.automation,
+                ...(result.status === "succeeded" && ["establish_evals", "instrument_observability"].includes(state.activeIntent.kind) &&
+                    interpretation.nextStage === "collecting_baseline"
+                    ? {
+                        baselineStartedAt: now,
+                        lastAssessmentAt: undefined,
+                        lastAssessmentTaskCount: state.automation.observedCompletedTasks,
+                        baselineMetricWindows: [],
+                    }
+                    : {}),
+                ...(evaluationSpend !== undefined ? { evaluationSpend } : {}),
                 ...(state.activeIntent.kind === "run_assessment"
                     ? { lastAssessmentTaskCount: state.automation.observedCompletedTasks, lastAssessmentAt: now }
                     : {}),
@@ -984,6 +1408,7 @@ export class OpenMeritCoordinator {
                 cycleId: nextState.cycle.id,
                 intentId: result.intentId,
                 data: {
+                    targetId: observability.targetId,
                     status: observability.status,
                     requiredMetricIds: observability.requiredMetricIds,
                     coveredMetricIds: observability.coveredMetricIds,
@@ -1016,21 +1441,54 @@ export class OpenMeritCoordinator {
     }
     async recordTaskObservation(details = {}) {
         const now = new Date().toISOString();
+        const state = await this.store.readState();
+        if (!state?.cycle.targetId)
+            throw new Error("A confirmed application LLM target is required.");
         const result = await this.recordAutomationSignal({
             protocolVersion: PROTOCOL_VERSION,
             id: `legacy-task-${crypto.randomUUID()}`,
             type: "product_task_completed",
             occurredAt: now,
+            targetId: state.cycle.targetId,
             modelId: typeof details.modelId === "string" ? details.modelId : "unknown",
-            contextTokens: typeof details.contextTokens === "number" || details.contextTokens === null
-                ? details.contextTokens
-                : undefined,
         });
         return {
             state: result.state,
-            assessmentDue: result.decision.action === "issue_intent" && result.decision.intentKind === "run_assessment",
+            assessmentDue: result.state.automation.lastAssessmentAt === now,
         };
     }
+    async assessBaseline(now = new Date().toISOString()) {
+        const state = await this.store.readState();
+        if (!state || state.cycle.stage !== "collecting_baseline" || state.activeIntent) {
+            throw new Error("Baseline assessment requires an idle collecting-baseline cycle.");
+        }
+        const config = await this.store.readConfig();
+        const profile = config?.taskProfiles.find((item) => item.id === config.activeTaskProfileId);
+        const errors = baselineAssessmentErrors(state, profile, state.automation.baselineMetricWindows ?? []);
+        const sufficient = errors.length === 0;
+        const decision = sufficient
+            ? { action: "issue_intent", intentKind: "discover_candidates", reason: "Required baseline metric windows meet the confirmed sample thresholds; discover candidates." }
+            : { action: "observe", reason: `Baseline evidence remains insufficient: ${errors.join("; ")}.` };
+        const nextState = {
+            ...state,
+            cycle: { ...state.cycle, baselineEvidenceSufficient: sufficient, stage: sufficient ? "baseline_ready" : "collecting_baseline" },
+            automation: {
+                ...state.automation,
+                lastAssessmentAt: now,
+                lastAssessmentTaskCount: state.automation.observedCompletedTasks,
+                pendingIntentKind: decision.action === "issue_intent" ? decision.intentKind : undefined,
+                pendingIntentReason: decision.action === "issue_intent" ? decision.reason : undefined,
+            },
+            updatedAt: now,
+        };
+        await this.store.writeState(nextState);
+        await this.store.appendEvent({
+            schemaVersion: 1, id: `event-${crypto.randomUUID()}`, type: "baseline_sufficiency_checked",
+            occurredAt: now, cycleId: state.cycle.id,
+            data: { sufficient, errors, metricIds: (state.automation.baselineMetricWindows ?? []).map((metric) => metric.metricId), source: "manual" },
+        });
+        return { state: nextState, errors, decision };
+    }
     async recordAutomationSignal(signal) {
         const state = await this.store.readState();
         if (!state)
@@ -1039,6 +1497,14 @@ export class OpenMeritCoordinator {
             throw new Error("Automation signal protocol version does not match.");
         if (!signal.id.trim())
             throw new Error("Automation signal ID is required.");
+        if (!state.cycle.targetId || signal.targetId !== state.cycle.targetId) {
+            throw new Error("Automation signal target does not match the confirmed application LLM target.");
+        }
+        if (signal.type === "metric_window_available" || signal.type === "verification_window_completed") {
+            const mismatched = (signal.metrics ?? []).find((metric) => metric.targetId !== state.cycle.targetId);
+            if (mismatched)
+                throw new Error(`Metric ${mismatched.metricId} target does not match the confirmed application LLM target.`);
+        }
         const processed = state.automation.processedSignalIds ?? [];
         if (processed.includes(signal.id)) {
             await this.store.appendEvent({
@@ -1053,12 +1519,31 @@ export class OpenMeritCoordinator {
         }
         const completedTasks = state.automation.observedCompletedTasks +
             (signal.type === "product_task_completed" ? 1 : 0);
+        const baselineMetricWindows = state.cycle.stage === "collecting_baseline"
+            ? updatedBaselineWindows(state.automation.baselineMetricWindows, signal)
+            : state.automation.baselineMetricWindows ?? [];
         const decisionState = {
             ...state,
-            automation: { ...state.automation, observedCompletedTasks: completedTasks },
+            automation: { ...state.automation, observedCompletedTasks: completedTasks, baselineMetricWindows },
         };
-        const decision = dueDecision(decisionState, signal, signal.occurredAt);
+        const due = dueDecision(decisionState, signal, signal.occurredAt);
+        const baselineCheckDue = due.action === "issue_intent" && due.intentKind === "run_assessment";
+        const config = baselineCheckDue ? await this.store.readConfig() : undefined;
+        const profile = config?.taskProfiles.find((item) => item.id === config.activeTaskProfileId);
+        const baselineErrors = baselineCheckDue ? baselineAssessmentErrors(state, profile, baselineMetricWindows) : [];
+        const baselineReady = baselineCheckDue && baselineErrors.length === 0;
+        const decision = baselineCheckDue
+            ? baselineReady
+                ? { action: "issue_intent", intentKind: "discover_candidates", reason: "Required baseline metric windows meet the confirmed sample thresholds; discover candidates." }
+                : { action: "observe", reason: `Baseline evidence remains insufficient: ${baselineErrors.join("; ")}.` }
+            : due;
         const latestMetricValues = { ...(state.automation.latestMetricValues ?? {}) };
+        if (signal.type === "product_task_completed" && signal.telemetry) {
+            latestMetricValues.input_token_consumption = signal.telemetry.inputTokens;
+            latestMetricValues.output_token_consumption = signal.telemetry.outputTokens;
+            latestMetricValues.total_task_cost = signal.telemetry.cost;
+            latestMetricValues.end_to_end_task_latency = signal.telemetry.durationMs;
+        }
         if (signal.type === "metric_window_available" || signal.type === "verification_window_completed") {
             for (const metric of signal.metrics ?? []) {
                 if (metric.state === "measured" && metric.value !== undefined)
@@ -1067,10 +1552,19 @@ export class OpenMeritCoordinator {
         }
         const nextState = {
             ...decisionState,
+            cycle: baselineCheckDue
+                ? { ...decisionState.cycle, baselineEvidenceSufficient: baselineReady, stage: baselineReady ? "baseline_ready" : "collecting_baseline" }
+                : decisionState.cycle,
             automation: {
                 ...decisionState.automation,
                 processedSignalIds: [...processed, signal.id].slice(-MAX_PROCESSED_SIGNAL_IDS),
                 latestMetricValues,
+                ...(baselineCheckDue ? {
+                    lastAssessmentAt: signal.occurredAt,
+                    lastAssessmentTaskCount: completedTasks,
+                    pendingIntentKind: decision.action === "issue_intent" ? decision.intentKind : undefined,
+                    pendingIntentReason: decision.action === "issue_intent" ? decision.reason : undefined,
+                } : {}),
                 ...(signal.type === "model_catalog_changed" ? {
                     modelCatalogFingerprint: signal.fingerprint,
                     ...(state.automation.modelCatalogFingerprint && state.automation.modelCatalogFingerprint !== signal.fingerprint
@@ -1099,6 +1593,21 @@ export class OpenMeritCoordinator {
                 ...auditSignalData(signal),
             },
         });
+        if (baselineCheckDue) {
+            await this.store.appendEvent({
+                schemaVersion: 1,
+                id: `event-${crypto.randomUUID()}`,
+                type: "baseline_sufficiency_checked",
+                occurredAt: signal.occurredAt,
+                cycleId: state.cycle.id,
+                data: {
+                    sufficient: baselineReady,
+                    errors: baselineErrors,
+                    metricIds: baselineMetricWindows.map((metric) => metric.metricId),
+                    sourceSignalId: signal.id,
+                },
+            });
+        }
         if (decision.action === "issue_intent") {
             await this.store.appendEvent({
                 schemaVersion: 1,
@@ -1115,6 +1624,8 @@ export class OpenMeritCoordinator {
         const state = await this.store.readState();
         if (!state)
             throw new Error("OpenMerit project is not initialized.");
+        if (!state.cycle.targetId)
+            throw new Error("A confirmed application LLM target is required.");
         const previousFingerprint = state.automation.modelCatalogFingerprint;
         const changed = previousFingerprint !== undefined && previousFingerprint !== fingerprint;
         if (previousFingerprint === fingerprint) {
@@ -1126,6 +1637,7 @@ export class OpenMeritCoordinator {
             id: `legacy-catalog-${crypto.randomUUID()}`,
             type: "model_catalog_changed",
             occurredAt: now,
+            targetId: state.cycle.targetId,
             fingerprint,
         });
         return {