openmerit 0.1.5 → 0.1.6-preview.0

package/dist/pi/src/index.js

@@ -1,8 +1,12 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
 import { createHash } from "node:crypto";
+import { basename, join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
 import { StringEnum } from "@earendil-works/pi-ai";
 import { nextImprovementAction, OpenMeritCoordinator, ProjectStore, } from "../../core/src/index.js";
 import { completionToolSchemaFor, OPENMERIT_SCHEMA_DIALECT, PROTOCOL_VERSION, MetricRecordSchema, } from "../../protocol/src/index.js";
 import { Type } from "typebox";
+import { piWakeupPlistPath, provisionPiWakeup, removePiWakeup } from "./scheduler.js";
 const INTENT_ENTRY_TYPE = "openmerit.intent";
 const RESULT_ENTRY_TYPE = "openmerit.result";
 const INTENT_MESSAGE_TYPE = "openmerit.intent-context";
@@ -34,34 +38,146 @@ export const PI_HARNESS_DESCRIPTOR = {
             "product_task_completed", "metric_window_available", "scheduled_tick",
             "model_catalog_changed", "verification_window_completed",
         ],
-        persistentScheduling: false,
-        backgroundExecution: false,
-        wakeupProvisioning: "none",
+        persistentScheduling: process.platform === "darwin",
+        backgroundExecution: process.platform === "darwin",
+        wakeupProvisioning: process.platform === "darwin" ? "infrastructure_change" : "none",
     },
 };
 export function createInitialUiState() {
     return {
         proactiveNudgesPaused: false,
+        applicationTarget: "not_detected",
         taskProfile: "not_configured",
         metricReadiness: "not_assessed",
         evaluationReadiness: "not_assessed",
         activeIntent: null,
         nextAssessment: null,
         lifecycleStage: "not_initialized",
+        incumbentCandidate: null,
+        candidateCount: 0,
+        assessmentCount: 0,
+        evaluationSpend: 0,
+        proposedCandidate: null,
     };
 }
 export function renderStatus(state) {
     return [
         "OpenMerit evidence readiness",
+        `application target : ${state.applicationTarget.replaceAll("_", " ")}`,
         `task profile       : ${state.taskProfile.replaceAll("_", " ")}`,
         `metric readiness   : ${state.metricReadiness.replaceAll("_", " ")}`,
         `evaluation status  : ${state.evaluationReadiness.replaceAll("_", " ")}`,
         `active work        : ${state.activeIntent ?? "none"}`,
         `next assessment    : ${state.nextAssessment ?? "not scheduled"}`,
         `frontier lifecycle : ${state.lifecycleStage.replaceAll("_", " ")}`,
+        `incumbent candidate: ${state.incumbentCandidate ?? "not confirmed"}`,
+        `candidate set      : ${state.candidateCount}`,
+        `assessments        : ${state.assessmentCount}`,
+        `evaluation spend   : $${state.evaluationSpend.toFixed(6)}`,
+        `proposed candidate : ${state.proposedCandidate ?? "none"}`,
         `proactive nudges   : ${state.proactiveNudgesPaused ? "paused" : "enabled"}`,
     ].join("\n");
 }
+const DETECTION_IGNORES = new Set([
+    ".git", ".openmerit", "node_modules", "dist", "build", "coverage", ".next", "vendor", "tests", "test", "__tests__",
+]);
+const DETECTION_EXTENSIONS = new Set([".js", ".jsx", ".mjs", ".cjs", ".ts", ".tsx", ".py", ".go", ".rs", ".java", ".kt"]);
+const APPLICATION_CALL_PATTERNS = [
+    { provider: "openai-compatible", pattern: /\b(?:chat\.completions|responses)\.create\s*\(/ },
+    { provider: "anthropic", pattern: /\bmessages\.create\s*\(/ },
+    { provider: "google", pattern: /\bgenerateContent(?:Stream)?\s*\(/ },
+    { provider: "vercel-ai", pattern: /\b(?:generateText|streamText|generateObject|streamObject)\s*\(/ },
+    { provider: "langchain", pattern: /\b(?:ChatOpenAI|ChatAnthropic|ChatGoogleGenerativeAI)\s*\(/ },
+    { provider: "openrouter", pattern: /openrouter\.ai\/api\/v1[\s\S]{0,5000}\/(?:chat\/completions|responses)/ },
+];
+function extensionOf(path) {
+    const match = path.match(/(\.[^.\/]+)$/);
+    return match?.[1]?.toLowerCase() ?? "";
+}
+/** Bounded, adapter-local detection. It requires an actual application call pattern, not only an SDK dependency. */
+export function detectApplicationLlmTargets(projectRoot) {
+    const files = [];
+    const pending = [projectRoot];
+    while (pending.length && files.length < 500) {
+        const directory = pending.shift();
+        let entries;
+        try {
+            entries = readdirSync(directory, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const entry of entries) {
+            if (DETECTION_IGNORES.has(entry.name))
+                continue;
+            const path = join(directory, entry.name);
+            if (entry.isDirectory())
+                pending.push(path);
+            else if (entry.isFile() && DETECTION_EXTENSIONS.has(extensionOf(path)))
+                files.push(path);
+            if (files.length >= 500)
+                break;
+        }
+    }
+    const applicationId = basename(projectRoot);
+    const targets = [];
+    for (const path of files) {
+        let source;
+        try {
+            if (statSync(path).size > 1_000_000)
+                continue;
+            source = readFileSync(path, "utf8");
+        }
+        catch {
+            continue;
+        }
+        for (const { provider, pattern } of APPLICATION_CALL_PATTERNS) {
+            const match = pattern.exec(source);
+            if (!match)
+                continue;
+            const projectPath = relative(projectRoot, path).replaceAll("\\", "/");
+            const line = source.slice(0, match.index).split("\n").length;
+            const identity = `${applicationId}:${projectPath}:${provider}`;
+            targets.push({
+                id: `app-llm-${createHash("sha256").update(identity).digest("hex").slice(0, 12)}`,
+                kind: "application_llm_call",
+                name: `${provider} application call in ${projectPath}`,
+                applicationId,
+                callSites: [`${projectPath}:${line}`],
+                routeKey: `${projectPath}#${provider}`,
+                ...((source.match(/(?:DEFAULT_MODEL|defaultModel|model)\s*=\s*["'`]([^"'`]+)["'`]/)?.[1])
+                    ? { configuredModelId: source.match(/(?:DEFAULT_MODEL|defaultModel|model)\s*=\s*["'`]([^"'`]+)["'`]/)[1] }
+                    : {}),
+            });
+        }
+    }
+    return targets;
+}
+function detectedTargetsConstraint(targets) {
+    return targets.map((target) => ({
+        id: target.id,
+        kind: target.kind,
+        name: target.name,
+        applicationId: target.applicationId,
+        callSites: [...target.callSites],
+        routeKey: target.routeKey,
+        ...(target.configuredModelId ? { configuredModelId: target.configuredModelId } : {}),
+    }));
+}
+function validateReportedMetrics(metrics, targetId) {
+    for (const metric of metrics) {
+        if (metric.targetId !== targetId) {
+            throw new Error(`${metric.metricId} target does not match the reported application LLM target.`);
+        }
+        if (metric.state === "measured" && (!metric.evidence.length || metric.value === undefined)) {
+            throw new Error(`Measured metric ${metric.metricId} requires a numeric value and evidence.`);
+        }
+        if (metric.metricId === "tool_use_performance" && metric.state === "measured" &&
+            !metric.evidence.some((item) => item.source === "harness_trace")) {
+            throw new Error("tool_use_performance requires harness-trace evidence; test coverage is not tool-use evidence.");
+        }
+    }
+}
 function modelCatalogFingerprint(ctx) {
     const catalogue = ctx.modelRegistry.getAvailable().map((model) => ({
         provider: model.provider,
@@ -80,56 +196,69 @@ function privateIntentMessage(intent) {
         "Perform the intelligent work using the current project, session context, and available tools.",
         "Use harness-configured fast decision resources such as Jev System One aggressively for bounded ranking, triage, and judgment when available; OpenMerit does not call those services itself.",
         "You, the coding harness, own all research, evaluation design, instrumentation, measurement, Pareto calculation, and authorized configuration work. OpenMerit only validates the returned contract and evidence.",
-        "Instrument every required metric selected by the user. Report explicit covered and missing metric IDs; never describe observability as ready while a required metric is missing.",
+        "The system under test is the user-confirmed application LLM target. Pi's own model, tokens, cost, latency, and session are orchestration data and must never be submitted as application evidence.",
+        intent.targetId
+            ? `Every result and metric must reference application target ${intent.targetId}.`
+            : "Before setup succeeds, identify an actual application LLM call or shared application route and ask the user to confirm its stable target ID, call sites, and route key.",
+        "For setup, infer task-relevant metrics from the application rather than requiring the user to invent them. Before any setup run, show the user the proposed metrics, rationale, sampling strategy, representative case count, repetitions per case, aggregation, true performance thresholds if any, and estimated cost, then obtain confirmation or edits. Rate, reliability, distribution, percentile, variance, and average claims require multiple runs. Distinguish repeating the same input for stochastic variance from running representative task instances for category performance; use both when the claim needs both. minimumSamples is evidence quantity and must never be copied into constraint.value. A constraint is optional and represents only a real metric threshold in that metric's unit. For an active-session demo, default baselineAssessment and postSwapVerification to afterCompletedTasks: 1 unless the user changes the cadence; never return an empty cadence. Instrument every confirmed required metric. For setup or instrumentation, actually run each task-specific grader and collector through the application path with working provider authentication, inspect the resulting metric values, and cite the run artifacts. Report explicit covered and missing metric IDs; never describe observability as ready when a required metric cannot be produced. A setup smoke run is verification, not a production baseline sample.",
+        "Capture cost, token use, and task latency from the application LLM provider or application observability. Do not substitute Pi provider telemetry or zero for unavailable application telemetry.",
+        intent.kind === "instrument_observability"
+            ? "For instrumentation, execute the confirmed application target itself and return one measured MetricRecord for every covered required metric. Map provider/application usage fields to input_token_consumption and cost_per_successful_task, use the exact metric IDs and objective aggregation, include sampleCount and valid startedAt/completedAt windows, and attach durable observability_record or harness_trace evidence. Do not merely describe telemetry or add arbitrary fields to the application response; those do not count until converted into MetricRecords in the completion result."
+            : "",
+        "Tool-use performance must be derived from tool-call traces and outcomes; code coverage is not tool-use evidence.",
         intent.constraints.modelMutationAllowed
-            ? "This intent authorizes only the model mutation described by the active OpenMerit cycle."
+            ? `This intent authorizes only the application model mutation for target ${intent.targetId ?? "(missing target)"}. Never change Pi's model.`
             : "Do not change Pi's model or application model configuration.",
         "Before finishing, call openmerit_complete_intent exactly once with the result and durable evidence references.",
         JSON.stringify(intent),
     ].join("\n\n");
 }
-function piHarnessAdapter(pi) {
+function piHarnessAdapter(pi, ctx) {
     return {
         descriptor: PI_HARNESS_DESCRIPTOR,
         async dispatch(intent) {
             pi.appendEntry(INTENT_ENTRY_TYPE, { intent });
-            pi.sendMessage({
-                customType: INTENT_MESSAGE_TYPE,
-                content: privateIntentMessage(intent),
-                display: false,
-                details: { intentId: intent.id, kind: intent.kind },
-            }, { deliverAs: "followUp", triggerTurn: true });
+            if (ctx.mode === "tui" || ctx.mode === "rpc") {
+                pi.sendMessage({
+                    customType: INTENT_MESSAGE_TYPE,
+                    content: privateIntentMessage(intent),
+                    display: false,
+                    details: { intentId: intent.id, kind: intent.kind },
+                }, { deliverAs: "followUp", triggerTurn: true });
+            }
             return { accepted: true, harnessJobId: intent.id };
         },
+        async scheduleWakeup(request) {
+            if (request.signal !== "scheduled_tick")
+                return { accepted: false, reason: "Pi only schedules OpenMerit ticks." };
+            if (!ctx.model)
+                return { accepted: false, reason: "Select an authenticated Pi model before enabling background checks." };
+            try {
+                const path = await provisionPiWakeup({
+                    projectRoot: ctx.cwd,
+                    extensionPath: fileURLToPath(import.meta.url),
+                    provider: ctx.model.provider,
+                    modelId: ctx.model.id,
+                });
+                return { accepted: true, harnessJobId: path };
+            }
+            catch (error) {
+                return { accepted: false, reason: error instanceof Error ? error.message : String(error) };
+            }
+        },
     };
 }
-function restoreState(ctx, state) {
-    state.activeIntent = null;
-    for (const entry of ctx.sessionManager.getEntries()) {
-        if (entry.type !== "custom")
-            continue;
-        if (entry.customType === INTENT_ENTRY_TYPE) {
-            const stored = entry.data;
-            if (stored?.intent?.id)
-                state.activeIntent = stored.intent.id;
-        }
-        if (entry.customType === RESULT_ENTRY_TYPE) {
-            const stored = entry.data;
-            if (stored?.result?.intentId === state.activeIntent)
-                state.activeIntent = null;
-        }
-    }
-}
-async function dispatchIntent(pi, ctx, state, store, kind, prepareCompletionTool) {
+async function dispatchIntent(pi, ctx, state, store, kind, prepareCompletionTool, additionalConstraints) {
     if (state.activeIntent) {
         ctx.ui.notify(`openmerit: work is already active (${state.activeIntent})`, "warning");
         return;
     }
     prepareCompletionTool(kind);
-    const issued = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).issue(kind);
+    const issued = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).issue(kind, { additionalConstraints });
     const { intent } = issued;
     state.activeIntent = intent.id;
     if (kind === "establish_evals") {
+        state.applicationTarget = "detected";
         state.taskProfile = "pending";
         state.metricReadiness = "pending";
         state.evaluationReadiness = "pending";
@@ -142,7 +271,38 @@ async function dispatchIntent(pi, ctx, state, store, kind, prepareCompletionTool
     ctx.ui.notify(`openmerit: ${label} started (${intent.id})`, "info");
 }
 async function reportAutomationSignal(pi, ctx, state, store, signal, prepareCompletionTool) {
-    const result = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).recordAutomationSignal(signal);
+    const result = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).recordAutomationSignal(signal);
+    const persistedState = state.activeIntent ? await store.readState() : undefined;
+    if (state.activeIntent && persistedState?.activeIntent?.kind === "instrument_observability" && signal.type === "metric_window_available") {
+        const projectConfig = await store.readConfig();
+        const profile = projectConfig?.taskProfiles.find((item) => item.id === projectConfig.activeTaskProfileId);
+        const requiredObjectives = profile?.objectives.filter((item) => item.required) ?? [];
+        const measuredMetricIds = new Set(signal.metrics
+            .filter((metric) => metric.targetId === signal.targetId && metric.state === "measured" && metric.value !== undefined && metric.sampleCount > 0 && metric.evidence.length > 0)
+            .map((metric) => metric.metricId));
+        const coveredMetricIds = requiredObjectives.filter((objective) => measuredMetricIds.has(objective.metricId)).map((objective) => objective.metricId);
+        const missingMetricIds = requiredObjectives.filter((objective) => !measuredMetricIds.has(objective.metricId)).map((objective) => objective.metricId);
+        const evidence = [...new Map(signal.metrics.flatMap((metric) => metric.evidence).map((item) => [item.id, item])).values()];
+        const completion = {
+            protocolVersion: PROTOCOL_VERSION,
+            intentId: state.activeIntent,
+            status: "succeeded",
+            completedAt: new Date().toISOString(),
+            summary: `Instrumentation verified ${coveredMetricIds.length} required metric(s); ${missingMetricIds.length} remain missing.`,
+            evidence,
+            outputs: {
+                targetId: signal.targetId,
+                coveredMetricIds,
+                missingMetricIds,
+            },
+        };
+        const accepted = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).acceptResult(completion);
+        state.activeIntent = null;
+        state.lifecycleStage = accepted.state.cycle.stage;
+        state.metricReadiness = accepted.state.observability?.status === "ready" ? "assessed" : "pending";
+        prepareCompletionTool("instrument_observability");
+        return;
+    }
     if (result.decision.action === "issue_intent" && !state.activeIntent) {
         await dispatchIntent(pi, ctx, state, store, result.decision.intentKind, prepareCompletionTool);
     }
@@ -150,7 +310,15 @@ async function reportAutomationSignal(pi, ctx, state, store, signal, prepareComp
 export default function openMeritExtension(pi) {
     const state = createInitialUiState();
     let store;
-    let suppressNextSettledObservation = false;
+    const queuedUserInputs = [];
+    const releaseQueuedUserInput = () => {
+        if (state.activeIntent || !queuedUserInputs.length)
+            return;
+        const queued = queuedUserInputs.splice(0);
+        const text = queued.map((item) => item.text).join("\n\n");
+        const images = queued.flatMap((item) => item.images ?? []);
+        pi.sendUserMessage(images.length ? [{ type: "text", text }, ...images] : text, { deliverAs: "followUp" });
+    };
     const registerCompletionTool = (kind) => {
         const parameters = kind
             ? completionToolSchemaFor(kind)
@@ -214,13 +382,27 @@ export default function openMeritExtension(pi) {
                         }
                         : {}),
                 };
-                const accepted = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).acceptResult(result);
+                const accepted = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).acceptResult(result);
+                if (accepted.intentKind === "establish_evals" &&
+                    (accepted.state.cycle.policy?.checkPolicy.execution.mode !== "persistent" ||
+                        !accepted.state.cycle.policy?.checkPolicy.execution.infrastructureChangesAllowed)) {
+                    try {
+                        await removePiWakeup(ctx.cwd);
+                    }
+                    catch (error) {
+                        ctx.ui.notify(`openmerit: could not remove prior wakeup: ${error instanceof Error ? error.message : String(error)}`, "warning");
+                    }
+                }
                 pi.appendEntry(RESULT_ENTRY_TYPE, { result });
-                suppressNextSettledObservation = true;
                 const { intentKind, state: nextState } = accepted;
                 const stage = nextState.cycle.stage;
                 state.activeIntent = null;
                 state.lifecycleStage = stage;
+                state.incumbentCandidate = nextState.cycle.activeCandidateId ?? null;
+                state.candidateCount = nextState.candidates?.length ?? 0;
+                state.assessmentCount = nextState.assessments?.length ?? 0;
+                state.evaluationSpend = nextState.automation.evaluationSpend ?? 0;
+                state.proposedCandidate = nextState.cycle.proposedCandidateId ?? null;
                 registerCompletionTool();
                 state.nextAssessment = accepted.automation.plan.nextWakeupAt ??
                     (accepted.automation.plan.scheduling === "external_scheduler_required" ? "external scheduler required" : "signal driven");
@@ -228,6 +410,7 @@ export default function openMeritExtension(pi) {
                     ctx.ui.notify(`openmerit automation: ${accepted.automation.plan.gaps.join(" ")}`, "warning");
                 }
                 if (input.status === "succeeded" && intentKind === "establish_evals") {
+                    state.applicationTarget = "confirmed";
                     state.taskProfile = "configured";
                     state.metricReadiness = nextState.observability?.status === "ready" ? "assessed" : "pending";
                     state.evaluationReadiness = "assessed";
@@ -244,6 +427,10 @@ export default function openMeritExtension(pi) {
                         ctx.ui.notify(`openmerit: ${decision.reason}`, "warning");
                     }
                 }
+                // A successful result can immediately issue another OpenMerit intent
+                // (for example, setup -> instrumentation). Release product work only
+                // after that chain has no active follow-up.
+                releaseQueuedUserInput();
                 return {
                     content: [{ type: "text", text: `Recorded OpenMerit intent ${input.intentId} as ${input.status}.` }],
                     details: { result },
@@ -255,46 +442,159 @@ export default function openMeritExtension(pi) {
     pi.registerTool({
         name: "openmerit_report_signal",
         label: "Report OpenMerit Signal",
-        description: "Report a verified observability metric window or completed post-swap verification window to OpenMerit.",
-        promptSnippet: "Report verified OpenMerit observability signals",
+        description: "Report a verified application LLM task, observability metric window, or completed post-swap verification window to OpenMerit.",
+        promptSnippet: "Report verified application-target OpenMerit signals",
         promptGuidelines: [
             "Use stable signalId values so retried deliveries remain idempotent.",
-            "Report only measured metrics backed by the referenced observability or evaluation artifacts.",
+            "Every signal and metric must reference the confirmed application LLM target.",
+            "Report only application-provider telemetry and measured metrics backed by referenced application observability or evaluation artifacts. Never report Pi's own model usage.",
         ],
         parameters: Type.Object({
             signalId: Type.String({ minLength: 1 }),
-            type: StringEnum(["metric_window_available", "verification_window_completed"]),
+            type: StringEnum(["product_task_completed", "metric_window_available", "verification_window_completed"]),
             occurredAt: Type.Optional(Type.String({ minLength: 1 })),
-            metrics: Type.Array(MetricRecordSchema),
+            targetId: Type.String({ minLength: 1 }),
+            metrics: Type.Optional(Type.Array(MetricRecordSchema)),
+            modelId: Type.Optional(Type.String({ minLength: 1 })),
+            evidence: Type.Optional(Type.Array(Type.Object({
+                id: Type.String({ minLength: 1 }),
+                source: StringEnum(["harness_trace", "evaluation_artifact", "observability_record", "external_source", "user_feedback"]),
+                uri: Type.String({ minLength: 1 }),
+                mediaType: Type.Optional(Type.String({ minLength: 1 })),
+                digest: Type.Optional(Type.String({ minLength: 1 })),
+            }, { additionalProperties: false }))),
+            telemetry: Type.Optional(Type.Object({
+                startedAt: Type.String({ minLength: 1 }),
+                completedAt: Type.String({ minLength: 1 }),
+                durationMs: Type.Number({ minimum: 0 }),
+                inputTokens: Type.Number({ minimum: 0 }),
+                outputTokens: Type.Number({ minimum: 0 }),
+                cacheReadTokens: Type.Number({ minimum: 0 }),
+                cacheWriteTokens: Type.Number({ minimum: 0 }),
+                cost: Type.Number({ minimum: 0 }),
+                currency: Type.Literal("USD"),
+            }, { additionalProperties: false })),
         }, { additionalProperties: false }),
         constrainedSampling: { type: "json_schema", strict: "prefer" },
         async execute(_toolCallId, input, _signal, _onUpdate, ctx) {
             store ??= new ProjectStore(ctx.cwd);
-            await reportAutomationSignal(pi, ctx, state, store, {
-                protocolVersion: PROTOCOL_VERSION,
-                id: input.signalId,
-                type: input.type,
-                occurredAt: input.occurredAt ?? new Date().toISOString(),
-                metrics: input.metrics,
-            }, registerCompletionTool);
+            if (input.type === "product_task_completed") {
+                if (!input.modelId || !input.evidence?.length) {
+                    throw new Error("A completed application task requires the application modelId and durable evidence.");
+                }
+                await reportAutomationSignal(pi, ctx, state, store, {
+                    protocolVersion: PROTOCOL_VERSION,
+                    id: input.signalId,
+                    type: input.type,
+                    occurredAt: input.occurredAt ?? new Date().toISOString(),
+                    targetId: input.targetId,
+                    modelId: input.modelId,
+                    evidence: input.evidence,
+                    ...(input.telemetry ? { telemetry: input.telemetry } : {}),
+                }, registerCompletionTool);
+            }
+            else {
+                if (!input.metrics)
+                    throw new Error(`${input.type} requires application-target metrics.`);
+                validateReportedMetrics(input.metrics, input.targetId);
+                await reportAutomationSignal(pi, ctx, state, store, {
+                    protocolVersion: PROTOCOL_VERSION,
+                    id: input.signalId,
+                    type: input.type,
+                    occurredAt: input.occurredAt ?? new Date().toISOString(),
+                    targetId: input.targetId,
+                    metrics: input.metrics,
+                }, registerCompletionTool);
+            }
             return {
                 content: [{ type: "text", text: `Recorded OpenMerit automation signal ${input.signalId}.` }],
                 details: { signalId: input.signalId, type: input.type },
             };
         },
     });
+    pi.on("input", (event, ctx) => {
+        if (ctx.mode !== "tui" || !state.activeIntent || event.source !== "interactive" || ctx.isIdle())
+            return;
+        queuedUserInputs.push({ text: event.text, ...(event.images ? { images: event.images } : {}) });
+        ctx.ui.notify(`openmerit: queued your request until active setup work finishes (${state.activeIntent})`, "info");
+        return { action: "handled" };
+    });
+    // A fresh demo can start in an empty directory. Re-scan only after Pi has
+    // settled the user's build turn, so setup sees a complete application call
+    // instead of racing partially written files.
+    pi.on("turn_end", async (_event, ctx) => {
+        if (!ctx.hasUI || state.proactiveNudgesPaused || state.activeIntent)
+            return;
+        store ??= new ProjectStore(ctx.cwd);
+        const projectState = await store.readState();
+        if (projectState?.cycle.targetId || (projectState && projectState.cycle.stage !== "unconfigured"))
+            return;
+        const detectedApplicationTargets = detectApplicationLlmTargets(ctx.cwd);
+        if (!detectedApplicationTargets.length)
+            return;
+        ctx.ui.notify(`openmerit: detected ${detectedApplicationTargets.length} application LLM target candidate(s) after the build; starting evidence setup`, "info");
+        await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool, {
+            applicationTargetRequired: true,
+            detectedApplicationTargets: detectedTargetsConstraint(detectedApplicationTargets),
+            ...(detectedApplicationTargets.length === 1 && detectedApplicationTargets[0].configuredModelId ? { expectedIncumbentModelId: detectedApplicationTargets[0].configuredModelId } : {}),
+        });
+    });
     pi.on("session_start", async (_event, ctx) => {
         store = new ProjectStore(ctx.cwd);
         let projectState = await store.readState();
         if (projectState) {
             state.lifecycleStage = projectState.cycle.stage;
+            state.proactiveNudgesPaused = projectState.automation.paused === true;
             state.activeIntent = projectState.activeIntent?.id ?? null;
-            if (projectState.activeIntent)
+            if (projectState.activeIntent) {
                 registerCompletionTool(projectState.activeIntent.kind);
+                if (ctx.hasUI || ctx.mode === "rpc") {
+                    pi.sendMessage({
+                        customType: INTENT_MESSAGE_TYPE,
+                        content: privateIntentMessage(projectState.activeIntent),
+                        display: false,
+                        details: { intentId: projectState.activeIntent.id, kind: projectState.activeIntent.kind, resumed: true },
+                    }, { deliverAs: "followUp", triggerTurn: true });
+                }
+            }
+            state.applicationTarget = projectState.cycle.targetId ? "confirmed" : "not_detected";
             state.taskProfile = projectState.cycle.taskProfileId ? "configured" : "not_configured";
             state.metricReadiness = projectState.observability?.status === "ready" ? "assessed" : "pending";
             state.evaluationReadiness = projectState.cycle.taskProfileId ? "assessed" : "not_assessed";
-            const automation = await new OpenMeritCoordinator(store, piHarnessAdapter(pi)).reconcileAutomation();
+            state.incumbentCandidate = projectState.cycle.activeCandidateId ?? null;
+            state.candidateCount = projectState.candidates?.length ?? 0;
+            state.assessmentCount = projectState.assessments?.length ?? 0;
+            state.evaluationSpend = projectState.automation.evaluationSpend ?? 0;
+            state.proposedCandidate = projectState.cycle.proposedCandidateId ?? null;
+            if (!projectState.cycle.targetId) {
+                if (ctx.hasUI && !state.proactiveNudgesPaused && projectState.cycle.stage === "unconfigured") {
+                    const detectedApplicationTargets = detectApplicationLlmTargets(ctx.cwd);
+                    if (detectedApplicationTargets.length) {
+                        await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool, {
+                            applicationTargetRequired: true,
+                            detectedApplicationTargets: detectedTargetsConstraint(detectedApplicationTargets),
+                            ...(detectedApplicationTargets.length === 1 && detectedApplicationTargets[0].configuredModelId ? { expectedIncumbentModelId: detectedApplicationTargets[0].configuredModelId } : {}),
+                        });
+                    }
+                }
+                else if (state.proactiveNudgesPaused) {
+                    ctx.ui.notify("openmerit: proactive checks remain paused; use /openmerit resume to re-enable them", "info");
+                }
+                else {
+                    ctx.ui.notify("openmerit: existing state has no confirmed application LLM target; use /openmerit setup", "warning");
+                }
+                return;
+            }
+            if (state.proactiveNudgesPaused || projectState.cycle.policy?.checkPolicy.execution.mode !== "persistent" ||
+                !projectState.cycle.policy?.checkPolicy.execution.infrastructureChangesAllowed) {
+                try {
+                    await removePiWakeup(ctx.cwd);
+                }
+                catch (error) {
+                    ctx.ui.notify(`openmerit: could not remove prior wakeup: ${error instanceof Error ? error.message : String(error)}`, "warning");
+                }
+            }
+            const automation = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).reconcileAutomation();
             state.nextAssessment = automation.plan.nextWakeupAt ??
                 (automation.plan.scheduling === "external_scheduler_required" ? "external scheduler required" : "signal driven");
             if (automation.plan.gaps.length) {
@@ -306,6 +606,7 @@ export default function openMeritExtension(pi) {
                 id: `signal-${crypto.randomUUID()}`,
                 type: "model_catalog_changed",
                 occurredAt: new Date().toISOString(),
+                targetId: projectState.cycle.targetId,
                 fingerprint,
             }, registerCompletionTool);
             if (!state.activeIntent) {
@@ -314,35 +615,30 @@ export default function openMeritExtension(pi) {
                     id: `signal-${crypto.randomUUID()}`,
                     type: "scheduled_tick",
                     occurredAt: new Date().toISOString(),
+                    targetId: projectState.cycle.targetId,
                 }, registerCompletionTool);
             }
         }
         else if (ctx.hasUI && !state.proactiveNudgesPaused) {
-            ctx.ui.notify("openmerit: no frontier profile found; asking Pi to establish evals and observability", "info");
-            await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool);
+            const detectedApplicationTargets = detectApplicationLlmTargets(ctx.cwd);
+            if (detectedApplicationTargets.length) {
+                ctx.ui.notify(`openmerit: detected ${detectedApplicationTargets.length} application LLM target candidate(s); asking Pi to confirm the target before setup`, "info");
+                await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool, {
+                    applicationTargetRequired: true,
+                    detectedApplicationTargets: detectedTargetsConstraint(detectedApplicationTargets),
+                    ...(detectedApplicationTargets.length === 1 && detectedApplicationTargets[0].configuredModelId ? { expectedIncumbentModelId: detectedApplicationTargets[0].configuredModelId } : {}),
+                });
+            }
+            else {
+                ctx.ui.notify("openmerit: no application LLM call detected yet; automatic setup will check again after Pi finishes a build turn (/openmerit setup is available for recovery)", "info");
+            }
         }
-        restoreState(ctx, state);
-    });
-    pi.on("agent_settled", async (_event, ctx) => {
-        if (suppressNextSettledObservation) {
-            suppressNextSettledObservation = false;
-            return;
+        if (projectState?.activeIntent) {
+            ctx.ui.notify(`openmerit: recovered interrupted work (${projectState.activeIntent.id}); use /openmerit cancel to abandon it`, "warning");
         }
-        store ??= new ProjectStore(ctx.cwd);
-        const current = await store.readState();
-        if (!current || current.activeIntent)
-            return;
-        await reportAutomationSignal(pi, ctx, state, store, {
-            protocolVersion: PROTOCOL_VERSION,
-            id: `signal-${crypto.randomUUID()}`,
-            type: "product_task_completed",
-            occurredAt: new Date().toISOString(),
-            modelId: ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : "unknown",
-            contextTokens: ctx.getContextUsage()?.tokens ?? null,
-        }, registerCompletionTool);
     });
     pi.registerCommand("openmerit", {
-        description: "OpenMerit: status, setup, assess, frontier, approve, pause/resume, logs, doctor",
+        description: "OpenMerit: status, setup, assess, frontier, approve, cancel, pause/resume, logs, doctor",
         handler: async (args, ctx) => {
             const subcommand = args.trim().split(/\s+/)[0]?.toLowerCase() || "status";
             switch (subcommand) {
@@ -351,11 +647,27 @@ export default function openMeritExtension(pi) {
                     return;
                 case "setup":
                     store ??= new ProjectStore(ctx.cwd);
-                    await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool);
+                    {
+                        const detectedApplicationTargets = detectApplicationLlmTargets(ctx.cwd);
+                        await dispatchIntent(pi, ctx, state, store, "establish_evals", registerCompletionTool, {
+                            applicationTargetRequired: true,
+                            detectedApplicationTargets: detectedTargetsConstraint(detectedApplicationTargets),
+                            ...(detectedApplicationTargets.length === 1 && detectedApplicationTargets[0].configuredModelId ? { expectedIncumbentModelId: detectedApplicationTargets[0].configuredModelId } : {}),
+                        });
+                    }
                     return;
                 case "assess":
                     store ??= new ProjectStore(ctx.cwd);
-                    await dispatchIntent(pi, ctx, state, store, "run_assessment", registerCompletionTool);
+                    {
+                        const assessment = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).assessBaseline();
+                        state.lifecycleStage = assessment.state.cycle.stage;
+                        ctx.ui.notify(assessment.errors.length
+                            ? `openmerit: baseline is not ready: ${assessment.errors.join("; ")}`
+                            : "openmerit: baseline meets the required metric thresholds; starting candidate discovery", assessment.errors.length ? "warning" : "info");
+                        if (assessment.decision.action === "issue_intent") {
+                            await dispatchIntent(pi, ctx, state, store, assessment.decision.intentKind, registerCompletionTool);
+                        }
+                    }
                     return;
                 case "frontier":
                     store ??= new ProjectStore(ctx.cwd);
@@ -386,22 +698,81 @@ export default function openMeritExtension(pi) {
                     await dispatchIntent(pi, ctx, state, store, "apply_model_swap", registerCompletionTool);
                     return;
                 }
-                case "pause":
+                case "cancel": {
+                    store ??= new ProjectStore(ctx.cwd);
+                    const projectState = await store.readState();
+                    if (!projectState?.activeIntent) {
+                        ctx.ui.notify("openmerit: there is no active work to cancel", "warning");
+                        return;
+                    }
+                    const cancelled = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).acceptResult({
+                        protocolVersion: PROTOCOL_VERSION,
+                        intentId: projectState.activeIntent.id,
+                        status: "cancelled",
+                        completedAt: new Date().toISOString(),
+                        summary: "Cancelled explicitly by the user after an interrupted harness run.",
+                        evidence: [],
+                    });
+                    state.activeIntent = null;
+                    state.lifecycleStage = cancelled.state.cycle.stage;
+                    registerCompletionTool();
+                    releaseQueuedUserInput();
+                    ctx.ui.notify(`openmerit: cancelled ${projectState.activeIntent.id}`, "info");
+                    return;
+                }
+                case "pause": {
+                    store ??= new ProjectStore(ctx.cwd);
+                    const existing = await store.readState();
+                    const now = new Date().toISOString();
+                    const paused = existing
+                        ? { ...existing, automation: { ...existing.automation, paused: true }, updatedAt: now }
+                        : {
+                            schemaVersion: 1,
+                            cycle: { id: `cycle-${crypto.randomUUID()}`, stage: "unconfigured", verifiedSwapCount: 0, baselineEvidenceSufficient: false },
+                            automation: { setupNudgeShown: false, observedCompletedTasks: 0, lastAssessmentTaskCount: 0, paused: true },
+                            updatedAt: now,
+                        };
+                    await store.writeState(paused);
                     state.proactiveNudgesPaused = true;
+                    state.nextAssessment = "paused";
+                    try {
+                        await removePiWakeup(ctx.cwd);
+                    }
+                    catch (error) {
+                        ctx.ui.notify(`openmerit: could not unload wakeup: ${error instanceof Error ? error.message : String(error)}`, "warning");
+                    }
                     ctx.ui.notify("openmerit: proactive nudges paused", "info");
                     return;
-                case "resume":
+                }
+                case "resume": {
+                    store ??= new ProjectStore(ctx.cwd);
+                    const existing = await store.readState();
+                    if (existing) {
+                        await store.writeState({ ...existing, automation: { ...existing.automation, paused: false }, updatedAt: new Date().toISOString() });
+                    }
                     state.proactiveNudgesPaused = false;
+                    if (existing?.cycle.targetId) {
+                        const automation = await new OpenMeritCoordinator(store, piHarnessAdapter(pi, ctx)).reconcileAutomation();
+                        state.nextAssessment = automation.plan.nextWakeupAt ??
+                            (automation.plan.scheduling === "external_scheduler_required" ? "external scheduler required" : "signal driven");
+                        if (automation.plan.gaps.length) {
+                            ctx.ui.notify(`openmerit automation: ${automation.plan.gaps.join(" ")}`, "warning");
+                        }
+                    }
                     ctx.ui.notify("openmerit: proactive nudges resumed", "info");
                     return;
+                }
                 case "doctor":
                     ctx.ui.notify([
                         "OpenMerit doctor",
                         "Pi adapter        : loaded",
                         "intent executor   : connected",
                         `audit log          : ${store?.eventsPath ?? `${ctx.cwd}/.openmerit/events.jsonl`}`,
-                        "model mutation    : disabled",
-                        "persistent schedule: unavailable (active-session ticks only)",
+                        "Pi model mutation : prohibited",
+                        "application swap  : target-bound after approval",
+                        `persistent schedule: ${process.platform === "darwin" ? "opt-in LaunchAgent when policy permits" : "unavailable on this platform"}`,
+                        `schedule file     : ${piWakeupPlistPath(ctx.cwd)}`,
+                        `schedule status   : ${ctx.cwd}/.openmerit/scheduler-status.json`,
                         `interactive UI     : ${ctx.hasUI ? "available" : "unavailable"}`,
                     ].join("\n"), "info");
                     return;
@@ -416,7 +787,7 @@ export default function openMeritExtension(pi) {
                     ].join("\n"), "info");
                     return;
                 default:
-                    ctx.ui.notify("openmerit: usage: /openmerit [status|setup|assess|frontier|approve|pause|resume|logs|doctor]", "warning");
+                    ctx.ui.notify("openmerit: usage: /openmerit [status|setup|assess|frontier|approve|cancel|pause|resume|logs|doctor]", "warning");
             }
         },
     });