@interf/compiler 0.22.2 → 0.33.0

package/dist/packages/runtime/runtime-verify-runs.js

@@ -0,0 +1,295 @@
+import { mkdirSync } from "node:fs";
+import { asProjectDataDir, projectContextGraphPath, } from "../contracts/lib/project-paths.js";
+import { buildPlanIdForProjectConfig, DEFAULT_BUILD_PLAN_ID, listProjectConfigs, loadSourceFolderConfig, } from "../projects/source-config.js";
+import { loadAgentsRegistry } from "./agents/registry.js";
+import { createRunEventId, createRunEventTimestamp, } from "./execution/events.js";
+import { listJsonFiles, newestFirst, readVerifyRunAt, verifyRunPath, verifyRunsRoot, writeJsonFile, } from "./runtime-persistence.js";
+import { createRunId } from "./runtime-resource-builders.js";
+import { LocalRunHandlerResultSchema, VerifyRunCreateRequestSchema, VerifyRunResourceSchema, } from "./schemas/index.js";
+import { readSavedBenchmarkRun } from "./verify/benchmark-run.js";
+import { createContextGraphTestTarget } from "./verify/verify-targets.js";
+function requestedAgentExecutorInfo(agentName, registry = loadAgentsRegistry()) {
+    const requested = agentName?.trim();
+    if (!requested)
+        return null;
+    const agent = registry.agents.find((entry) => entry.name === requested && entry.available !== false);
+    if (!agent) {
+        throw new Error(`Benchmark agent "${requested}" is not connected.`);
+    }
+    return {
+        kind: "local-agent",
+        name: agent.name,
+        display_name: agent.display_name,
+        command: agent.command,
+        model: agent.prefs?.model ?? null,
+        effort: agent.prefs?.effort ?? null,
+    };
+}
+export function benchmarkFanoutAgentNames(registry) {
+    return [...new Set(registry.agents
+            .filter((agent) => agent.available !== false)
+            .map((agent) => agent.name))];
+}
+function isTerminalVerifyStatus(status) {
+    return status === "succeeded" || status === "failed" || status === "cancelled";
+}
+export function latestTopLevelVerifyRun(runs) {
+    return runs.find((run) => !run.parent_run_id) ?? runs[0] ?? null;
+}
+export function summarizeVerifyBatch(childRunIds, children) {
+    return {
+        child_run_ids: childRunIds,
+        total_agents: childRunIds.length,
+        completed_agents: children.filter((run) => isTerminalVerifyStatus(run.status)).length,
+        succeeded_agents: children.filter((run) => run.status === "succeeded").length,
+        failed_agents: children.filter((run) => run.status === "failed" || run.status === "cancelled").length,
+    };
+}
+function statusForBatch(summary) {
+    if (summary.completed_agents < summary.total_agents)
+        return "running";
+    return summary.failed_agents > 0 ? "failed" : "succeeded";
+}
+function latestContextGraphPathForProject(runtime, projectDataDir, projectName) {
+    const project = listProjectConfigs(loadSourceFolderConfig(projectDataDir))
+        .find((entry) => entry.name === projectName);
+    const buildRuns = runtime.listBuildRunsForProject(projectDataDir, projectName);
+    const selected = project?.latest_context_graph_run_id
+        ? buildRuns.find((run) => run.run_id === project.latest_context_graph_run_id && run.status === "succeeded") ?? null
+        : null;
+    return selected?.context_graph_path ??
+        buildRuns.find((run) => run.status === "succeeded")?.context_graph_path ??
+        projectContextGraphPath(asProjectDataDir(projectDataDir), projectName);
+}
+export function listVerifyRuns(runtime, projectDataDir) {
+    return newestFirst(listProjectConfigs(loadSourceFolderConfig(projectDataDir))
+        .flatMap((project) => listVerifyRunsForProject(runtime, projectDataDir, project.name)));
+}
+export function listVerifyRunsForProject(runtime, projectDataDir, projectName) {
+    return runtime.verifyRunCache.get(projectDataDir, projectName, () => {
+        const contextGraphPath = latestContextGraphPathForProject(runtime, projectDataDir, projectName);
+        return newestFirst(listJsonFiles(verifyRunsRoot(contextGraphPath))
+            .map(readVerifyRunAt)
+            .filter((run) => run !== null));
+    }, (run) => run.run_id);
+}
+export function getVerifyRun(runtime, projectDataDir, runId) {
+    const known = runtime.verifyRunCache.projectFor(projectDataDir, runId);
+    if (known) {
+        const found = listVerifyRunsForProject(runtime, projectDataDir, known).find((run) => run.run_id === runId);
+        if (found)
+            return found;
+    }
+    return listVerifyRuns(runtime, projectDataDir).find((run) => run.run_id === runId) ?? null;
+}
+export async function createVerifyRun(runtime, projectDataDir, requestValue) {
+    return createVerifyRunForRegistry(runtime, projectDataDir, requestValue, loadAgentsRegistry());
+}
+export async function createVerifyRunForRegistry(runtime, projectDataDir, requestValue, registry) {
+    const request = VerifyRunCreateRequestSchema.parse(requestValue);
+    if (!request.agent_name) {
+        const agentNames = benchmarkFanoutAgentNames(registry);
+        if (agentNames.length > 1) {
+            return createVerifyBatchRun(runtime, projectDataDir, request, agentNames, registry);
+        }
+        if (agentNames.length === 1) {
+            return createSingleVerifyRun(runtime, projectDataDir, {
+                ...request,
+                agent_name: agentNames[0],
+            }, { registry });
+        }
+    }
+    return createSingleVerifyRun(runtime, projectDataDir, request, { registry });
+}
+async function createVerifyBatchRun(runtime, projectDataDir, request, agentNames, registry) {
+    const projectConfig = runtime.resolveProjectConfig(projectDataDir, request.project);
+    const contextGraphPath = latestContextGraphPathForProject(runtime, projectDataDir, projectConfig.name);
+    const contextGraphTarget = createContextGraphTestTarget(contextGraphPath, projectConfig.name, buildPlanIdForProjectConfig(projectConfig) ?? DEFAULT_BUILD_PLAN_ID);
+    const runId = createRunId("verify");
+    const childRunIds = agentNames.map(() => createRunId("verify"));
+    const now = new Date().toISOString();
+    const initial = VerifyRunResourceSchema.parse({
+        run_id: runId,
+        status: "running",
+        project: projectConfig.name,
+        target: request.target,
+        source_path: projectDataDir,
+        context_graph_path: contextGraphTarget.eligible ? contextGraphPath : null,
+        started_at: now,
+        agent_label: `${agentNames.length} agents`,
+        batch: summarizeVerifyBatch(childRunIds, []),
+        benchmark_run: null,
+        events: [],
+    });
+    writeVerifyRun(runtime, projectDataDir, contextGraphPath, initial);
+    await Promise.all(agentNames.map((agentName, index) => createSingleVerifyRun(runtime, projectDataDir, {
+        ...request,
+        agent_name: agentName,
+    }, {
+        parentRunId: runId,
+        registry,
+        runId: childRunIds[index],
+        startedAt: now,
+    })));
+    return readVerifyRunAt(verifyRunPath(contextGraphPath, runId)) ?? initial;
+}
+async function createSingleVerifyRun(runtime, projectDataDir, request, options = {}) {
+    const projectConfig = runtime.resolveProjectConfig(projectDataDir, request.project);
+    const contextGraphPath = latestContextGraphPathForProject(runtime, projectDataDir, projectConfig.name);
+    const contextGraphTarget = createContextGraphTestTarget(contextGraphPath, projectConfig.name, buildPlanIdForProjectConfig(projectConfig) ?? DEFAULT_BUILD_PLAN_ID);
+    const runId = options.runId ?? createRunId("verify");
+    const now = options.startedAt ?? new Date().toISOString();
+    const requestedExecutor = requestedAgentExecutorInfo(request.agent_name, options.registry);
+    const initial = VerifyRunResourceSchema.parse({
+        run_id: runId,
+        status: "running",
+        project: projectConfig.name,
+        target: request.target,
+        source_path: projectDataDir,
+        context_graph_path: contextGraphTarget.eligible ? contextGraphPath : null,
+        started_at: now,
+        ...(requestedExecutor ? {
+            executor: requestedExecutor,
+            agent_label: requestedExecutor.display_name,
+        } : {}),
+        ...(options.parentRunId ? { parent_run_id: options.parentRunId } : {}),
+        benchmark_run: null,
+        events: [],
+    });
+    writeVerifyRun(runtime, projectDataDir, contextGraphPath, initial);
+    void runVerifyInBackground(runtime, projectDataDir, request, {
+        runId,
+        sourcePath: projectDataDir,
+        contextGraphPath,
+        projectConfig,
+    }, initial);
+    return initial;
+}
+async function runVerifyInBackground(runtime, projectDataDir, request, context, initial) {
+    let finalRun = null;
+    runtime.beginActiveRun();
+    try {
+        if (!runtime.handlers.createVerifyRun) {
+            throw new Error("No verify-run handler is configured for this local service.");
+        }
+        const result = LocalRunHandlerResultSchema.parse(await runtime.handlers.createVerifyRun(request, context));
+        const benchmarkRun = result.benchmark_run ?? null;
+        const executor = result.executor ?? initial.executor ?? null;
+        const resultEvent = benchmarkRun
+            ? checksEvaluatedEvent(context.runId, benchmarkRun)
+            : null;
+        const nextWithoutReadiness = VerifyRunResourceSchema.parse({
+            ...initial,
+            status: result.ok ? "succeeded" : "failed",
+            finished_at: new Date().toISOString(),
+            ...(executor ? {
+                executor,
+                agent_label: result.agent_label ?? executor.display_name,
+            } : {}),
+            benchmark_run: benchmarkRun,
+            events: resultEvent ? [resultEvent] : [],
+            ...(!result.ok ? { error: result.error ?? "Benchmark check failed." } : {}),
+        });
+        writeVerifyRun(runtime, projectDataDir, context.contextGraphPath, nextWithoutReadiness);
+        const readiness = runtime.computeProjectReadiness(projectDataDir, context.projectConfig);
+        const next = VerifyRunResourceSchema.parse({
+            ...nextWithoutReadiness,
+            readiness,
+            events: [
+                ...nextWithoutReadiness.events,
+                readinessUpdatedEvent(context.runId, context.projectConfig.name, readiness),
+            ],
+        });
+        writeVerifyRun(runtime, projectDataDir, context.contextGraphPath, next);
+        finalRun = next;
+    }
+    catch (error) {
+        const failedWithoutReadiness = VerifyRunResourceSchema.parse({
+            ...initial,
+            status: "failed",
+            finished_at: new Date().toISOString(),
+            error: error instanceof Error ? error.message : String(error),
+        });
+        writeVerifyRun(runtime, projectDataDir, context.contextGraphPath, failedWithoutReadiness);
+        const readiness = runtime.computeProjectReadiness(projectDataDir, context.projectConfig);
+        const next = VerifyRunResourceSchema.parse({
+            ...failedWithoutReadiness,
+            readiness,
+            events: [readinessUpdatedEvent(context.runId, context.projectConfig.name, readiness)],
+        });
+        writeVerifyRun(runtime, projectDataDir, context.contextGraphPath, next);
+        finalRun = next;
+    }
+    finally {
+        if (finalRun?.parent_run_id) {
+            updateVerifyBatchRun(runtime, projectDataDir, context.contextGraphPath, finalRun.parent_run_id);
+        }
+        runtime.endActiveRun();
+    }
+}
+function updateVerifyBatchRun(runtime, projectDataDir, contextGraphPath, parentRunId) {
+    const parent = readVerifyRunAt(verifyRunPath(contextGraphPath, parentRunId));
+    if (!parent?.batch)
+        return;
+    const childRunIds = parent.batch.child_run_ids;
+    const children = childRunIds
+        .map((childRunId) => readVerifyRunAt(verifyRunPath(contextGraphPath, childRunId)))
+        .filter((run) => run !== null);
+    const batch = summarizeVerifyBatch(childRunIds, children);
+    const status = statusForBatch(batch);
+    const terminal = isTerminalVerifyStatus(status);
+    const nextWithoutReadiness = VerifyRunResourceSchema.parse({
+        ...parent,
+        status,
+        batch,
+        ...(terminal
+            ? {
+                finished_at: new Date().toISOString(),
+            }
+            : {}),
+        ...(terminal && batch.failed_agents > 0
+            ? { error: `${batch.failed_agents}/${batch.total_agents} benchmark agent${batch.failed_agents === 1 ? "" : "s"} failed.` }
+            : {}),
+    });
+    writeVerifyRun(runtime, projectDataDir, contextGraphPath, nextWithoutReadiness);
+    if (!terminal)
+        return;
+    const readiness = runtime.computeProjectReadiness(projectDataDir, runtime.resolveProjectConfig(projectDataDir, parent.project));
+    const next = VerifyRunResourceSchema.parse({
+        ...nextWithoutReadiness,
+        readiness,
+        events: [readinessUpdatedEvent(parent.run_id, parent.project, readiness)],
+    });
+    writeVerifyRun(runtime, projectDataDir, contextGraphPath, next);
+}
+export function writeVerifyRun(runtime, projectDataDir, contextGraphPath, run) {
+    mkdirSync(verifyRunsRoot(contextGraphPath), { recursive: true });
+    writeJsonFile(verifyRunPath(contextGraphPath, run.run_id), VerifyRunResourceSchema.parse(run));
+    runtime.verifyRunCache.invalidateProject(projectDataDir, run.project);
+    runtime.readinessCache.invalidateProject(projectDataDir, run.project);
+}
+export function readLatestBenchmarkRun(projectDataDir, projectName) {
+    return readSavedBenchmarkRun(projectDataDir, projectName);
+}
+export function checksEvaluatedEvent(runId, benchmarkRun) {
+    const target = benchmarkRun.context_graph ?? benchmarkRun.source_files;
+    return {
+        type: "checks.evaluated",
+        event_id: createRunEventId("event"),
+        run_id: runId,
+        timestamp: createRunEventTimestamp(),
+        passed: target?.passed_cases ?? 0,
+        total: target?.total_cases ?? 0,
+        test_run_id: runId,
+    };
+}
+export function readinessUpdatedEvent(runId, project, readiness) {
+    return {
+        type: "readiness.updated",
+        event_id: createRunEventId("event"),
+        run_id: runId,
+        timestamp: createRunEventTimestamp(),
+        project,
+        readiness,
+    };
+}