@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/adapters/repository-command-evidence.js

@@ -0,0 +1,102 @@
+import { stripVTControlCharacters } from "node:util";
+import { repositoryVitestTestEvidenceV1 } from "./repository-vitest-reporter.js";
+/** Process completion and evidence completeness are prerequisites, not grades. */
+export const repositoryCommandExecutionIncompleteV1 = (result) => result.timedOut || result.exitCode === null || result.outputTruncated;
+export const repositoryCommandTestEvidenceV1 = (result, recipeId) => {
+    const unknown = (detail) => ({ outcome: "unknown", detail });
+    if (result === undefined ||
+        result.recipeId !== recipeId ||
+        !result.gradeExecuted ||
+        result.stage !== "grade") {
+        return unknown("the reviewed grading command did not execute");
+    }
+    if (repositoryCommandExecutionIncompleteV1(result)) {
+        return unknown("grading did not produce a complete, non-timeout result");
+    }
+    if (result.trustedPreparationResults.some((preparation) => preparation.exitCode !== 0 || repositoryCommandExecutionIncompleteV1(preparation))) {
+        return unknown("trusted preparation did not complete before grading");
+    }
+    if (result.candidateValidationResults.some((validation) => validation.exitCode !== 0 || repositoryCommandExecutionIncompleteV1(validation))) {
+        return unknown("candidate validation did not pass before grading");
+    }
+    const output = stripVTControlCharacters(`${result.stdout}\n${result.stderr}`);
+    if (result.testReporter === "vitest-v1") {
+        return (repositoryVitestTestEvidenceV1(result, output) ??
+            unknown("the host-selected Vitest reporter did not emit complete execution evidence"));
+    }
+    if (result.testReporter !== undefined)
+        return unknown("unsupported host-selected test reporter");
+    // Node's TAP and spec reporters expose completed test counts. Other reporters
+    // need their own host-selected evidence adapter; exit status alone is not an oracle.
+    // Reporter evidence does not prove that an assertion covers the target
+    // behavior: that still requires independent review of fixtures and controls.
+    const summaries = [
+        ...output.matchAll(/^(?:#|ℹ) (tests|pass|fail|cancelled|skipped|todo) (\d+)\s*$/gmu)
+    ];
+    const counts = new Map(summaries.map((match) => [match[1], Number(match[2])]));
+    const tests = counts.get("tests");
+    const passed = counts.get("pass");
+    const failed = counts.get("fail");
+    const cancelled = counts.get("cancelled");
+    const skipped = counts.get("skipped");
+    const todo = counts.get("todo");
+    const namedTests = [
+        ...output.matchAll(/^(?:\s*# Subtest: |\s*[✔✖] )(.+?)(?: \([\d.]+m?s\))?$/gmu)
+    ].some((match) => !/\.[cm]?[jt]sx?$/u.test(match[1]) && match[1] !== "failing tests:");
+    if (summaries.length !== 6 ||
+        counts.size !== 6 ||
+        tests === undefined ||
+        passed === undefined ||
+        failed === undefined ||
+        cancelled === undefined ||
+        skipped === undefined ||
+        todo === undefined ||
+        tests !== passed + failed + cancelled + skipped + todo ||
+        passed + failed === 0 ||
+        cancelled > 0 ||
+        !namedTests) {
+        return unknown("no complete, unambiguous Node test discovery and execution evidence");
+    }
+    if (result.exitCode === 0 && failed === 0) {
+        return { outcome: "pass", detail: `executed=${String(passed)}; failed=0; reporter=node` };
+    }
+    const assertionFailures = [...output.matchAll(/^\s+code: ['"]ERR_ASSERTION['"],?\s*$/gmu)].length;
+    const tap = /^TAP version 13$/mu.test(output);
+    const assertionErrors = tap
+        ? [...output.matchAll(/^\s+name: ['"]AssertionError['"]\s*$/gmu)].length
+        : [...output.matchAll(/^\s+AssertionError \[ERR_ASSERTION\]:/gmu)].length;
+    // A failed nested test also fails its parent. Attribute leaf failures rather
+    // than mistaking the parent's propagated failure for a second runtime error.
+    const leafFailures = [...output.matchAll(/^\s+failureType: ['"]([^'"]+)['"]\s*$/gmu)].filter((match) => match[1] !== "subtestsFailed");
+    if (tap &&
+        result.exitCode === 1 &&
+        failed > 0 &&
+        assertionFailures > 0 &&
+        leafFailures.every((match) => match[1] === "testCodeFailure") &&
+        assertionFailures <= failed &&
+        assertionFailures === leafFailures.length &&
+        assertionErrors === assertionFailures) {
+        return {
+            outcome: "fail",
+            detail: `executed=${String(passed + failed)}; assertionFailures=${String(assertionFailures)}; reporter=node`
+        };
+    }
+    const testBodyFailure = tap &&
+        result.exitCode === 1 &&
+        leafFailures.length > 0 &&
+        leafFailures.every((match) => match[1] === "testCodeFailure") &&
+        !/^\s+(?:exitCode|signal):/mu.test(output);
+    return {
+        ...unknown("grading failed without attributable test-body assertion failures for every failed test"),
+        ...(testBodyFailure ? { inconclusiveReason: "unattributed-test-failure" } : {})
+    };
+};
+/** A diagnostic probe uses the same evidence rules as admission, never exit status alone. */
+export const repositoryCommandProbeOutcomeV1 = (results) => {
+    const evidence = results.map((result) => repositoryCommandTestEvidenceV1(result, result.recipeId));
+    if (evidence.length === 0 || evidence.some((entry) => entry.outcome === "unknown"))
+        return "environment_blocked";
+    return evidence.every((entry) => entry.outcome === "pass")
+        ? "completed"
+        : "scientific_rejection";
+};

package/dist/adapters/repository-command-runner.d.ts

@@ -0,0 +1,238 @@
+import { Context, Effect } from "effect";
+import { RepositoryFoundryError } from "../errors.js";
+import type { RepositoryCommandRecipeV1 } from "../repository-behavior-protocol.js";
+import type { RepositoryCommandFileOverrideV1, RepositoryCommandInvocationResultV1 } from "../repository-execution-protocol.js";
+export type { RepositoryCommandExecutionStageV1, RepositoryCommandFileOverrideV1, RepositoryCommandInvocationResultV1, RepositoryCommandResultV1 } from "../repository-execution-protocol.js";
+/** Optional platform command executor. Checkout, overlays, recipes and grading remain owned here. */
+export type RepositoryCommandExecutionPortV1 = {
+    readonly run: (input: {
+        readonly operationId: string;
+        readonly cwd: string;
+        readonly executable: string;
+        readonly args: readonly string[];
+        readonly environment: Readonly<Record<string, string>>;
+        readonly timeoutMs: number;
+    }) => Effect.Effect<{
+        readonly exitCode: number | null;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly timedOut?: boolean;
+    }, RepositoryFoundryError>;
+};
+export declare const RepositoryCommandExecutionPort: Context.Reference<RepositoryCommandExecutionPortV1 | undefined>;
+/** Host-provided, versioned pnpm store populated before an offline invocation. */
+export declare const RepositoryReadOnlyPnpmStore: Context.Reference<string | undefined>;
+type IsolatedRepositoryCheckout = {
+    readonly root: string;
+    readonly checkout: string;
+    readonly home: string;
+    readonly temporary: string;
+    readonly candidateSandboxProfile?: string;
+    readonly trustedPreparationSandboxProfile?: string;
+    readonly trustedPnpmStore?: string;
+    readonly trustedPnpmStoreReadOnly?: boolean;
+    readonly materializationDurationMs: number;
+    readonly timings: {
+        checkoutDurationMs: number;
+        overlayDurationMs: number;
+        cleanupDurationMs: number;
+    };
+};
+type PreparedRepositoryTemplate = {
+    readonly isolated: IsolatedRepositoryCheckout;
+    readonly key: string;
+    readonly preparationResults: readonly RepositoryCommandInvocationResultV1[];
+    readonly relocatedFiles: readonly string[];
+};
+/** Scoped, pristine installation resources; never contains a candidate execution. */
+export type RepositoryPreparationCacheV1 = {
+    readonly get: (input: {
+        readonly repositoryRoot: string;
+        readonly commit: string;
+        readonly recipes: readonly RepositoryCommandRecipeV1[];
+    }) => Effect.Effect<PreparedRepositoryTemplate | undefined, RepositoryFoundryError>;
+};
+export type RepositoryCommandOverlayV1 = {
+    readonly path: string;
+    readonly sourceCommit: string;
+};
+export type RepositoryCommandContentOverlayV1 = {
+    readonly path: string;
+    readonly content: string;
+};
+export type RepositoryPathInspectionKindV1 = "file" | "directory" | "symlink" | "missing";
+export type RepositoryPathInspectionV1 = {
+    readonly commit: string;
+    readonly repetition: number;
+    readonly path: string;
+    readonly kind: RepositoryPathInspectionKindV1;
+};
+/** Installation reuse is bounded to these immutable commits and this owning Effect scope. */
+export declare const makeRepositoryPreparationCacheV1: (input: {
+    readonly repositoryRoot: string;
+    readonly commits: readonly string[];
+    readonly trustedPreparationRecipes: readonly RepositoryCommandRecipeV1[];
+}) => Effect.Effect<RepositoryPreparationCacheV1, RepositoryFoundryError, import("effect/Scope").Scope>;
+export declare const replayRepositoryCommandsV1: (input: {
+    readonly repositoryRoot: string;
+    readonly commit: string;
+    readonly recipes: readonly RepositoryCommandRecipeV1[];
+    readonly repetitions?: number;
+    /** Diagnostic sequences may need post-failure controls in the same checkout. */
+    readonly continueAfterIncompleteGrade?: boolean;
+    /** Preserve the scientific repetition identity when a durable caller resumes mid-phase. */
+    readonly repetitionStart?: number;
+    readonly overlays?: readonly RepositoryCommandOverlayV1[];
+    readonly contentOverlays?: readonly RepositoryCommandContentOverlayV1[];
+    readonly fileOverrides?: readonly RepositoryCommandFileOverrideV1[];
+    readonly patch?: string;
+    readonly trustedPreparationRecipes?: readonly RepositoryCommandRecipeV1[];
+    readonly candidateValidationRecipes?: readonly RepositoryCommandRecipeV1[];
+    readonly preparationCache?: RepositoryPreparationCacheV1;
+}) => Effect.Effect<readonly {
+    readonly recipeId: string;
+    readonly commit: string;
+    readonly repetition: number;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly exitCode: number | null;
+    readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+    readonly timedOut: boolean;
+    readonly outputTruncated: boolean;
+    readonly durationMs: number;
+    readonly trustedPreparationResults: readonly {
+        readonly recipeId: string;
+        readonly commit: string;
+        readonly repetition: number;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly exitCode: number | null;
+        readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+        readonly timedOut: boolean;
+        readonly outputTruncated: boolean;
+        readonly durationMs: number;
+        readonly testReporter?: "vitest-v1" | undefined;
+    }[];
+    readonly candidateValidationResults: readonly {
+        readonly recipeId: string;
+        readonly commit: string;
+        readonly repetition: number;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly exitCode: number | null;
+        readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+        readonly timedOut: boolean;
+        readonly outputTruncated: boolean;
+        readonly durationMs: number;
+        readonly testReporter?: "vitest-v1" | undefined;
+    }[];
+    readonly gradeExecuted: boolean;
+    readonly testReporter?: "vitest-v1" | undefined;
+    readonly isolationTimings?: {
+        readonly checkoutDurationMs: number;
+        readonly overlayDurationMs: number;
+        readonly cleanupDurationMs: number;
+    } | undefined;
+    readonly preparation?: {
+        readonly kind: "pristine-install-template";
+        readonly templateKey: string;
+        readonly sourceCommit: string;
+        readonly cloneDurationMs: number;
+        readonly sharedResults: readonly {
+            readonly recipeId: string;
+            readonly commit: string;
+            readonly repetition: number;
+            readonly stdout: string;
+            readonly stderr: string;
+            readonly exitCode: number | null;
+            readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+            readonly timedOut: boolean;
+            readonly outputTruncated: boolean;
+            readonly durationMs: number;
+            readonly testReporter?: "vitest-v1" | undefined;
+        }[];
+    } | undefined;
+}[], RepositoryFoundryError, never>;
+export declare const replayRepositoryCommandV1: (input: {
+    readonly repositoryRoot: string;
+    readonly commit: string;
+    readonly recipe: RepositoryCommandRecipeV1;
+    readonly repetitions?: number;
+    readonly overlays?: readonly RepositoryCommandOverlayV1[];
+    readonly contentOverlays?: readonly RepositoryCommandContentOverlayV1[];
+    readonly fileOverrides?: readonly RepositoryCommandFileOverrideV1[];
+    readonly patch?: string;
+    readonly trustedPreparationRecipes?: readonly RepositoryCommandRecipeV1[];
+    readonly candidateValidationRecipes?: readonly RepositoryCommandRecipeV1[];
+}) => Effect.Effect<readonly {
+    readonly recipeId: string;
+    readonly commit: string;
+    readonly repetition: number;
+    readonly stdout: string;
+    readonly stderr: string;
+    readonly exitCode: number | null;
+    readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+    readonly timedOut: boolean;
+    readonly outputTruncated: boolean;
+    readonly durationMs: number;
+    readonly trustedPreparationResults: readonly {
+        readonly recipeId: string;
+        readonly commit: string;
+        readonly repetition: number;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly exitCode: number | null;
+        readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+        readonly timedOut: boolean;
+        readonly outputTruncated: boolean;
+        readonly durationMs: number;
+        readonly testReporter?: "vitest-v1" | undefined;
+    }[];
+    readonly candidateValidationResults: readonly {
+        readonly recipeId: string;
+        readonly commit: string;
+        readonly repetition: number;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly exitCode: number | null;
+        readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+        readonly timedOut: boolean;
+        readonly outputTruncated: boolean;
+        readonly durationMs: number;
+        readonly testReporter?: "vitest-v1" | undefined;
+    }[];
+    readonly gradeExecuted: boolean;
+    readonly testReporter?: "vitest-v1" | undefined;
+    readonly isolationTimings?: {
+        readonly checkoutDurationMs: number;
+        readonly overlayDurationMs: number;
+        readonly cleanupDurationMs: number;
+    } | undefined;
+    readonly preparation?: {
+        readonly kind: "pristine-install-template";
+        readonly templateKey: string;
+        readonly sourceCommit: string;
+        readonly cloneDurationMs: number;
+        readonly sharedResults: readonly {
+            readonly recipeId: string;
+            readonly commit: string;
+            readonly repetition: number;
+            readonly stdout: string;
+            readonly stderr: string;
+            readonly exitCode: number | null;
+            readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+            readonly timedOut: boolean;
+            readonly outputTruncated: boolean;
+            readonly durationMs: number;
+            readonly testReporter?: "vitest-v1" | undefined;
+        }[];
+    } | undefined;
+}[], RepositoryFoundryError, never>;
+export declare const inspectRepositorySolutionPathsV1: (input: {
+    readonly repositoryRoot: string;
+    readonly commit: string;
+    readonly paths: readonly string[];
+    readonly repetitions?: number;
+    readonly fileOverrides?: readonly RepositoryCommandFileOverrideV1[];
+    readonly patch?: string;
+}) => Effect.Effect<RepositoryPathInspectionV1[], RepositoryFoundryError, never>;