@velum-labs/routekit-eval-setup 1.4.0 → 1.5.0

package/dist/services/specification-critic/service.js

@@ -0,0 +1,172 @@
+import { Context, Effect, Layer } from "effect";
+import { captureGitTreeSnapshotV1 } from "../../adapters/git-task-history.js";
+import { assertRepositorySpecificationCritiqueV1 } from "../../repository-specification-critique-protocol.js";
+const visibleText = (visible) => visible.lane === "repository-agent"
+    ? [visible.requestText, ...visible.constraints].join("\n")
+    : visible.request.bodyJson;
+const implementationDetailLanguage = /\b(?:hidden (?:grader|test)|reference (?:patch|solution)|post-change|mutation|expected answer|gold answer)\b/iu;
+const actionableLanguage = /\b(?:fix|repair|implement|preserve|update|add|remove|change|prevent|ensure)\b/iu;
+export class SpecificationCritic extends Context.Service()("@velum-labs/routekit-eval-setup/SpecificationCritic") {
+}
+export const critiqueRepositoryTaskSpecificationV1 = Effect.fn("SpecificationCritic.critique")(function* (input) {
+    const snapshot = yield* captureGitTreeSnapshotV1({
+        repositoryRoot: input.repositoryRoot,
+        requestedRef: input.seed.initialState.commit
+    });
+    const text = visibleText(input.visible);
+    const visibleEvidencePaths = [
+        ...new Set(input.seed.capabilityEvidence
+            .map((evidence) => evidence.path)
+            .filter((path) => path !== undefined))
+    ];
+    const unavailablePaths = visibleEvidencePaths.filter((path) => !snapshot.files.includes(path));
+    const criticalIds = new Set(input.seed.targetBehavior.filter((clause) => clause.critical).map((clause) => clause.id));
+    const reviewCoverageFindings = input.reviews.flatMap((review) => {
+        const coverage = review.criticalBehaviorCoverage;
+        if (coverage === undefined)
+            return [];
+        const ids = coverage.map((entry) => entry.behaviorId);
+        return ids.length !== criticalIds.size || new Set(ids).size !== ids.length ||
+            ids.some((id) => !criticalIds.has(id)) ||
+            coverage.some((entry) => entry.outcome !== "covered" || entry.detail.trim().length === 0)
+            ? [{
+                    axis: "missing-precondition",
+                    severity: "blocker",
+                    detail: `Reviewer ${review.reviewerId} did not establish coverage of every critical seed requirement.`
+                }]
+            : [];
+    });
+    const dimensionFindings = input.requestedDimension === undefined
+        ? []
+        : input.reviews.flatMap((review) => {
+            const fit = review.dimensionFit;
+            return fit === undefined || fit.requestedDimension !== input.requestedDimension ||
+                fit.outcome !== "fit" || fit.detail.trim().length === 0 || fit.evidencePaths.length === 0 ||
+                fit.evidencePaths.some((path) => !snapshot.files.includes(path))
+                ? [{
+                        axis: "unsupported-assumption",
+                        severity: "blocker",
+                        detail: `Reviewer ${review.reviewerId} did not establish fit to requested dimension ${JSON.stringify(input.requestedDimension)}.`
+                    }]
+                : [];
+        });
+    const mechanicalFindings = [
+        ...reviewCoverageFindings,
+        ...dimensionFindings,
+        ...(text.trim().length >= 100 && actionableLanguage.test(text)
+            ? []
+            : [
+                {
+                    axis: "missing-precondition",
+                    severity: "blocker",
+                    detail: "The visible request does not contain enough actionable task context."
+                }
+            ]),
+        ...(input.visible.lane === input.family.lane
+            ? []
+            : [
+                {
+                    axis: "unsupported-assumption",
+                    severity: "blocker",
+                    detail: "The visible task lane does not match its task family."
+                }
+            ]),
+        ...(input.visible.lane !== "repository-agent" ||
+            input.visible.checkoutCommit === input.seed.initialState.commit
+            ? []
+            : [
+                {
+                    axis: "unsupported-assumption",
+                    severity: "blocker",
+                    detail: "The repository-agent task does not expose the replay-qualified pre-task checkout."
+                }
+            ]),
+        ...(input.family.seedIds.includes(input.seed.id)
+            ? []
+            : [
+                {
+                    axis: "unsupported-assumption",
+                    severity: "blocker",
+                    detail: "The task family does not contain the task seed."
+                }
+            ]),
+        ...(unavailablePaths.length === 0
+            ? []
+            : [
+                {
+                    axis: "unavailable-context",
+                    severity: "blocker",
+                    detail: `The pre-task checkout does not contain referenced paths: ${unavailablePaths.join(", ")}`
+                }
+            ]),
+        ...(input.seed.referenceCommit === undefined ||
+            !text.includes(input.seed.referenceCommit)
+            ? []
+            : [
+                {
+                    axis: "hidden-answer-leakage",
+                    severity: "blocker",
+                    detail: "The visible task contains the post-change commit."
+                }
+            ]),
+        ...(implementationDetailLanguage.test(text)
+            ? [
+                {
+                    axis: "unrealistic-phrasing",
+                    severity: "blocker",
+                    detail: "The visible task uses evaluation-only or hidden-answer terminology."
+                }
+            ]
+            : []),
+        ...input.seed.targetBehavior.flatMap((clause) => clause.critical &&
+            !text.toLowerCase().includes(clause.description.toLowerCase()) &&
+            input.reviews.filter((review) => review.independentlyProduced &&
+                review.criticalBehaviorCoverage?.some((entry) => entry.behaviorId === clause.id &&
+                    entry.outcome === "covered" && entry.detail.trim().length > 0)).length < 2
+            ? [
+                {
+                    axis: "multiple-interpretations",
+                    severity: "blocker",
+                    detail: `Critical behavior is neither stated directly nor independently established in the request: ${clause.description}`
+                }
+            ]
+            : [])
+    ];
+    const independentApprovalCount = input.reviews.filter((review) => review.independentlyProduced && review.verdict === "sufficient").length;
+    const blocker = mechanicalFindings.some((finding) => finding.severity === "blocker") ||
+        input.reviews.some((review) => review.verdict === "insufficient" ||
+            review.findings.some((finding) => finding.severity === "blocker"));
+    const outcome = blocker
+        ? "fail"
+        : independentApprovalCount >= 2
+            ? "pass"
+            : "review-required";
+    const rejectionReasons = [
+        ...mechanicalFindings
+            .filter((finding) => finding.severity === "blocker")
+            .map((finding) => `${finding.axis}: ${finding.detail}`),
+        ...input.reviews
+            .filter((review) => review.verdict === "insufficient")
+            .map((review) => `reviewer ${JSON.stringify(review.reviewerId)} found the task insufficient: ${review.detail}`),
+        ...(blocker || independentApprovalCount >= 2
+            ? []
+            : [
+                `only ${String(independentApprovalCount)} independent reviewers found the task sufficiently specified; two are required`
+            ])
+    ];
+    const critique = {
+        version: 1,
+        caseId: input.visible.id,
+        outcome,
+        mechanicalFindings,
+        reviews: input.reviews,
+        independentApprovalCount,
+        rejectionReasons
+    };
+    assertRepositorySpecificationCritiqueV1(critique);
+    return critique;
+});
+export const makeSpecificationCritic = Effect.succeed(SpecificationCritic.of({
+    critique: critiqueRepositoryTaskSpecificationV1
+}));
+export const SpecificationCriticLive = Layer.effect(SpecificationCritic, makeSpecificationCritic);

package/dist/services/task-family/service.d.ts

@@ -0,0 +1,40 @@
+import { Context, Effect, Layer } from "effect";
+import type { RepositoryBehaviorMapV1 } from "../../repository-behavior-protocol.js";
+import { type RepositoryTaskFamilyV1 } from "../../repository-task-family-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+export type TaskFamilyShape = {
+    readonly derive: (input: {
+        readonly map: RepositoryBehaviorMapV1;
+        readonly seeds: readonly RepositoryTaskSeedV1[];
+    }) => Effect.Effect<readonly RepositoryTaskFamilyV1[]>;
+};
+declare const TaskFamily_base: Context.ServiceClass<TaskFamily, "@velum-labs/routekit-eval-setup/TaskFamily", TaskFamilyShape>;
+export declare class TaskFamily extends TaskFamily_base {
+}
+export declare const deriveRepositoryTaskFamiliesV1: (input: {
+    readonly map: RepositoryBehaviorMapV1;
+    readonly seeds: readonly RepositoryTaskSeedV1[];
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly id: string;
+    readonly capabilityId: string;
+    readonly jobStatement: string;
+    readonly lane: "request" | "repository-agent";
+    readonly seedIds: readonly string[];
+    readonly inputStateFacets: readonly string[];
+    readonly requestedChangeFacets: readonly string[];
+    readonly observableFacets: readonly string[];
+    readonly admissibleSolutionPolicy: "behavior-equivalent" | "behavior-and-structure" | "exact-artifact";
+    readonly fixtureAxes: readonly string[];
+    readonly adversaryFamilies: readonly string[];
+    readonly excludedNearMisses: readonly string[];
+    readonly repositoryCoverage: readonly {
+        readonly kind: "symbol" | "package" | "public-surface" | "protocol" | "schema" | "test" | "command" | "history-episode" | "workload-signal" | "documentation";
+        readonly id: string;
+        readonly path?: string | undefined;
+    }[];
+    readonly confidence: "high" | "medium" | "review-required";
+}[], never, never>;
+export declare const makeTaskFamily: Effect.Effect<TaskFamilyShape, never, never>;
+export declare const TaskFamilyLive: Layer.Layer<TaskFamily, never, never>;
+export {};

package/dist/services/task-family/service.js

@@ -0,0 +1,55 @@
+import { Context, Effect, Layer } from "effect";
+import { assertRepositoryTaskFamilyV1 } from "../../repository-task-family-protocol.js";
+const slug = (value) => value
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/gu, "-")
+    .replace(/^-|-$/gu, "")
+    .slice(0, 80);
+const episodeId = (seed) => seed.source.kind === "historical-fix" ||
+    seed.source.kind === "historical-feature" ||
+    seed.source.kind === "historical-refactor"
+    ? seed.source.changeEpisodeId
+    : undefined;
+export class TaskFamily extends Context.Service()("@velum-labs/routekit-eval-setup/TaskFamily") {
+}
+export const deriveRepositoryTaskFamiliesV1 = Effect.fn("TaskFamily.derive")(function* (input) {
+    const qualified = input.seeds.filter((seed) => seed.status === "qualified");
+    const families = qualified.map((seed) => {
+        const sourceEpisodeId = episodeId(seed);
+        const signal = input.map.workloadSignals.find((entry) => sourceEpisodeId !== undefined &&
+            entry.episodeIds.includes(sourceEpisodeId));
+        const capabilityId = signal === undefined
+            ? `capability-${slug(seed.summary)}`
+            : `capability-${slug(signal.label)}`;
+        const family = {
+            version: 1,
+            id: `family-${slug(seed.summary)}`,
+            capabilityId,
+            jobStatement: seed.summary,
+            lane: "repository-agent",
+            seedIds: [seed.id],
+            inputStateFacets: ["historical-pre-change-checkout"],
+            requestedChangeFacets: ["repair-observed-regression"],
+            observableFacets: seed.targetBehavior.map((clause) => clause.description),
+            admissibleSolutionPolicy: "behavior-equivalent",
+            fixtureAxes: ["historical-failing-state"],
+            adversaryFamilies: [
+                "historical-defect",
+                "partial-fix",
+                "integration-near-miss"
+            ],
+            excludedNearMisses: [
+                "documentation-only response",
+                "test-only expectation weakening"
+            ],
+            repositoryCoverage: seed.capabilityEvidence,
+            confidence: seed.confidence
+        };
+        assertRepositoryTaskFamilyV1(family);
+        return family;
+    });
+    return families;
+});
+export const makeTaskFamily = Effect.succeed(TaskFamily.of({ derive: deriveRepositoryTaskFamiliesV1 }));
+export const TaskFamilyLive = Layer.effect(TaskFamily, makeTaskFamily);