@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/task-specification/service.d.ts

@@ -0,0 +1,27 @@
+import { Context, Effect, Layer } from "effect";
+import type { RepositoryCandidateVisibleTaskV1 } from "../../repository-benchmark-protocol.js";
+import type { RepositoryTaskFamilyV1 } from "../../repository-task-family-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+export type TaskSpecificationShape = {
+    readonly construct: (input: {
+        readonly seed: RepositoryTaskSeedV1;
+        readonly family: RepositoryTaskFamilyV1;
+    }) => Effect.Effect<RepositoryCandidateVisibleTaskV1>;
+};
+declare const TaskSpecification_base: Context.ServiceClass<TaskSpecification, "@velum-labs/routekit-eval-setup/TaskSpecification", TaskSpecificationShape>;
+export declare class TaskSpecification extends TaskSpecification_base {
+}
+export declare const constructRepositoryAgentTaskV1: (input: {
+    readonly seed: RepositoryTaskSeedV1;
+    readonly family: RepositoryTaskFamilyV1;
+}) => Effect.Effect<{
+    lane: "repository-agent";
+    id: string;
+    requestText: string;
+    checkoutCommit: string;
+    allowedTools: ("test" | "read" | "search" | "edit" | "shell")[];
+    constraints: string[];
+}, never, never>;
+export declare const makeTaskSpecification: Effect.Effect<TaskSpecificationShape, never, never>;
+export declare const TaskSpecificationLive: Layer.Layer<TaskSpecification, never, never>;
+export {};

package/dist/services/task-specification/service.js

@@ -0,0 +1,40 @@
+import { Context, Effect, Layer } from "effect";
+const visiblePaths = (seed) => [
+    ...new Set(seed.capabilityEvidence
+        .map((evidence) => evidence.path)
+        .filter((path) => path !== undefined))
+];
+export class TaskSpecification extends Context.Service()("@velum-labs/routekit-eval-setup/TaskSpecification") {
+}
+export const constructRepositoryAgentTaskV1 = Effect.fn("TaskSpecification.construct")(function* (input) {
+    const paths = visiblePaths(input.seed);
+    const scope = paths.length === 0
+        ? ""
+        : ` The observed regression involves ${paths
+            .slice(0, 6)
+            .map((path) => `\`${path}\``)
+            .join(", ")}.`;
+    return {
+        lane: "repository-agent",
+        id: `case-${input.seed.id}`,
+        requestText: [
+            input.seed.summary.endsWith(".")
+                ? input.seed.summary
+                : `${input.seed.summary}.`,
+            scope.trim(),
+            "Repair the underlying production behavior in the supplied checkout.",
+            "Preserve unrelated behavior and the repository's public contracts.",
+            "Use the repository's normal tests to verify the change."
+        ]
+            .filter(Boolean)
+            .join(" "),
+        checkoutCommit: input.seed.initialState.commit,
+        allowedTools: ["read", "search", "edit", "shell", "test"],
+        constraints: [
+            "Do not weaken or delete existing tests to make the task pass.",
+            "Do not change unrelated public behavior."
+        ]
+    };
+});
+export const makeTaskSpecification = Effect.succeed(TaskSpecification.of({ construct: constructRepositoryAgentTaskV1 }));
+export const TaskSpecificationLive = Layer.effect(TaskSpecification, makeTaskSpecification);

package/dist/services/trajectory-policy/service.d.ts

@@ -0,0 +1,110 @@
+import { Context, Effect, Layer, Schema } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import type { RepositoryCandidateVisibleTaskV1 } from "../../repository-benchmark-protocol.js";
+import type { RepositoryFoundryModelCallV1, RepositoryFoundryModelPlanV1 } from "../../repository-language-model-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+import { type RepositoryTrajectoryPolicyV1 } from "../../repository-trajectory-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+declare const TrajectoryPolicyReview: Schema.Struct<{
+    readonly verdict: Schema.Literals<readonly ["approve", "reject"]>;
+    readonly detail: Schema.String;
+    readonly unverifiableRequirements: Schema.$Array<Schema.String>;
+    readonly styleBiases: Schema.$Array<Schema.String>;
+}>;
+export type RepositoryTrajectoryPolicyReviewV1 = typeof TrajectoryPolicyReview.Type;
+declare const TrajectoryPolicyProposal: Schema.Struct<{
+    readonly thresholds: Schema.Struct<{
+        readonly maximumToolCalls: Schema.Int;
+        readonly maximumFailedToolCalls: Schema.Int;
+        readonly maximumRepeatedFailedActions: Schema.Int;
+    }>;
+    readonly rationale: Schema.String;
+}>;
+export type RepositoryTrajectoryPolicyProposalV1 = typeof TrajectoryPolicyProposal.Type;
+export type RepositoryTrajectoryRevisionAttemptV1 = {
+    readonly sequence: number;
+    readonly beforePolicy: RepositoryTrajectoryPolicyV1;
+    readonly beforeReview: RepositoryTrajectoryPolicyReviewV1;
+    readonly proposal: RepositoryTrajectoryPolicyProposalV1;
+    readonly candidatePolicy: RepositoryTrajectoryPolicyV1;
+    readonly candidateReview: RepositoryTrajectoryPolicyReviewV1;
+    readonly decision: "accepted" | "rejected";
+};
+export type RepositoryTrajectoryAuthoringResultV1 = {
+    readonly version: 1;
+    readonly policy: RepositoryTrajectoryPolicyV1;
+    readonly review: RepositoryTrajectoryPolicyReviewV1;
+    readonly revisionAttempts: readonly RepositoryTrajectoryRevisionAttemptV1[];
+    readonly modelCalls: readonly RepositoryFoundryModelCallV1[];
+};
+export type TrajectoryPolicyShape = {
+    readonly authorReviewed: (input: {
+        readonly operationId: string;
+        readonly caseId: string;
+        readonly visible: RepositoryCandidateVisibleTaskV1;
+        readonly targetBehavior: RepositoryTaskSeedV1["targetBehavior"];
+        readonly reviewedImplementationPathCount: number;
+        readonly validationRecipeIds: readonly string[];
+        readonly protectedPaths: readonly string[];
+        readonly reviewedCommands: readonly {
+            readonly id: string;
+            readonly kind: string;
+            readonly timeoutMs: number;
+        }[];
+        readonly modelPlan: RepositoryFoundryModelPlanV1;
+    }) => Effect.Effect<RepositoryTrajectoryAuthoringResultV1, RepositoryFoundryError>;
+};
+declare const TrajectoryPolicy_base: Context.ServiceClass<TrajectoryPolicy, "@velum-labs/routekit-eval-setup/TrajectoryPolicy", TrajectoryPolicyShape>;
+export declare class TrajectoryPolicy extends TrajectoryPolicy_base {
+}
+export declare const authorReviewedRepositoryTrajectoryPolicyV1: (input: {
+    readonly operationId: string;
+    readonly caseId: string;
+    readonly visible: RepositoryCandidateVisibleTaskV1;
+    readonly targetBehavior: RepositoryTaskSeedV1["targetBehavior"];
+    readonly reviewedImplementationPathCount: number;
+    readonly validationRecipeIds: readonly string[];
+    readonly protectedPaths: readonly string[];
+    readonly reviewedCommands: readonly {
+        readonly id: string;
+        readonly kind: string;
+        readonly timeoutMs: number;
+    }[];
+    readonly modelPlan: RepositoryFoundryModelPlanV1;
+}) => Effect.Effect<{
+    version: 1;
+    policy: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly purpose: "routing-signal";
+        readonly validationRecipeIds: readonly string[];
+        readonly protectedPaths: readonly string[];
+        readonly thresholds: {
+            readonly maximumToolCalls: number;
+            readonly maximumFailedToolCalls: number;
+            readonly maximumRepeatedFailedActions: number;
+        };
+        readonly requirements: {
+            readonly passingValidationAfterLastMutation: boolean;
+            readonly noProtectedPathMutations: boolean;
+        };
+        readonly rationale: string;
+    };
+    review: {
+        readonly verdict: "approve" | "reject";
+        readonly detail: string;
+        readonly unverifiableRequirements: readonly string[];
+        readonly styleBiases: readonly string[];
+    };
+    revisionAttempts: RepositoryTrajectoryRevisionAttemptV1[];
+    modelCalls: {
+        readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+        readonly role: "oracle-critic" | "repository-analyst" | "specification-writer" | "specification-critic-a" | "specification-critic-b" | "trajectory-designer" | "trajectory-critic" | "oracle-designer" | "valid-solution-generator-a" | "valid-solution-generator-b" | "adversary" | "solution-critic-a" | "solution-critic-b" | "held-out-adversary" | "held-out-solution-critic-a" | "held-out-solution-critic-b" | "quality-reviewer" | "hill-climb-planner";
+        readonly model: string;
+        readonly operationId: string;
+        readonly callId?: string | undefined;
+    }[];
+}, RepositoryFoundryError, RepositoryFoundryLanguageModel>;
+export declare const makeTrajectoryPolicy: Effect.Effect<TrajectoryPolicyShape, never, RepositoryFoundryLanguageModel>;
+export declare const TrajectoryPolicyLive: Layer.Layer<TrajectoryPolicy, never, RepositoryFoundryLanguageModel>;
+export {};

package/dist/services/trajectory-policy/service.js

@@ -0,0 +1,216 @@
+import { Context, Effect, Layer, Schema } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import { repositoryFoundryRoleOutputCeilingV1 } from "../../repository-foundry-plan-protocol.js";
+import { REPOSITORY_FOUNDRY_TRAJECTORY_REPAIR_ATTEMPTS } from "../../repository-foundry-plan-protocol.js";
+import { assertRepositoryTrajectoryPolicyV1 } from "../../repository-trajectory-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+const TrajectoryPolicyReview = Schema.Struct({
+    verdict: Schema.Literals(["approve", "reject"]),
+    detail: Schema.String,
+    unverifiableRequirements: Schema.Array(Schema.String),
+    styleBiases: Schema.Array(Schema.String)
+});
+const TrajectoryPolicyProposal = Schema.Struct({
+    thresholds: Schema.Struct({
+        maximumToolCalls: Schema.Int,
+        maximumFailedToolCalls: Schema.Int,
+        maximumRepeatedFailedActions: Schema.Int
+    }),
+    rationale: Schema.String
+});
+export class TrajectoryPolicy extends Context.Service()("@velum-labs/routekit-eval-setup/TrajectoryPolicy") {
+}
+const DESIGN_INSTRUCTIONS = `You are the trajectory-policy designer in a repository coding benchmark foundry.
+Define bounded, task-appropriate tool-use thresholds for routing evidence. The policy is a routing signal, never
+a replacement for executable final-patch correctness. Require a passing reviewed validation after the last
+observed repository mutation and prohibit mutations to protected harness controls. Reviewed shell/test tools
+invoke immutable recipe IDs rather than arbitrary candidate-supplied shell text; every edit, shell, and test
+event records before/after working-tree state and changed paths. Visible repository tests are not protected
+merely because they are tests: independently applied hidden overlays and deterministic grading judge the final
+behavior. Allow reasonable exploration, debugging, recovery, and regression-test additions; do not prescribe
+an exact file-reading order, implementation strategy, edit count, number of reasoning steps, or imitation of
+the historical solution. The engine binds exact validation recipes and protected paths deterministically;
+propose only total-call and failure-related routing ceilings plus a non-normative rationale for those numbers.
+Use at least 16 total calls, 4 failed calls, and 2 repeated identical failed actions as lower bounds so normal
+debugging is not rejected. Return only the requested structured object.`;
+const CRITIC_INSTRUCTIONS = `You are an independent trajectory-policy critic in a repository coding benchmark
+foundry. Reject policies that reward a particular implementation style, require an exact tool order, penalize
+reasonable debugging, rely on hidden information, use unverifiable requirements, or substitute process signals
+for final-patch correctness. Approve only when every normative requirement is mechanically observable from the
+bounded tool trace, command mutation evidence, exact recipe/path bindings, and supplied measurement semantics,
+and the soft ceilings are proportionate to the visible repository task. The rationale may explain numeric
+headroom but is not itself an additional requirement; reject it only when it attempts to add a normative rule
+or reveals a style preference. Reviewed shell/test events invoke immutable recipe IDs, not arbitrary command
+text. Visible tests are editable and are independently restored or overlaid for hidden final grading. Reject
+edit-count ceilings or any equivalent tool-style preference. Do not rewrite the policy. Return only the
+requested structured review.`;
+const REPAIR_INSTRUCTIONS = `You are the GPT-5.6 Sol trajectory-policy repair role in a repository coding
+benchmark foundry. An independent critic rejected the first proposal. Produce one complete replacement proposal
+that resolves every cited unverifiable requirement and style bias without weakening the two compiler-owned hard
+requirements: passing reviewed validation after the last mutation and no protected harness path mutations. The
+model controls only the three soft numeric ceilings. Its rationale is advisory generation evidence: the compiler
+publishes a separate non-normative policy rationale, so do not attempt to add requirements through this field.
+Explain only why the numeric headroom is proportionate; do not create rules about test legitimacy, implementation
+strategy, file order, edit count, reasoning, style, or intent. Visible repository tests are editable;
+independently restored hidden overlays and deterministic final-patch grading establish correctness. Resolve all
+supplied prior critiques, keep reasonable debugging headroom, and return only the requested structured object.`;
+const failure = (detail, cause) => new RepositoryFoundryError({
+    operation: "compile-reviewed-case",
+    detail,
+    ...(cause === undefined ? {} : { cause })
+});
+const reviewRejected = (review) => review.verdict !== "approve" ||
+    review.unverifiableRequirements.length > 0 ||
+    review.styleBiases.length > 0;
+const detailOf = (cause) => cause instanceof Error && cause.message.trim().length > 0 ? cause.message : String(cause);
+export const authorReviewedRepositoryTrajectoryPolicyV1 = Effect.fn("TrajectoryPolicy.authorReviewed")(function* (input) {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const policyFrom = (proposal) => ({
+        version: 1,
+        caseId: input.caseId,
+        purpose: "routing-signal",
+        validationRecipeIds: input.validationRecipeIds,
+        protectedPaths: input.protectedPaths,
+        thresholds: proposal.thresholds,
+        requirements: {
+            passingValidationAfterLastMutation: true,
+            noProtectedPathMutations: true
+        },
+        rationale: [
+            `The GPT-5.6 trajectory designer selected soft ceilings of ${proposal.thresholds.maximumToolCalls} total tool calls,`,
+            `${proposal.thresholds.maximumFailedToolCalls} failed calls, and ${proposal.thresholds.maximumRepeatedFailedActions} repeated identical failed actions`,
+            `for a task spanning ${input.reviewedImplementationPathCount} reviewed implementation paths and ${input.validationRecipeIds.length} reviewed validation recipes.`,
+            "These routing signals allow exploration and debugging but never override executable final-patch correctness."
+        ].join(" ")
+    });
+    const measurementSemantics = {
+        toolCall: "one recorded bounded read, search, edit, shell, or test event",
+        failedToolCall: "a recorded event whose outcome is fail",
+        repeatedFailedAction: "each failed event after the first failure with the identical tool and input",
+        reviewedCommand: "a shell or test event invokes one immutable reviewed recipe ID; the candidate does not supply arbitrary shell text",
+        repositoryMutation: "an edit, shell, or test event whose recorded before and after repository-state digests differ; changedPaths identifies the changed versionable or protected paths",
+        passingValidationAfterLastMutation: "a passing, non-mutating shell or test event after the final observed mutation whose input is one of policy.validationRecipeIds",
+        protectedPathMutation: "a direct edit targeting policy.protectedPaths or an edit, shell, or test event whose changedPaths includes one",
+        visibleTestTreatment: "ordinary visible tests are editable and independently restored or overlaid for hidden final grading",
+        correctnessSeparation: "trajectory signals are routing-only and never override hidden final-patch correctness",
+        rationale: "policy.rationale explains numeric headroom but is not an additional normative requirement"
+    };
+    const reviewInput = (policy) => ({
+        visible: input.visible,
+        targetBehavior: input.targetBehavior,
+        allowedTools: input.visible.lane === "repository-agent" ? input.visible.allowedTools : [],
+        reviewedCommandIds: input.reviewedCommands.map((recipe) => recipe.id),
+        measurementSemantics,
+        policy
+    });
+    const initial = yield* languageModel.generateStructured({
+        plan: input.modelPlan,
+        role: "trajectory-designer",
+        operationId: `${input.operationId}:trajectory-designer`,
+        instructions: DESIGN_INSTRUCTIONS,
+        input: {
+            visible: input.visible,
+            targetBehavior: input.targetBehavior,
+            reviewedImplementationPathCount: input.reviewedImplementationPathCount,
+            protectedPathCount: input.protectedPaths.length,
+            reviewedCommands: input.reviewedCommands
+        },
+        schemaName: "routekit_repository_trajectory_policy_proposal_v1",
+        outputSchema: TrajectoryPolicyProposal,
+        maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("trajectory-designer")
+    });
+    let policy = policyFrom(initial.value);
+    let review = yield* languageModel.generateStructured({
+        plan: input.modelPlan,
+        role: "trajectory-critic",
+        operationId: `${input.operationId}:trajectory-critic`,
+        instructions: CRITIC_INSTRUCTIONS,
+        input: reviewInput(policy),
+        schemaName: "routekit_repository_trajectory_policy_review_v1",
+        outputSchema: TrajectoryPolicyReview,
+        maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("trajectory-critic")
+    });
+    const modelCalls = [initial.call, review.call];
+    const revisionAttempts = [];
+    for (let sequence = 1; reviewRejected(review.value) &&
+        sequence <= REPOSITORY_FOUNDRY_TRAJECTORY_REPAIR_ATTEMPTS; sequence += 1) {
+        const beforePolicy = policy;
+        const beforeReview = review.value;
+        const revision = yield* languageModel.generateStructured({
+            plan: input.modelPlan,
+            role: "trajectory-designer",
+            operationId: `${input.operationId}:trajectory-designer:repair-${sequence}`,
+            instructions: REPAIR_INSTRUCTIONS,
+            input: {
+                visible: input.visible,
+                targetBehavior: input.targetBehavior,
+                measurementSemantics,
+                beforePolicy,
+                independentCritique: beforeReview,
+                priorRejectedAttempts: revisionAttempts.map((attempt) => ({
+                    sequence: attempt.sequence,
+                    proposal: attempt.proposal,
+                    independentCritique: attempt.candidateReview
+                }))
+            },
+            schemaName: "routekit_repository_trajectory_policy_proposal_v1",
+            outputSchema: TrajectoryPolicyProposal,
+            maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("trajectory-designer")
+        });
+        const candidatePolicy = policyFrom(revision.value);
+        const candidateReview = yield* languageModel.generateStructured({
+            plan: input.modelPlan,
+            role: "trajectory-critic",
+            operationId: `${input.operationId}:trajectory-critic:repair-${sequence}`,
+            instructions: CRITIC_INSTRUCTIONS,
+            input: reviewInput(candidatePolicy),
+            schemaName: "routekit_repository_trajectory_policy_review_v1",
+            outputSchema: TrajectoryPolicyReview,
+            maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("trajectory-critic")
+        });
+        modelCalls.push(revision.call, candidateReview.call);
+        const accepted = !reviewRejected(candidateReview.value);
+        revisionAttempts.push({
+            sequence,
+            beforePolicy,
+            beforeReview,
+            proposal: revision.value,
+            candidatePolicy,
+            candidateReview: candidateReview.value,
+            decision: accepted ? "accepted" : "rejected"
+        });
+        policy = candidatePolicy;
+        review = candidateReview;
+    }
+    yield* Effect.try({
+        try: () => {
+            assertRepositoryTrajectoryPolicyV1(policy);
+            if (policy.caseId !== input.caseId) {
+                throw new Error("trajectory policy case id does not match the generated case");
+            }
+            if (reviewRejected(review.value)) {
+                throw new Error([
+                    "independent trajectory-policy review rejected the generated policy",
+                    review.value.detail,
+                    ...review.value.unverifiableRequirements.map((requirement) => `unverifiable=${requirement}`),
+                    ...review.value.styleBiases.map((bias) => `style-bias=${bias}`)
+                ].join("; "));
+            }
+        },
+        catch: (cause) => failure(`trajectory policy failed validation: ${detailOf(cause)}`, cause)
+    });
+    return {
+        version: 1,
+        policy,
+        review: review.value,
+        revisionAttempts,
+        modelCalls
+    };
+});
+export const makeTrajectoryPolicy = Effect.gen(function* () {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    return TrajectoryPolicy.of({
+        authorReviewed: (input) => authorReviewedRepositoryTrajectoryPolicyV1(input).pipe(Effect.provideService(RepositoryFoundryLanguageModel, languageModel))
+    });
+});
+export const TrajectoryPolicyLive = Layer.effect(TrajectoryPolicy, makeTrajectoryPolicy);

package/dist/test/agentic-capabilities-protocol.test.d.ts

@@ -0,0 +1 @@
+export {};