@velum-labs/routekit-eval-setup 1.4.0 → 1.5.0

package/dist/repository-semantic-calibration-protocol.js

@@ -0,0 +1,276 @@
+import { Schema } from "effect";
+import { RepositoryCandidateVisibleTaskV1 } from "./repository-benchmark-protocol.js";
+import { RepositoryFoundryModelPlanV1, RepositoryFoundryReasoningEffortV1 } from "./repository-language-model-protocol.js";
+import { RepositoryBehaviorClauseV1 } from "./repository-task-seed-protocol.js";
+export const REPOSITORY_SEMANTIC_CALIBRATION_POLICY_VERSION = 1;
+export const REPOSITORY_SEMANTIC_CALIBRATION_MAXIMUM_EXAMPLES = 32;
+export const REPOSITORY_SEMANTIC_CALIBRATION_MAXIMUM_REPETITIONS = 5;
+const Count = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => Number.isSafeInteger(value) && value >= 0 ? undefined : "expected a non-negative safe integer")));
+const Source = Schema.Struct({ path: Schema.String, content: Schema.String });
+const Role = Schema.Literals(["solution-critic-a", "solution-critic-b"]);
+const Classification = Schema.Literals(["valid", "wrong", "uncertain"]);
+const Independence = Schema.Literals(["independent", "correlated", "uncertain"]);
+const Rate = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && value <= 1 ? undefined : "expected a rate from zero to one")));
+/** Declared before execution; these are not admission policy or a computed pass. */
+export const RepositorySemanticCalibrationThresholdsV1 = Schema.Struct({
+    minimumScenarios: Count,
+    minimumValidControls: Count,
+    minimumWrongControls: Count,
+    maximumFalseValidRate: Rate,
+    maximumFalseWrongRate: Rate,
+    maximumUncertaintyRate: Rate,
+    maximumInstabilityRate: Rate,
+    requireCompleteExecution: Schema.Literal(true)
+});
+/**
+ * Whitelisted source-only context, shaped like the normal generation packet.
+ * There is no arbitrary metadata bag into which adjudications can be copied.
+ */
+export const RepositorySemanticCalibrationInputV1 = Schema.Struct({
+    requestedDimension: Schema.optionalKey(Schema.String),
+    repositoryEnvironment: Schema.Struct({
+        commit: Schema.String,
+        sources: Schema.Array(Source),
+        localImports: Schema.Struct({
+            commit: Schema.String,
+            sources: Schema.Array(Source),
+            omitted: Schema.Array(Schema.Struct({
+                path: Schema.String,
+                reason: Schema.Literals(["file-size", "total-size", "file-count", "depth"])
+            })),
+            unresolved: Schema.Array(Schema.Struct({ from: Schema.String, specifier: Schema.String })),
+            dependencyContextVersion: Schema.optionalKey(Schema.Literals([1, 2])),
+            dependencies: Schema.optionalKey(Schema.Array(Schema.Struct({
+                from: Schema.String,
+                specifier: Schema.String,
+                symbols: Schema.Union([Schema.Array(Schema.String), Schema.Null]),
+                resolution: Schema.Literals(["resolved", "external", "unsupported", "unresolved"]),
+                paths: Schema.Array(Schema.String)
+            })))
+        })
+    }),
+    behavioralScope: Schema.optionalKey(Schema.Array(Schema.Struct({
+        id: Schema.String,
+        kind: Schema.Literals(["changed", "preserved"]),
+        behaviorIds: Schema.Array(Schema.String),
+        description: Schema.String
+    }))),
+    visible: RepositoryCandidateVisibleTaskV1,
+    targetBehavior: Schema.Array(RepositoryBehaviorClauseV1),
+    preChangeSources: Schema.Array(Source),
+    solutions: Schema.Array(Schema.Struct({
+        id: Schema.String,
+        family: Schema.String,
+        fileOverrides: Schema.Array(Source)
+    })),
+    /** Frozen generation-comparison membership; not derived from expected labels. */
+    comparisonSolutionIds: Schema.Array(Schema.String)
+});
+/** Raw development-check stdout must decode to this independently produced report. */
+export const RepositorySemanticCalibrationExecutableCheckV1 = Schema.Struct({
+    version: Schema.Literal(1),
+    purpose: Schema.Literal("development-label-adjudication"),
+    inputDigest: Schema.String,
+    checkerId: Schema.String,
+    checkerSourceDigest: Schema.String,
+    checks: Schema.Array(Schema.Struct({
+        solutionId: Schema.String,
+        classification: Schema.Literals(["valid", "wrong"]),
+        repetitions: Count,
+        confirmed: Schema.Literal(true),
+        violatedBehaviorIds: Schema.Array(Schema.String)
+    }))
+});
+export const RepositorySemanticCalibrationExampleV1 = Schema.Struct({
+    id: Schema.String,
+    split: Schema.Literal("development"),
+    repository: Schema.Struct({
+        id: Schema.String,
+        commit: Schema.String,
+        sourceDigests: Schema.Array(Schema.Struct({ path: Schema.String, sha256: Schema.String }))
+    }),
+    input: RepositorySemanticCalibrationInputV1,
+    adjudication: Schema.Struct({
+        kind: Schema.Literals([
+            "independent-human-development-adjudication",
+            "independent-executable-development-adjudication"
+        ]),
+        reservation: Schema.Literal("not-reserved-holdout"),
+        adjudicatorIds: Schema.Array(Schema.String),
+        generatorIdentities: Schema.Array(Schema.String),
+        completedAt: Schema.String,
+        inputDigest: Schema.String,
+        evidence: Schema.Array(Schema.Union([
+            Schema.Struct({
+                id: Schema.String,
+                kind: Schema.Literal("human-review"),
+                content: Schema.String,
+                sha256: Schema.String
+            }),
+            Schema.Struct({
+                id: Schema.String,
+                kind: Schema.Literal("executable-development-check"),
+                /** Exact checker and stdout bytes are private and never model-visible. */
+                checkerSource: Schema.String,
+                checkerSourceSha256: Schema.String,
+                content: Schema.String,
+                sha256: Schema.String
+            })
+        ])),
+        labels: Schema.Array(Schema.Struct({
+            solutionId: Schema.String,
+            expectedClass: Schema.Literals(["valid", "wrong"]),
+            rationale: Schema.String,
+            violatedBehaviorIds: Schema.Array(Schema.String),
+            evidenceIds: Schema.Array(Schema.String)
+        }))
+    }),
+    exampleDigest: Schema.String
+});
+export const RepositorySemanticCalibrationManifestV1 = Schema.Struct({
+    version: Schema.Literal(1),
+    purpose: Schema.Literal("development-semantic-calibration"),
+    thresholds: RepositorySemanticCalibrationThresholdsV1,
+    examples: Schema.Array(RepositorySemanticCalibrationExampleV1)
+});
+export const RepositorySemanticCalibrationPlanV1 = Schema.Struct({
+    version: Schema.Literal(1),
+    policyVersion: Schema.Literal(REPOSITORY_SEMANTIC_CALIBRATION_POLICY_VERSION),
+    purpose: Schema.Literal("development-semantic-calibration"),
+    admissionCredit: Schema.Literal(false),
+    manifest: RepositorySemanticCalibrationManifestV1,
+    manifestDigest: Schema.String,
+    modelPlan: RepositoryFoundryModelPlanV1,
+    validControlReviewVersion: Schema.Union([Schema.Null, Schema.Literals([1, 2])]),
+    repetitions: Count,
+    callCount: Count,
+    maximumInputBytesPerCall: Count,
+    requiredTotalInputTokens: Count,
+    requiredTotalOutputTokens: Count,
+    maximumOutputTokensPerCall: Count,
+    preparedCritics: Schema.Array(Schema.Struct({
+        exampleId: Schema.String,
+        role: Role,
+        model: Schema.String,
+        reasoningEffort: RepositoryFoundryReasoningEffortV1,
+        inputDigest: Schema.String,
+        instructionsDigest: Schema.String,
+        schemaName: Schema.String,
+        schemaDigest: Schema.String,
+        maximumOutputTokens: Count
+    })),
+    planDigest: Schema.String
+});
+const Review = Schema.Struct({
+    solutionId: Schema.String,
+    classification: Classification,
+    violatedBehaviorIds: Schema.Array(Schema.String)
+});
+export const RepositorySemanticCalibrationOutcomeV1 = Schema.Union([
+    Schema.Struct({
+        kind: Schema.Literal("classified"),
+        reviews: Schema.Array(Review),
+        independence: Schema.optionalKey(Independence),
+        callId: Schema.optionalKey(Schema.String)
+    }),
+    Schema.Struct({
+        kind: Schema.Literal("malformed"),
+        reason: Schema.Literals([
+            "invalid-json",
+            "normalization",
+            "schema",
+            "review-population",
+            "review-rationale",
+            "behavior-reference",
+            "independence-review"
+        ])
+    }),
+    Schema.Struct({
+        kind: Schema.Literal("execution-failure"),
+        /** No provider error prose, payload, credentials, or inferred billing outcome. */
+        errorOperation: Schema.String
+    }),
+    Schema.Struct({
+        kind: Schema.Literal("not-run"),
+        reason: Schema.Literal("prior-execution-failure")
+    })
+]);
+export const RepositorySemanticCalibrationObservationV1 = Schema.Struct({
+    exampleId: Schema.String,
+    exampleDigest: Schema.String,
+    role: Role,
+    repetition: Count,
+    operationId: Schema.String,
+    model: Schema.String,
+    reasoningEffort: RepositoryFoundryReasoningEffortV1,
+    inputDigest: Schema.String,
+    instructionsDigest: Schema.String,
+    schemaDigest: Schema.String,
+    outcome: RepositorySemanticCalibrationOutcomeV1
+});
+export const RepositorySemanticCalibrationCountsV1 = Schema.Struct({
+    scheduledCalls: Count,
+    classifiedCalls: Count,
+    malformedCalls: Count,
+    executionFailureCalls: Count,
+    notRunCalls: Count,
+    scheduledControls: Count,
+    unscoredControls: Count,
+    validAsValid: Count,
+    validAsWrong: Count,
+    validAsUncertain: Count,
+    wrongAsValid: Count,
+    wrongAsWrong: Count,
+    wrongAsUncertain: Count,
+    independenceIndependent: Count,
+    independenceCorrelated: Count,
+    independenceUncertain: Count,
+    repeatabilityGroups: Count,
+    stableClassificationGroups: Count,
+    changedClassificationGroups: Count,
+    incompleteClassificationGroups: Count
+});
+export const RepositorySemanticCalibrationReportV1 = Schema.Struct({
+    version: Schema.Literal(1),
+    policyVersion: Schema.Literal(REPOSITORY_SEMANTIC_CALIBRATION_POLICY_VERSION),
+    purpose: Schema.Literal("development-semantic-calibration"),
+    admissionCredit: Schema.Literal(false),
+    planDigest: Schema.String,
+    manifestDigest: Schema.String,
+    operationId: Schema.String,
+    coverage: Schema.Struct({
+        semanticCorrectness: Schema.Literal("development-only"),
+        mechanismIndependence: Schema.Literal("unadjudicated"),
+        fullValidControlAcceptance: Schema.Literal("not-established")
+    }),
+    thresholds: RepositorySemanticCalibrationThresholdsV1,
+    denominators: Schema.Struct({
+        scenarios: Count,
+        models: Count,
+        critics: Count,
+        repetitions: Count,
+        distinctControls: Count,
+        distinctValidControls: Count,
+        distinctWrongControls: Count,
+        scheduledCalls: Count,
+        scheduledControlClassifications: Count,
+        scheduledValidClassifications: Count,
+        scheduledWrongClassifications: Count
+    }),
+    controls: Schema.Array(Schema.Struct({
+        exampleId: Schema.String,
+        exampleDigest: Schema.String,
+        solutionId: Schema.String,
+        controlDigest: Schema.String,
+        expectedClass: Schema.Literals(["valid", "wrong"]),
+        evidenceDigests: Schema.Array(Schema.String)
+    })),
+    observations: Schema.Array(RepositorySemanticCalibrationObservationV1),
+    counts: RepositorySemanticCalibrationCountsV1,
+    byRole: Schema.Array(Schema.Struct({
+        role: Role,
+        counts: RepositorySemanticCalibrationCountsV1
+    })),
+    limitations: Schema.Array(Schema.String),
+    reportDigest: Schema.String
+});

package/dist/repository-semantic-calibration.d.ts

@@ -0,0 +1,163 @@
+import { Effect } from "effect";
+import { RepositoryFoundryError } from "./errors.js";
+import { RepositoryFoundryModelPlanV1 } from "./repository-language-model-protocol.js";
+import { type RepositorySemanticCalibrationInputV1 as CalibrationInput, type RepositorySemanticCalibrationExampleV1 as Example, RepositorySemanticCalibrationManifestV1, RepositorySemanticCalibrationObservationV1, RepositorySemanticCalibrationPlanV1, RepositorySemanticCalibrationReportV1 } from "./repository-semantic-calibration-protocol.js";
+type Plan = RepositorySemanticCalibrationPlanV1;
+type Observation = RepositorySemanticCalibrationObservationV1;
+type Report = RepositorySemanticCalibrationReportV1;
+export declare const repositorySemanticCalibrationInputDigestV1: (input: CalibrationInput) => string;
+export declare const repositorySemanticCalibrationExampleDigestV1: (example: Omit<Example, "exampleDigest">) => string;
+export declare const compileRepositorySemanticCalibrationPlanV1: (input: {
+    readonly manifest: RepositorySemanticCalibrationManifestV1;
+    readonly modelPlan: RepositoryFoundryModelPlanV1;
+    readonly validControlReviewVersion: null | 1 | 2;
+    readonly repetitions: number;
+}) => Plan;
+export declare function assertRepositorySemanticCalibrationPlanV1(value: unknown): asserts value is Plan;
+export declare const compileRepositorySemanticCalibrationReportV1: (input: {
+    readonly plan: Plan;
+    readonly operationId: string;
+    readonly observations: readonly Observation[];
+}) => Report;
+export declare function assertRepositorySemanticCalibrationReportV1(input: {
+    readonly plan: Plan;
+    readonly report: unknown;
+}): asserts input is {
+    readonly plan: Plan;
+    readonly report: Report;
+};
+/**
+ * Composition only: the CLI owns immutable execution claims, strict submission
+ * authority, wall time, durable observation writes, and final evidence closure.
+ * No retries, repairs, generated labels, admission, or direct provider transport.
+ */
+export declare const executeRepositorySemanticCalibrationPlanV1: <ObservationError = never, ObservationContext = never>(input: {
+    readonly plan: Plan;
+    readonly operationId: string;
+    readonly recordObservation?: (observation: Observation) => Effect.Effect<void, ObservationError, ObservationContext>;
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly policyVersion: 1;
+    readonly purpose: "development-semantic-calibration";
+    readonly admissionCredit: false;
+    readonly planDigest: string;
+    readonly manifestDigest: string;
+    readonly operationId: string;
+    readonly coverage: {
+        readonly semanticCorrectness: "development-only";
+        readonly mechanismIndependence: "unadjudicated";
+        readonly fullValidControlAcceptance: "not-established";
+    };
+    readonly thresholds: {
+        readonly minimumScenarios: number;
+        readonly minimumValidControls: number;
+        readonly minimumWrongControls: number;
+        readonly maximumFalseValidRate: number;
+        readonly maximumFalseWrongRate: number;
+        readonly maximumUncertaintyRate: number;
+        readonly maximumInstabilityRate: number;
+        readonly requireCompleteExecution: true;
+    };
+    readonly denominators: {
+        readonly scenarios: number;
+        readonly models: number;
+        readonly critics: number;
+        readonly repetitions: number;
+        readonly distinctControls: number;
+        readonly distinctValidControls: number;
+        readonly distinctWrongControls: number;
+        readonly scheduledCalls: number;
+        readonly scheduledControlClassifications: number;
+        readonly scheduledValidClassifications: number;
+        readonly scheduledWrongClassifications: number;
+    };
+    readonly controls: readonly {
+        readonly exampleId: string;
+        readonly exampleDigest: string;
+        readonly solutionId: string;
+        readonly controlDigest: string;
+        readonly expectedClass: "valid" | "wrong";
+        readonly evidenceDigests: readonly string[];
+    }[];
+    readonly observations: readonly {
+        readonly exampleId: string;
+        readonly exampleDigest: string;
+        readonly role: "solution-critic-a" | "solution-critic-b";
+        readonly repetition: number;
+        readonly operationId: string;
+        readonly model: string;
+        readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+        readonly inputDigest: string;
+        readonly instructionsDigest: string;
+        readonly schemaDigest: string;
+        readonly outcome: {
+            readonly kind: "classified";
+            readonly reviews: readonly {
+                readonly solutionId: string;
+                readonly classification: "valid" | "wrong" | "uncertain";
+                readonly violatedBehaviorIds: readonly string[];
+            }[];
+            readonly callId?: string | undefined;
+            readonly independence?: "uncertain" | "independent" | "correlated" | undefined;
+        } | {
+            readonly kind: "malformed";
+            readonly reason: "schema" | "invalid-json" | "normalization" | "review-population" | "review-rationale" | "behavior-reference" | "independence-review";
+        } | {
+            readonly kind: "execution-failure";
+            readonly errorOperation: string;
+        } | {
+            readonly kind: "not-run";
+            readonly reason: "prior-execution-failure";
+        };
+    }[];
+    readonly counts: {
+        readonly scheduledCalls: number;
+        readonly classifiedCalls: number;
+        readonly malformedCalls: number;
+        readonly executionFailureCalls: number;
+        readonly notRunCalls: number;
+        readonly scheduledControls: number;
+        readonly unscoredControls: number;
+        readonly validAsValid: number;
+        readonly validAsWrong: number;
+        readonly validAsUncertain: number;
+        readonly wrongAsValid: number;
+        readonly wrongAsWrong: number;
+        readonly wrongAsUncertain: number;
+        readonly independenceIndependent: number;
+        readonly independenceCorrelated: number;
+        readonly independenceUncertain: number;
+        readonly repeatabilityGroups: number;
+        readonly stableClassificationGroups: number;
+        readonly changedClassificationGroups: number;
+        readonly incompleteClassificationGroups: number;
+    };
+    readonly byRole: readonly {
+        readonly role: "solution-critic-a" | "solution-critic-b";
+        readonly counts: {
+            readonly scheduledCalls: number;
+            readonly classifiedCalls: number;
+            readonly malformedCalls: number;
+            readonly executionFailureCalls: number;
+            readonly notRunCalls: number;
+            readonly scheduledControls: number;
+            readonly unscoredControls: number;
+            readonly validAsValid: number;
+            readonly validAsWrong: number;
+            readonly validAsUncertain: number;
+            readonly wrongAsValid: number;
+            readonly wrongAsWrong: number;
+            readonly wrongAsUncertain: number;
+            readonly independenceIndependent: number;
+            readonly independenceCorrelated: number;
+            readonly independenceUncertain: number;
+            readonly repeatabilityGroups: number;
+            readonly stableClassificationGroups: number;
+            readonly changedClassificationGroups: number;
+            readonly incompleteClassificationGroups: number;
+        };
+    }[];
+    readonly limitations: readonly string[];
+    readonly reportDigest: string;
+}, RepositoryFoundryError | ObservationError, import("./effect-api.js").RepositoryFoundryLanguageModel | ObservationContext>;
+export {};