@velum-labs/routekit-eval-setup 1.0.6 → 1.0.8

package/dist/project-authoring.js

@@ -127,6 +127,72 @@ export function selectProjectAuthoringSourceFiles(input) {
 const DimensionsOutput = Schema.Struct({
     dimensions: Schema.Array(WorkloadDimension)
 });
+function schemaRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+/**
+ * Anthropic removes unsupported structured-output constraints from its wire
+ * schema. Enforce those deferred constraints against the decoded response so
+ * authoring validation remains provider-independent.
+ */
+function assertDeferredSchemaConstraints(schema, value, path = "$") {
+    const record = schemaRecord(schema);
+    if (record === undefined)
+        return;
+    if (typeof value === "number") {
+        if (typeof record.minimum === "number" && value < record.minimum) {
+            throw new Error(`${path} must be greater than or equal to ${String(record.minimum)}`);
+        }
+        if (typeof record.maximum === "number" && value > record.maximum) {
+            throw new Error(`${path} must be less than or equal to ${String(record.maximum)}`);
+        }
+        if (typeof record.exclusiveMinimum === "number" && value <= record.exclusiveMinimum) {
+            throw new Error(`${path} must be greater than ${String(record.exclusiveMinimum)}`);
+        }
+        if (typeof record.exclusiveMaximum === "number" && value >= record.exclusiveMaximum) {
+            throw new Error(`${path} must be less than ${String(record.exclusiveMaximum)}`);
+        }
+        if (typeof record.multipleOf === "number" &&
+            record.multipleOf !== 0 &&
+            Math.abs(value / record.multipleOf - Math.round(value / record.multipleOf)) >
+                Number.EPSILON * Math.max(1, Math.abs(value / record.multipleOf)) * 8) {
+            throw new Error(`${path} must be a multiple of ${String(record.multipleOf)}`);
+        }
+    }
+    if (typeof value === "string") {
+        const length = [...value].length;
+        if (typeof record.minLength === "number" && length < record.minLength) {
+            throw new Error(`${path} must contain at least ${String(record.minLength)} characters`);
+        }
+        if (typeof record.maxLength === "number" && length > record.maxLength) {
+            throw new Error(`${path} must contain at most ${String(record.maxLength)} characters`);
+        }
+    }
+    if (Array.isArray(value)) {
+        if (typeof record.minItems === "number" &&
+            record.minItems > 1 &&
+            value.length < record.minItems) {
+            throw new Error(`${path} must contain at least ${String(record.minItems)} items`);
+        }
+        if (typeof record.maxItems === "number" && value.length > record.maxItems) {
+            throw new Error(`${path} must contain at most ${String(record.maxItems)} items`);
+        }
+        for (const [index, item] of value.entries()) {
+            assertDeferredSchemaConstraints(record.items, item, `${path}[${String(index)}]`);
+        }
+    }
+    const valueRecord = schemaRecord(value);
+    const properties = schemaRecord(record.properties);
+    if (valueRecord !== undefined && properties !== undefined) {
+        for (const [key, propertySchema] of Object.entries(properties)) {
+            if (Object.hasOwn(valueRecord, key)) {
+                assertDeferredSchemaConstraints(propertySchema, valueRecord[key], `${path}.${key}`);
+            }
+        }
+    }
+}
 const DIMENSIONS_JSON_SCHEMA = {
     type: "object",
     additionalProperties: false,
@@ -350,11 +416,14 @@ export const makeEvalProjectAuthor = Effect.gen(function* () {
         });
         const decoded = yield* Schema.decodeUnknownEffect(DimensionsOutput)(yield* parseJson("authoring-dimensions", text)).pipe(Effect.mapError((cause) => failure("authoring-dimensions", "dimension proposal failed validation", cause)));
         yield* Effect.try({
-            try: () => assertRoutingBasis({
-                version: 2,
-                basisDigest: "authoring-validation",
-                dimensions: decoded.dimensions
-            }),
+            try: () => {
+                assertDeferredSchemaConstraints(DIMENSIONS_JSON_SCHEMA, decoded);
+                assertRoutingBasis({
+                    version: 2,
+                    basisDigest: "authoring-validation",
+                    dimensions: decoded.dimensions
+                });
+            },
             catch: (cause) => failure("authoring-dimensions", "dimension proposal failed validation", cause)
         });
         return decoded.dimensions;
@@ -377,6 +446,10 @@ export const makeEvalProjectAuthor = Effect.gen(function* () {
                 maximumOutputTokens: 16_384
             });
             const suite = yield* Schema.decodeUnknownEffect(EvalDimensionSuiteSchema)(yield* parseJson("authoring-evaluations", text)).pipe(Effect.mapError((cause) => failure("authoring-evaluations", `suite for ${JSON.stringify(dimension.id)} failed validation`, cause)));
+            yield* Effect.try({
+                try: () => assertDeferredSchemaConstraints(SUITE_JSON_SCHEMA, suite),
+                catch: (cause) => failure("authoring-evaluations", `suite for ${JSON.stringify(dimension.id)} failed validation`, cause)
+            });
             if (suite.dimensionId !== dimension.id ||
                 suite.cases.length !== EVAL_AUTHORING_CASES_PER_DIMENSION ||
                 new Set(suite.cases.map((testCase) => testCase.id)).size !== suite.cases.length) {
@@ -399,6 +472,10 @@ export const makeEvalProjectAuthor = Effect.gen(function* () {
             maximumOutputTokens: 16_384
         });
         const decompositionBenchmark = yield* Schema.decodeUnknownEffect(EvalDecompositionBenchmarkSchema)(yield* parseJson("authoring-evaluations", decompositionText)).pipe(Effect.mapError((cause) => failure("authoring-evaluations", "decomposition benchmark failed validation", cause)));
+        yield* Effect.try({
+            try: () => assertDeferredSchemaConstraints(decompositionBenchmarkJsonSchema(dimensionIds), decompositionBenchmark),
+            catch: (cause) => failure("authoring-evaluations", "decomposition benchmark failed validation", cause)
+        });
         for (const benchmarkCase of decompositionBenchmark.cases) {
             yield* Effect.try({
                 try: () => assertDecompositionResult(benchmarkCase.expected, input.basis),
@@ -419,6 +496,10 @@ export const makeEvalProjectAuthor = Effect.gen(function* () {
             maximumOutputTokens: 16_384
         });
         const compositionSuite = yield* Schema.decodeUnknownEffect(EvalCompositionSuiteSchema)(yield* parseJson("authoring-evaluations", compositionText)).pipe(Effect.mapError((cause) => failure("authoring-evaluations", "composition benchmark failed validation", cause)));
+        yield* Effect.try({
+            try: () => assertDeferredSchemaConstraints(compositionSuiteJsonSchema(dimensionIds), compositionSuite),
+            catch: (cause) => failure("authoring-evaluations", "composition benchmark failed validation", cause)
+        });
         for (const compositionCase of compositionSuite.cases) {
             yield* Effect.try({
                 try: () => assertDecompositionResult(compositionCase.decomposition, input.basis),

package/dist/test/project-authoring.test.js

@@ -6,7 +6,8 @@ import test from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { Effect, Layer } from "effect";
 import { EvalProjectAuthoringError } from "../errors.js";
-import { EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, readProjectAuthoringSources } from "../project-authoring.js";
+import { EVAL_AUTHORING_CASES_PER_DIMENSION, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, readProjectAuthoringSources } from "../project-authoring.js";
+import { EVAL_PROJECT_VERSION } from "../project-contracts.js";
 const withRepository = async (use) => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-author-sources-"));
     const outside = await mkdtemp(path.join(os.tmpdir(), "routekit-author-outside-"));
@@ -52,6 +53,81 @@ const proposeDimensions = (root, output, requests = []) => Effect.runPromise(Eff
         return JSON.stringify(output);
     })
 }))), Layer.provide(NodeServicesLayer)))));
+const basis = {
+    version: 2,
+    basisDigest: "basis-digest",
+    dimensions: dimensions(5)
+};
+const dimensionCases = () => Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => ({
+    id: `dimension-case-${String(index + 1)}`,
+    prompt: `Answer production request ${String(index + 1)}.`,
+    context: "Reference context.",
+    rubric: "State the expected production behavior."
+}));
+const dimensionSuite = (dimensionId = basis.dimensions[0].id) => ({
+    version: EVAL_PROJECT_VERSION,
+    dimensionId,
+    maximumOutputTokens: 256,
+    cases: dimensionCases()
+});
+const decompositionWeights = () => basis.dimensions.map((dimension) => ({
+    dimensionId: dimension.id,
+    weight: 1 / basis.dimensions.length
+}));
+const decompositionBenchmark = () => ({
+    maximumVectorL1Error: 0.25,
+    cases: Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => ({
+        id: `decomposition-case-${String(index + 1)}`,
+        request: `Classify production request ${String(index + 1)}.`,
+        expected: {
+            weights: decompositionWeights(),
+            unknownWeight: 0
+        }
+    }))
+});
+const compositionSuite = () => ({
+    maximumOutputTokens: 256,
+    minimumWinnerScoreGap: 0.05,
+    minimumWinnerAgreement: 0.8,
+    cases: Array.from({ length: EVAL_AUTHORING_CASES_PER_DIMENSION }, (_, index) => ({
+        id: `composition-case-${String(index + 1)}`,
+        prompt: `Compose production response ${String(index + 1)}.`,
+        context: "Reference context.",
+        rubric: "State the expected composed production behavior.",
+        decomposition: {
+            weights: decompositionWeights(),
+            unknownWeight: 0
+        },
+        requirements: {
+            endpoint: "chat",
+            requiresTools: false,
+            requiresVision: false,
+            inputTokens: 128,
+            maxOutputTokens: 256
+        }
+    }))
+});
+const proposeEvaluations = (root, outputs) => Effect.runPromise(Effect.gen(function* () {
+    const author = yield* EvalProjectAuthor;
+    return yield* author.proposeEvaluations({
+        operationId: "eng-833",
+        repositoryRoot: root,
+        sourceInventory: ["source.md"],
+        configuration,
+        basis
+    });
+}).pipe(Effect.provide(EvalProjectAuthorLive.pipe(Layer.provide(Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+    complete: (input) => Effect.sync(() => {
+        if (input.schemaName === "routekit_dimension_suite") {
+            const dimension = basis.dimensions.find((candidate) => input.operationId.endsWith(`:${candidate.id}`));
+            return JSON.stringify(outputs.suite ?? dimensionSuite(dimension?.id ?? basis.dimensions[0].id));
+        }
+        if (input.schemaName === "routekit_decomposition_benchmark") {
+            return JSON.stringify(outputs.decomposition ?? decompositionBenchmark());
+        }
+        return JSON.stringify(outputs.composition ?? compositionSuite());
+    })
+}))), Layer.provide(NodeServicesLayer)))));
 const collectSchemaNumberKeyword = (value, keyword) => {
     if (typeof value !== "object" || value === null)
         return [];
@@ -142,3 +218,43 @@ test("dimension authoring enforces routing basis counts after structured output
         });
     });
 });
+test("evaluation authoring enforces Anthropic-deferred bounds after parsing", async () => {
+    await withRepository(async ({ root }) => {
+        for (const maximumOutputTokens of [0, 16_385]) {
+            await assert.rejects(proposeEvaluations(root, {
+                suite: { ...dimensionSuite(), maximumOutputTokens }
+            }), (error) => {
+                assert.ok(error instanceof EvalProjectAuthoringError);
+                assert.match(error.detail, /suite .* failed validation/u);
+                assert.match(String(error.cause), /maximumOutputTokens/u);
+                return true;
+            });
+        }
+        await assert.rejects(proposeEvaluations(root, {
+            suite: {
+                ...dimensionSuite(),
+                cases: dimensionCases().slice(0, EVAL_AUTHORING_CASES_PER_DIMENSION - 1)
+            }
+        }), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.match(String(error.cause), /cases must contain at least 20 items/u);
+            return true;
+        });
+        await assert.rejects(proposeEvaluations(root, {
+            decomposition: { ...decompositionBenchmark(), maximumVectorL1Error: 3 }
+        }), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.equal(error.detail, "decomposition benchmark failed validation");
+            assert.match(String(error.cause), /maximumVectorL1Error/u);
+            return true;
+        });
+        await assert.rejects(proposeEvaluations(root, {
+            composition: { ...compositionSuite(), minimumWinnerAgreement: 2 }
+        }), (error) => {
+            assert.ok(error instanceof EvalProjectAuthoringError);
+            assert.equal(error.detail, "composition benchmark failed validation");
+            assert.match(String(error.cause), /minimumWinnerAgreement/u);
+            return true;
+        });
+    });
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.6",
+  "version": "1.0.8",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.6",
-    "@velum-labs/routekit-runtime": "1.0.6"
+    "@velum-labs/routekit-eval-contracts": "1.0.8",
+    "@velum-labs/routekit-runtime": "1.0.8"
   },
   "keywords": [
     "routekit",