@orq-ai/evaluators 1.0.0-12

package/README.md

@@ -0,0 +1,48 @@
+# @orq-ai/evaluators
+
+Reusable evaluators for AI evaluation frameworks. This package provides a collection of pre-built evaluators that can be imported and used in your `.eval` files.
+
+## Installation
+
+```bash
+npm install @orq-ai/evaluators
+```
+
+## Usage
+
+### Cosine Similarity Evaluator
+
+Compare semantic similarity between output and expected text using OpenAI embeddings:
+
+```typescript
+import { 
+  cosineSimilarityEvaluator, 
+  cosineSimilarityThresholdEvaluator,
+  simpleCosineSimilarity 
+} from "@orq-ai/evaluators";
+
+// Simple usage - returns similarity score (0-1)
+const evaluator = simpleCosineSimilarity("The capital of France is Paris");
+
+// With threshold - returns boolean based on threshold
+const thresholdEvaluator = cosineSimilarityThresholdEvaluator({
+  expectedText: "The capital of France is Paris",
+  threshold: 0.8,
+  name: "semantic-match"
+});
+
+// Advanced configuration
+const customEvaluator = cosineSimilarityEvaluator({
+  expectedText: "Expected output text",
+  model: "text-embedding-3-large", // optional: custom embedding model
+  name: "custom-similarity"
+});
+```
+
+#### Environment Variables
+
+The cosine similarity evaluator requires one of:
+- `OPENAI_API_KEY` - For direct OpenAI API access
+- `ORQ_API_KEY` - For Orq proxy access (automatically uses `https://api.orq.ai/v2/proxy`)
+
+When using Orq proxy, models should be prefixed with `openai/` (e.g., `openai/text-embedding-3-small`).

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./lib/cosine-similarity-evaluator.js";
+export * from "./lib/vector-utils.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,sCAAsC,CAAC;AACrD,cAAc,uBAAuB,CAAC"}

package/dist/index.js

@@ -0,0 +1,2 @@
+export * from "./lib/cosine-similarity-evaluator.js";
+export * from "./lib/vector-utils.js";

package/dist/lib/comparison-evaluators.d.ts

@@ -0,0 +1,26 @@
+import type { Evaluator } from "@orq-ai/evaluatorq";
+/**
+ * Validates exact equality between output and expected output
+ */
+export declare const exactMatch: Evaluator;
+/**
+ * Validates fuzzy equality (case-insensitive, trimmed strings)
+ */
+export declare const fuzzyMatch: Evaluator;
+/**
+ * Validates that a numeric output is within a tolerance of the expected value
+ */
+export declare function withinTolerance(tolerance: number): Evaluator;
+/**
+ * Validates that a numeric output is greater than a threshold
+ */
+export declare function greaterThan(threshold: number): Evaluator;
+/**
+ * Validates that a numeric output is less than a threshold
+ */
+export declare function lessThan(threshold: number): Evaluator;
+/**
+ * Validates that a numeric output is within a range (inclusive)
+ */
+export declare function inRange(min: number, max: number): Evaluator;
+//# sourceMappingURL=comparison-evaluators.d.ts.map

package/dist/lib/comparison-evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-evaluators.d.ts","sourceRoot":"","sources":["../../src/lib/comparison-evaluators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,SAmBxB,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,SAqBxB,CAAC;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAgC5D;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAuBxD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAuBrD;AAED;;GAEG;AACH,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,CAuB3D"}

package/dist/lib/comparison-evaluators.js

@@ -0,0 +1,148 @@
+/**
+ * Validates exact equality between output and expected output
+ */
+export const exactMatch = {
+    name: "exact-match",
+    scorer: async ({ output, data }) => {
+        if (data.expectedOutput === undefined) {
+            return {
+                value: true,
+                explanation: "No expected output provided, skipping validation",
+            };
+        }
+        const matches = JSON.stringify(output) === JSON.stringify(data.expectedOutput);
+        return {
+            value: matches,
+            explanation: matches
+                ? "Output exactly matches expected"
+                : "Output does not match expected",
+        };
+    },
+};
+/**
+ * Validates fuzzy equality (case-insensitive, trimmed strings)
+ */
+export const fuzzyMatch = {
+    name: "fuzzy-match",
+    scorer: async ({ output, data }) => {
+        if (data.expectedOutput === undefined) {
+            return {
+                value: true,
+                explanation: "No expected output provided, skipping validation",
+            };
+        }
+        const outputStr = String(output).trim().toLowerCase();
+        const expectedStr = String(data.expectedOutput).trim().toLowerCase();
+        const matches = outputStr === expectedStr;
+        return {
+            value: matches,
+            explanation: matches
+                ? "Output matches expected (case-insensitive)"
+                : "Output does not match expected",
+        };
+    },
+};
+/**
+ * Validates that a numeric output is within a tolerance of the expected value
+ */
+export function withinTolerance(tolerance) {
+    return {
+        name: `within-tolerance-${tolerance}`,
+        scorer: async ({ output, data }) => {
+            if (data.expectedOutput === undefined) {
+                return {
+                    value: true,
+                    explanation: "No expected output provided, skipping validation",
+                };
+            }
+            const outputNum = Number(output);
+            const expectedNum = Number(data.expectedOutput);
+            if (Number.isNaN(outputNum) || Number.isNaN(expectedNum)) {
+                return {
+                    value: false,
+                    explanation: "Output or expected value is not a valid number",
+                };
+            }
+            const difference = Math.abs(outputNum - expectedNum);
+            const isWithinTolerance = difference <= tolerance;
+            return {
+                value: isWithinTolerance,
+                explanation: isWithinTolerance
+                    ? `Value ${outputNum} is within ${tolerance} of expected ${expectedNum}`
+                    : `Value ${outputNum} differs by ${difference} from expected ${expectedNum} (tolerance: ${tolerance})`,
+            };
+        },
+    };
+}
+/**
+ * Validates that a numeric output is greater than a threshold
+ */
+export function greaterThan(threshold) {
+    return {
+        name: `greater-than-${threshold}`,
+        scorer: async ({ output }) => {
+            const value = Number(output);
+            if (Number.isNaN(value)) {
+                return {
+                    value: false,
+                    explanation: "Output is not a valid number",
+                };
+            }
+            const isGreater = value > threshold;
+            return {
+                value: isGreater,
+                explanation: isGreater
+                    ? `Value ${value} is greater than ${threshold}`
+                    : `Value ${value} is not greater than ${threshold}`,
+            };
+        },
+    };
+}
+/**
+ * Validates that a numeric output is less than a threshold
+ */
+export function lessThan(threshold) {
+    return {
+        name: `less-than-${threshold}`,
+        scorer: async ({ output }) => {
+            const value = Number(output);
+            if (Number.isNaN(value)) {
+                return {
+                    value: false,
+                    explanation: "Output is not a valid number",
+                };
+            }
+            const isLess = value < threshold;
+            return {
+                value: isLess,
+                explanation: isLess
+                    ? `Value ${value} is less than ${threshold}`
+                    : `Value ${value} is not less than ${threshold}`,
+            };
+        },
+    };
+}
+/**
+ * Validates that a numeric output is within a range (inclusive)
+ */
+export function inRange(min, max) {
+    return {
+        name: `in-range-${min}-${max}`,
+        scorer: async ({ output }) => {
+            const value = Number(output);
+            if (Number.isNaN(value)) {
+                return {
+                    value: false,
+                    explanation: "Output is not a valid number",
+                };
+            }
+            const isInRange = value >= min && value <= max;
+            return {
+                value: isInRange,
+                explanation: isInRange
+                    ? `Value ${value} is within range [${min}, ${max}]`
+                    : `Value ${value} is outside range [${min}, ${max}]`,
+            };
+        },
+    };
+}

package/dist/lib/cosine-similarity-evaluator.d.ts

@@ -0,0 +1,62 @@
+import type { Evaluator } from "@orq-ai/evaluatorq";
+/**
+ * Configuration options for the cosine similarity evaluator
+ */
+export interface CosineSimilarityConfig {
+    /**
+     * The expected text to compare against the output
+     */
+    expectedText: string;
+    /**
+     * The embedding model to use
+     * @default "text-embedding-3-small" for OpenAI, "openai/text-embedding-3-small" for Orq
+     */
+    model?: string;
+    /**
+     * Optional name for the evaluator
+     * @default "cosine-similarity"
+     */
+    name?: string;
+}
+/**
+ * Configuration options for the cosine similarity threshold evaluator
+ */
+export interface CosineSimilarityThresholdConfig extends CosineSimilarityConfig {
+    /**
+     * Threshold for similarity score (0-1)
+     * The evaluator will return true if similarity meets the threshold
+     */
+    threshold: number;
+}
+/**
+ * Creates a cosine similarity evaluator that returns the raw similarity score
+ * between the output and expected text using OpenAI embeddings
+ *
+ * @example
+ * ```typescript
+ * const evaluator = cosineSimilarityEvaluator({
+ *   expectedText: "The capital of France is Paris"
+ * });
+ * ```
+ */
+export declare function cosineSimilarityEvaluator(config: CosineSimilarityConfig): Evaluator;
+/**
+ * Creates a cosine similarity evaluator that returns a boolean based on
+ * whether the similarity meets a threshold
+ *
+ * @example
+ * ```typescript
+ * const evaluator = cosineSimilarityThresholdEvaluator({
+ *   expectedText: "The capital of France is Paris",
+ *   threshold: 0.8
+ * });
+ * ```
+ */
+export declare function cosineSimilarityThresholdEvaluator(config: CosineSimilarityThresholdConfig): Evaluator;
+/**
+ * Creates a simple cosine similarity evaluator with default settings
+ * @param expectedText The expected text to compare against
+ * @returns An evaluator that returns the cosine similarity score (0-1)
+ */
+export declare function simpleCosineSimilarity(expectedText: string): Evaluator;
+//# sourceMappingURL=cosine-similarity-evaluator.d.ts.map

package/dist/lib/cosine-similarity-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cosine-similarity-evaluator.d.ts","sourceRoot":"","sources":["../../src/lib/cosine-similarity-evaluator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAgCpD;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,+BACf,SAAQ,sBAAsB;IAC9B;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,sBAAsB,GAC7B,SAAS,CA2DX;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kCAAkC,CAChD,MAAM,EAAE,+BAA+B,GACtC,SAAS,CAmEX;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,CAEtE"}

package/dist/lib/cosine-similarity-evaluator.js

@@ -0,0 +1,162 @@
+import OpenAI from "openai";
+import { cosineSimilarity } from "./vector-utils.js";
+/**
+ * Creates an OpenAI client configured for either direct OpenAI API access or Orq proxy
+ * @throws {Error} If neither OPENAI_API_KEY nor ORQ_API_KEY is defined
+ */
+function createOpenAIClient() {
+    const orqApiKey = process.env.ORQ_API_KEY;
+    const openaiApiKey = process.env.OPENAI_API_KEY;
+    if (orqApiKey) {
+        // Use Orq proxy when ORQ_API_KEY is available
+        return new OpenAI({
+            baseURL: "https://api.orq.ai/v2/proxy",
+            apiKey: orqApiKey,
+        });
+    }
+    if (openaiApiKey) {
+        // Use direct OpenAI API
+        return new OpenAI({
+            apiKey: openaiApiKey,
+        });
+    }
+    throw new Error("Cosine similarity evaluator requires either ORQ_API_KEY or OPENAI_API_KEY environment variable to be set for embeddings");
+}
+/**
+ * Creates a cosine similarity evaluator that returns the raw similarity score
+ * between the output and expected text using OpenAI embeddings
+ *
+ * @example
+ * ```typescript
+ * const evaluator = cosineSimilarityEvaluator({
+ *   expectedText: "The capital of France is Paris"
+ * });
+ * ```
+ */
+export function cosineSimilarityEvaluator(config) {
+    const { expectedText, model: userModel, name = "cosine-similarity" } = config;
+    // Lazy initialization of OpenAI client
+    let openaiClient = null;
+    const getClient = () => {
+        if (!openaiClient) {
+            openaiClient = createOpenAIClient();
+        }
+        return openaiClient;
+    };
+    // Determine the appropriate model based on the environment
+    const getModel = () => {
+        if (userModel)
+            return userModel;
+        const isOrq = !!process.env.ORQ_API_KEY;
+        return isOrq ? "openai/text-embedding-3-small" : "text-embedding-3-small";
+    };
+    return {
+        name,
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: 0,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            const outputText = String(output);
+            const client = getClient(); // This will throw if no API keys
+            const model = getModel();
+            // Get embeddings for both texts
+            const [outputEmbedding, expectedEmbedding] = await Promise.all([
+                client.embeddings.create({
+                    input: outputText,
+                    model,
+                }),
+                client.embeddings.create({
+                    input: expectedText,
+                    model,
+                }),
+            ]);
+            // Extract the embedding vectors
+            const outputVector = outputEmbedding.data[0].embedding;
+            const expectedVector = expectedEmbedding.data[0].embedding;
+            // Calculate cosine similarity
+            const similarity = cosineSimilarity(outputVector, expectedVector);
+            return {
+                value: similarity,
+                explanation: `Cosine similarity: ${similarity.toFixed(3)}`,
+            };
+        },
+    };
+}
+/**
+ * Creates a cosine similarity evaluator that returns a boolean based on
+ * whether the similarity meets a threshold
+ *
+ * @example
+ * ```typescript
+ * const evaluator = cosineSimilarityThresholdEvaluator({
+ *   expectedText: "The capital of France is Paris",
+ *   threshold: 0.8
+ * });
+ * ```
+ */
+export function cosineSimilarityThresholdEvaluator(config) {
+    const { expectedText, threshold, model: userModel, name = "cosine-similarity-threshold", } = config;
+    // Lazy initialization of OpenAI client
+    let openaiClient = null;
+    const getClient = () => {
+        if (!openaiClient) {
+            openaiClient = createOpenAIClient();
+        }
+        return openaiClient;
+    };
+    // Determine the appropriate model based on the environment
+    const getModel = () => {
+        if (userModel)
+            return userModel;
+        const isOrq = !!process.env.ORQ_API_KEY;
+        return isOrq ? "openai/text-embedding-3-small" : "text-embedding-3-small";
+    };
+    return {
+        name,
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: false,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            const outputText = String(output);
+            const client = getClient(); // This will throw if no API keys
+            const model = getModel();
+            // Get embeddings for both texts
+            const [outputEmbedding, expectedEmbedding] = await Promise.all([
+                client.embeddings.create({
+                    input: outputText,
+                    model,
+                }),
+                client.embeddings.create({
+                    input: expectedText,
+                    model,
+                }),
+            ]);
+            // Extract the embedding vectors
+            const outputVector = outputEmbedding.data[0].embedding;
+            const expectedVector = expectedEmbedding.data[0].embedding;
+            // Calculate cosine similarity
+            const similarity = cosineSimilarity(outputVector, expectedVector);
+            const meetsThreshold = similarity >= threshold;
+            return {
+                value: meetsThreshold,
+                explanation: meetsThreshold
+                    ? `Similarity (${similarity.toFixed(3)}) meets threshold (${threshold})`
+                    : `Similarity (${similarity.toFixed(3)}) below threshold (${threshold})`,
+            };
+        },
+    };
+}
+/**
+ * Creates a simple cosine similarity evaluator with default settings
+ * @param expectedText The expected text to compare against
+ * @returns An evaluator that returns the cosine similarity score (0-1)
+ */
+export function simpleCosineSimilarity(expectedText) {
+    return cosineSimilarityEvaluator({ expectedText });
+}

package/dist/lib/evaluators.d.ts

@@ -0,0 +1,2 @@
+export declare function evaluators(): string;
+//# sourceMappingURL=evaluators.d.ts.map

package/dist/lib/evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluators.d.ts","sourceRoot":"","sources":["../../src/lib/evaluators.ts"],"names":[],"mappings":"AAAA,wBAAgB,UAAU,IAAI,MAAM,CAEnC"}

package/dist/lib/evaluators.js

@@ -0,0 +1,3 @@
+export function evaluators() {
+    return "evaluators";
+}

package/dist/lib/json-evaluators.d.ts

@@ -0,0 +1,25 @@
+import type { Evaluator } from "@orq-ai/evaluatorq";
+/**
+ * Validates that the output is valid JSON
+ */
+export declare const isValidJson: Evaluator;
+/**
+ * Validates that the output contains specific JSON fields
+ */
+export declare function hasJsonFields(requiredFields: string[]): Evaluator;
+/**
+ * Validates JSON schema compliance
+ */
+export declare function matchesJsonStructure(validator: (obj: any) => {
+    valid: boolean;
+    message?: string;
+}): Evaluator;
+/**
+ * Validates that JSON array has a specific length
+ */
+export declare function jsonArrayLength(expectedLength: number): Evaluator;
+/**
+ * Validates that JSON array contains specific number of items within a range
+ */
+export declare function jsonArrayLengthInRange(min: number, max: number): Evaluator;
+//# sourceMappingURL=json-evaluators.d.ts.map

package/dist/lib/json-evaluators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-evaluators.d.ts","sourceRoot":"","sources":["../../src/lib/json-evaluators.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD;;GAEG;AACH,eAAO,MAAM,WAAW,EAAE,SAwBzB,CAAC;AAEF;;GAEG;AACH,wBAAgB,aAAa,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,CA2CjE;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CACnC,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D,SAAS,CA2BX;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,cAAc,EAAE,MAAM,GAAG,SAAS,CAqCjE;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,CAqC1E"}

package/dist/lib/json-evaluators.js

@@ -0,0 +1,177 @@
+/**
+ * Validates that the output is valid JSON
+ */
+export const isValidJson = {
+    name: "is-valid-json",
+    scorer: async ({ output }) => {
+        if (output === undefined || output === null) {
+            return {
+                value: false,
+                explanation: "Output is null or undefined",
+            };
+        }
+        try {
+            const str = typeof output === "string" ? output : JSON.stringify(output);
+            JSON.parse(str);
+            return {
+                value: true,
+                explanation: "Output is valid JSON",
+            };
+        }
+        catch {
+            return {
+                value: false,
+                explanation: "Output is not valid JSON",
+            };
+        }
+    },
+};
+/**
+ * Validates that the output contains specific JSON fields
+ */
+export function hasJsonFields(requiredFields) {
+    const fieldList = requiredFields.join("-");
+    return {
+        name: `has-fields-${fieldList.slice(0, 30)}`,
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: false,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            try {
+                const obj = typeof output === "string" ? JSON.parse(output) : output;
+                if (typeof obj !== "object" || Array.isArray(obj)) {
+                    return {
+                        value: false,
+                        explanation: "Output is not a JSON object",
+                    };
+                }
+                const missingFields = requiredFields.filter((field) => !(field in obj));
+                if (missingFields.length === 0) {
+                    return {
+                        value: true,
+                        explanation: `All required fields present: ${requiredFields.join(", ")}`,
+                    };
+                }
+                return {
+                    value: false,
+                    explanation: `Missing fields: ${missingFields.join(", ")}`,
+                };
+            }
+            catch {
+                return {
+                    value: false,
+                    explanation: "Output is not valid JSON",
+                };
+            }
+        },
+    };
+}
+/**
+ * Validates JSON schema compliance
+ */
+export function matchesJsonStructure(validator) {
+    return {
+        name: "matches-json-structure",
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: false,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            try {
+                const obj = typeof output === "string" ? JSON.parse(output) : output;
+                const result = validator(obj);
+                return {
+                    value: result.valid,
+                    explanation: result.message || (result.valid ? "JSON structure is valid" : "JSON structure is invalid"),
+                };
+            }
+            catch (error) {
+                return {
+                    value: false,
+                    explanation: `Failed to parse JSON: ${error instanceof Error ? error.message : "Unknown error"}`,
+                };
+            }
+        },
+    };
+}
+/**
+ * Validates that JSON array has a specific length
+ */
+export function jsonArrayLength(expectedLength) {
+    return {
+        name: `json-array-length-${expectedLength}`,
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: false,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            try {
+                const arr = typeof output === "string" ? JSON.parse(output) : output;
+                if (!Array.isArray(arr)) {
+                    return {
+                        value: false,
+                        explanation: "Output is not a JSON array",
+                    };
+                }
+                const hasExpectedLength = arr.length === expectedLength;
+                return {
+                    value: hasExpectedLength,
+                    explanation: hasExpectedLength
+                        ? `Array has expected length of ${expectedLength}`
+                        : `Array length ${arr.length} does not match expected ${expectedLength}`,
+                };
+            }
+            catch {
+                return {
+                    value: false,
+                    explanation: "Output is not valid JSON",
+                };
+            }
+        },
+    };
+}
+/**
+ * Validates that JSON array contains specific number of items within a range
+ */
+export function jsonArrayLengthInRange(min, max) {
+    return {
+        name: `json-array-length-${min}-${max}`,
+        scorer: async ({ output }) => {
+            if (output === undefined || output === null) {
+                return {
+                    value: false,
+                    explanation: "Output is null or undefined",
+                };
+            }
+            try {
+                const arr = typeof output === "string" ? JSON.parse(output) : output;
+                if (!Array.isArray(arr)) {
+                    return {
+                        value: false,
+                        explanation: "Output is not a JSON array",
+                    };
+                }
+                const isInRange = arr.length >= min && arr.length <= max;
+                return {
+                    value: isInRange,
+                    explanation: isInRange
+                        ? `Array length ${arr.length} is within range [${min}, ${max}]`
+                        : `Array length ${arr.length} is outside range [${min}, ${max}]`,
+                };
+            }
+            catch {
+                return {
+                    value: false,
+                    explanation: "Output is not valid JSON",
+                };
+            }
+        },
+    };
+}