@hamza1331/aieval 0.0.0-stage → 0.1.0

package/dist/checks.js

@@ -0,0 +1,396 @@
+export const makeFailure = (code, message, severity = "error", options = {}) => ({
+    code,
+    message,
+    severity,
+    ...options,
+});
+export const requiredFields = (fields) => ({
+    name: "requiredFields",
+    description: "Ensures required keys are present in an object output.",
+    run: (value, _context) => {
+        if (value == null || typeof value !== "object" || Array.isArray(value)) {
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("schema_violation", "Output must be an object for required-field validation.", "error"),
+                ],
+                warnings: [],
+            };
+        }
+        const missing = fields.filter((field) => !(field in value));
+        if (missing.length === 0) {
+            return { passed: true, failures: [], warnings: [] };
+        }
+        return {
+            passed: false,
+            failures: missing.map((field) => makeFailure("missing_field", `Missing required field: ${field}`, "error", {
+                path: field,
+            })),
+            warnings: [],
+        };
+    },
+});
+export const noExtraFields = (allowedFields, options = {}) => ({
+    name: "noExtraFields",
+    description: "Ensures the object does not contain additional keys beyond the allowed set.",
+    run: (value, _context) => {
+        if (value == null || typeof value !== "object" || Array.isArray(value)) {
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("schema_violation", "Output must be an object for extra-field validation.", "error"),
+                ],
+                warnings: [],
+            };
+        }
+        const keys = Object.keys(value);
+        const unexpected = keys.filter((key) => !allowedFields.includes(key));
+        if (unexpected.length === 0) {
+            return { passed: true, failures: [], warnings: [] };
+        }
+        return {
+            passed: false,
+            failures: unexpected.map((key) => makeFailure("schema_violation", `Unexpected field: ${key}`, options.severity ?? "error", {
+                path: key,
+                metadata: { allowedFields },
+            })),
+            warnings: [],
+        };
+    },
+});
+export const enumCheck = (field, allowedValues) => ({
+    name: "enumCheck",
+    description: "Ensures a field matches one of a set of allowed values.",
+    run: (value, _context) => {
+        if (value == null || typeof value !== "object" || Array.isArray(value)) {
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("schema_violation", `Field ${field} cannot be checked because the value is not an object.`, "error"),
+                ],
+                warnings: [],
+            };
+        }
+        const actual = value[field];
+        if (actual === undefined) {
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("missing_field", `Missing field: ${field}`, "error", {
+                        path: field,
+                    }),
+                ],
+                warnings: [],
+            };
+        }
+        if (allowedValues.includes(actual)) {
+            return { passed: true, failures: [], warnings: [] };
+        }
+        return {
+            passed: false,
+            failures: [
+                makeFailure("invalid_enum", `Field ${field} must be one of: ${allowedValues.join(", ")}`, "error", {
+                    path: field,
+                    metadata: { allowedValues, actualValue: actual },
+                }),
+            ],
+            warnings: [],
+        };
+    },
+});
+export const fieldTypeCheck = (field, expectedType) => ({
+    name: "fieldTypeCheck",
+    description: "Ensures a field has the expected runtime type.",
+    run: (value, _context) => {
+        if (value == null || typeof value !== "object" || Array.isArray(value)) {
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("schema_violation", `Field ${field} cannot be checked because the value is not an object.`, "error"),
+                ],
+                warnings: [],
+            };
+        }
+        const actual = value[field];
+        const actualType = actual === null
+            ? "object"
+            : Array.isArray(actual)
+                ? "array"
+                : typeof actual;
+        if (actualType === expectedType) {
+            return { passed: true, failures: [], warnings: [] };
+        }
+        return {
+            passed: false,
+            failures: [
+                makeFailure("invalid_type", `Field ${field} expected type ${expectedType} but received ${actualType}.`, "error", {
+                    path: field,
+                    metadata: { expectedType, actualType },
+                }),
+            ],
+            warnings: [],
+        };
+    },
+});
+export const schemaCheck = (schema) => ({
+    name: "schemaCheck",
+    description: "Runs a Zod schema against the output.",
+    run: (value, _context) => {
+        const result = schema.safeParse(value);
+        if (result.success) {
+            return { passed: true, failures: [], warnings: [] };
+        }
+        return {
+            passed: false,
+            failures: result.error.issues.map((issue) => makeFailure("schema_violation", issue.message, "error", {
+                path: issue.path.join("."),
+                metadata: { issueCode: issue.code },
+            })),
+            warnings: [],
+        };
+    },
+});
+const typeOf = (value) => value === null ? "null" : Array.isArray(value) ? "array" : typeof value;
+const resolveToolArgs = (candidate) => {
+    const raw = candidate.args ?? candidate.arguments;
+    if (raw === undefined) {
+        return { args: {} };
+    }
+    if (typeof raw === "string") {
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed == null ||
+                typeof parsed !== "object" ||
+                Array.isArray(parsed)) {
+                return {
+                    failure: makeFailure("tool_call_mismatch", "Tool call arguments must be a JSON object.", "error", {
+                        path: "arguments",
+                    }),
+                };
+            }
+            return { args: parsed };
+        }
+        catch {
+            return {
+                failure: makeFailure("invalid_json", "Tool call arguments are not valid JSON.", "error", { path: "arguments" }),
+            };
+        }
+    }
+    if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+        return {
+            failure: makeFailure("tool_call_mismatch", "Tool call args must be an object.", "error", { path: "args" }),
+        };
+    }
+    return { args: raw };
+};
+const validateToolCall = (value, definition) => {
+    if (value == null || typeof value !== "object" || Array.isArray(value)) {
+        return [
+            makeFailure("tool_call_mismatch", "Tool call must be an object.", "error"),
+        ];
+    }
+    const candidate = value;
+    const failures = [];
+    if (candidate.name !== definition.name) {
+        failures.push(makeFailure("tool_call_mismatch", `Tool call name must be ${definition.name}.`, "error", {
+            path: "name",
+            metadata: {
+                expectedName: definition.name,
+                actualName: candidate.name,
+            },
+        }));
+    }
+    const resolved = resolveToolArgs(candidate);
+    if ("failure" in resolved) {
+        failures.push(resolved.failure);
+        return failures;
+    }
+    const { args } = resolved;
+    for (const arg of definition.requiredArgs ?? []) {
+        if (!(arg in args)) {
+            failures.push(makeFailure("missing_field", `Tool call is missing required argument: ${arg}`, "error", {
+                path: `args.${arg}`,
+            }));
+        }
+    }
+    const argTypes = definition.argTypes ?? {};
+    const known = new Set([
+        ...(definition.allowedArgs ?? []),
+        ...Object.keys(argTypes),
+        ...(definition.requiredArgs ?? []),
+    ]);
+    if (!definition.allowExtraArgs && known.size > 0) {
+        for (const key of Object.keys(args)) {
+            if (!known.has(key)) {
+                failures.push(makeFailure("schema_violation", `Unexpected tool argument: ${key}`, "error", { path: `args.${key}` }));
+            }
+        }
+    }
+    for (const [key, expectedType] of Object.entries(argTypes)) {
+        const actual = args[key];
+        if (actual === undefined)
+            continue;
+        const actualType = typeOf(actual);
+        if (actualType !== expectedType) {
+            failures.push(makeFailure("invalid_type", `Tool argument ${key} expected type ${expectedType} but received ${actualType}.`, "error", {
+                path: `args.${key}`,
+                metadata: { expectedType, actualType },
+            }));
+        }
+    }
+    return failures;
+};
+const resultFrom = (failures) => ({
+    passed: failures.length === 0,
+    failures,
+    warnings: [],
+});
+/**
+ * Validates a tool call shaped `{ name, args }` (or `{ name, arguments: "<json>" }`).
+ */
+export const toolCallCheck = (definition) => ({
+    name: "toolCallCheck",
+    description: "Validates tool-call structure, required args, and argument types.",
+    run: (value, _context) => resultFrom(validateToolCall(value, definition)),
+});
+/**
+ * Validates a tool call against a set of allowed tools. Unknown tool names fail.
+ */
+toolCallCheck.oneOf = (definitions) => ({
+    name: "toolCallCheck.oneOf",
+    description: "Validates a tool call against one of several allowed tool definitions.",
+    run: (value, _context) => {
+        const name = value != null && typeof value === "object"
+            ? value.name
+            : undefined;
+        const definition = definitions.find((candidate) => candidate.name === name);
+        if (!definition) {
+            return resultFrom([
+                makeFailure("tool_call_mismatch", `Unsupported tool: ${String(name)}`, "error", {
+                    path: "name",
+                    metadata: {
+                        allowedTools: definitions.map((candidate) => candidate.name),
+                        actualName: name,
+                    },
+                }),
+            ]);
+        }
+        return resultFrom(validateToolCall(value, definition));
+    },
+});
+/**
+ * Parses a JSON string output and emits `invalid_json` on failure.
+ * Use to guard string outputs before running object-level checks.
+ */
+export const jsonCheck = () => ({
+    name: "jsonCheck",
+    description: "Ensures a string output is valid JSON.",
+    run: (value, _context) => {
+        if (typeof value !== "string") {
+            return resultFrom([
+                makeFailure("invalid_json", "Output must be a JSON string.", "error"),
+            ]);
+        }
+        try {
+            JSON.parse(value);
+            return resultFrom([]);
+        }
+        catch (error) {
+            return resultFrom([
+                makeFailure("invalid_json", `Output is not valid JSON: ${error instanceof Error ? error.message : "parse error"}`, "error"),
+            ]);
+        }
+    },
+});
+const getField = (value, field) => {
+    if (value == null || typeof value !== "object" || Array.isArray(value)) {
+        return {
+            ok: false,
+            failure: makeFailure("schema_violation", `Field ${field} cannot be checked because the value is not an object.`, "error"),
+        };
+    }
+    const actual = value[field];
+    if (actual === undefined) {
+        return {
+            ok: false,
+            failure: makeFailure("missing_field", `Missing field: ${field}`, "error", { path: field }),
+        };
+    }
+    return { ok: true, actual };
+};
+export const nonEmpty = (field) => ({
+    name: "nonEmpty",
+    description: "Ensures a field is not an empty string, array, or object.",
+    run: (value, _context) => {
+        const found = getField(value, field);
+        if (!found.ok)
+            return resultFrom([found.failure]);
+        const { actual } = found;
+        const empty = actual === null ||
+            (typeof actual === "string" && actual.trim() === "") ||
+            (Array.isArray(actual) && actual.length === 0) ||
+            (typeof actual === "object" &&
+                !Array.isArray(actual) &&
+                Object.keys(actual).length === 0);
+        return resultFrom(empty
+            ? [
+                makeFailure("policy_violation", `Field ${field} must not be empty.`, "error", { path: field }),
+            ]
+            : []);
+    },
+});
+export const regexCheck = (field, pattern) => ({
+    name: "regexCheck",
+    description: "Ensures a string field matches a regular expression.",
+    run: (value, _context) => {
+        const found = getField(value, field);
+        if (!found.ok)
+            return resultFrom([found.failure]);
+        const { actual } = found;
+        if (typeof actual !== "string") {
+            return resultFrom([
+                makeFailure("invalid_type", `Field ${field} expected type string but received ${typeOf(actual)}.`, "error", {
+                    path: field,
+                }),
+            ]);
+        }
+        // Reset lastIndex so global/sticky patterns behave deterministically.
+        pattern.lastIndex = 0;
+        return resultFrom(pattern.test(actual)
+            ? []
+            : [
+                makeFailure("policy_violation", `Field ${field} does not match ${pattern}.`, "error", {
+                    path: field,
+                    metadata: { pattern: String(pattern) },
+                }),
+            ]);
+    },
+});
+export const rangeCheck = (field, bounds) => ({
+    name: "rangeCheck",
+    description: "Ensures a numeric field falls within optional min/max bounds.",
+    run: (value, _context) => {
+        const found = getField(value, field);
+        if (!found.ok)
+            return resultFrom([found.failure]);
+        const { actual } = found;
+        if (typeof actual !== "number" || Number.isNaN(actual)) {
+            return resultFrom([
+                makeFailure("invalid_type", `Field ${field} expected type number but received ${typeOf(actual)}.`, "error", {
+                    path: field,
+                }),
+            ]);
+        }
+        const tooLow = bounds.min !== undefined && actual < bounds.min;
+        const tooHigh = bounds.max !== undefined && actual > bounds.max;
+        return resultFrom(tooLow || tooHigh
+            ? [
+                makeFailure("policy_violation", `Field ${field} must be between ${bounds.min ?? "-∞"} and ${bounds.max ?? "∞"}.`, "error", { path: field, metadata: { ...bounds, actualValue: actual } }),
+            ]
+            : []);
+    },
+});
+export const customCheck = (name, fn) => ({
+    name,
+    run: fn,
+});

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,20 @@
+#!/usr/bin/env node
+import { runCli } from "./cliMain.js";
+const readStdin = () => new Promise((resolvePromise, reject) => {
+    let chunk = "";
+    process.stdin.setEncoding("utf8");
+    process.stdin.on("data", (data) => {
+        chunk += data;
+    });
+    process.stdin.on("end", () => resolvePromise(chunk));
+    process.stdin.on("error", reject);
+});
+process.exitCode = await runCli(process.argv.slice(2), {
+    stdout: (text) => {
+        process.stdout.write(text);
+    },
+    stderr: (text) => {
+        process.stderr.write(text);
+    },
+    readStdin,
+});

package/dist/cliMain.d.ts

@@ -0,0 +1,7 @@
+export interface CliIO {
+    stdout: (text: string) => void;
+    stderr: (text: string) => void;
+    readStdin: () => Promise<string>;
+}
+/** Runs the CLI in-process and returns the exit code. */
+export declare const runCli: (argv: string[], io: CliIO) => Promise<number>;

package/dist/cliMain.js

@@ -0,0 +1,170 @@
+import { readFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+import { buildChecks } from "./registry.js";
+import { UsageError, loadFixtures, parseSeverity, runAll, runFixture, } from "./runner.js";
+const USAGE = `Usage:
+  aieval run <file|dir>... [--fail-on <severity>] [--format text|json]
+  aieval check [output.json] --checks <checks.json> [--fail-on <severity>] [--format text|json]
+  aieval --help | --version
+
+Commands:
+  run     Run fixture files ({ name, output, checks, expected? }) or directories of them.
+          With "expected" a case is a regression test; without it, the case passes if validation passes.
+  check   Validate one output (file or stdin) against a JSON array of check specs.
+
+Options:
+  --fail-on <severity>  info | warning | error | critical (default: error)
+  --format <format>     text (default) or json
+  -h, --help            Show this help
+  -v, --version         Show version
+
+Exit codes: 0 all passed, 1 at least one case failed, 2 usage or input error.
+`;
+const readVersion = () => {
+    const raw = readFileSync(new URL("../package.json", import.meta.url), "utf8");
+    return JSON.parse(raw).version;
+};
+const formatCase = (result) => {
+    const lines = [`${result.passed ? "PASS" : "FAIL"} ${result.name}`];
+    if (result.mismatch) {
+        lines.push(`  ! ${result.mismatch}`);
+    }
+    if (!result.passed || result.mismatch) {
+        for (const failure of result.evaluation.failures) {
+            lines.push(`  - ${failure.code}${failure.path ? ` (${failure.path})` : ""}: ${failure.message}`);
+        }
+    }
+    for (const warning of result.evaluation.warnings) {
+        lines.push(`  ~ ${warning.code}${warning.path ? ` (${warning.path})` : ""}: ${warning.message}`);
+    }
+    return lines;
+};
+const formatText = (summary) => {
+    const lines = summary.cases.flatMap(formatCase);
+    lines.push("", `${summary.total - summary.failed} passed, ${summary.failed} failed, ${summary.total} total`);
+    return `${lines.join("\n")}\n`;
+};
+const formatJson = (summary) => `${JSON.stringify({
+    passed: summary.passed,
+    total: summary.total,
+    failed: summary.failed,
+    cases: summary.cases.map((result) => ({
+        file: result.file,
+        name: result.name,
+        passed: result.passed,
+        mismatch: result.mismatch,
+        failures: result.evaluation.failures,
+        warnings: result.evaluation.warnings,
+    })),
+}, null, 2)}\n`;
+const parseJsonText = (text, label) => {
+    if (!text.trim()) {
+        throw new UsageError(`No input provided for ${label}.`);
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch (error) {
+        throw new UsageError(`Invalid JSON in ${label}: ${error instanceof Error ? error.message : "parse error"}`);
+    }
+};
+const readFileText = (path) => {
+    try {
+        return readFileSync(path, "utf8");
+    }
+    catch {
+        throw new UsageError(`Cannot read file: ${path}`);
+    }
+};
+/** Runs the CLI in-process and returns the exit code. */
+export const runCli = async (argv, io) => {
+    try {
+        let parsed;
+        try {
+            parsed = parseArgs({
+                args: argv,
+                allowPositionals: true,
+                options: {
+                    "fail-on": { type: "string" },
+                    format: { type: "string", default: "text" },
+                    checks: { type: "string" },
+                    help: { type: "boolean", short: "h" },
+                    version: { type: "boolean", short: "v" },
+                },
+            });
+        }
+        catch (error) {
+            throw new UsageError(error instanceof Error ? error.message : String(error));
+        }
+        const { values, positionals } = parsed;
+        if (values.help || (positionals.length === 0 && !values.version)) {
+            (values.help ? io.stdout : io.stderr)(USAGE);
+            return values.help ? 0 : 2;
+        }
+        if (values.version) {
+            io.stdout(`${readVersion()}\n`);
+            return 0;
+        }
+        const format = values.format;
+        if (format !== "text" && format !== "json") {
+            throw new UsageError(`Invalid format "${String(format)}". Expected text or json.`);
+        }
+        const failOn = values["fail-on"] !== undefined
+            ? parseSeverity(values["fail-on"])
+            : undefined;
+        const [command, ...rest] = positionals;
+        let summary;
+        if (command === "run") {
+            if (rest.length === 0) {
+                throw new UsageError("`run` needs at least one file or directory.");
+            }
+            const loaded = loadFixtures(rest);
+            if (loaded.length === 0) {
+                throw new UsageError("No fixtures found.");
+            }
+            summary = await runAll(loaded, { failOn });
+        }
+        else if (command === "check") {
+            if (!values.checks) {
+                throw new UsageError("`check` requires --checks <checks.json>.");
+            }
+            if (rest.length > 1) {
+                throw new UsageError("`check` accepts at most one output file.");
+            }
+            const specs = parseJsonText(readFileText(values.checks), values.checks);
+            try {
+                buildChecks(specs);
+            }
+            catch (error) {
+                throw new UsageError(`${values.checks}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            const label = rest[0] ?? "stdin";
+            const output = parseJsonText(rest[0] ? readFileText(rest[0]) : await io.readStdin(), label);
+            const loaded = {
+                file: label,
+                index: 0,
+                fixture: { name: label, output, checks: specs },
+            };
+            const result = await runFixture(loaded, { failOn });
+            summary = {
+                passed: result.passed,
+                total: 1,
+                failed: result.passed ? 0 : 1,
+                cases: [result],
+            };
+        }
+        else {
+            throw new UsageError(`Unknown command "${command}".`);
+        }
+        io.stdout(format === "json" ? formatJson(summary) : formatText(summary));
+        return summary.passed ? 0 : 1;
+    }
+    catch (error) {
+        if (error instanceof UsageError) {
+            io.stderr(`${error.message}\n\n${USAGE}`);
+            return 2;
+        }
+        io.stderr(`${error instanceof Error ? error.message : String(error)}\n`);
+        return 2;
+    }
+};

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+export * from "./types.js";
+export * from "./checks.js";
+export * from "./registry.js";
+export * from "./runner.js";
+export * from "./semantic.js";
+import type { Check, EvaluationOptions, EvaluationResult } from "./types.js";
+export declare class EvaluationError<T = unknown> extends Error {
+    readonly result: EvaluationResult<T>;
+    constructor(result: EvaluationResult<T>);
+}
+export declare const evaluate: <T = unknown>({ input, output, checks, metadata, failOn, }: EvaluationOptions<T>) => Promise<EvaluationResult<T>>;
+export declare const validate: <T = unknown>(output: T, checks: Check<T>[], metadata?: Record<string, unknown>) => Promise<EvaluationResult<T>>;
+export declare const assert: <T = unknown>(output: T, checks: Check<T>[], metadata?: Record<string, unknown>) => Promise<void>;

package/dist/index.js

@@ -0,0 +1,83 @@
+export * from "./types.js";
+export * from "./checks.js";
+export * from "./registry.js";
+export * from "./runner.js";
+export * from "./semantic.js";
+import { makeFailure } from "./checks.js";
+const SEVERITY_RANK = {
+    info: 0,
+    warning: 1,
+    error: 2,
+    critical: 3,
+};
+export class EvaluationError extends Error {
+    result;
+    constructor(result) {
+        super(result.failures.map((failure) => failure.message).join("; ") ||
+            "Validation failed");
+        this.name = "EvaluationError";
+        this.result = result;
+    }
+}
+export const evaluate = async ({ input, output, checks, metadata, failOn = "error", }) => {
+    const threshold = SEVERITY_RANK[failOn];
+    const allFailures = [];
+    const allWarnings = [];
+    const checkMetadata = {};
+    const startedAt = Date.now();
+    const withCheckName = (failure, checkName) => ({
+        ...failure,
+        metadata: { checkName, ...failure.metadata },
+    });
+    for (const check of checks) {
+        let result;
+        try {
+            result = await check.run(output, { input, output, metadata });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            allFailures.push(makeFailure("custom_check", `Check "${check.name}" threw: ${message}`, "critical", {
+                metadata: { checkName: check.name },
+            }));
+            continue;
+        }
+        for (const failure of result.failures) {
+            const tagged = withCheckName(failure, check.name);
+            (SEVERITY_RANK[failure.severity] >= threshold
+                ? allFailures
+                : allWarnings).push(tagged);
+        }
+        for (const warning of result.warnings) {
+            allWarnings.push(withCheckName(warning, check.name));
+        }
+        if (result.metadata) {
+            let key = check.name;
+            for (let suffix = 2; key in checkMetadata; suffix++) {
+                key = `${check.name}#${suffix}`;
+            }
+            checkMetadata[key] = result.metadata;
+        }
+    }
+    return {
+        passed: allFailures.length === 0,
+        failures: allFailures,
+        warnings: allWarnings,
+        output,
+        metadata: {
+            ...metadata,
+            durationMs: Date.now() - startedAt,
+            ...(Object.keys(checkMetadata).length > 0
+                ? { checks: checkMetadata }
+                : {}),
+        },
+    };
+};
+export const validate = async (output, checks, metadata) => {
+    return evaluate({ output, checks, metadata });
+};
+export const assert = async (output, checks, metadata) => {
+    const result = await validate(output, checks, metadata);
+    if (!result.passed) {
+        throw new EvaluationError(result);
+    }
+};