@hamza1331/aieval 0.0.0-stage → 0.1.0

package/dist/registry.d.ts

@@ -0,0 +1,46 @@
+import type { Check, Severity } from "./types.js";
+import { type ToolCallDefinition } from "./checks.js";
+/**
+ * JSON-serialisable description of a check. Used by the CLI and fixture files.
+ * `schemaCheck` is intentionally absent: Zod schemas cannot be serialised.
+ */
+export type CheckSpec = {
+    type: "requiredFields";
+    fields: string[];
+} | {
+    type: "noExtraFields";
+    allowed: string[];
+    severity?: Severity;
+} | {
+    type: "enum";
+    field: string;
+    values: Array<string | number>;
+} | {
+    type: "fieldType";
+    field: string;
+    expected: "string" | "number" | "boolean" | "object" | "array";
+} | ({
+    type: "toolCall";
+} & ToolCallDefinition) | {
+    type: "toolCallOneOf";
+    tools: ToolCallDefinition[];
+} | {
+    type: "json";
+} | {
+    type: "nonEmpty";
+    field: string;
+} | {
+    type: "regex";
+    field: string;
+    pattern: string;
+    flags?: string;
+} | {
+    type: "range";
+    field: string;
+    min?: number;
+    max?: number;
+};
+type AnyCheck = Check<any>;
+export declare const buildCheck: (spec: unknown, index?: number) => AnyCheck;
+export declare const buildChecks: (specs: unknown) => AnyCheck[];
+export {};

package/dist/registry.js

@@ -0,0 +1,82 @@
+import { enumCheck, fieldTypeCheck, jsonCheck, noExtraFields, nonEmpty, rangeCheck, regexCheck, requiredFields, toolCallCheck, } from "./checks.js";
+const fail = (index, message) => {
+    throw new Error(`Invalid check spec at index ${index}: ${message}`);
+};
+const requireString = (spec, key, index) => {
+    const value = spec[key];
+    if (typeof value !== "string" || value === "")
+        fail(index, `"${key}" must be a non-empty string.`);
+    return value;
+};
+const requireStringArray = (spec, key, index) => {
+    const value = spec[key];
+    if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) {
+        fail(index, `"${key}" must be an array of strings.`);
+    }
+    return value;
+};
+export const buildCheck = (spec, index = 0) => {
+    if (spec == null || typeof spec !== "object" || Array.isArray(spec)) {
+        return fail(index, "spec must be an object.");
+    }
+    const s = spec;
+    switch (s.type) {
+        case "requiredFields":
+            return requiredFields(requireStringArray(s, "fields", index));
+        case "noExtraFields":
+            return noExtraFields(requireStringArray(s, "allowed", index), {
+                severity: s.severity,
+            });
+        case "enum": {
+            if (!Array.isArray(s.values))
+                fail(index, `"values" must be an array.`);
+            return enumCheck(requireString(s, "field", index), s.values);
+        }
+        case "fieldType":
+            return fieldTypeCheck(requireString(s, "field", index), requireString(s, "expected", index));
+        case "toolCall": {
+            const { type: _type, ...definition } = s;
+            requireString(s, "name", index);
+            return toolCallCheck(definition);
+        }
+        case "toolCallOneOf": {
+            if (!Array.isArray(s.tools) || s.tools.length === 0)
+                fail(index, `"tools" must be a non-empty array.`);
+            return toolCallCheck.oneOf(s.tools);
+        }
+        case "json":
+            return jsonCheck();
+        case "nonEmpty":
+            return nonEmpty(requireString(s, "field", index));
+        case "regex": {
+            const field = requireString(s, "field", index);
+            const pattern = requireString(s, "pattern", index);
+            try {
+                return regexCheck(field, new RegExp(pattern, typeof s.flags === "string" ? s.flags : undefined));
+            }
+            catch (error) {
+                return fail(index, `invalid regex: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        case "range": {
+            const min = s.min;
+            const max = s.max;
+            if ((min !== undefined && typeof min !== "number") ||
+                (max !== undefined && typeof max !== "number")) {
+                fail(index, `"min" and "max" must be numbers.`);
+            }
+            return rangeCheck(requireString(s, "field", index), {
+                min: min,
+                max: max,
+            });
+        }
+        default:
+            return fail(index, `unknown check type "${String(s.type)}".`);
+    }
+};
+export const buildChecks = (specs) => {
+    if (!Array.isArray(specs)) {
+        throw new Error("Checks must be an array of check specs.");
+    }
+    return specs.map((spec, index) => buildCheck(spec, index));
+};

package/dist/runner.d.ts

@@ -0,0 +1,47 @@
+import type { EvaluationResult, Severity } from "./types.js";
+/** Thrown for problems with how the runner was invoked or with fixture contents. Maps to CLI exit code 2. */
+export declare class UsageError extends Error {
+    constructor(message: string);
+}
+export interface FixtureExpectation {
+    passed: boolean;
+    codes?: string[];
+    warningCodes?: string[];
+}
+export interface Fixture {
+    name?: string;
+    output: unknown;
+    checks: unknown[];
+    failOn?: Severity;
+    /** When present the case is a regression test: the evaluation must match this expectation. */
+    expected?: FixtureExpectation;
+}
+export interface LoadedFixture {
+    file: string;
+    index: number;
+    fixture: Fixture;
+}
+export interface CaseResult {
+    file: string;
+    name: string;
+    /** Whether the case itself succeeded (matched `expected`, or the evaluation passed when no `expected`). */
+    passed: boolean;
+    evaluation: EvaluationResult;
+    mismatch?: string;
+}
+export interface RunSummary {
+    passed: boolean;
+    total: number;
+    failed: number;
+    cases: CaseResult[];
+}
+export declare const parseSeverity: (value: string) => Severity;
+/** Loads fixtures from files and/or directories. Directories are scanned non-recursively for `*.json`. */
+export declare const loadFixtures: (paths: string[]) => LoadedFixture[];
+/** Runs a single fixture. `failOn` (if given) overrides the fixture's own value. */
+export declare const runFixture: (loaded: LoadedFixture, options?: {
+    failOn?: Severity;
+}) => Promise<CaseResult>;
+export declare const runAll: (loaded: LoadedFixture[], options?: {
+    failOn?: Severity;
+}) => Promise<RunSummary>;

package/dist/runner.js

@@ -0,0 +1,143 @@
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { evaluate } from "./index.js";
+import { buildChecks } from "./registry.js";
+/** Thrown for problems with how the runner was invoked or with fixture contents. Maps to CLI exit code 2. */
+export class UsageError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UsageError";
+    }
+}
+const SEVERITIES = ["info", "warning", "error", "critical"];
+export const parseSeverity = (value) => {
+    if (!SEVERITIES.includes(value)) {
+        throw new UsageError(`Invalid severity "${value}". Expected one of: ${SEVERITIES.join(", ")}.`);
+    }
+    return value;
+};
+const readJson = (file) => {
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch {
+        throw new UsageError(`Cannot read file: ${file}`);
+    }
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        throw new UsageError(`Invalid JSON in ${file}: ${error instanceof Error ? error.message : "parse error"}`);
+    }
+};
+const validateFixture = (value, file, index) => {
+    const where = `${file}${index > 0 ? ` (case ${index + 1})` : ""}`;
+    if (value == null || typeof value !== "object" || Array.isArray(value)) {
+        throw new UsageError(`Fixture in ${where} must be an object.`);
+    }
+    const candidate = value;
+    if (!("output" in candidate)) {
+        throw new UsageError(`Fixture in ${where} is missing "output".`);
+    }
+    if (!Array.isArray(candidate.checks)) {
+        throw new UsageError(`Fixture in ${where} must have a "checks" array.`);
+    }
+    if (candidate.failOn !== undefined) {
+        parseSeverity(String(candidate.failOn));
+    }
+    if (candidate.expected !== undefined) {
+        const expected = candidate.expected;
+        if (expected == null ||
+            typeof expected !== "object" ||
+            typeof expected.passed !== "boolean") {
+            throw new UsageError(`Fixture in ${where} has an invalid "expected" (needs a boolean "passed").`);
+        }
+    }
+    return candidate;
+};
+/** Loads fixtures from files and/or directories. Directories are scanned non-recursively for `*.json`. */
+export const loadFixtures = (paths) => {
+    const files = [];
+    for (const path of paths) {
+        const absolute = resolve(path);
+        let stats;
+        try {
+            stats = statSync(absolute);
+        }
+        catch {
+            throw new UsageError(`Path not found: ${path}`);
+        }
+        if (stats.isDirectory()) {
+            const entries = readdirSync(absolute)
+                .filter((entry) => entry.endsWith(".json"))
+                .sort()
+                .map((entry) => join(absolute, entry));
+            files.push(...entries);
+        }
+        else {
+            files.push(absolute);
+        }
+    }
+    const loaded = [];
+    for (const file of files) {
+        const data = readJson(file);
+        const items = Array.isArray(data) ? data : [data];
+        items.forEach((item, index) => {
+            loaded.push({ file, index, fixture: validateFixture(item, file, index) });
+        });
+    }
+    return loaded;
+};
+const sameCodes = (actual, expected) => JSON.stringify([...actual].sort()) === JSON.stringify([...expected].sort());
+/** Runs a single fixture. `failOn` (if given) overrides the fixture's own value. */
+export const runFixture = async (loaded, options = {}) => {
+    const { fixture, file, index } = loaded;
+    const name = fixture.name ?? `${file}${index > 0 ? `#${index + 1}` : ""}`;
+    let checks;
+    try {
+        checks = buildChecks(fixture.checks);
+    }
+    catch (error) {
+        throw new UsageError(`${file}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    const evaluation = await evaluate({
+        output: fixture.output,
+        checks,
+        failOn: options.failOn ?? fixture.failOn,
+    });
+    if (!fixture.expected) {
+        return { file, name, passed: evaluation.passed, evaluation };
+    }
+    const { expected } = fixture;
+    const problems = [];
+    if (evaluation.passed !== expected.passed) {
+        problems.push(`expected passed=${expected.passed} but got passed=${evaluation.passed}`);
+    }
+    const actualCodes = evaluation.failures.map((failure) => failure.code);
+    const expectedCodes = expected.codes ?? [];
+    if (!sameCodes(actualCodes, expectedCodes)) {
+        problems.push(`expected failure codes [${expectedCodes.join(", ")}] but got [${actualCodes.join(", ")}]`);
+    }
+    if (expected.warningCodes) {
+        const actualWarnings = evaluation.warnings.map((warning) => warning.code);
+        if (!sameCodes(actualWarnings, expected.warningCodes)) {
+            problems.push(`expected warning codes [${expected.warningCodes.join(", ")}] but got [${actualWarnings.join(", ")}]`);
+        }
+    }
+    return {
+        file,
+        name,
+        passed: problems.length === 0,
+        evaluation,
+        mismatch: problems.length > 0 ? problems.join("; ") : undefined,
+    };
+};
+export const runAll = async (loaded, options = {}) => {
+    const cases = [];
+    for (const item of loaded) {
+        cases.push(await runFixture(item, options));
+    }
+    const failed = cases.filter((result) => !result.passed).length;
+    return { passed: failed === 0, total: cases.length, failed, cases };
+};

package/dist/semantic.d.ts

@@ -0,0 +1,36 @@
+import type { Check, Severity } from "./types.js";
+export interface JudgeRequest {
+    input: unknown;
+    output: unknown;
+    criteria: string;
+}
+export interface JudgeVerdict {
+    /** Score between 0 and 1 inclusive. */
+    score: number;
+    reasoning?: string;
+    metadata?: Record<string, unknown>;
+}
+/** A caller-supplied function that scores an output against criteria, typically by calling an LLM. */
+export type SemanticJudge = (request: JudgeRequest) => Promise<JudgeVerdict> | JudgeVerdict;
+export interface SemanticCheckOptions {
+    criteria: string;
+    /** Minimum passing score. Defaults to 0.5. */
+    threshold?: number;
+    /** Severity of the failure when the score is below the threshold. Defaults to "warning" (advisory). */
+    severity?: Severity;
+    /** Check name used in failures and result metadata. Defaults to "semanticCheck". */
+    name?: string;
+}
+/**
+ * Optional LLM-judged check. Scores are advisory by default: a low score is reported as a warning
+ * unless `severity` is raised or `failOn` is lowered. Results are tagged `metadata.kind = "semantic"`.
+ */
+export declare const semanticCheck: <T = unknown>(judge: SemanticJudge, options: SemanticCheckOptions) => Check<T>;
+export interface MockJudge extends SemanticJudge {
+    calls: JudgeRequest[];
+}
+/**
+ * Deterministic judge for tests and examples. Accepts a fixed score, a sequence of scores
+ * (the last one repeats), or a function mapping a request to a score or verdict.
+ */
+export declare const mockJudge: (scores: number | number[] | ((request: JudgeRequest) => number | JudgeVerdict)) => MockJudge;

package/dist/semantic.js

@@ -0,0 +1,74 @@
+import { makeFailure } from "./checks.js";
+/**
+ * Optional LLM-judged check. Scores are advisory by default: a low score is reported as a warning
+ * unless `severity` is raised or `failOn` is lowered. Results are tagged `metadata.kind = "semantic"`.
+ */
+export const semanticCheck = (judge, options) => {
+    const threshold = options.threshold ?? 0.5;
+    const severity = options.severity ?? "warning";
+    return {
+        name: options.name ?? "semanticCheck",
+        description: `Semantic judge: ${options.criteria}`,
+        run: async (value, context) => {
+            const verdict = await judge({
+                input: context.input,
+                output: value,
+                criteria: options.criteria,
+            });
+            if (verdict == null ||
+                typeof verdict.score !== "number" ||
+                !Number.isFinite(verdict.score) ||
+                verdict.score < 0 ||
+                verdict.score > 1) {
+                return {
+                    passed: false,
+                    failures: [
+                        makeFailure("custom_check", `Semantic judge returned an invalid score: ${String(verdict?.score)}. Expected a number between 0 and 1.`, "critical", {
+                            metadata: { kind: "semantic", criteria: options.criteria },
+                        }),
+                    ],
+                    warnings: [],
+                    metadata: { kind: "semantic", criteria: options.criteria },
+                };
+            }
+            const metadata = {
+                kind: "semantic",
+                score: verdict.score,
+                threshold,
+                criteria: options.criteria,
+                reasoning: verdict.reasoning,
+                ...verdict.metadata,
+            };
+            if (verdict.score >= threshold) {
+                return { passed: true, failures: [], warnings: [], metadata };
+            }
+            return {
+                passed: false,
+                failures: [
+                    makeFailure("unsupported_claim", `Semantic score ${verdict.score} is below threshold ${threshold} for: ${options.criteria}`, severity, { metadata }),
+                ],
+                warnings: [],
+                metadata,
+            };
+        },
+    };
+};
+/**
+ * Deterministic judge for tests and examples. Accepts a fixed score, a sequence of scores
+ * (the last one repeats), or a function mapping a request to a score or verdict.
+ */
+export const mockJudge = (scores) => {
+    const calls = [];
+    const judge = ((request) => {
+        const index = calls.length;
+        calls.push(request);
+        const raw = typeof scores === "function"
+            ? scores(request)
+            : Array.isArray(scores)
+                ? scores[Math.min(index, scores.length - 1)]
+                : scores;
+        return typeof raw === "number" ? { score: raw } : raw;
+    });
+    judge.calls = calls;
+    return judge;
+};

package/dist/types.d.ts

@@ -0,0 +1,42 @@
+export type Severity = "info" | "warning" | "error" | "critical";
+export type FailureCode = "missing_field" | "invalid_type" | "invalid_enum" | "schema_violation" | "invalid_json" | "tool_call_mismatch" | "unsupported_claim" | "policy_violation" | "custom_check" | "unknown";
+export interface Failure {
+    code: FailureCode;
+    message: string;
+    severity: Severity;
+    path?: string;
+    metadata?: Record<string, unknown>;
+    suggestedRepair?: string;
+}
+export interface CheckResult {
+    passed: boolean;
+    failures: Failure[];
+    warnings: Failure[];
+    metadata?: Record<string, unknown>;
+}
+export type CheckContext = {
+    input?: unknown;
+    output?: unknown;
+    metadata?: Record<string, unknown>;
+};
+export type CheckFn<T = unknown> = (value: T, context: CheckContext) => CheckResult | Promise<CheckResult>;
+export interface Check<T = unknown> {
+    name: string;
+    description?: string;
+    run: CheckFn<T>;
+}
+export interface EvaluationOptions<T = unknown> {
+    input?: unknown;
+    output?: T;
+    checks: Check<T>[];
+    metadata?: Record<string, unknown>;
+    /** Minimum severity that counts as a failure. Lower severities become warnings. Defaults to "error". */
+    failOn?: Severity;
+}
+export interface EvaluationResult<T = unknown> {
+    passed: boolean;
+    failures: Failure[];
+    warnings: Failure[];
+    output?: T;
+    metadata?: Record<string, unknown>;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,79 @@
 {
   "name": "@hamza1331/aieval",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "TypeScript-first validation toolkit for LLM outputs and agent actions.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hamza1331/aieval.git"
+  },
+  "homepage": "https://github.com/hamza1331/aieval#readme",
+  "bugs": {
+    "url": "https://github.com/hamza1331/aieval/issues"
+  },
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "aieval": "dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./cli": {
+      "import": "./dist/cli.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.test.json",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "dev": "vitest",
+    "ci": "npm run lint && npm run format:check && npm run typecheck && npm run build && npm run test:coverage",
+    "prepublishOnly": "npm run build && npm test"
+  },
+  "keywords": [
+    "ai",
+    "llm",
+    "evaluation",
+    "validation",
+    "agents",
+    "typescript"
+  ],
+  "author": "Hamza Ali <hamxa1331@gmail.com>",
+  "license": "MIT",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=20"
+  },
+  "peerDependencies": {
+    "zod": "^4.0.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/node": "^24.0.0",
+    "@vitest/coverage-v8": "^3.2.7",
+    "eslint": "^10.12.0",
+    "prettier": "^3.9.9",
+    "tsx": "^4.23.15",
+    "typescript": "^5.8.3",
+    "typescript-eslint": "^8.71.1",
+    "vitest": "^3.2.4",
+    "zod": "^4.6.5"
+  }
+}