@alisio/core 0.5.2 → 0.7.0

package/dist/eval/stats.js

@@ -0,0 +1,50 @@
+/** Pure statistics of the reliability lab (specs/010 D6, D17). No I/O, no dependencies. */
+/** z of a two-sided 95 % interval. */
+const Z95 = 1.96;
+/**
+ * Wilson score interval of a proportion (default 95 %). Unlike the normal approximation it stays
+ * inside [0, 1] and is informative at 0/n and n/n. With no trials it carries no information.
+ */
+export function wilsonInterval(successes, trials, z = Z95) {
+    if (!Number.isInteger(trials) || trials < 0)
+        throw new RangeError(`trials must be a non-negative integer, got ${trials}.`);
+    if (!Number.isInteger(successes) || successes < 0 || successes > trials)
+        throw new RangeError(`successes must be an integer in [0, ${trials}], got ${successes}.`);
+    if (trials === 0)
+        return { low: 0, high: 1 };
+    const p = successes / trials;
+    const z2 = z * z;
+    const denominator = 1 + z2 / trials;
+    const centre = (p + z2 / (2 * trials)) / denominator;
+    const half = (z * Math.sqrt((p * (1 - p)) / trials + z2 / (4 * trials * trials))) / denominator;
+    // Floating point can leave 1e-17 residue at the extremes; the bounds there are exact.
+    return {
+        low: successes === 0 ? 0 : Math.max(0, centre - half),
+        high: successes === trials ? 1 : Math.min(1, centre + half),
+    };
+}
+/** Median; null for an empty set. Does not mutate the input. */
+export function median(values) {
+    if (values.length === 0)
+        return null;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2
+        ? sorted[mid]
+        : (sorted[mid - 1] + sorted[mid]) / 2;
+}
+/** Sample variance (n - 1), the same convention as `scripts/eval-baseline.ts`; null below two samples. */
+export function sampleVariance(values) {
+    if (values.length < 2)
+        return null;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    return values.reduce((a, b) => a + (b - mean) ** 2, 0) / (values.length - 1);
+}
+/** pass@k: at least one of the k repetitions succeeded. No repetitions is not a pass. */
+export function passAtK(results) {
+    return results.some(Boolean);
+}
+/** pass^k: every one of the k repetitions succeeded (consistency). No repetitions is not a pass. */
+export function passPowK(results) {
+    return results.length > 0 && results.every(Boolean);
+}

package/dist/eval/suite.d.ts

@@ -0,0 +1,20 @@
+/** Loader and validator of reliability-lab suites (specs/010 D1). */
+import type { EvalSuite } from "@alisio/sdk";
+export declare class EvalSuiteError extends Error {
+    constructor(message: string);
+}
+export interface LoadedSuite {
+    suite: EvalSuite;
+    /** Absolute suite directory. */
+    dir: string;
+    /** Absolute fixture directory per task id. */
+    fixtureDirs: Record<string, string>;
+}
+/** Upper bound of a `command` verifier timeout: a suite cannot park a run for hours. */
+export declare const MAX_VERIFIER_TIMEOUT_MS = 600000;
+/**
+ * Reads `<dir>/suite.json` and proves the suite is internally consistent: schema, unique ids,
+ * existing fixture directories and every path confined to the suite (or, for verifiers, to the
+ * task working directory). Every failure names the task, the field and how to fix it.
+ */
+export declare function loadSuite(dir: string): LoadedSuite;

package/dist/eval/suite.js

@@ -0,0 +1,165 @@
+/** Loader and validator of reliability-lab suites (specs/010 D1). */
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { z } from "zod";
+import { EvalPathError, resolveInside, unsafeRelativeReason } from "./paths.js";
+export class EvalSuiteError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "EvalSuiteError";
+    }
+}
+const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
+/** Upper bound of a `command` verifier timeout: a suite cannot park a run for hours. */
+export const MAX_VERIFIER_TIMEOUT_MS = 600_000;
+/** Effects a task policy may add to its mode; MCP and agents are never grantable by a suite. */
+const TASK_ALLOW = ["write", "process", "external", "analysis"];
+const id = z.string().min(1);
+const verifier = z.discriminatedUnion("kind", [
+    z.strictObject({
+        kind: z.literal("command"),
+        id,
+        command: z.array(z.string()).min(1),
+        timeoutMs: z
+            .number()
+            .int()
+            .positive()
+            .max(MAX_VERIFIER_TIMEOUT_MS, `must be at most ${MAX_VERIFIER_TIMEOUT_MS} ms; split the check or lower timeoutMs`),
+        expectExit: z.number().int().optional(),
+        expectStdoutIncludes: z.string().optional(),
+    }),
+    z.strictObject({
+        kind: z.literal("file-equals"),
+        id,
+        path: id,
+        sha256: z.string().regex(/^[0-9a-f]{64}$/, "must be a lowercase hex sha256"),
+    }),
+    z.strictObject({ kind: z.literal("file-contains"), id, path: id, text: z.string() }),
+    z.strictObject({ kind: z.literal("file-absent"), id, path: id }),
+    z.strictObject({ kind: z.literal("tree-diff-within"), id, allowed: z.array(id) }),
+    z.strictObject({
+        kind: z.literal("tool-sequence"),
+        id,
+        rule: z.discriminatedUnion("type", [
+            z.strictObject({ type: z.literal("before"), first: id, then: id }),
+            z.strictObject({ type: z.literal("never"), tool: id }),
+            z.strictObject({
+                type: z.literal("max-calls"),
+                tool: id,
+                max: z.number().int().nonnegative(),
+            }),
+            z.strictObject({ type: z.literal("no-repeat-failure"), tool: id }),
+        ]),
+    }),
+]);
+const task = z.strictObject({
+    id,
+    prompt: id,
+    fixture: id,
+    agent: z.string().optional(),
+    budget: z.strictObject({
+        maxTurns: z.number().int().positive(),
+        maxTokens: z.number().int().positive().optional(),
+        maxWallMs: z.number().int().positive(),
+        maxToolCalls: z.number().int().positive().optional(),
+    }),
+    policy: z.strictObject({
+        mode: z.enum(["ask", "auto", "full"]),
+        readOnly: z.boolean().optional(),
+        allow: z
+            .array(z.enum(TASK_ALLOW, {
+            error: `must be one of ${TASK_ALLOW.join(", ")} (MCP and agents cannot be granted by a suite)`,
+        }))
+            .optional(),
+    }),
+    verifiers: z.array(verifier),
+    tags: z.array(z.string()).optional(),
+    fixtureGit: z.boolean().optional(),
+    replayNormalize: z
+        .array(z.strictObject({
+        pattern: z.string().superRefine((value, ctx) => {
+            try {
+                new RegExp(value, "g");
+            }
+            catch {
+                ctx.addIssue({ code: "custom", message: "must be a valid regular expression" });
+            }
+        }),
+        replace: z.string(),
+    }))
+        .optional(),
+});
+const suiteSchema = z.strictObject({
+    schemaVersion: z.literal(1),
+    suiteId: id,
+    suiteVersion: z.string().regex(SEMVER, "must be a semver version such as 1.0.0"),
+    description: z.string(),
+    tasks: z.array(task).min(1, "must declare at least one task"),
+});
+/**
+ * Reads `<dir>/suite.json` and proves the suite is internally consistent: schema, unique ids,
+ * existing fixture directories and every path confined to the suite (or, for verifiers, to the
+ * task working directory). Every failure names the task, the field and how to fix it.
+ */
+export function loadSuite(dir) {
+    const suiteDir = resolve(dir);
+    const file = join(suiteDir, "suite.json");
+    if (!existsSync(file))
+        throw new EvalSuiteError(`suite.json not found in ${suiteDir}. Create it or pass the suite directory.`);
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch {
+        throw new EvalSuiteError(`${file} is not valid JSON. Fix the syntax error and retry.`);
+    }
+    const version = raw?.schemaVersion;
+    if (version !== 1)
+        throw new EvalSuiteError(`Unsupported schemaVersion in ${file}: expected 1, found ${JSON.stringify(version)}.`);
+    const parsed = suiteSchema.safeParse(raw);
+    if (!parsed.success) {
+        const issues = parsed.error.issues
+            .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
+            .join("; ");
+        throw new EvalSuiteError(`Invalid suite ${file}: ${issues}`);
+    }
+    const suite = parsed.data;
+    const seen = new Set();
+    const fixtureDirs = {};
+    for (const t of suite.tasks) {
+        if (seen.has(t.id))
+            throw new EvalSuiteError(`Duplicate task id "${t.id}" in ${file}. Task ids must be unique.`);
+        seen.add(t.id);
+        let fixtureDir;
+        try {
+            fixtureDir = resolveInside(suiteDir, t.fixture, "fixture");
+        }
+        catch (error) {
+            if (!(error instanceof EvalPathError))
+                throw error;
+            throw new EvalSuiteError(`Task "${t.id}": fixture "${t.fixture}" must be a relative path inside the suite directory (${error.message})`);
+        }
+        if (!existsSync(fixtureDir))
+            throw new EvalSuiteError(`Task "${t.id}": fixture "${t.fixture}" does not exist (looked for ${fixtureDir}). Create the directory or fix the path.`);
+        if (!statSync(fixtureDir).isDirectory())
+            throw new EvalSuiteError(`Task "${t.id}": fixture "${t.fixture}" is not a directory.`);
+        fixtureDirs[t.id] = fixtureDir;
+        const verifierIds = new Set();
+        for (const v of t.verifiers) {
+            if (verifierIds.has(v.id))
+                throw new EvalSuiteError(`Task "${t.id}": duplicate verifier id "${v.id}".`);
+            verifierIds.add(v.id);
+            const paths = v.kind === "tree-diff-within"
+                ? { field: "allowed", values: v.allowed }
+                : "path" in v
+                    ? { field: "path", values: [v.path] }
+                    : undefined;
+            for (const value of paths?.values ?? []) {
+                const reason = unsafeRelativeReason(value);
+                if (reason)
+                    throw new EvalSuiteError(`Task "${t.id}": verifier "${v.id}" ${paths?.field} "${value}" ${reason}. Verifier paths are relative to the task working directory.`);
+            }
+        }
+    }
+    return { suite, dir: suiteDir, fixtureDirs };
+}

package/dist/eval/variants.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Configuration variants of the reliability lab (specs/010 D15, section 7). A variant is a set of
+ * dotted-key overrides over the effective configuration. Keys are validated against the REAL
+ * config schema (strict objects: an unknown key is rejected and named), never applied as code.
+ * The overrides are applied BEFORE `runTask`, which hashes what it writes, so a variant gets its
+ * own `configHash`.
+ */
+import { type Config } from "../config.ts";
+export declare class EvalVariantError extends Error {
+    constructor(message: string);
+}
+/**
+ * `base` with `overrides` applied (dotted keys, e.g. `{"compaction.threshold": 0.6}`); `base` is
+ * not modified. Throws `EvalVariantError` naming the key for an unknown key, a value the schema
+ * refuses, or a key the lab never measures (provider, plugins, MCP, paths: they are outside the
+ * configuration hash, so overriding them would change behavior without changing `configHash`).
+ */
+export declare function applyConfigOverrides(base: Config, overrides: Record<string, unknown>): Config;
+export interface VariantsFile {
+    suite: string;
+    variants: Array<{
+        id: string;
+        overrides: Record<string, unknown>;
+    }>;
+}
+/** Reads and validates a `compare.json` (`{ suite, variants: [{ id, overrides }] }`). */
+export declare function loadVariantsFile(path: string): VariantsFile;
+/**
+ * Applies every variant's overrides to `base`, all-or-nothing: one bad key rejects the file before
+ * anything runs, and the error names the variant and the key.
+ */
+export declare function resolveVariants(file: VariantsFile, base: Config): Array<{
+    id: string;
+    config: Config;
+}>;

package/dist/eval/variants.js

@@ -0,0 +1,137 @@
+/**
+ * Configuration variants of the reliability lab (specs/010 D15, section 7). A variant is a set of
+ * dotted-key overrides over the effective configuration. Keys are validated against the REAL
+ * config schema (strict objects: an unknown key is rejected and named), never applied as code.
+ * The overrides are applied BEFORE `runTask`, which hashes what it writes, so a variant gets its
+ * own `configHash`.
+ */
+import { readFileSync } from "node:fs";
+import { z } from "zod";
+import { configSchema } from "../config.js";
+import { CONFIG_HASH_COVERAGE } from "./engine.js";
+export class EvalVariantError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "EvalVariantError";
+    }
+}
+const FORBIDDEN_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]);
+const isRecord = (value) => !!value && typeof value === "object" && !Array.isArray(value);
+function knownKeysAt(root, path) {
+    let cursor = root;
+    for (const segment of path) {
+        if (!isRecord(cursor) || !Object.hasOwn(cursor, segment))
+            return [];
+        cursor = cursor[segment];
+    }
+    return isRecord(cursor) ? Object.keys(cursor).sort() : [];
+}
+function unknownKey(key, base, parent) {
+    const siblings = knownKeysAt(base, parent);
+    const where = parent.length > 0 ? `section "${parent.join(".")}"` : "the configuration";
+    return new EvalVariantError(`Unknown config key "${key}". ${siblings.length > 0
+        ? `Known keys in ${where}: ${siblings.join(", ")}.`
+        : `Use a dotted key of the Alisio configuration, for example "compaction.threshold".`}`);
+}
+/**
+ * `base` with `overrides` applied (dotted keys, e.g. `{"compaction.threshold": 0.6}`); `base` is
+ * not modified. Throws `EvalVariantError` naming the key for an unknown key, a value the schema
+ * refuses, or a key the lab never measures (provider, plugins, MCP, paths: they are outside the
+ * configuration hash, so overriding them would change behavior without changing `configHash`).
+ */
+export function applyConfigOverrides(base, overrides) {
+    const { mcpSources: _sources, ...plain } = base;
+    const draft = JSON.parse(JSON.stringify(plain));
+    const reference = JSON.parse(JSON.stringify(plain));
+    for (const [key, value] of Object.entries(overrides)) {
+        const segments = key.split(".");
+        if (segments.some((s) => s === "" || FORBIDDEN_SEGMENTS.has(s)))
+            throw unknownKey(key, reference, []);
+        const top = segments[0];
+        if (!Object.hasOwn(CONFIG_HASH_COVERAGE, top))
+            throw unknownKey(key, reference, []);
+        if (CONFIG_HASH_COVERAGE[top] === "excluded")
+            throw new EvalVariantError(`Config key "${key}" cannot be overridden by a variant: "${top}" is not part of the measured configuration (it holds endpoints, secrets, plugins or paths), so the change would not show up in configHash. Compare models with --model instead.`);
+        let cursor = draft;
+        for (const [index, segment] of segments.slice(0, -1).entries()) {
+            const next = Object.hasOwn(cursor, segment) ? cursor[segment] : undefined;
+            if (next === undefined)
+                cursor[segment] = {};
+            else if (!isRecord(next))
+                throw unknownKey(key, reference, segments.slice(0, index + 1));
+            cursor = cursor[segment];
+        }
+        cursor[segments[segments.length - 1]] = JSON.parse(JSON.stringify(value ?? null));
+    }
+    const parsed = configSchema.safeParse(draft);
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        if (issue && issue.code === "unrecognized_keys") {
+            const parent = issue.path.map(String);
+            const bad = issue.keys[0];
+            throw unknownKey([...parent, bad].join("."), reference, parent);
+        }
+        const path = issue?.path.map(String).join(".") ?? "";
+        throw new EvalVariantError(`Invalid value for config key "${path}": ${issue?.message ?? "rejected by the configuration schema"}.`);
+    }
+    return parsed.data;
+}
+const variantsSchema = z
+    .object({
+    suite: z.string().min(1),
+    variants: z
+        .array(z
+        .object({
+        id: z
+            .string()
+            .regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/, "variant ids use letters, digits, dot, underscore or dash"),
+        overrides: z.record(z.string(), z.unknown()).default({}),
+    })
+        .strict())
+        .min(2, "a variants file needs at least two variants (a baseline and a candidate)"),
+})
+    .strict();
+/** Reads and validates a `compare.json` (`{ suite, variants: [{ id, overrides }] }`). */
+export function loadVariantsFile(path) {
+    let text;
+    try {
+        text = readFileSync(path, "utf8");
+    }
+    catch (error) {
+        throw new EvalVariantError(`Cannot read the variants file "${path}": ${error instanceof Error ? error.message : String(error)}`);
+    }
+    let json;
+    try {
+        json = JSON.parse(text);
+    }
+    catch {
+        throw new EvalVariantError(`The variants file "${path}" is not valid JSON.`);
+    }
+    const parsed = variantsSchema.safeParse(json);
+    if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        const where = issue?.path.length ? ` at "${issue.path.join(".")}"` : "";
+        throw new EvalVariantError(`Invalid variants file "${path}"${where}: ${issue?.message ?? "invalid"}.`);
+    }
+    const ids = parsed.data.variants.map((v) => v.id);
+    const duplicate = ids.find((id, index) => ids.indexOf(id) !== index);
+    if (duplicate)
+        throw new EvalVariantError(`Invalid variants file "${path}": duplicate variant id "${duplicate}".`);
+    return parsed.data;
+}
+/**
+ * Applies every variant's overrides to `base`, all-or-nothing: one bad key rejects the file before
+ * anything runs, and the error names the variant and the key.
+ */
+export function resolveVariants(file, base) {
+    return file.variants.map((variant) => {
+        try {
+            return { id: variant.id, config: applyConfigOverrides(base, variant.overrides) };
+        }
+        catch (error) {
+            if (error instanceof EvalVariantError)
+                throw new EvalVariantError(`Variant "${variant.id}": ${error.message}`);
+            throw error;
+        }
+    });
+}

package/dist/eval/verifiers.d.ts

@@ -0,0 +1,49 @@
+/**
+ * Deterministic, local verifiers of the reliability lab (specs/010 D4). A verifier answers
+ * `pass`, `fail` or `error`; `error` means it could not run (bad path, missing input, command that
+ * cannot start) and is never folded into `fail`. Details never echo file contents.
+ *
+ * `command` runs a subprocess with a clean environment and a timeout. A subprocess is NOT a
+ * sandbox: suites are trusted repository code and the command runs with the user's privileges.
+ */
+import type { EvalVerifier } from "@alisio/sdk";
+export type VerifierStatus = "pass" | "fail" | "error";
+export interface VerifierResult {
+    id: string;
+    status: VerifierStatus;
+    detail?: string;
+}
+/** One tool call observed in a run, as the `tool-sequence` verifier needs it. */
+export interface ToolEvent {
+    tool: string;
+    ok: boolean;
+    /** Canonical key of the call arguments; two calls with the same key are "identical". */
+    argsKey?: string;
+}
+/** Content hashes of a tree (relative POSIX path -> sha256) plus one hash of the whole. */
+export interface TreeSnapshot {
+    files: Record<string, string>;
+    hash: string;
+}
+export interface VerifierContext {
+    /** Working directory of the run; every verifier path is confined to it. */
+    cwd: string;
+    /** Snapshot of the fixture before the run; required by `tree-diff-within`. */
+    baseline?: TreeSnapshot;
+    /** Tool calls of the run in order; required by `tool-sequence`. */
+    toolEvents?: ToolEvent[];
+    /**
+     * Run-private temporary directory. When set, `command` verifiers get HOME and TMPDIR pointing at
+     * it so tooling caches and temp files stay inside the run; unset keeps the minimal environment.
+     */
+    tmpDir?: string;
+}
+/**
+ * Hashes every file under `dir` (symlinks by their target text, never followed). `.git` and
+ * `node_modules` are skipped at any depth: they are tooling state, not task output.
+ */
+export declare function snapshotTree(dir: string): TreeSnapshot;
+/** Runs one verifier; anything that prevents a verdict (including exceptions) is `error`. */
+export declare function runVerifier(v: EvalVerifier, ctx: VerifierContext): Promise<VerifierResult>;
+/** Runs the verifiers in order (sequentially: `command` verifiers may share the working directory). */
+export declare function runVerifiers(verifiers: EvalVerifier[], ctx: VerifierContext): Promise<VerifierResult[]>;