@cogitator-ai/evals 0.11.0 → 0.13.0

package/dist/runs/schema.js

@@ -0,0 +1,171 @@
+import { z } from 'zod';
+import { EvalCaseSchema } from '../schema.js';
+/** The `format` every run file carries */
+export const EVAL_RUN_FORMAT = 'cogitator.eval-run';
+/** The version of the run file format this package writes and reads */
+export const EVAL_RUN_VERSION = 1;
+/** A figure JSON writes as `null` when it is not a finite number, such as an unscored mean */
+const figure = z
+    .number()
+    .nullable()
+    .transform((value) => value ?? Number.NaN);
+export const RunPromptSchema = z.object({
+    key: z.string().min(1),
+    versionId: z.string().optional(),
+    version: z.number().optional(),
+    abTest: z.object({ id: z.string(), variant: z.enum(['control', 'treatment']) }).optional(),
+});
+const UsageSchema = z.object({
+    inputTokens: z.number(),
+    outputTokens: z.number(),
+    totalTokens: z.number(),
+    cost: z.number(),
+});
+const ScoreSchema = z.object({
+    name: z.string(),
+    score: z.number(),
+    details: z.string().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+    error: z.string().optional(),
+    usage: UsageSchema.optional(),
+});
+const ToolCallSchema = z.object({
+    id: z.string(),
+    name: z.string(),
+    arguments: z.record(z.string(), z.unknown()),
+});
+const TurnSchema = z.object({
+    input: z.string(),
+    output: z.string(),
+    toolCalls: z.array(ToolCallSchema).optional(),
+    traceId: z.string().optional(),
+    prompt: RunPromptSchema.optional(),
+    error: z.string().optional(),
+});
+const CaseResultSchema = z.object({
+    case: EvalCaseSchema,
+    output: z.string(),
+    duration: z.number(),
+    error: z.string().optional(),
+    attempts: z.number().int().optional(),
+    usage: UsageSchema.extend({ duration: z.number() }).optional(),
+    toolCalls: z.array(ToolCallSchema).optional(),
+    toolResults: z
+        .array(z.object({
+        callId: z.string(),
+        name: z.string(),
+        result: z.unknown().optional(),
+        error: z.string().optional(),
+    }))
+        .optional(),
+    retrieved: z
+        .array(z.object({
+        id: z.string().optional(),
+        content: z.string(),
+        score: z.number().optional(),
+        source: z.string().optional(),
+    }))
+        .optional(),
+    turns: z.array(TurnSchema).optional(),
+    traceId: z.string().optional(),
+    prompt: RunPromptSchema.optional(),
+    truncated: z.boolean().optional(),
+    iterationLimitReached: z.boolean().optional(),
+    paused: z.boolean().optional(),
+    scores: z.array(ScoreSchema),
+});
+const RunCaseSchema = CaseResultSchema.extend({
+    /** SHA-256 of the canonical JSON of the case's `{ input, turns, expected }` */
+    caseKey: z.string().regex(/^[0-9a-f]{64}$/),
+    trials: z.array(CaseResultSchema).optional(),
+});
+const RunConfigSchema = z.object({
+    metrics: z.array(z.string()),
+    statisticalMetrics: z.array(z.string()),
+    judge: z
+        .object({
+        model: z.string(),
+        temperature: z.number(),
+        samples: z.number().int().positive().optional(),
+    })
+        .optional(),
+    trials: z.number().int().min(1),
+    concurrency: z.number().int().min(1),
+    target: z.object({
+        kind: z.enum(['fn', 'agent']),
+        /** The agent's name, for an agent target */
+        name: z.string().optional(),
+        /** The agent's model, for an agent target that sets one */
+        model: z.string().optional(),
+    }),
+});
+const AggregatedSchema = z.object({
+    name: z.string(),
+    mean: figure,
+    median: figure,
+    min: figure,
+    max: figure,
+    stdDev: figure,
+    p50: figure,
+    p95: figure,
+    p99: figure,
+    count: z.number().int().optional(),
+    metadata: z.record(z.string(), z.unknown()).optional(),
+});
+const AssertionSchema = z.object({
+    name: z.string(),
+    passed: z.boolean(),
+    message: z.string(),
+    actual: figure.optional(),
+    expected: figure.optional(),
+});
+const StatsSchema = z.object({
+    total: z.number().int(),
+    errors: z.number().int(),
+    metricErrors: z.number().int(),
+    duration: z.number(),
+    cost: z.number(),
+    targetCost: z.number(),
+    judgeCost: z.number(),
+});
+/** JSON-serializable metadata: object keys holding `undefined` are dropped, as JSON drops them */
+export const JsonMetadataSchema = z.preprocess(withoutUndefined, z.record(z.string(), z.json()));
+function withoutUndefined(value) {
+    if (Array.isArray(value))
+        return value.map(withoutUndefined);
+    if (typeof value !== 'object' || value === null)
+        return value;
+    if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) {
+        return value;
+    }
+    return Object.fromEntries(Object.entries(value)
+        .filter(([, item]) => item !== undefined)
+        .map(([key, item]) => [key, withoutUndefined(item)]));
+}
+export const EvalRunSchema = z.object({
+    format: z.literal(EVAL_RUN_FORMAT),
+    version: z.literal(EVAL_RUN_VERSION),
+    runId: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/),
+    startedAt: z.iso.datetime(),
+    finishedAt: z.iso.datetime(),
+    name: z.string().optional(),
+    metadata: JsonMetadataSchema.optional(),
+    dataset: z.object({
+        /** SHA-256 of the canonical JSON of the cases run, in order */
+        fingerprint: z.string().regex(/^[0-9a-f]{64}$/),
+        cases: z.number().int().min(0),
+    }),
+    config: RunConfigSchema,
+    /** The distinct versioned instructions the agent runs used */
+    prompts: z.array(RunPromptSchema),
+    results: z.array(RunCaseSchema),
+    aggregated: z.record(z.string(), AggregatedSchema),
+    assertions: z.array(AssertionSchema),
+    stats: StatsSchema,
+});
+/** The options of a suite that describe the experiment it runs */
+export const ExperimentOptionsSchema = z.object({
+    name: z.string().trim().min(1, 'name must not be empty').optional(),
+    metadata: JsonMetadataSchema.optional(),
+});
+//# sourceMappingURL=schema.js.map

package/dist/runs/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/runs/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,CAAC,MAAM,eAAe,GAAG,oBAAoB,CAAC;AAEpD,uEAAuE;AACvE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,8FAA8F;AAC9F,MAAM,MAAM,GAAG,CAAC;KACb,MAAM,EAAE;KACR,QAAQ,EAAE;KACV,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;AAE7C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3F,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;CACjB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,KAAK,EAAE,WAAW,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;CAC7C,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC7C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,cAAc;IACpB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IACrC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE;IAC9D,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE;IAC7C,WAAW,EAAE,CAAC;SACX,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC9B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,SAAS,EAAE,CAAC;SACT,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;QACP,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACzB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC9B,CAAC,CACH;SACA,QAAQ,EAAE;IACb,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAE;IACrC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IACjC,qBAAqB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC7C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;CAC7B,CAAC,CAAC;AAEH,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC;IAC5C,+EAA+E;IAC/E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;IAC3C,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CAC7C,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC5B,kBAAkB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACvC,KAAK,EAAE,CAAC;SACL,MAAM,CAAC;QACN,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;QACvB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,QAAQ,EAAE;IACb,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,4CAA4C;QAC5C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC3B,2DAA2D;QAC3D,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC;CACH,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,MAAM;IACZ,MAAM,EAAE,MAAM;IACd,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,MAAM,EAAE,MAAM;IACd,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE;IAClC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE;IACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACvB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACxB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IAC9B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;CACtB,CAAC,CAAC;AAEH,kGAAkG;AAClG,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAEjG,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/F,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,8BAA8B,CAAC;IACvD,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC3B,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE;IACvC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,+DAA+D;QAC/D,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAC/C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;KAC/B,CAAC;IACF,MAAM,EAAE,eAAe;IACvB,8DAA8D;IAC9D,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;IACjC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC;IAC/B,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC;IAClD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;IACpC,KAAK,EAAE,WAAW;CACnB,CAAC,CAAC;AAWH,kEAAkE;AAClE,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC,QAAQ,EAAE;IACnE,QAAQ,EAAE,kBAAkB,CAAC,QAAQ,EAAE;CACxC,CAAC,CAAC"}

package/dist/runs/store.d.ts

@@ -0,0 +1,57 @@
+import type { EvalRun, JsonMetadata } from './schema.js';
+import type { EvalRunSource } from './run.js';
+/** A run as `RunStore.list()` shows it */
+export interface RunSummary {
+    id: string;
+    name?: string;
+    metadata?: JsonMetadata;
+    startedAt: string;
+    finishedAt: string;
+    /** Cases run */
+    cases: number;
+    /** The mean of each aggregated metric, NaN for a metric no case was scored on */
+    metrics: Record<string, number>;
+    /** The run file */
+    path: string;
+}
+/** Picks runs: by `name`, or by a predicate over their summaries */
+export type RunFilter = {
+    name?: string;
+} | ((run: RunSummary) => boolean);
+/** A file `list()` and `latest()` passed over: what it is and why it was not read */
+export interface SkippedRunFile {
+    path: string;
+    reason: string;
+}
+export interface RunStoreOptions {
+    /**
+     * Called for each `.json` entry `list()` and `latest()` skip: a directory, a file that cannot
+     * be read or is not JSON, or a run file that is not valid (of another format version, or edited
+     * by hand). Default: a process warning with the code `COGITATOR_EVALS_RUN_SKIPPED`
+     */
+    onSkip?: (skipped: SkippedRunFile) => void;
+}
+/**
+ * Saved suite runs in a directory, one `<runId>.json` run file each. `list()` and `latest()` pass
+ * over JSON files that are not run files, such as a baseline, and report the `.json` entries they
+ * cannot read through `onSkip`, so one bad file never hides the other runs. `load(id)` is strict
+ * and throws for a run file it cannot read
+ */
+export declare class RunStore {
+    readonly directory: string;
+    private readonly onSkip;
+    constructor(directory: string, options?: RunStoreOptions);
+    /** Writes the run file of a suite result or a run, returning its path */
+    save(source: EvalRunSource): string;
+    /** The runs of the directory, newest first */
+    list(filter?: RunFilter): RunSummary[];
+    /**
+     * The run with this id. Throws when the directory has none, and when its file cannot be read
+     * or is not a valid run file
+     */
+    load(id: string): EvalRun;
+    /** The newest run, of those the filter picks, undefined when there is none */
+    latest(filter?: RunFilter): EvalRun | undefined;
+    private entries;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/runs/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/runs/store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAE3C,0CAA0C;AAC1C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,oEAAoE;AACpE,MAAM,MAAM,SAAS,GAAG;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,UAAU,KAAK,OAAO,CAAC,CAAC;AAyB3E,qFAAqF;AACrF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B;;;;OAIG;IACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,IAAI,CAAC;CAC5C;AAyDD;;;;;GAKG;AACH,qBAAa,QAAQ;IAIjB,QAAQ,CAAC,SAAS,EAAE,MAAM;IAH5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;gBAGhD,SAAS,EAAE,MAAM,EAC1B,OAAO,GAAE,eAAoB;IAK/B,yEAAyE;IACzE,IAAI,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM;IAInC,8CAA8C;IAC9C,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,UAAU,EAAE;IAMtC;;;OAGG;IACH,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAOzB,8EAA8E;IAC9E,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS;IAO/C,OAAO,CAAC,OAAO;CAsBhB"}

package/dist/runs/store.js

@@ -0,0 +1,146 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { EVAL_RUN_FORMAT } from './schema.js';
+import { loadRun, parseRun, runOf, writeRun } from './run.js';
+const RUN_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+function summaryOf(run, path) {
+    const metrics = {};
+    for (const [name, aggregated] of Object.entries(run.aggregated))
+        metrics[name] = aggregated.mean;
+    return {
+        id: run.runId,
+        ...(run.name !== undefined && { name: run.name }),
+        ...(run.metadata !== undefined && { metadata: run.metadata }),
+        startedAt: run.startedAt,
+        finishedAt: run.finishedAt,
+        cases: run.dataset.cases,
+        metrics,
+        path,
+    };
+}
+function matches(summary, filter) {
+    if (filter === undefined)
+        return true;
+    if (typeof filter === 'function')
+        return filter(summary);
+    return filter.name === undefined || summary.name === filter.name;
+}
+function isGone(error) {
+    return error.code === 'ENOENT';
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function warnSkipped({ path, reason }) {
+    process.emitWarning(`RunStore skipped ${path}: ${reason}`, {
+        code: 'COGITATOR_EVALS_RUN_SKIPPED',
+    });
+}
+/** What a run file path holds: a valid run, JSON of another kind, or something unreadable */
+function readRunFile(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf-8');
+    }
+    catch (error) {
+        if (isGone(error))
+            return { kind: 'gone' };
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        return { kind: 'bad', reason: 'not JSON' };
+    }
+    const format = typeof value === 'object' && value !== null ? value.format : null;
+    if (format !== EVAL_RUN_FORMAT)
+        return { kind: 'other' };
+    try {
+        return { kind: 'run', run: parseRun(value, path) };
+    }
+    catch (error) {
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+}
+/** Whether a listed entry is still a file: `null` when it is, the reason to skip it otherwise */
+function notAFile(path) {
+    try {
+        return statSync(path).isFile() ? null : { kind: 'bad', reason: 'not a file' };
+    }
+    catch (error) {
+        if (isGone(error))
+            return { kind: 'gone' };
+        return { kind: 'bad', reason: errorMessage(error) };
+    }
+}
+/**
+ * Saved suite runs in a directory, one `<runId>.json` run file each. `list()` and `latest()` pass
+ * over JSON files that are not run files, such as a baseline, and report the `.json` entries they
+ * cannot read through `onSkip`, so one bad file never hides the other runs. `load(id)` is strict
+ * and throws for a run file it cannot read
+ */
+export class RunStore {
+    directory;
+    onSkip;
+    constructor(directory, options = {}) {
+        this.directory = directory;
+        this.onSkip = options.onSkip ?? warnSkipped;
+    }
+    /** Writes the run file of a suite result or a run, returning its path */
+    save(source) {
+        return writeRun(runOf(source), `${this.directory}/`);
+    }
+    /** The runs of the directory, newest first */
+    list(filter) {
+        return this.entries()
+            .map(({ run, path }) => summaryOf(run, path))
+            .filter((summary) => matches(summary, filter));
+    }
+    /**
+     * The run with this id. Throws when the directory has none, and when its file cannot be read
+     * or is not a valid run file
+     */
+    load(id) {
+        if (!RUN_ID.test(id))
+            throw new Error(`'${id}' is not a run id`);
+        const path = join(this.directory, `${id}.json`);
+        if (!existsSync(path))
+            throw new Error(`No run '${id}' in ${this.directory}`);
+        return loadRun(path);
+    }
+    /** The newest run, of those the filter picks, undefined when there is none */
+    latest(filter) {
+        for (const { run, path } of this.entries()) {
+            if (matches(summaryOf(run, path), filter))
+                return run;
+        }
+        return undefined;
+    }
+    entries() {
+        let files;
+        try {
+            files = readdirSync(this.directory);
+        }
+        catch (error) {
+            if (isGone(error))
+                return [];
+            throw error;
+        }
+        const entries = [];
+        for (const file of files.sort()) {
+            if (!file.endsWith('.json'))
+                continue;
+            const path = join(this.directory, file);
+            const read = notAFile(path) ?? readRunFile(path);
+            if (read.kind === 'run')
+                entries.push({ run: read.run, path });
+            else if (read.kind === 'bad')
+                this.onSkip({ path, reason: read.reason });
+        }
+        return entries.sort((a, b) => Date.parse(b.run.startedAt) - Date.parse(a.run.startedAt) ||
+            b.run.runId.localeCompare(a.run.runId));
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/runs/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/runs/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAC;AAqB3D,MAAM,MAAM,GAAG,8BAA8B,CAAC;AAE9C,SAAS,SAAS,CAAC,GAAY,EAAE,IAAY;IAC3C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC;IACjG,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,KAAK;QACb,GAAG,CAAC,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,GAAG,CAAC,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7D,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK;QACxB,OAAO;QACP,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,OAAmB,EAAE,MAA6B;IACjE,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACtC,IAAI,OAAO,MAAM,KAAK,UAAU;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC;IACzD,OAAO,MAAM,CAAC,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC;AACnE,CAAC;AAuBD,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAQ,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC5D,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAkB;IACnD,OAAO,CAAC,WAAW,CAAC,oBAAoB,IAAI,KAAK,MAAM,EAAE,EAAE;QACzD,IAAI,EAAE,6BAA6B;KACpC,CAAC,CAAC;AACL,CAAC;AAED,6FAA6F;AAC7F,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;IACD,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GACV,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAA8B,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9F,IAAI,MAAM,KAAK,eAAe;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACzD,IAAI,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,iGAAiG;AACjG,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAChF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC3C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,QAAQ;IAIR;IAHM,MAAM,CAAoC;IAE3D,YACW,SAAiB,EAC1B,UAA2B,EAAE;QADpB,cAAS,GAAT,SAAS,CAAQ;QAG1B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,WAAW,CAAC;IAC9C,CAAC;IAED,yEAAyE;IACzE,IAAI,CAAC,MAAqB;QACxB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,8CAA8C;IAC9C,IAAI,CAAC,MAAkB;QACrB,OAAO,IAAI,CAAC,OAAO,EAAE;aAClB,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;aAC5C,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,IAAI,CAAC,EAAU;QACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,mBAAmB,CAAC,CAAC;QACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,QAAQ,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9E,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,MAAkB;QACvB,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAE,OAAO,GAAG,CAAC;QACxD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,OAAO;QACb,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACH,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,CAAC,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,OAAO,GAA0C,EAAE,CAAC;QAC1D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;iBAC1D,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK;gBAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CACjB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;YACzD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CACzC,CAAC;IACJ,CAAC;CACF"}

package/dist/schema.d.ts

@@ -30,6 +30,7 @@ export interface JudgeCogitator {
     run(agent: unknown, options: {
         input: string;
         useMemory?: boolean;
+        signal?: AbortSignal;
     }): Promise<{
         output: string;
         structured?: unknown;
@@ -54,12 +55,27 @@ export declare const JudgeConfigSchema: z.ZodObject<{
     cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
     cache: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<JudgeCache, JudgeCache>]>>;
     concurrency: z.ZodOptional<z.ZodNumber>;
+    samples: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export declare const PairwiseOptionsSchema: z.ZodObject<{
+    criteria: z.ZodOptional<z.ZodString>;
+    name: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
 export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
     trials: z.ZodDefault<z.ZodNumber>;
+    alpha: z.ZodDefault<z.ZodNumber>;
+    test: z.ZodDefault<z.ZodEnum<{
+        t: "t";
+        wilcoxon: "wilcoxon";
+    }>>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+    pairwise: z.ZodOptional<z.ZodObject<{
+        criteria: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>>;
 }, z.core.$strip>;
 export type ExpectedToolCall = z.output<typeof ExpectedToolCallSchema>;
 export type EvalCase = z.output<typeof EvalCaseSchema>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;iBAKrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GACpE,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;;iBAsB5B,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;iBAGhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;iBAYrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -49,11 +49,28 @@ export const JudgeConfigSchema = z.object({
      * metrics. Default: no limit of its own
      */
     concurrency: z.number().int().positive().optional(),
+    /**
+     * Verdicts per case and judge metric, a positive integer, 1 by default. The score is the mean
+     * of the verdicts the judge gave, their spread is in the score's `details`. Each verdict is
+     * cached on its own, so use a `temperature` above 0 for verdicts that differ
+     */
+    samples: z.number().int().positive().optional(),
+});
+export const PairwiseOptionsSchema = z.object({
+    criteria: z.string().trim().min(1, 'criteria must not be empty').optional(),
+    name: z.string().trim().min(1, 'name must not be empty').optional(),
 });
 export const EvalComparisonConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
     trials: z.number().int().min(1).default(1),
+    /** Significance level, after Holm's correction across the compared metrics */
+    alpha: z.number().gt(0).lt(1).default(0.05),
+    /** Test for metrics that are not binary, and for latency and cost */
+    test: z.enum(['t', 'wilcoxon']).default('t'),
+    /** Seeds the bootstrap of the confidence intervals, so they are the same on every run */
+    seed: z.union([z.number(), z.string()]).optional(),
+    pairwise: PairwiseOptionsSchema.optional(),
 });
 //# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACnD;;;;OAIG;IACH,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAChD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,4BAA4B,CAAC,CAAC,QAAQ,EAAE;IAC3E,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,wBAAwB,CAAC,CAAC,QAAQ,EAAE;CACpE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1C,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3C,qEAAqE;IACrE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5C,yFAAyF;IACzF,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IAClD,QAAQ,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC"}

package/dist/stats/bootstrap.d.ts

@@ -0,0 +1,22 @@
+import { z } from 'zod';
+export declare const BootstrapOptionsSchema: z.ZodObject<{
+    confidence: z.ZodDefault<z.ZodNumber>;
+    resamples: z.ZodDefault<z.ZodNumber>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+}, z.core.$strip>;
+export type BootstrapOptions = z.input<typeof BootstrapOptionsSchema>;
+export interface BootstrapInterval {
+    /** Lower bound of the interval */
+    low: number;
+    /** Upper bound of the interval */
+    high: number;
+    /** Mean of the values themselves */
+    mean: number;
+}
+/**
+ * Percentile bootstrap confidence interval of the mean: the values are resampled with
+ * replacement `resamples` times and the interval spans the middle `confidence` share of the
+ * resampled means. With `seed` the interval is the same on every run. All NaN for no values
+ */
+export declare function bootstrapCI(values: readonly number[], options?: BootstrapOptions): BootstrapInterval;
+//# sourceMappingURL=bootstrap.d.ts.map

package/dist/stats/bootstrap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.d.ts","sourceRoot":"","sources":["../../src/stats/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,eAAO,MAAM,sBAAsB;;;;iBAIjC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,MAAM,WAAW,iBAAiB;IAChC,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,kCAAkC;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,oCAAoC;IACpC,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,OAAO,GAAE,gBAAqB,GAC7B,iBAAiB,CAiBnB"}

package/dist/stats/bootstrap.js

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+import { mean, percentile } from './percentiles.js';
+import { seededRandom } from './random.js';
+export const BootstrapOptionsSchema = z.object({
+    confidence: z.number().gt(0).lt(1).default(0.95),
+    resamples: z.number().int().positive().default(2000),
+    seed: z.union([z.number(), z.string()]).optional(),
+});
+/**
+ * Percentile bootstrap confidence interval of the mean: the values are resampled with
+ * replacement `resamples` times and the interval spans the middle `confidence` share of the
+ * resampled means. With `seed` the interval is the same on every run. All NaN for no values
+ */
+export function bootstrapCI(values, options = {}) {
+    const { confidence, resamples, seed } = BootstrapOptionsSchema.parse(options);
+    if (values.length === 0)
+        return { low: Number.NaN, high: Number.NaN, mean: Number.NaN };
+    const random = seededRandom(seed);
+    const n = values.length;
+    const means = new Array(resamples);
+    for (let r = 0; r < resamples; r++) {
+        let sum = 0;
+        for (let i = 0; i < n; i++)
+            sum += values[Math.floor(random() * n)];
+        means[r] = sum / n;
+    }
+    const tail = (1 - confidence) / 2;
+    return {
+        low: percentile(means, tail),
+        high: percentile(means, 1 - tail),
+        mean: mean([...values]),
+    };
+}
+//# sourceMappingURL=bootstrap.js.map

package/dist/stats/bootstrap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bootstrap.js","sourceRoot":"","sources":["../../src/stats/bootstrap.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAChD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC,CAAC;AAaH;;;;GAIG;AACH,MAAM,UAAU,WAAW,CACzB,MAAyB,EACzB,UAA4B,EAAE;IAE9B,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,sBAAsB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC9E,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;IACxF,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAS,SAAS,CAAC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACpE,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAClC,OAAO;QACL,GAAG,EAAE,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC;QAC5B,IAAI,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;QACjC,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;KACxB,CAAC;AACJ,CAAC"}

package/dist/stats/correlation.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pearson's correlation of paired values, from -1 to 1. NaN when there are fewer than 2 pairs
+ * or either side does not vary
+ */
+export declare function pearson(x: readonly number[], y: readonly number[]): number;
+/**
+ * Spearman's rank correlation of paired values: Pearson's correlation of their ranks, ties
+ * sharing the mean of their ranks. NaN when Pearson's would be
+ */
+export declare function spearman(x: readonly number[], y: readonly number[]): number;
+//# sourceMappingURL=correlation.d.ts.map

package/dist/stats/correlation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation.d.ts","sourceRoot":"","sources":["../../src/stats/correlation.ts"],"names":[],"mappings":"AAQA;;;GAGG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAkB1E;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAG3E"}

package/dist/stats/correlation.js

@@ -0,0 +1,42 @@
+import { mean } from './percentiles.js';
+import { averageRanks } from './ranks.js';
+import { isConstant, withoutNoise } from './precision.js';
+function assertPaired(x, y) {
+    if (x.length !== y.length)
+        throw new Error('Sample arrays must have equal length');
+}
+/**
+ * Pearson's correlation of paired values, from -1 to 1. NaN when there are fewer than 2 pairs
+ * or either side does not vary
+ */
+export function pearson(x, y) {
+    assertPaired(x, y);
+    if (x.length < 2)
+        return Number.NaN;
+    if (isConstant(x.map(withoutNoise)) || isConstant(y.map(withoutNoise)))
+        return Number.NaN;
+    const mx = mean([...x]);
+    const my = mean([...y]);
+    let sxy = 0;
+    let sxx = 0;
+    let syy = 0;
+    for (let i = 0; i < x.length; i++) {
+        const dx = x[i] - mx;
+        const dy = y[i] - my;
+        sxy += dx * dy;
+        sxx += dx * dx;
+        syy += dy * dy;
+    }
+    if (sxx === 0 || syy === 0)
+        return Number.NaN;
+    return Math.max(-1, Math.min(1, sxy / Math.sqrt(sxx * syy)));
+}
+/**
+ * Spearman's rank correlation of paired values: Pearson's correlation of their ranks, ties
+ * sharing the mean of their ranks. NaN when Pearson's would be
+ */
+export function spearman(x, y) {
+    assertPaired(x, y);
+    return pearson(averageRanks(x.map(withoutNoise)), averageRanks(y.map(withoutNoise)));
+}
+//# sourceMappingURL=correlation.js.map

package/dist/stats/correlation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"correlation.js","sourceRoot":"","sources":["../../src/stats/correlation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEvD,SAAS,YAAY,CAAC,CAAoB,EAAE,CAAoB;IAC9D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;AACrF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,CAAoB,EAAE,CAAoB;IAChE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IACpC,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC1F,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrB,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrB,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;QACf,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC;IACjB,CAAC;IACD,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,CAAoB,EAAE,CAAoB;IACjE,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnB,OAAO,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;AACvF,CAAC"}

package/dist/stats/effect-size.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Cohen's d for paired samples (dz): the mean of the paired differences over their standard
+ * deviation, the differences rounded to 12 significant digits. NaN with fewer than 2 differences,
+ * plus or minus Infinity when the differences are all the same and not 0, 0 when they are all 0
+ */
+export declare function cohensD(differences: readonly number[]): number;
+//# sourceMappingURL=effect-size.d.ts.map

package/dist/stats/effect-size.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"effect-size.d.ts","sourceRoot":"","sources":["../../src/stats/effect-size.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAO9D"}