@prismatic-io/lux 0.0.1

package/src/core/rubric.ts

@@ -0,0 +1,52 @@
+import { z } from "zod";
+import { AssertionSchema } from "./case.js";
+import { formatZodIssues } from "./zod.js";
+
+export const RubricDefinitionSchema = z
+  .object({
+    schemaVersion: z.literal(1),
+    /** Stable rubric identity, normally the canonical case id. */
+    id: z.string().min(1),
+    assertions: z.array(AssertionSchema).min(1),
+    description: z.string().optional(),
+    meta: z.record(z.string(), z.unknown()).optional(),
+  })
+  .strict()
+  .superRefine((rubric, ctx) => {
+    const ids = new Set<string>();
+    for (const [index, assertion] of rubric.assertions.entries()) {
+      if (!assertion.id) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["assertions", index, "id"],
+          message: "grader-alignment rubrics require an explicit stable assertion id",
+        });
+        continue;
+      }
+      if (!/^[a-z0-9][a-z0-9_-]*$/.test(assertion.id)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["assertions", index, "id"],
+          message: "stable assertion ids must be lowercase safe slugs",
+        });
+      }
+      if (ids.has(assertion.id)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["assertions", index, "id"],
+          message: `duplicate assertion id '${assertion.id}'`,
+        });
+      }
+      ids.add(assertion.id);
+    }
+  });
+export type RubricDefinitionInput = z.input<typeof RubricDefinitionSchema>;
+export type RubricDefinition = z.output<typeof RubricDefinitionSchema>;
+
+export const defineRubric = (input: RubricDefinitionInput): RubricDefinition => {
+  const parsed = RubricDefinitionSchema.safeParse(input);
+  if (!parsed.success) {
+    throw new Error(`Invalid Lux rubric: ${formatZodIssues(parsed.error)}`);
+  }
+  return parsed.data;
+};

package/src/core/run.ts

@@ -0,0 +1,144 @@
+import { z } from "zod";
+import { EvalCaseSchema } from "./case.js";
+import { ArtifactSchema } from "./driver.js";
+import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
+import { ToolOutcomeSummarySchema } from "./tool-events.js";
+import { RunUsageSchema } from "./usage.js";
+
+export const RUN_SCHEMA_VERSION = 1 as const;
+
+/**
+ * Wall-clock split of a run. `agent` is the driver working, `grading` is the
+ * assertions running (an LLM judge dominates it), `collect` is artifact capture.
+ */
+export const PhaseDurationsSchema = z.object({
+  agent: z.number(),
+  collect: z.number(),
+  grading: z.number(),
+});
+export type PhaseDurations = z.infer<typeof PhaseDurationsSchema>;
+
+export const RunProvenanceSchema = z.object({
+  runtime: z.object({
+    platform: z.string(),
+    arch: z.string(),
+    node: z.string(),
+    bun: z.exactOptional(z.string()),
+  }),
+  source: z.exactOptional(
+    z.object({
+      /** Case path relative to its repository root, when one was found. */
+      casePath: z.string(),
+      git: z.exactOptional(
+        z.object({
+          commit: z.string(),
+          dirty: z.boolean(),
+          /**
+           * Content identity for tracked changes and untracked, non-ignored
+           * files. Absent on clean trees and legacy/unhashable dirty trees.
+           */
+          workingTreeHash: z.exactOptional(z.string()),
+        }),
+      ),
+    }),
+  ),
+  caseHash: z.exactOptional(z.string()),
+  assertionsHash: z.exactOptional(z.string()),
+  /** Hash of schema-defaulted/transformed assertion configs actually graded. */
+  validatedAssertionsHash: z.exactOptional(z.string()),
+  fixturesHash: z.exactOptional(z.string()),
+  subjectHash: z.exactOptional(z.string()),
+  profileId: z.exactOptional(z.string()),
+  experimentId: z.exactOptional(z.string()),
+  variantId: z.exactOptional(z.string()),
+  candidateId: z.exactOptional(z.string()),
+  driverConfigHash: z.exactOptional(z.string()),
+  answererConfigHash: z.exactOptional(z.string()),
+  /** Grader-only identity shared with offline regrade snapshots. */
+  graderConfigHash: z.exactOptional(z.string()),
+  evaluatorConfigHash: z.exactOptional(z.string()),
+});
+export type RunProvenance = z.infer<typeof RunProvenanceSchema>;
+
+export const OperationalFailureSchema = z
+  .object({
+    phase: z.enum([
+      "setup",
+      "answerer",
+      "driver-transport",
+      "trace",
+      "collection",
+      "grading",
+      "close",
+      "abort",
+    ]),
+    reason: z.string(),
+  })
+  .strict();
+export type OperationalFailure = z.infer<typeof OperationalFailureSchema>;
+
+export const RunMetadataSchema = z.object({
+  schemaVersion: z.literal(RUN_SCHEMA_VERSION),
+  luxVersion: z.string(),
+  /** The `runSuite` invocation this case attempt belongs to, when any. */
+  suiteId: z.exactOptional(z.string()),
+  startedAt: z.string(),
+  /** Time the subject agent and artifact collection finished, before grading. */
+  agentEndedAt: z.exactOptional(z.string()),
+  /** Time the complete run, including grading, finished. */
+  endedAt: z.string(),
+  exitReason: z.union([z.literal("done"), ErrorEventReasonSchema]),
+  driver: ReadyStateSchema.extend({
+    name: z.string(),
+    version: z.exactOptional(z.string()),
+  }),
+  answerer: z.object({
+    name: z.string(),
+    version: z.exactOptional(z.string()),
+    provider: z.exactOptional(z.string()),
+    model: z.exactOptional(z.string()),
+  }),
+  /**
+   * Tokens by source. `usage.agent` is the subject under test; the rest is
+   * eval overhead.
+   */
+  usage: RunUsageSchema,
+  modelAccounting: z.exactOptional(
+    z.object({
+      agentCalls: z.number().int().nonnegative(),
+      answererCalls: z.number().int().nonnegative(),
+      reportedCostUsd: z.number().nonnegative(),
+      unpricedModelCalls: z.number().int().nonnegative(),
+    }),
+  ),
+  cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
+  interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
+  /** Legacy raw count of progress events whose kind is `tool-call`. */
+  toolCallCount: z.exactOptional(z.number()),
+  toolOutcomes: z.exactOptional(ToolOutcomeSummarySchema),
+  artifacts: z.exactOptional(
+    z.object({
+      count: z.number().int().nonnegative(),
+      bytes: z.exactOptional(z.number().int().nonnegative()),
+    }),
+  ),
+  /** Wall-clock per phase, derived from the lifecycle event stream. */
+  phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
+  /**
+   * Harness/infrastructure failure that invalidates this run as candidate
+   * evidence. A driver-reported subject failure intentionally leaves this
+   * absent so experiments may score it as product behavior.
+   */
+  operationalFailure: z.exactOptional(OperationalFailureSchema),
+  provenance: z.exactOptional(RunProvenanceSchema),
+});
+export type RunMetadata = z.infer<typeof RunMetadataSchema>;
+
+export const RunSchema = z.object({
+  case: EvalCaseSchema,
+  metadata: RunMetadataSchema,
+  events: z.array(RunEventSchema),
+  artifacts: z.array(ArtifactSchema),
+  runDir: z.string(),
+});
+export type Run = z.infer<typeof RunSchema>;

package/src/core/statistics.ts

@@ -0,0 +1,194 @@
+import { z } from "zod";
+
+/**
+ * Upper bound on synchronous random draws performed by one percentile
+ * bootstrap. This keeps authored resample and sample-count limits from
+ * multiplying into work that can block the event loop for an extreme duration.
+ */
+export const MAX_PERCENTILE_BOOTSTRAP_SAMPLE_DRAWS = 10_000_000;
+
+export const ConfidenceConfigSchema = z
+  .object({
+    /**
+     * The interval must match the estimand: Wilson is only valid for Bernoulli
+     * observations, Hoeffding covers arbitrary means bounded to [0, 1], and the
+     * percentile bootstrap is for unbounded means.
+     */
+    method: z.enum(["wilson", "hoeffding", "percentile-bootstrap"]),
+    level: z.number().min(0.5).max(0.999).default(0.95),
+    resamples: z.number().int().min(100).max(100_000).default(2_000),
+    seed: z.number().int().default(0),
+    minSamples: z.number().int().min(2).max(1_000_000).default(5),
+  })
+  .strict();
+export type ConfidenceConfig = z.output<typeof ConfidenceConfigSchema>;
+
+export const ConfidenceIntervalSchema = z
+  .object({
+    method: z.enum(["wilson", "hoeffding", "percentile-bootstrap"]),
+    level: z.number().min(0.5).max(0.999),
+    estimate: z.number(),
+    lower: z.number(),
+    upper: z.number(),
+    n: z.number().int().nonnegative(),
+    resamples: z.number().int().nonnegative(),
+  })
+  .strict();
+export type ConfidenceInterval = z.infer<typeof ConfidenceIntervalSchema>;
+
+const average = (values: readonly number[]): number =>
+  values.reduce((sum, value) => sum + value, 0) / values.length;
+
+const seededRandom = (seed: number): (() => number) => {
+  let state = seed >>> 0;
+  return () => {
+    state += 0x6d2b79f5;
+    let value = state;
+    value = Math.imul(value ^ (value >>> 15), value | 1);
+    value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
+    return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296;
+  };
+};
+
+const quantile = (sorted: readonly number[], probability: number): number => {
+  if (sorted.length === 1) return sorted[0] ?? 0;
+  const position = Math.max(0, Math.min(sorted.length - 1, probability * (sorted.length - 1)));
+  const lower = Math.floor(position);
+  const upper = Math.ceil(position);
+  const fraction = position - lower;
+  return (sorted[lower] ?? 0) * (1 - fraction) + (sorted[upper] ?? 0) * fraction;
+};
+
+const polynomial = (coefficients: readonly number[], value: number): number =>
+  coefficients.reduce((result, coefficient) => result * value + coefficient, 0);
+
+const standardNormalQuantile = (probability: number): number => {
+  const low = 0.02425;
+  const high = 1 - low;
+  const a = [
+    -39.69683028665376, 220.9460984245205, -275.9285104469687, 138.357751867269, -30.66479806614716,
+    2.506628277459239,
+  ];
+  const b = [
+    -54.47609879822406, 161.5858368580409, -155.6989798598866, 66.80131188771972,
+    -13.28068155288572,
+  ];
+  const c = [
+    -0.007784894002430293, -0.3223964580411365, -2.400758277161838, -2.549732539343734,
+    4.374664141464968, 2.938163982698783,
+  ];
+  const d = [0.007784695709041462, 0.3224671290700398, 2.445134137142996, 3.754408661907416];
+
+  if (probability < low) {
+    const q = Math.sqrt(-2 * Math.log(probability));
+    return polynomial(c, q) / (polynomial(d, q) * q + 1);
+  }
+  if (probability > high) {
+    const q = Math.sqrt(-2 * Math.log(1 - probability));
+    return -polynomial(c, q) / (polynomial(d, q) * q + 1);
+  }
+  const q = probability - 0.5;
+  const r = q * q;
+  return (polynomial(a, r) * q) / (polynomial(b, r) * r + 1);
+};
+
+const wilsonInterval = (
+  values: readonly number[],
+  config: ConfidenceConfig,
+): ConfidenceInterval => {
+  const estimate = average(values);
+  const normalQuantile = standardNormalQuantile(0.5 + config.level / 2);
+  const zSquared = normalQuantile * normalQuantile;
+  const denominator = 1 + zSquared / values.length;
+  const center = (estimate + zSquared / (2 * values.length)) / denominator;
+  const margin =
+    (normalQuantile / denominator) *
+    Math.sqrt(
+      (estimate * (1 - estimate)) / values.length + zSquared / (4 * values.length * values.length),
+    );
+  return ConfidenceIntervalSchema.parse({
+    method: "wilson",
+    level: config.level,
+    estimate,
+    lower: Math.max(0, center - margin),
+    upper: Math.min(1, center + margin),
+    n: values.length,
+    resamples: 0,
+  });
+};
+
+const hoeffdingInterval = (
+  values: readonly number[],
+  config: ConfidenceConfig,
+): ConfidenceInterval => {
+  const estimate = average(values);
+  const alpha = 1 - config.level;
+  const margin = Math.sqrt(Math.log(2 / alpha) / (2 * values.length));
+  return ConfidenceIntervalSchema.parse({
+    method: "hoeffding",
+    level: config.level,
+    estimate,
+    lower: Math.max(0, estimate - margin),
+    upper: Math.min(1, estimate + margin),
+    n: values.length,
+    resamples: 0,
+  });
+};
+
+/**
+ * Confidence interval for a sample mean. The caller chooses a method that
+ * matches the metric's estimand rather than inferring it from observed values.
+ */
+export const bootstrapMeanConfidenceInterval = (
+  values: readonly number[],
+  input: z.input<typeof ConfidenceConfigSchema> = { method: "percentile-bootstrap" },
+): ConfidenceInterval => {
+  if (values.length === 0) throw new Error("confidence interval requires at least one sample");
+  if (values.some((value) => !Number.isFinite(value))) {
+    throw new Error("confidence interval samples must be finite");
+  }
+  const config = ConfidenceConfigSchema.parse(input);
+  if (values.length < config.minSamples) {
+    throw new Error(
+      `confidence interval requires at least ${config.minSamples} independent samples (received ${values.length})`,
+    );
+  }
+  if (config.method === "wilson") {
+    if (values.some((value) => value !== 0 && value !== 1)) {
+      throw new Error(
+        "Wilson confidence intervals require Bernoulli samples containing only 0 or 1",
+      );
+    }
+    return wilsonInterval(values, config);
+  }
+  if (config.method === "hoeffding") {
+    if (values.some((value) => value < 0 || value > 1)) {
+      throw new Error("Hoeffding confidence intervals require samples bounded to [0, 1]");
+    }
+    return hoeffdingInterval(values, config);
+  }
+  const sampleDraws = values.length * config.resamples;
+  if (sampleDraws > MAX_PERCENTILE_BOOTSTRAP_SAMPLE_DRAWS) {
+    throw new Error(
+      `percentile bootstrap requires ${sampleDraws} sample draws (${values.length} samples x ${config.resamples} resamples), exceeding the limit of ${MAX_PERCENTILE_BOOTSTRAP_SAMPLE_DRAWS}`,
+    );
+  }
+  const random = seededRandom(config.seed);
+  const means = Array.from({ length: config.resamples }, () => {
+    let sum = 0;
+    for (let index = 0; index < values.length; index++) {
+      sum += values[Math.floor(random() * values.length)] ?? 0;
+    }
+    return sum / values.length;
+  }).sort((a, b) => a - b);
+  const tail = (1 - config.level) / 2;
+  return ConfidenceIntervalSchema.parse({
+    method: "percentile-bootstrap",
+    level: config.level,
+    estimate: average(values),
+    lower: quantile(means, tail),
+    upper: quantile(means, 1 - tail),
+    n: values.length,
+    resamples: config.resamples,
+  });
+};

package/src/core/subject.ts

@@ -0,0 +1,50 @@
+import { isAbsolute, normalize, resolve } from "node:path";
+
+export const SUBJECT_ROOT_TOKEN = "{{lux.subjectRoot}}";
+
+export const subjectPath = (relativePath: string = ""): string => {
+  if (isAbsolute(relativePath)) throw new Error("subjectPath requires a relative path");
+  const normalized = normalize(relativePath).replaceAll("\\", "/");
+  if (normalized === ".." || normalized.startsWith("../")) {
+    throw new Error("subjectPath may not escape the subject root");
+  }
+  return normalized === "." || normalized === ""
+    ? SUBJECT_ROOT_TOKEN
+    : `${SUBJECT_ROOT_TOKEN}/${normalized}`;
+};
+
+const bindValue = <T>(
+  value: T,
+  subjectRoot: string | undefined,
+  seen: WeakMap<object, object>,
+): T => {
+  if (typeof value === "string") {
+    if (!value.includes(SUBJECT_ROOT_TOKEN)) return value;
+    if (!subjectRoot) {
+      throw new Error(
+        `case uses ${SUBJECT_ROOT_TOKEN} but no subjectRoot is configured or supplied`,
+      );
+    }
+    return value.replaceAll(SUBJECT_ROOT_TOKEN, resolve(subjectRoot)) as T;
+  }
+  if (Array.isArray(value)) {
+    const cached = seen.get(value);
+    if (cached) return cached as T;
+    const bound: unknown[] = [];
+    seen.set(value, bound);
+    for (const item of value) bound.push(bindValue(item, subjectRoot, seen));
+    return bound as T;
+  }
+  if (!value || typeof value !== "object") return value;
+  const cached = seen.get(value);
+  if (cached) return cached as T;
+  const bound: Record<string, unknown> = {};
+  seen.set(value, bound);
+  for (const [key, item] of Object.entries(value)) {
+    bound[key] = bindValue(item, subjectRoot, seen);
+  }
+  return bound as T;
+};
+
+export const bindSubjectRoot = <T>(value: T, subjectRoot?: string): T =>
+  bindValue(value, subjectRoot, new WeakMap());