@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.32

package/src/core/evaluation-clusters.ts

@@ -0,0 +1,35 @@
+import { z } from "zod";
+
+export const EvaluationMetadataSchema = z
+  .object({
+    /** Related perturbations share one independent statistical unit. */
+    cluster: z.string().trim().min(1).optional(),
+  })
+  .catchall(z.unknown());
+
+export const clusterKey = (id: string, cluster: string | undefined): string =>
+  JSON.stringify(cluster === undefined ? ["case", id] : ["cluster", cluster]);
+
+export const independentClusterCount = (
+  items: readonly { id: string; cluster?: string | undefined }[],
+): number => new Set(items.map((item) => clusterKey(item.id, item.cluster))).size;
+
+export const assertDisjointClusters = (
+  splits: readonly {
+    name: string;
+    items: readonly { id: string; cluster?: string | undefined }[];
+  }[],
+): void => {
+  const owners = new Map<string, string>();
+  for (const split of splits) {
+    for (const item of split.items) {
+      if (item.cluster === undefined) continue;
+      const owner = owners.get(item.cluster);
+      if (owner !== undefined && owner !== split.name)
+        throw new Error(
+          `evaluation cluster '${item.cluster}' appears in both ${owner} and ${split.name} splits`,
+        );
+      owners.set(item.cluster, split.name);
+    }
+  }
+};

package/src/core/events.ts

@@ -57,11 +57,22 @@ export const AnswerSchema = z.discriminatedUnion("kind", [
 ]);
 export type Answer = z.infer<typeof AnswerSchema>;
 
+export const AgentAttributionSchema = z.object({
+  id: z.string().min(1),
+  role: z.enum(["main", "subagent"]),
+  parentId: z.exactOptional(z.string().min(1)),
+  spawnToolUseId: z.exactOptional(z.string().min(1)),
+  name: z.exactOptional(z.string().min(1)),
+  model: z.exactOptional(z.string().min(1)),
+});
+export type AgentAttribution = z.infer<typeof AgentAttributionSchema>;
+
 export const ProgressSchema = z.object({
   id: z.string(),
   kind: z.string(),
   payload: z.unknown(),
   ts: z.number(),
+  agent: z.exactOptional(AgentAttributionSchema),
 });
 export type Progress = z.infer<typeof ProgressSchema>;
 
@@ -69,8 +80,8 @@ export const DoneSchema = z.object({
   exitReason: z.literal("done"),
   /**
    * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
-   * `modelCalls`, and `unpricedModelCalls` when present while preserving room
-   * for domain-specific measurements.
+   * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
+   * preserving room for other domain-specific evidence.
    */
   summary: z.unknown().optional(),
 });
@@ -82,6 +93,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
 export const ErrorEventSchema = z.object({
   exitReason: ErrorEventReasonSchema,
   reason: z.string(),
+  summary: z.unknown().optional(),
 });
 export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
 

package/src/core/experiment.ts

@@ -1,4 +1,6 @@
 import { z } from "zod";
+import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
+import { DurationSchema } from "./duration.js";
 import { ConfidenceConfigSchema } from "./statistics.js";
 import { TokenUsageSchema } from "./usage.js";
 import { formatZodIssues } from "./zod.js";
@@ -45,7 +47,7 @@ export const FileSetSubjectSchema = z
              * External paths and URLs are rejected by experiment preflight.
              */
             args: z.array(z.string()).default([]),
-            timeoutMs: z.number().int().positive().default(120_000),
+            timeout: DurationSchema.default("2 minutes"),
             /**
              * Validation launches a host process against model-generated source.
              * Authors must explicitly acknowledge that trust boundary. Commands
@@ -83,6 +85,8 @@ export type ExperimentVariant = z.output<typeof ExperimentVariantSchema>;
 export const ExperimentConstraintSchema = z
   .object({
     metric: z.string().min(1),
+    /** Bounds apply to paired candidate-minus-baseline case means. */
+    relativeTo: z.exactOptional(z.literal("baseline")),
     min: z.number().optional(),
     max: z.number().optional(),
     profile: z.string().optional(),
@@ -150,6 +154,47 @@ const DEFAULT_EXPERIMENT_BUDGET: ExperimentBudget = {
   maxWallTimeMs: 3_600_000,
 };
 
+const validateConstraintConfidence = (
+  constraint: ExperimentConstraint,
+  index: number,
+  boundedMetrics: Set<string>,
+  ctx: z.RefinementCtx,
+): void => {
+  const issue = (message: string): void =>
+    ctx.addIssue({ code: "custom", path: ["objectives", "constraints", index], message });
+  if (constraint.relativeTo && constraint.confidence?.method === "wilson") {
+    issue(
+      "paired differences are not Bernoulli observations; use Hoeffding or percentile-bootstrap confidence",
+    );
+  }
+  if (
+    constraint.relativeTo &&
+    ["subject.bytes", "quality.caseRegressions"].includes(constraint.metric)
+  ) {
+    issue("relative constraints require a metric with per-case observations");
+  }
+  const bounded = boundedMetrics.has(constraint.metric);
+  if (
+    (constraint.confidence?.method === "hoeffding" || constraint.confidence?.method === "wilson") &&
+    !bounded
+  ) {
+    issue(
+      `metric '${constraint.metric}' has no declared [0,1] support; use percentile-bootstrap confidence`,
+    );
+  }
+  const minimum = constraint.relativeTo ? -1 : 0;
+  if (
+    bounded &&
+    [constraint.min, constraint.max].some(
+      (bound) => bound !== undefined && (bound < minimum || bound > 1),
+    )
+  ) {
+    issue(
+      `metric '${constraint.metric}' has support [${minimum},1]; constraint bounds must stay in that interval`,
+    );
+  }
+};
+
 export const ExperimentCampaignSchema = z
   .object({
     id: z.string().min(1),
@@ -214,6 +259,8 @@ export const ExperimentCampaignSchema = z
       "grader.macroFalseNegativeRate",
       "grader.coverage",
       "grader.abstentionRate",
+      "grader.insufficientEvidenceRate",
+      "grader.operationalErrorRate",
       "grader.scoreMae",
     ]);
     const unitIntervalMetrics = graderConfidenceMetrics;
@@ -288,29 +335,7 @@ export const ExperimentCampaignSchema = z
       profiles.add(profile);
     }
     for (const [index, constraint] of campaign.objectives.constraints.entries()) {
-      if (
-        (constraint.confidence?.method === "hoeffding" ||
-          constraint.confidence?.method === "wilson") &&
-        !unitIntervalMetrics.has(constraint.metric)
-      ) {
-        ctx.addIssue({
-          code: "custom",
-          path: ["objectives", "constraints", index, "confidence", "method"],
-          message: `metric '${constraint.metric}' has no declared [0,1] support; use percentile-bootstrap confidence`,
-        });
-      }
-      if (
-        unitIntervalMetrics.has(constraint.metric) &&
-        [constraint.min, constraint.max].some(
-          (bound) => bound !== undefined && (bound < 0 || bound > 1),
-        )
-      ) {
-        ctx.addIssue({
-          code: "custom",
-          path: ["objectives", "constraints", index],
-          message: `metric '${constraint.metric}' has support [0,1]; constraint bounds must stay in that interval`,
-        });
-      }
+      validateConstraintConfidence(constraint, index, unitIntervalMetrics, ctx);
       if (constraint.profile && !profiles.has(constraint.profile)) {
         ctx.addIssue({
           code: "custom",
@@ -415,6 +440,7 @@ export const CandidateEvaluationSchema = z.object({
   passRate: z.number().min(0).max(1),
   casePassRates: z.record(z.string(), z.number().min(0).max(1)),
   metrics: z.record(z.string(), z.number()),
+  metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
   metricCalls: z.exactOptional(z.number().int().nonnegative()),
   judgeCalls: z.exactOptional(z.number().int().nonnegative()),
   unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),
@@ -422,6 +448,20 @@ export const CandidateEvaluationSchema = z.object({
   costUsd: z.exactOptional(z.number().nonnegative()),
   /** Raw observations used by confidence-aware constraints. */
   samples: z.exactOptional(z.record(z.string(), z.array(z.number()))),
+  /** Independent cases with repetition coverage retained for paired comparisons. */
+  caseSamples: z.exactOptional(
+    z.record(
+      z.string(),
+      z.object({
+        identityHash: z.string().nullable(),
+        clusterId: z.exactOptional(z.string().min(1)),
+        metrics: z.record(
+          z.string(),
+          z.object({ mean: z.number().finite(), n: z.number().int().positive() }),
+        ),
+      }),
+    ),
+  ),
   failedCases: z.array(
     z.object({
       caseId: z.string(),

package/src/core/file-tree.ts

@@ -0,0 +1,35 @@
+import { existsSync } from "node:fs";
+import { readdir } from "node:fs/promises";
+import { join, relative, resolve } from "node:path";
+
+/**
+ * Walk authored trees without silently following or omitting a symlink that
+ * can change outside the experiment identity. Exact ignored paths support
+ * generated state directories whose names are project-configurable.
+ */
+export const walkAuthoredFiles = async (
+  root: string,
+  ignoredDirectoryNames: ReadonlySet<string> = new Set(),
+  ignoredPaths: ReadonlySet<string> = new Set(),
+): Promise<string[]> => {
+  if (!existsSync(root)) return [];
+  const out: string[] = [];
+  const stack = [root];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    if (!dir) continue;
+    for (const entry of await readdir(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name);
+      if (ignoredPaths.has(resolve(full))) continue;
+      if (entry.isDirectory() && ignoredDirectoryNames.has(entry.name)) continue;
+      if (entry.isSymbolicLink()) {
+        throw new Error(
+          `authored evaluation trees cannot contain symlinks: ${relative(root, full)}`,
+        );
+      }
+      if (entry.isDirectory()) stack.push(full);
+      else if (entry.isFile()) out.push(relative(root, full));
+    }
+  }
+  return out.sort();
+};

package/src/core/global-id.ts

@@ -0,0 +1,26 @@
+import { z } from "zod";
+
+export interface GlobalIdParts {
+  type: string;
+  id: string;
+}
+
+const UuidSchema = z.uuid();
+
+export const tryParseGlobalId = (globalId: string): GlobalIdParts | null => {
+  const decoded = Buffer.from(globalId, "base64").toString("utf8");
+  const parts = decoded.split(":");
+  const type = parts[0];
+  const id = parts[1];
+  if (
+    parts.length !== 2 ||
+    !type ||
+    !id ||
+    !UuidSchema.safeParse(id).success ||
+    Buffer.from(decoded, "utf8").toString("base64") !== globalId
+  ) {
+    return null;
+  }
+
+  return { type, id };
+};

package/src/core/harness-catalog.ts

@@ -0,0 +1,100 @@
+import { z } from "zod";
+
+export const HARNESS_EFFORTS = [
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max",
+  "ultra",
+] as const;
+export const ReasoningEffortSchema = z.enum(HARNESS_EFFORTS);
+export type ReasoningEffort = z.infer<typeof ReasoningEffortSchema>;
+
+export type CatalogModels = Readonly<Record<string, readonly ReasoningEffort[]>>;
+
+export type HarnessCatalog<
+  D extends string = string,
+  E extends readonly ReasoningEffort[] = readonly ReasoningEffort[],
+  M extends CatalogModels | undefined = CatalogModels | undefined,
+> = {
+  readonly driver: D;
+  readonly verified: string;
+  readonly sources: readonly string[];
+  readonly efforts: E;
+  readonly models: M;
+};
+
+export type ClosedHarnessCatalog = HarnessCatalog<
+  string,
+  readonly ReasoningEffort[],
+  CatalogModels
+>;
+
+export const defineHarnessCatalog = <
+  const D extends string,
+  const E extends readonly ReasoningEffort[],
+  const M extends CatalogModels | undefined,
+>(
+  catalog: HarnessCatalog<D, E, M>,
+): HarnessCatalog<D, E, M> => catalog;
+
+export const isClosedCatalog = (catalog: HarnessCatalog): catalog is ClosedHarnessCatalog =>
+  catalog.models !== undefined;
+
+export const catalogModelIds = (catalog: HarnessCatalog): string[] =>
+  Object.keys(catalog.models ?? {});
+
+export const catalogEfforts = (
+  catalog: HarnessCatalog,
+  model: string,
+): readonly ReasoningEffort[] | undefined => catalog.models?.[model];
+
+export const defaultCatalogEffort = (
+  catalog: HarnessCatalog,
+  model: string,
+): ReasoningEffort | undefined => {
+  if (!isClosedCatalog(catalog)) return undefined;
+  const accepted = catalog.models[model] ?? catalog.efforts;
+  return accepted.includes("low") ? "low" : accepted[0];
+};
+
+export const describeAcceptedEfforts = (
+  catalog: HarnessCatalog,
+  model: string,
+  accepted: readonly ReasoningEffort[],
+): string =>
+  accepted.length === 0
+    ? `model '${model}' has no reasoning effort in ${catalog.driver}`
+    : `model '${model}' accepts reasoning effort ${accepted.join(", ")} in ${catalog.driver}`;
+
+export const effortIssue = (
+  catalog: HarnessCatalog,
+  model: string,
+  effort: ReasoningEffort | undefined,
+): string | undefined => {
+  if (effort === undefined) return undefined;
+  const accepted = catalogEfforts(catalog, model);
+  if (accepted === undefined || accepted.includes(effort)) return undefined;
+  return describeAcceptedEfforts(catalog, model, accepted);
+};
+
+export type CatalogEffortConfig = {
+  model: string;
+  reasoningEffort?: ReasoningEffort | undefined;
+};
+
+export const applyCatalogEffort =
+  (catalog: HarnessCatalog) =>
+  <C extends CatalogEffortConfig>(config: C, ctx: z.RefinementCtx): C => {
+    const issue = effortIssue(catalog, config.model, config.reasoningEffort);
+    if (issue) {
+      ctx.addIssue({ code: "custom", path: ["reasoningEffort"], message: issue });
+      return config;
+    }
+    if (config.reasoningEffort !== undefined) return config;
+    const effort = defaultCatalogEffort(catalog, config.model);
+    return effort === undefined ? config : { ...config, reasoningEffort: effort };
+  };

package/src/core/harness-catalogs/antigravity.ts

@@ -0,0 +1,24 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const flash = ["low", "medium", "high"] as const;
+
+export const antigravityCatalog = defineHarnessCatalog({
+  driver: "antigravity",
+  verified: "2026-09-18",
+  sources: [
+    "https://antigravity.google/docs/models/",
+    "https://antigravity.google/docs/cli/headless/",
+    "agy models (needs a login)",
+  ],
+  efforts: flash,
+  models: {
+    "gemini-3.8-flash": flash,
+    "gemini-3.7-flash": flash,
+    "gemini-3.6-flash": flash,
+    "gemini-3.5-flash": flash,
+    "gemini-3.1-pro": ["low", "high"],
+    "claude-sonnet-4-6": [],
+    "claude-opus-4-6-thinking": [],
+    "gpt-oss-120b-medium": [],
+  },
+});

package/src/core/harness-catalogs/claude-code.ts

@@ -0,0 +1,25 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const full = ["low", "medium", "high", "xhigh", "max"] as const;
+const withoutXhigh = ["low", "medium", "high", "max"] as const;
+
+export const claudeCodeCatalog = defineHarnessCatalog({
+  driver: "claude-code",
+  verified: "2026-09-18",
+  sources: [
+    "https://code.claude.com/docs/en/model-config",
+    "https://platform.claude.com/docs/en/about-claude/models/overview",
+  ],
+  efforts: full,
+  models: {
+    "claude-fable-5-1": full,
+    "claude-fable-5": full,
+    "claude-opus-5": full,
+    "claude-opus-4-8": full,
+    "claude-opus-4-7": full,
+    "claude-sonnet-5": full,
+    "claude-opus-4-6": withoutXhigh,
+    "claude-sonnet-4-6": withoutXhigh,
+    "claude-haiku-4-5": [],
+  },
+});

package/src/core/harness-catalogs/codex.ts

@@ -0,0 +1,27 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const throughUltra = ["low", "medium", "high", "xhigh", "max", "ultra"] as const;
+const throughMax = ["low", "medium", "high", "xhigh", "max"] as const;
+const throughXhigh = ["low", "medium", "high", "xhigh"] as const;
+const noneThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
+
+export const codexCatalog = defineHarnessCatalog({
+  driver: "codex",
+  verified: "2026-09-18",
+  sources: [
+    "https://learn.chatgpt.com/docs/models",
+    "https://developers.openai.com/api/docs/guides/reasoning",
+    "codex app-server model/list (mirrored in $CODEX_HOME/models_cache.json)",
+  ],
+  efforts: ["none", "low", "medium", "high", "xhigh", "max", "ultra"],
+  models: {
+    "gpt-6-astra": throughUltra,
+    "gpt-5.6-sol": throughUltra,
+    "gpt-5.6-terra": throughUltra,
+    "gpt-5.6-luna": throughMax,
+    "gpt-5.5": throughXhigh,
+    "gpt-5.4": noneThroughXhigh,
+    "gpt-5.4-mini": noneThroughXhigh,
+    "gpt-5.3-codex": throughXhigh,
+  },
+});

package/src/core/harness-catalogs/copilot.ts

@@ -0,0 +1,46 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const claude = ["low", "medium", "high", "xhigh", "max"] as const;
+const claudeWithoutXhigh = ["low", "medium", "high", "max"] as const;
+const gptThroughMax = ["none", "low", "medium", "high", "xhigh", "max"] as const;
+const gptThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
+const fixed = [] as const;
+
+export const copilotCatalog = defineHarnessCatalog({
+  driver: "copilot",
+  verified: "2026-09-18",
+  sources: [
+    "copilot help config (model values)",
+    "https://docs.github.com/en/copilot/reference/ai-models/supported-models",
+    "https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-programmatic-reference",
+  ],
+  efforts: ["none", "minimal", "low", "medium", "high", "xhigh", "max"],
+  models: {
+    "claude-sonnet-5": claude,
+    "claude-fable-5.1": claude,
+    "claude-fable-5": claude,
+    "claude-opus-5": claude,
+    "claude-opus-4.8": claude,
+    "claude-opus-4.8-fast": claude,
+    "claude-opus-4.7": claude,
+    "claude-sonnet-4.6": claudeWithoutXhigh,
+    "claude-haiku-4.5": fixed,
+    "gpt-5.6-sol": gptThroughMax,
+    "gpt-5.6-terra": gptThroughMax,
+    "gpt-5.6-luna": gptThroughMax,
+    "gpt-5.5": gptThroughXhigh,
+    "gpt-5.4": gptThroughXhigh,
+    "gpt-5.4-mini": gptThroughXhigh,
+    "gpt-5.3-codex": ["low", "medium", "high", "xhigh"],
+    "gpt-5-mini": fixed,
+    "mai-code-1.1-flash": fixed,
+    "mai-code-1-flash-picker": fixed,
+    "gemini-3.8-flash": fixed,
+    "gemini-3.7-flash": fixed,
+    "gemini-3.6-flash": fixed,
+    "gemini-3.5-flash": fixed,
+    "grok-4.5": fixed,
+    "kimi-k3": fixed,
+    "kimi-k2.7-code": fixed,
+  },
+});

package/src/core/harness-catalogs/cursor.ts

@@ -0,0 +1,13 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+export const cursorCatalog = defineHarnessCatalog({
+  driver: "cursor",
+  verified: "2026-09-18",
+  sources: [
+    "cursor-agent --list-models (account-scoped, needs a login)",
+    "https://cursor.com/docs/models",
+    "https://cursor.com/docs/cli/acp",
+  ],
+  efforts: ["low", "medium", "high", "xhigh"],
+  models: undefined,
+});

package/src/core/harness-catalogs/grok-build.ts

@@ -0,0 +1,19 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const efforts = ["low", "medium", "high", "xhigh"] as const;
+
+export const grokBuildCatalog = defineHarnessCatalog({
+  driver: "grok-build",
+  verified: "2026-09-24",
+  sources: [
+    "grok 1.0.40: models and ACP initialize modelState",
+    "https://github.com/xai-org/grok-build/blob/main/crates/codegen/xai-grok-pager/docs/user-guide/15-agent-mode.md",
+  ],
+  efforts,
+  models: {
+    "grok-4.7": efforts,
+    "grok-4.7-build-fast": efforts,
+    "grok-4.6": efforts,
+    "grok-4.5": ["low", "medium", "high"],
+  },
+});

package/src/core/harness-catalogs/index.ts

@@ -0,0 +1,31 @@
+import type { HarnessCatalog } from "../harness-catalog.js";
+import { antigravityCatalog } from "./antigravity.js";
+import { claudeCodeCatalog } from "./claude-code.js";
+import { codexCatalog } from "./codex.js";
+import { copilotCatalog } from "./copilot.js";
+import { cursorCatalog } from "./cursor.js";
+import { grokBuildCatalog } from "./grok-build.js";
+
+export {
+  antigravityCatalog,
+  claudeCodeCatalog,
+  codexCatalog,
+  copilotCatalog,
+  cursorCatalog,
+  grokBuildCatalog,
+};
+
+export const builtInHarnessCatalogs = [
+  claudeCodeCatalog,
+  codexCatalog,
+  copilotCatalog,
+  cursorCatalog,
+  antigravityCatalog,
+  grokBuildCatalog,
+] as const;
+
+export type BuiltInHarnessCatalog = (typeof builtInHarnessCatalogs)[number];
+export type BuiltInHarnessDriver = BuiltInHarnessCatalog["driver"];
+
+export const harnessCatalogFor = (driver: string): HarnessCatalog | undefined =>
+  builtInHarnessCatalogs.find((catalog) => catalog.driver === driver);