@prismatic-io/lux 0.0.1

package/src/core/experiment.ts

@@ -0,0 +1,467 @@
+import { z } from "zod";
+import { ConfidenceConfigSchema } from "./statistics.js";
+import { TokenUsageSchema } from "./usage.js";
+import { formatZodIssues } from "./zod.js";
+
+export const ExperimentSelectorSchema = z
+  .object({
+    patterns: z.array(z.string()).default([]),
+    tags: z.array(z.string()).default([]),
+  })
+  .strict();
+export type ExperimentSelector = z.output<typeof ExperimentSelectorSchema>;
+
+export const FileSetSubjectSchema = z
+  .object({
+    root: z.string(),
+    mutable: z.array(z.string()).min(1),
+    exclude: z
+      .array(z.string())
+      .default([
+        ".git/**",
+        ".lux-runs/**",
+        "**/.lux-runs/**",
+        ".env",
+        ".env.*",
+        "**/.env",
+        "**/.env.*",
+        ".npmrc",
+        "**/.npmrc",
+        ".pypirc",
+        "**/.pypirc",
+        ".claude/worktrees/**",
+        "node_modules/**",
+        "**/node_modules/**",
+        "lib/**",
+        "dist/**",
+      ]),
+    validate: z
+      .array(
+        z
+          .object({
+            command: z.string(),
+            /**
+             * Relative paths resolve inside the identity-bound candidate.
+             * External paths and URLs are rejected by experiment preflight.
+             */
+            args: z.array(z.string()).default([]),
+            timeoutMs: z.number().int().positive().default(120_000),
+            /**
+             * Validation launches a host process against model-generated source.
+             * Authors must explicitly acknowledge that trust boundary. Commands
+             * are observational and must not mutate the materialized subject.
+             */
+            unsafeHostExecution: z.literal(true),
+          })
+          .strict(),
+      )
+      .default([]),
+  })
+  .strict();
+export type FileSetSubject = z.output<typeof FileSetSubjectSchema>;
+
+export const CandidateIdSchema = z
+  .string()
+  .min(1)
+  .max(120)
+  .regex(
+    /^[a-z0-9][a-z0-9_-]*$/,
+    "candidate id must be a lowercase filesystem-safe slug using letters, numbers, '_' or '-'",
+  );
+
+export const ExperimentVariantSchema = z
+  .object({
+    id: CandidateIdSchema,
+    name: z.string().optional(),
+    parentIds: z.array(CandidateIdSchema).default([]),
+    /** Full replacement contents keyed by a mutable path relative to the subject root. */
+    components: z.record(z.string(), z.string()).default({}),
+  })
+  .strict();
+export type ExperimentVariant = z.output<typeof ExperimentVariantSchema>;
+
+export const ExperimentConstraintSchema = z
+  .object({
+    metric: z.string().min(1),
+    min: z.number().optional(),
+    max: z.number().optional(),
+    profile: z.string().optional(),
+    split: z.enum(["train", "validation", "test"]).default("validation"),
+    /** Require a confidence bound, rather than only the point estimate. */
+    confidence: ConfidenceConfigSchema.optional(),
+  })
+  .strict()
+  .refine((constraint) => constraint.min !== undefined || constraint.max !== undefined, {
+    message: "constraint requires min or max",
+  });
+export type ExperimentConstraint = z.output<typeof ExperimentConstraintSchema>;
+
+export const ExperimentObjectiveSchema = z
+  .object({
+    constraints: z.array(ExperimentConstraintSchema).default([]),
+    minimize: z
+      .array(
+        z
+          .object({
+            metric: z.string().min(1),
+            profile: z.string().optional(),
+            split: z.enum(["train", "validation", "test"]).default("validation"),
+          })
+          .strict(),
+      )
+      .default([]),
+  })
+  .strict();
+export type ExperimentObjective = z.output<typeof ExperimentObjectiveSchema>;
+
+export const ExperimentBudgetSchema = z
+  .object({
+    maxCandidates: z.number().int().min(1).default(5),
+    patience: z.number().int().min(1).default(2),
+    /** Individual case/annotation evaluations across candidates, profiles, and repetitions. */
+    maxMetricCalls: z.number().int().min(1).default(10_000),
+    /** Individual LLM-judge assertion calls made while scoring metric calls. */
+    maxJudgeCalls: z.number().int().min(1).default(100_000),
+    /** Reflection model calls, independent of proposals returned per call. */
+    maxProposalCalls: z.number().int().min(1).default(50),
+    /** Aggregate model tokens touched by evaluation and proposal calls. */
+    maxTokens: z.number().int().min(1).default(10_000_000),
+    /** Cap costs reported by the selected driver/model CLI; unavailable provider prices stay explicit. */
+    maxCostUsd: z.number().positive().default(100),
+    /**
+     * Explicit allowance for calls whose model provider reports no price.
+     * Zero fails closed; raise it deliberately for an unpriced provider.
+     */
+    maxUnpricedModelCalls: z.number().int().nonnegative().default(0),
+    maxWallTimeMs: z.number().int().positive().default(3_600_000),
+  })
+  .strict();
+export type ExperimentBudget = z.infer<typeof ExperimentBudgetSchema>;
+
+const DEFAULT_EXPERIMENT_BUDGET: ExperimentBudget = {
+  maxCandidates: 5,
+  patience: 2,
+  maxMetricCalls: 10_000,
+  maxJudgeCalls: 100_000,
+  maxProposalCalls: 50,
+  maxTokens: 10_000_000,
+  maxCostUsd: 100,
+  maxUnpricedModelCalls: 0,
+  maxWallTimeMs: 3_600_000,
+};
+
+export const ExperimentCampaignSchema = z
+  .object({
+    id: z.string().min(1),
+    subject: FileSetSubjectSchema,
+    evaluation: z
+      .discriminatedUnion("kind", [
+        z.object({ kind: z.literal("agent") }).strict(),
+        z
+          .object({
+            kind: z.literal("grader"),
+            /** Human-labeled evidence used to align candidate assertions/rubrics. */
+            annotations: z.string().min(1),
+            /** Explicit rubric-path to assertion IDs that candidates may add. */
+            allowedAssertionAdditions: z.record(z.string(), z.array(z.string().min(1))).default({}),
+            /** Absolute error accepted when a human labels a continuous score. */
+            scoreTolerance: z.number().min(0).max(1).default(0.1),
+          })
+          .strict(),
+      ])
+      .default({ kind: "agent" }),
+    profiles: z.array(z.string()).min(1),
+    splits: z
+      .object({
+        train: ExperimentSelectorSchema.optional(),
+        validation: ExperimentSelectorSchema,
+        test: ExperimentSelectorSchema.optional(),
+      })
+      .strict(),
+    repetitions: z
+      .object({
+        search: z.number().int().positive().default(1),
+        promotion: z.number().int().positive().default(3),
+      })
+      .strict()
+      .default({ search: 1, promotion: 3 }),
+    variants: z.array(ExperimentVariantSchema).default([]),
+    objectives: ExperimentObjectiveSchema.default({ constraints: [], minimize: [] }),
+    optimizer: z
+      .object({
+        algorithm: z.literal("reflective"),
+        config: z.unknown().optional(),
+        /** Pareto search preserves candidates that specialize on different cases. */
+        candidateSelection: z.enum(["pareto", "current-best"]).default("pareto"),
+        /** Let reflection combine complementary components from frontier lineages. */
+        mergeFrontier: z.boolean().default(true),
+        /** Re-evaluate several search leaders before choosing a held-out test candidate. */
+        promotionTopK: z.number().int().min(1).max(20).default(3),
+      })
+      .strict()
+      .optional(),
+    budget: ExperimentBudgetSchema.default(DEFAULT_EXPERIMENT_BUDGET),
+    meta: z.record(z.string(), z.unknown()).optional(),
+  })
+  .strict()
+  .superRefine((campaign, ctx) => {
+    const graderConfidenceMetrics = new Set([
+      "quality.passRate",
+      "grader.agreement",
+      "grader.caseAgreement",
+      "grader.assertionAgreement",
+      "grader.macroFalsePositiveRate",
+      "grader.macroFalseNegativeRate",
+      "grader.coverage",
+      "grader.abstentionRate",
+      "grader.scoreMae",
+    ]);
+    const unitIntervalMetrics = graderConfidenceMetrics;
+    if (campaign.optimizer && !campaign.splits.train) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["splits", "train"],
+        message: "optimized experiments require a train split for proposal feedback",
+      });
+    }
+    if (campaign.optimizer && !campaign.splits.test) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["splits", "test"],
+        message: "optimized experiments require a held-out test split",
+      });
+    }
+    if (campaign.optimizer && campaign.repetitions.promotion <= campaign.repetitions.search) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["repetitions", "promotion"],
+        message:
+          "optimized experiments require promotion repetitions greater than search repetitions",
+      });
+    }
+    if (
+      campaign.splits.test &&
+      !campaign.objectives.constraints.some(
+        (constraint) => constraint.split === "test" && constraint.metric !== "subject.bytes",
+      )
+    ) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["objectives", "constraints"],
+        message: "experiments with a test split require an evidence-dependent test constraint",
+      });
+    }
+    if (
+      campaign.evaluation.kind === "agent" &&
+      campaign.repetitions.promotion > 1 &&
+      campaign.objectives.constraints.some(
+        (constraint) => constraint.confidence?.method === "wilson",
+      )
+    ) {
+      ctx.addIssue({
+        code: "custom",
+        path: ["objectives", "constraints"],
+        message:
+          "agent promotion with repeated case runs produces clustered means; use Hoeffding or percentile-bootstrap confidence",
+      });
+    }
+    const ids = new Set(["baseline"]);
+    for (const [index, variant] of campaign.variants.entries()) {
+      if (ids.has(variant.id)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["variants", index, "id"],
+          message: `duplicate or reserved variant id '${variant.id}'`,
+        });
+      }
+      ids.add(variant.id);
+    }
+    const profiles = new Set<string>();
+    for (const [index, profile] of campaign.profiles.entries()) {
+      if (profiles.has(profile)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["profiles", index],
+          message: `duplicate experiment profile '${profile}'`,
+        });
+      }
+      profiles.add(profile);
+    }
+    for (const [index, constraint] of campaign.objectives.constraints.entries()) {
+      if (
+        (constraint.confidence?.method === "hoeffding" ||
+          constraint.confidence?.method === "wilson") &&
+        !unitIntervalMetrics.has(constraint.metric)
+      ) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "confidence", "method"],
+          message: `metric '${constraint.metric}' has no declared [0,1] support; use percentile-bootstrap confidence`,
+        });
+      }
+      if (
+        unitIntervalMetrics.has(constraint.metric) &&
+        [constraint.min, constraint.max].some(
+          (bound) => bound !== undefined && (bound < 0 || bound > 1),
+        )
+      ) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index],
+          message: `metric '${constraint.metric}' has support [0,1]; constraint bounds must stay in that interval`,
+        });
+      }
+      if (constraint.profile && !profiles.has(constraint.profile)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "profile"],
+          message: `constraint references undeclared profile '${constraint.profile}'`,
+        });
+      }
+      if (constraint.split === "train" && !campaign.splits.train) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "split"],
+          message: "constraint references a missing train split",
+        });
+      }
+      if (constraint.split === "test" && !campaign.splits.test) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "split"],
+          message: "constraint references a missing test split",
+        });
+      }
+      if (
+        campaign.evaluation.kind === "grader" &&
+        constraint.confidence &&
+        constraint.metric.startsWith("grader.") &&
+        !graderConfidenceMetrics.has(constraint.metric)
+      ) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "confidence"],
+          message: `grader metric '${constraint.metric}' does not expose an annotation-cluster sample mean for confidence gating`,
+        });
+      }
+      if (
+        campaign.evaluation.kind === "grader" &&
+        constraint.confidence?.method === "wilson" &&
+        graderConfidenceMetrics.has(constraint.metric)
+      ) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "constraints", index, "confidence", "method"],
+          message: `grader metric '${constraint.metric}' is an annotation-cluster mean; use Hoeffding or percentile-bootstrap confidence`,
+        });
+      }
+    }
+    for (const [index, objective] of campaign.objectives.minimize.entries()) {
+      if (objective.profile && !profiles.has(objective.profile)) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "minimize", index, "profile"],
+          message: `objective references undeclared profile '${objective.profile}'`,
+        });
+      }
+      if (objective.split === "train" && !campaign.splits.train) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "minimize", index, "split"],
+          message: "objective references a missing train split",
+        });
+      }
+      if (objective.split === "test" && !campaign.splits.test) {
+        ctx.addIssue({
+          code: "custom",
+          path: ["objectives", "minimize", index, "split"],
+          message: "objective references a missing test split",
+        });
+      }
+    }
+  });
+
+export type ExperimentCampaignInput = z.input<typeof ExperimentCampaignSchema>;
+export type ExperimentCampaign = z.output<typeof ExperimentCampaignSchema>;
+
+export const defineExperiment = (input: ExperimentCampaignInput): ExperimentCampaign => {
+  const parsed = ExperimentCampaignSchema.safeParse(input);
+  if (!parsed.success) {
+    throw new Error(`Invalid Lux experiment: ${formatZodIssues(parsed.error)}`);
+  }
+  return parsed.data;
+};
+
+export const CandidateSchema = z.object({
+  schemaVersion: z.literal(1),
+  id: CandidateIdSchema,
+  parentIds: z.array(CandidateIdSchema),
+  createdAt: z.string(),
+  proposer: z.string(),
+  baseSourceHash: z.string(),
+  candidateHash: z.string(),
+  components: z.record(z.string(), z.string()),
+  sourceBytes: z.number().int().nonnegative(),
+  rationale: z.exactOptional(z.string()),
+  proposalUsage: z.exactOptional(TokenUsageSchema),
+});
+export type Candidate = z.infer<typeof CandidateSchema>;
+
+export const CandidateEvaluationSchema = z.object({
+  profileId: z.string(),
+  split: z.enum(["train", "validation", "test"]),
+  suiteDir: z.exactOptional(z.string()),
+  evidencePath: z.exactOptional(z.string()),
+  passRate: z.number().min(0).max(1),
+  casePassRates: z.record(z.string(), z.number().min(0).max(1)),
+  metrics: z.record(z.string(), z.number()),
+  metricCalls: z.exactOptional(z.number().int().nonnegative()),
+  judgeCalls: z.exactOptional(z.number().int().nonnegative()),
+  unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),
+  usage: z.exactOptional(TokenUsageSchema),
+  costUsd: z.exactOptional(z.number().nonnegative()),
+  /** Raw observations used by confidence-aware constraints. */
+  samples: z.exactOptional(z.record(z.string(), z.array(z.number()))),
+  failedCases: z.array(
+    z.object({
+      caseId: z.string(),
+      score: z.number().nullable(),
+      feedback: z.array(z.string()),
+    }),
+  ),
+});
+export type CandidateEvaluation = z.infer<typeof CandidateEvaluationSchema>;
+
+export const CandidateProposalSchema = z
+  .object({
+    updates: z.record(z.string(), z.string()),
+    rationale: z.string().optional(),
+    usage: TokenUsageSchema.optional(),
+  })
+  .strict();
+export type CandidateProposal = z.infer<typeof CandidateProposalSchema>;
+
+export type OptimizerInput = {
+  campaign: ExperimentCampaign;
+  candidate: Candidate;
+  /** Complementary Pareto candidates available to a merge-aware proposer. */
+  alternatives?: Candidate[] | undefined;
+  /** Train observations explaining which examples each alternative preserves. */
+  alternativeObservations?: Record<string, CandidateEvaluation[]> | undefined;
+  observations: CandidateEvaluation[];
+  remainingCandidates: number;
+  cwd: string;
+  abortSignal?: AbortSignal | undefined;
+  /** Persist accounting immediately after each completed proposal-model call. */
+  recordModelCall?: (accounting: {
+    usage: z.infer<typeof TokenUsageSchema>;
+    costUsd?: number;
+  }) => Promise<void>;
+  /** Durable provider-attempt markers for hard-kill-safe proposal accounting. */
+  recordModelAttemptStart?: (kind: "proposal") => Promise<void>;
+  completeModelAttempts?: () => Promise<void>;
+};
+
+export interface Optimizer {
+  propose(input: OptimizerInput): Promise<CandidateProposal[]>;
+}

package/src/core/glob.ts

@@ -0,0 +1,31 @@
+const globRegex = (pattern: string): RegExp => {
+  const normalized = pattern.replaceAll("\\", "/").replace(/^\.\//, "");
+  const directoryGlob = normalized.endsWith("/**");
+  const input = directoryGlob ? normalized.slice(0, -3) : normalized;
+  let source = "";
+  for (let index = 0; index < input.length; index++) {
+    const char = input[index];
+    const next = input[index + 1];
+    const after = input[index + 2];
+    if (char === "*" && next === "*" && after === "/") {
+      source += "(?:.*/)?";
+      index += 2;
+    } else if (char === "*" && next === "*") {
+      source += ".*";
+      index++;
+    } else if (char === "*") {
+      source += "[^/]*";
+    } else if (char === "?") {
+      source += "[^/]";
+    } else {
+      source += char?.replace(/[|\\{}()[\]^$+?.]/g, "\\$&") ?? "";
+    }
+  }
+  return new RegExp(`^${source}${directoryGlob ? "(?:/.*)?" : ""}$`);
+};
+
+export const matchesGlob = (path: string, pattern: string): boolean =>
+  globRegex(pattern).test(path.replaceAll("\\", "/").replace(/^\.\//, ""));
+
+export const matchesAnyGlob = (path: string, patterns: readonly string[]): boolean =>
+  patterns.some((pattern) => matchesGlob(path, pattern));