@prismatic-io/lux 0.0.1

package/src/orchestrator/grading.ts

@@ -0,0 +1,260 @@
+import type { z } from "zod";
+import {
+  type Assertion,
+  type AssertionImpl,
+  type AssertionProgress,
+  AssertionResultSchema,
+  type GradingAssertionEntry,
+  type GradingReport,
+  guardRunObserver,
+  type Registry,
+  type Run,
+  type RunObserver,
+  valueHash,
+} from "../core/index.js";
+import { validatePluginConfigAsync } from "./validate-plugin-config.js";
+
+const FRAMEWORK_KEYS = new Set(["id", "type", "name", "weight", "threshold"]);
+
+const shapeOf = (schema: z.ZodTypeAny): Record<string, unknown> | null => {
+  const shape = (schema as { shape?: unknown }).shape;
+  return shape && typeof shape === "object" ? (shape as Record<string, unknown>) : null;
+};
+
+/**
+ * A `.strict()` plugin schema must not choke on framework keys it never
+ * declared, while one that does declare them (rubric's `threshold`,
+ * tool-called's `name`) keeps receiving them. Non-object schemas expose no
+ * shape to consult, so they get the assertion as-is.
+ */
+export const assertionConfigForSchema = (a: Assertion, schema: z.ZodTypeAny): unknown => {
+  const shape = shapeOf(schema);
+  if (!shape) return a;
+  return Object.fromEntries(
+    Object.entries(a).filter(([key]) => !FRAMEWORK_KEYS.has(key) || key in shape),
+  );
+};
+
+export type AssertionPluginIdentity = {
+  type: string;
+  pluginVersion: string | null;
+};
+
+export type AssertionEvaluatorIdentity = AssertionPluginIdentity & {
+  evaluate: AssertionImpl["evaluate"];
+};
+
+export const assertionEvaluatorIdentities = (
+  assertions: readonly Assertion[],
+  registry: Registry,
+): AssertionEvaluatorIdentity[] =>
+  [...new Set(assertions.map((assertion) => assertion.type))].sort().map((type) => {
+    const implementation = registry.getAssertion(type);
+    if (!implementation) throw new Error(`unknown assertion type: ${type}`);
+    return {
+      type,
+      pluginVersion:
+        "pluginVersion" in implementation && typeof implementation.pluginVersion === "string"
+          ? implementation.pluginVersion
+          : null,
+      evaluate: implementation.evaluate,
+    };
+  });
+
+export const assertionPluginIdentities = (
+  assertions: readonly Assertion[],
+  registry: Registry,
+): AssertionPluginIdentity[] =>
+  assertionEvaluatorIdentities(assertions, registry).map(({ type, pluginVersion }) => ({
+    type,
+    pluginVersion,
+  }));
+
+export const graderConfigHash = (input: {
+  luxVersion: string;
+  implementationHash: string;
+  assertions: readonly AssertionEvaluatorIdentity[];
+  validatedAssertionsHash: string;
+  graderContext?: unknown;
+}): string =>
+  valueHash({
+    luxVersion: input.luxVersion,
+    implementationHash: input.implementationHash,
+    assertions: input.assertions,
+    validatedAssertionsHash: input.validatedAssertionsHash,
+    graderContext: input.graderContext ?? null,
+  });
+
+export const validateAssertionPlan = async (
+  assertions: readonly Assertion[],
+  registry: Registry,
+): Promise<unknown[]> => {
+  const validated: unknown[] = [];
+  // Async refinements may allocate workers or other bounded resources.
+  // Validate in authored order so an unbounded assertion list cannot fan
+  // those resources out before the subject driver even starts.
+  for (const [index, assertion] of assertions.entries()) {
+    const implementation = registry.getAssertion(assertion.type);
+    if (!implementation) {
+      throw new Error(
+        `invalid assertion plan at assertions[${index}]: no assertion implementation registered for type '${assertion.type}'`,
+      );
+    }
+    validated.push(
+      await validatePluginConfigAsync(
+        implementation.configSchema,
+        implementation.configSchema
+          ? assertionConfigForSchema(assertion, implementation.configSchema)
+          : assertion,
+        `assertion '${assertion.type}' at assertions[${index}]`,
+      ),
+    );
+  }
+  return validated;
+};
+
+const describeAssertion = (a: Assertion): string => {
+  if (a.name) return a.name;
+  const fields = Object.entries(a)
+    .filter(([k, v]) => !FRAMEWORK_KEYS.has(k) && typeof v !== "function")
+    .map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`)
+    .join(" ");
+  return fields ? `${a.type} ${fields}` : a.type;
+};
+
+export const gradeRun = async (
+  run: Run,
+  registry: Registry,
+  observer?: RunObserver,
+  abortSignal?: AbortSignal,
+  recordAssertion?: (entry: GradingAssertionEntry, index: number) => void | Promise<void>,
+  modelAttempts?: {
+    start: (kind: "judge") => string | Promise<string>;
+    complete: (attemptIds: readonly string[]) => void | Promise<void>;
+  },
+  validatedConfigs?: readonly unknown[],
+): Promise<GradingReport> => {
+  const guardedObserver = guardRunObserver(observer);
+  const assertions = run.case.assertions ?? [];
+  if (assertions.length === 0) {
+    return { schemaVersion: 1, casePassed: null, caseScore: null, assertions: [] };
+  }
+
+  const entries: GradingAssertionEntry[] = [];
+  let weightedScoreSum = 0;
+  let weightSum = 0;
+  let allPassed = true;
+  let gradingIncomplete = false;
+
+  for (const [index, a] of assertions.entries()) {
+    // Grading is the last cancellation point — an LLM judge can run for
+    // minutes per assertion. An aborted grade can't claim everything passed.
+    if (abortSignal?.aborted) {
+      gradingIncomplete = true;
+      break;
+    }
+    const impl = registry.getAssertion(a.type);
+    const weight = a.weight ?? 1;
+    // `threshold` is an OPTIONAL extra score floor layered on top of the
+    // assertion's own verdict (`result.passed`). Default 0 means "no floor",
+    // so a graded assertion like `rubric` passes on its own bar (judge
+    // score >= 0.8). A non-zero default here would force every sub-1.0
+    // graded score to fail even when the assertion reports passed.
+    const threshold = a.threshold ?? 0;
+    const id = a.id ?? `${a.type}:${index + 1}`;
+    const name = describeAssertion(a);
+    const progress: AssertionProgress = {
+      index,
+      total: assertions.length,
+      id,
+      type: a.type,
+      name,
+    };
+    guardedObserver?.assertionStart?.(progress);
+    const startedAt = Date.now();
+    let entry: GradingAssertionEntry;
+    const assertionAttemptIds: string[] = [];
+
+    try {
+      if (!impl) throw new Error(`no assertion implementation registered for type '${a.type}'`);
+      const validated = validatedConfigs
+        ? validatedConfigs[index]
+        : await validatePluginConfigAsync(
+            impl.configSchema,
+            impl.configSchema ? assertionConfigForSchema(a, impl.configSchema) : a,
+            `assertion '${a.type}'`,
+          );
+      if (validatedConfigs && index >= validatedConfigs.length) {
+        throw new Error(`validated assertion plan is missing assertions[${index}]`);
+      }
+      // Pass the validated config so schema defaults (e.g. timeoutMs)
+      // reach evaluate(); the raw assertion object has none applied.
+      const result = AssertionResultSchema.parse(
+        await impl.evaluate(validated as typeof a, run, {
+          ...(abortSignal ? { abortSignal } : {}),
+          ...(modelAttempts
+            ? {
+                recordModelAttemptStart: async (_kind: "judge") => {
+                  assertionAttemptIds.push(await modelAttempts.start("judge"));
+                },
+              }
+            : {}),
+        }),
+      );
+      const passed =
+        result.errorMessage === undefined && result.passed && result.score >= threshold;
+      entry = {
+        id,
+        type: a.type,
+        name,
+        passed,
+        score: result.score,
+        weight,
+        threshold,
+        durationMs: Date.now() - startedAt,
+        ...(result.comment !== undefined ? { comment: result.comment } : {}),
+        detail: result.detail,
+        ...(result.errorMessage !== undefined ? { errorMessage: result.errorMessage } : {}),
+        ...(result.usage ? { usage: result.usage } : {}),
+      };
+      if (result.errorMessage !== undefined) {
+        gradingIncomplete = true;
+      } else {
+        if (!passed) allPassed = false;
+        weightedScoreSum += weight * result.score;
+        weightSum += weight;
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      entry = {
+        id,
+        type: a.type,
+        name,
+        passed: false,
+        score: 0,
+        weight,
+        threshold,
+        durationMs: Date.now() - startedAt,
+        errorMessage: message,
+      };
+      gradingIncomplete = true;
+    }
+
+    entries.push(entry);
+    guardedObserver?.assertionEnd?.(progress, entry);
+    await recordAssertion?.(entry, index);
+    await modelAttempts?.complete(assertionAttemptIds);
+  }
+
+  let caseScore: number | null = null;
+  if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
+  return {
+    schemaVersion: 1,
+    casePassed: gradingIncomplete ? null : allPassed,
+    caseScore,
+    assertions: entries,
+    ...(gradingIncomplete
+      ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
+      : {}),
+  };
+};

package/src/orchestrator/index.ts

@@ -0,0 +1,186 @@
+// Public orchestration surface. Keep this list explicit: several orchestrator
+// modules export helpers solely for focused contract tests, and a wildcard
+// here would silently turn the next such helper into package API.
+export {
+  assertDisjointAnnotationSplits,
+  type LoadedGraderAnnotations,
+  LoadedGraderAnnotationsSchema,
+  loadGraderAnnotations,
+  selectGraderAnnotations,
+} from "./annotation-loader.js";
+export { upsertGraderAnnotation } from "./annotation-store.js";
+export {
+  applyCandidate,
+  captureSeedCandidate,
+  createCandidate,
+  hashSubjectTree,
+  materializeCandidate,
+  validateCandidate,
+} from "./candidates.js";
+export {
+  type ComparisonInput,
+  compareRuns,
+  formatComparisonText,
+  type GradingDelta,
+  hasRegression,
+  loadRunBundle,
+  type RunBundle,
+  type RunComparison,
+} from "./compare.js";
+export {
+  buildRegistryFromConfig,
+  type ExperimentProfile,
+  ExperimentProfileSchema,
+  findConfigFile,
+  type LuxConfig,
+  loadLuxConfig,
+  type ResolvedLuxConfig,
+} from "./config.js";
+export {
+  type CaseFilter,
+  type DiscoveredCase,
+  type DiscoveryProblem,
+  type DiscoveryResult,
+  discoverCases,
+  selectCases,
+} from "./discover.js";
+export {
+  type DoctorCampaignReport,
+  DoctorCampaignReportSchema,
+  type DoctorDiagnostic,
+  DoctorDiagnosticSchema,
+  type DoctorOptions,
+  DoctorOptionsSchema,
+  type DoctorReport,
+  DoctorReportSchema,
+  doctorProject,
+  formatDoctorReport,
+} from "./doctor.js";
+export {
+  type CampaignExecutionOptions,
+  type CampaignExecutionResult,
+  CampaignExecutionResultSchema,
+  deriveExperimentSelection,
+  type ExperimentCampaignPlan,
+  ExperimentCampaignPlanSchema,
+  type ExperimentDecisionCandidate,
+  ExperimentDecisionCandidateSchema,
+  optimizeExperimentCampaign,
+  paretoFrontier,
+  planExperimentCampaign,
+  rankCandidates,
+  runExperimentCampaign,
+} from "./experiment.js";
+export {
+  type ExperimentCommandMode,
+  ExperimentCommandModeSchema,
+  experimentCampaignMode,
+  type LoadedExperimentCampaign,
+  LoadedExperimentCampaignSchema,
+  loadExperimentCampaign,
+  resolveExperimentCampaignPath,
+} from "./experiment-loader.js";
+export {
+  createExperimentDecisionReport,
+  type ExperimentDecisionReport,
+  ExperimentDecisionReportSchema,
+  formatExperimentDecisionMarkdown,
+  loadExperimentDecisionReport,
+  loadVerifiedExperimentDecisionSnapshot,
+  type VerifiedExperimentDecisionSnapshot,
+  VerifiedExperimentDecisionSnapshotSchema,
+  verifyExperimentDecisionIntegrity,
+} from "./experiment-report.js";
+export {
+  type ExperimentBudgetUsage,
+  type ExperimentEvaluation,
+  type ExperimentRun,
+  ExperimentRunSchema,
+  type ExperimentSelection,
+  ExperimentSelectionSchema,
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  isExperimentRun,
+  listExperimentRuns,
+  loadExperimentCandidate,
+  loadExperimentCandidates,
+  loadExperimentRun,
+  migrateExperimentCandidateIntegrity,
+  verifyExperimentRunIntegrity,
+} from "./experiment-runs.js";
+export {
+  compareGraderAnnotation,
+  evaluateGraderCandidate,
+  type GraderEvaluationEvidence,
+  GraderEvaluationEvidenceSchema,
+  type GraderJudgment,
+  summarizeGraderEvidence,
+} from "./grader-experiment.js";
+export { gradeRun, validateAssertionPlan } from "./grading.js";
+export {
+  formatRunDetail,
+  formatRunsTable,
+  type ListedRun,
+  listRuns,
+  loadRunDetail,
+  type RunDetail,
+} from "./list-runs.js";
+export { loadEvalCase } from "./loader.js";
+// The resolution and run-directory helpers exported by orchestrator.ts exist
+// for its own contract tests; the class and observer/request types are the
+// supported programmatic runner.
+export {
+  type CaseObserver,
+  Orchestrator,
+  type OrchestratorOptions,
+  type RunRequest,
+  type RunResult,
+  type SuiteCaseResult,
+  type SuiteItem,
+  type SuiteObserver,
+} from "./orchestrator.js";
+export {
+  GRADING_SNAPSHOT_SCHEMA_VERSION,
+  type GradingSnapshot,
+  GradingSnapshotSchema,
+  loadFrozenRegradableRun,
+  loadRegradableRun,
+  regradeRun,
+  releaseFrozenRegradableRun,
+} from "./regrade.js";
+export { loadRubricDefinition } from "./rubric-loader.js";
+export {
+  compareSuiteRuns,
+  compareSuiteSummaries,
+  formatSuiteComparisonText,
+  hasSuiteRegression,
+  type SuiteCaseComparison,
+  SuiteCaseComparisonSchema,
+  type SuiteComparison,
+  SuiteComparisonSchema,
+} from "./suite-compare.js";
+// openSuiteRun is persistence plumbing; callers create suites through
+// Orchestrator and consume them through these read APIs.
+export {
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  listSuiteRuns,
+  loadSuiteRun,
+  type SuiteRun,
+  type SuiteRunItem,
+  SuiteRunSchema,
+} from "./suite-runs.js";
+export {
+  buildSuiteSummary,
+  type CaseSummary,
+  loadSuiteSummary,
+  type MetricSummary,
+  MetricSummarySchema,
+  type OutcomeSummary,
+  type SuiteSample,
+  SuiteSampleSchema,
+  type SuiteSummary,
+  SuiteSummarySchema,
+  summarizeNumbers,
+  writeSuiteSummary,
+} from "./suite-summary.js";

package/src/orchestrator/lifecycle-hook-execution.ts

@@ -0,0 +1,90 @@
+import type { LifecycleHookCleanup, LifecycleHooksPlugin } from "../core/index.js";
+
+type RegisteredLifecycleCleanup = {
+  label: string;
+  cleanup: LifecycleHookCleanup;
+};
+
+const errorMessage = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const lifecycleError = (label: string, phase: "setup" | "cleanup", error: unknown): Error =>
+  new Error(`${label} ${phase} failed: ${errorMessage(error)}`, { cause: error });
+
+const cleanupLifecycleHooks = async (
+  cleanups: readonly RegisteredLifecycleCleanup[],
+): Promise<void> => {
+  const failures: Error[] = [];
+  for (const { label, cleanup } of [...cleanups].reverse()) {
+    try {
+      await cleanup();
+    } catch (error) {
+      failures.push(lifecycleError(label, "cleanup", error));
+    }
+  }
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) {
+    throw new AggregateError(failures, "multiple lifecycle hook cleanups failed");
+  }
+};
+
+export const setupLifecycleHooks = async (
+  hooks: readonly LifecycleHooksPlugin[],
+  scope: "suite" | "case",
+  invoke: (
+    hook: LifecycleHooksPlugin,
+  ) => LifecycleHookCleanup | undefined | Promise<LifecycleHookCleanup | undefined>,
+  abortSignal?: AbortSignal,
+): Promise<LifecycleHookCleanup> => {
+  const cleanups: RegisteredLifecycleCleanup[] = [];
+  try {
+    for (const hook of hooks) {
+      abortSignal?.throwIfAborted();
+      const label = `lifecycle hook '${hook.name}' ${scope}`;
+      let cleanup: LifecycleHookCleanup | undefined;
+      try {
+        cleanup = await invoke(hook);
+      } catch (error) {
+        throw lifecycleError(label, "setup", error);
+      }
+      if (cleanup !== undefined && typeof cleanup !== "function") {
+        throw new TypeError(`${label} setup returned a non-function cleanup`);
+      }
+      if (cleanup) cleanups.push({ label, cleanup });
+      abortSignal?.throwIfAborted();
+    }
+  } catch (error) {
+    let cleanupError: unknown;
+    try {
+      await cleanupLifecycleHooks(cleanups);
+    } catch (cleanupFailure) {
+      cleanupError = cleanupFailure;
+    }
+    const setupError =
+      error instanceof Error &&
+      (/lifecycle hook '.+' (suite|case) setup/.test(error.message) ||
+        /returned a non-function cleanup/.test(error.message))
+        ? error
+        : lifecycleError(`lifecycle hook ${scope}`, "setup", error);
+    if (cleanupError) {
+      throw new AggregateError(
+        [setupError, cleanupError],
+        `${scope} lifecycle setup and rollback failed`,
+      );
+    }
+    throw setupError;
+  }
+
+  let cleaned = false;
+  return async () => {
+    if (cleaned) return;
+    cleaned = true;
+    await cleanupLifecycleHooks(cleanups);
+  };
+};
+
+export const combineLifecycleFailures = (
+  primary: unknown,
+  cleanup: unknown,
+  message: string,
+): AggregateError => new AggregateError([primary, cleanup], message);