@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.30

package/src/authoring.ts

@@ -1,3 +1,346 @@
+// Keep authoring independent of bundled drivers and orchestration while exposing
+// the same supported core contracts as the package root.
 export { z } from "zod";
-export { defineEvalCase } from "./assertions/authoring.js";
-export * from "./core/index.js";
+export { type BuiltinAssertion, defineEvalCase } from "./assertions/authoring.js";
+export {
+  type ActionCategory,
+  ActionCategorySchema,
+  type ActionProgressPayload,
+  ActionProgressPayloadSchema,
+  type ActionTiming,
+  ActionTimingSchema,
+  actionTiming,
+} from "./core/action-events.js";
+export {
+  type AgentActivity,
+  AgentActivitySchema,
+  summarizeAgentActivity,
+} from "./core/agent-activity.js";
+export {
+  AGENT_COMPLETED_KIND,
+  AGENT_SPAWN_KIND,
+  type AgentCompletedPayload,
+  AgentCompletedPayloadSchema,
+  type AgentSpawnPayload,
+  AgentSpawnPayloadSchema,
+} from "./core/agent-events.js";
+export {
+  defineGraderAnnotations,
+  type GraderAnnotation,
+  type GraderAnnotationDataset,
+  type GraderAnnotationDatasetInput,
+  GraderAnnotationDatasetSchema,
+  GraderAnnotationSchema,
+} from "./core/annotation.js";
+export {
+  type Answerer,
+  type AnswererContext,
+  type AnswererModelAccounting,
+  AnswererModelAccountingSchema,
+  type AnswererModelCallAccounting,
+  AnswererModelCallAccountingSchema,
+  type AnswererPlugin,
+  type AnswerHistoryEntry,
+  answererRef,
+  defineAnswerer,
+} from "./core/answerer.js";
+export {
+  inspectRunArtifact,
+  type ReadRunArtifactTextOptions,
+  type RunArtifactInspection,
+  type RunArtifactIssueReason,
+  readRunArtifactText,
+} from "./core/artifact-evidence.js";
+export {
+  ARTIFACT_SNAPSHOT_MANIFEST,
+  type ArtifactSnapshotContent,
+  type ArtifactSnapshotFile,
+  ArtifactSnapshotFileSchema,
+  type ArtifactSnapshotFiles,
+  type ArtifactSnapshotManifest,
+  ArtifactSnapshotManifestSchema,
+  readArtifactSnapshotManifest,
+  type WriteArtifactSnapshotOptions,
+  writeArtifactSnapshot,
+} from "./core/artifact-snapshot.js";
+export {
+  type AssertionImpl,
+  type AssertionPlugin,
+  type AssertionPluginDefinition,
+  type AssertionResult,
+  AssertionResultSchema,
+  defineAssertion,
+  defineAssertionRegistry,
+  type EvaluateContext,
+  type GradingAssertionEntry,
+  GradingAssertionEntrySchema,
+  type GradingReport,
+  GradingReportSchema,
+} from "./core/assertion.js";
+export {
+  type AnswererRef,
+  AnswererRefSchema,
+  type Assertion,
+  AssertionSchema,
+  type DriverRef,
+  DriverRefSchema,
+  type EvalCase,
+  type EvalCaseAuthoringOptions,
+  type EvalCaseDefinition,
+  type EvalCaseInput,
+  EvalCaseSchema,
+  type FixturesRef,
+  FixturesRefSchema,
+  type PersistedCase,
+  PersistedCaseSchema,
+} from "./core/case.js";
+export {
+  CONVERSATION_TURN_REQUEST,
+  type ConversationTurnRequest,
+  ConversationTurnRequestSchema,
+  type ConversationTurnResponse,
+  ConversationTurnResponseSchema,
+  conversationTurnInterrupt,
+  conversationTurnResponse,
+  isConversationTurnInterrupt,
+} from "./core/conversation-turn.js";
+export {
+  type AgentDriver,
+  type Artifact,
+  ArtifactSchema,
+  type DriverCapability,
+  DriverCapabilitySchema,
+  type DriverEventMap,
+  type DriverExperiment,
+  type DriverPlugin,
+  type DriverRuntime,
+  defineDriver,
+  driverRef,
+  type ResolvedFixtures,
+  ResolvedFixturesSchema,
+  type StartContext,
+} from "./core/driver.js";
+export {
+  type ContainerExecution,
+  ContainerExecutionSchema,
+  type DriverExecution,
+  DriverExecutionSchema,
+} from "./core/driver-execution.js";
+export {
+  type DriverMetric,
+  type DriverMetricDescriptor,
+  DriverMetricDescriptorSchema,
+  DriverMetricSchema,
+  type DriverMetrics,
+  DriverMetricsSchema,
+  type DriverMetricUnit,
+  DriverMetricUnitSchema,
+  normalizeDriverMetrics,
+  type ReportedDriverMetrics,
+  ReportedDriverMetricsSchema,
+} from "./core/driver-metrics.js";
+export { DurationSchema, durationMs } from "./core/duration.js";
+export { type EventHandler, TypedEmitter } from "./core/emitter.js";
+export { type EnvironmentDeclaration, EnvironmentDeclarationSchema } from "./core/environment.js";
+export {
+  type AgentAttribution,
+  AgentAttributionSchema,
+  type Answer,
+  AnswerSchema,
+  ApproveAnswerSchema,
+  ApproveInterruptSchema,
+  AskAnswerSchema,
+  AskInterruptSchema,
+  CustomAnswerSchema,
+  CustomInterruptSchema,
+  type Done,
+  DoneSchema,
+  type ErrorEvent,
+  type ErrorEventReason,
+  ErrorEventReasonSchema,
+  ErrorEventSchema,
+  type Interrupt,
+  type InterruptKind,
+  InterruptKindSchema,
+  InterruptSchema,
+  type LifecycleStage,
+  LifecycleStageSchema,
+  type Progress,
+  ProgressSchema,
+  type ReadyState,
+  ReadyStateSchema,
+  type RunEvent,
+  RunEventSchema,
+} from "./core/events.js";
+export {
+  type ExecOptions,
+  type ExecResult,
+  exec,
+  type ManagedProcess,
+  type ManagedProcessExit,
+  type ManagedProcessOptions,
+  spawnManagedProcess,
+} from "./core/exec.js";
+export {
+  type Candidate,
+  type CandidateEvaluation,
+  CandidateEvaluationSchema,
+  CandidateIdSchema,
+  type CandidateProposal,
+  CandidateProposalSchema,
+  CandidateSchema,
+  defineExperiment,
+  type ExperimentBudget,
+  ExperimentBudgetSchema,
+  type ExperimentCampaign,
+  type ExperimentCampaignInput,
+  ExperimentCampaignSchema,
+  type ExperimentConstraint,
+  ExperimentConstraintSchema,
+  type ExperimentObjective,
+  ExperimentObjectiveSchema,
+  type ExperimentSelector,
+  ExperimentSelectorSchema,
+  type ExperimentVariant,
+  ExperimentVariantSchema,
+  type FileSetSubject,
+  FileSetSubjectSchema,
+  type Optimizer,
+  type OptimizerInput,
+} from "./core/experiment.js";
+export { type GlobalIdParts, tryParseGlobalId } from "./core/global-id.js";
+export {
+  type HarnessCatalog,
+  type ReasoningEffort,
+  ReasoningEffortSchema,
+} from "./core/harness-catalog.js";
+export type {
+  BuiltInHarnessMatrixEntry,
+  CustomHarnessMatrixEntry,
+  HarnessMatrixEntry,
+} from "./core/harness-matrix.js";
+export {
+  defineFixtures,
+  type LifecycleFixture,
+  type LifecycleFixturesDefinition,
+  type LifecycleFixtureUse,
+} from "./core/lifecycle-fixtures.js";
+export {
+  type CaseLifecycleHookContext,
+  defineLifecycleHooks,
+  type FixtureValue,
+  type FixtureValueBinding,
+  FixtureValueSchema,
+  type LifecycleHookCleanup,
+  type LifecycleHookSetupResult,
+  type LifecycleHooksPlugin,
+  lifecycleHooksPluginIssue,
+  type SuiteLifecycleHookContext,
+  type SuiteLifecycleHookItem,
+} from "./core/lifecycle-hooks.js";
+export {
+  hasMcpInit,
+  hasMcpToolsList,
+  isMcpInitEvent,
+  isMcpToolsListEvent,
+  type ListedTool,
+  listedTools,
+  MCP_INIT_KIND,
+  MCP_TOOLS_LIST_KIND,
+  type McpInitPayload,
+  type McpToolsListPayload,
+  mcpInit,
+} from "./core/mcp-events.js";
+export {
+  DEFAULT_CLAUDE_MODEL,
+  DEFAULT_CODEX_MODEL,
+  defaultHarnessModel,
+  type HarnessModelConfig,
+  HarnessModelConfigSchema,
+  type ModelCliProvider,
+  ModelCliProviderSchema,
+} from "./core/model-cli.js";
+export {
+  type AssertionProgress,
+  AssertionProgressSchema,
+  type RunObserver,
+} from "./core/observer.js";
+export { Registry } from "./core/registry.js";
+export {
+  defineRubric,
+  type RubricDefinition,
+  type RubricDefinitionInput,
+  RubricDefinitionSchema,
+} from "./core/rubric.js";
+export {
+  type OperationalFailure,
+  OperationalFailureSchema,
+  type PhaseDurations,
+  PhaseDurationsSchema,
+  RUN_SCHEMA_VERSION,
+  type Run,
+  type RunMetadata,
+  RunMetadataSchema,
+  type RunProvenance,
+  RunProvenanceSchema,
+  RunSchema,
+} from "./core/run.js";
+export {
+  bootstrapMeanConfidenceInterval,
+  type ConfidenceConfig,
+  ConfidenceConfigSchema,
+  type ConfidenceInterval,
+  ConfidenceIntervalSchema,
+} from "./core/statistics.js";
+export { SUBJECT_ROOT_TOKEN, subjectPath } from "./core/subject.js";
+export {
+  correlateToolEvents,
+  createToolOutcomeAccumulator,
+  isToolCallEvent,
+  isToolResultEvent,
+  malformedToolEventCount,
+  summarizeToolOutcomes,
+  TOOL_CALL_KIND,
+  TOOL_RESULT_KIND,
+  type ToolCallPayload,
+  ToolCallPayloadSchema,
+  type ToolEventCorrelation,
+  type ToolInvocation,
+  type ToolInvocationOutcome,
+  type ToolOutcomeAccumulator,
+  type ToolOutcomeSummary,
+  ToolOutcomeSummarySchema,
+  type ToolResultPayload,
+  ToolResultPayloadSchema,
+  toolCallProgress,
+  toolEventCalls,
+  toolInvocationOutcome,
+  toolInvocations,
+  toolResultProgress,
+  toolResults,
+  toolResultValue,
+} from "./core/tool-events.js";
+export {
+  addUsage,
+  emptyRunUsage,
+  emptyUsage,
+  harnessUsage,
+  isUsageEmpty,
+  type RunUsage,
+  RunUsageSchema,
+  sumUsage,
+  type TokenUsage,
+  TokenUsageSchema,
+  type TurnUsage,
+  TurnUsageSchema,
+  totalTokens,
+} from "./core/usage.js";
+export {
+  type FormatAnswerFn,
+  formatWireAnswer,
+  type ParsedDriverEvent,
+  type ParseEventFn,
+  parseWireEvent,
+  type WireEvent,
+  WireEventSchema,
+} from "./core/wire.js";

package/src/cli/bin.ts

@@ -1,14 +1,4 @@
 #!/usr/bin/env node
-import { buildProgram } from "./program.js";
+import { buildCli } from "./program.js";
 
-try {
-  await buildProgram().parseAsync(process.argv);
-} catch (error) {
-  // A raw stack trace is noise to a CLI user; keep it behind DEBUG.
-  const message = error instanceof Error ? error.message : String(error);
-  process.stderr.write(`lux: ${message}\n`);
-  if (process.env.DEBUG && error instanceof Error && error.stack) {
-    process.stderr.write(`${error.stack}\n`);
-  }
-  process.exitCode = 1;
-}
+await buildCli().serve();

package/src/cli/command-runtime.ts

@@ -1,25 +1,36 @@
 import { existsSync } from "node:fs";
 import { stat } from "node:fs/promises";
 import { isAbsolute, relative, resolve, sep } from "node:path";
+import { Errors } from "incur";
 
-export const abortOnInterrupt = (): AbortController => {
+export const abortOnInterrupt = (): AbortController & Disposable => {
   const controller = new AbortController();
   const abort = (): void => controller.abort();
   process.on("SIGINT", abort);
   process.on("SIGTERM", abort);
-  return controller;
+  return Object.assign(controller, {
+    [Symbol.dispose]: () => {
+      process.off("SIGINT", abort);
+      process.off("SIGTERM", abort);
+    },
+  });
 };
 
-export const failWith = (code: number): void => {
-  // Setting the code lets Node drain buffered output before exiting.
-  process.exitCode = code;
+export const failWith = (code: number): never => {
+  throw new Errors.IncurError({
+    code: code === 130 ? "INTERRUPTED" : "LUX_COMMAND_FAILED",
+    message: code === 130 ? "The command was interrupted." : "The command did not succeed.",
+    exitCode: code,
+  });
 };
 
-export const ensureDirectory = async (command: string, path: string): Promise<boolean> => {
+export const ensureDirectory = async (command: string, path: string): Promise<true> => {
   if (existsSync(path) && (await stat(path)).isDirectory()) return true;
-  process.stderr.write(`lux ${command}: ${path} is not a directory\n`);
-  failWith(1);
-  return false;
+  throw new Errors.IncurError({
+    code: "DIRECTORY_NOT_FOUND",
+    message: `lux ${command}: ${path} is not a directory`,
+    exitCode: 1,
+  });
 };
 
 export const relativePathWithin = (root: string, path: string): string | null => {

package/src/cli/doctor.ts

@@ -0,0 +1,17 @@
+import { Errors } from "incur";
+import { doctorProject, formatDoctorReport } from "../orchestrator/index.js";
+
+export const doctorCommand = async (
+  campaigns: string[],
+  options: { static?: boolean | undefined },
+) => {
+  const report = await doctorProject({ cwd: process.cwd(), campaigns, ...options });
+  if (!report.ok) {
+    throw new Errors.IncurError({
+      code: "PREFLIGHT_FAILED",
+      message: formatDoctorReport(report),
+      exitCode: 1,
+    });
+  }
+  return report;
+};

package/src/cli/harness-plan.ts

@@ -0,0 +1,117 @@
+import {
+  casePermutations,
+  type DriverRef,
+  driverNameOf,
+  focusHarnessPermutations,
+  type HarnessFocus,
+  type HarnessMatrixEntry,
+  type HarnessPermutation,
+  hasHarnessFocus,
+  permutationDriverRef,
+  permutationId,
+  type Registry,
+} from "../core/index.js";
+import { mergeProcessEnvironment, resolveExecutable } from "../core/platform-process.js";
+import { referencedDriverRuntime } from "../orchestrator/driver-identity.js";
+import { loadEvalCase, type SuiteItem } from "../orchestrator/index.js";
+import { resolveDriverReference } from "../orchestrator/orchestrator.js";
+
+export type HarnessPlanOptions = {
+  matrix?: readonly HarnessMatrixEntry[] | undefined;
+  defaultDriver?: DriverRef | undefined;
+  registry: Registry;
+  focus: HarnessFocus;
+};
+
+export type HarnessPlan = {
+  items: SuiteItem[];
+  /** Cases pinned to a driver the focus cannot apply to. */
+  excluded: string[];
+  unavailable: { id: string; commands: string[] }[];
+};
+
+const permutationLabel = (caseId: string, id: string): string => `${caseId} [${id}]`;
+
+const bindPermutation = (
+  permutation: HarnessPermutation,
+  options: HarnessPlanOptions,
+): DriverRef => {
+  const plugin = options.registry.getDriver(permutation.driver);
+  if (!plugin) throw new Error(`unknown driver '${permutation.driver}'`);
+  return permutationDriverRef(permutation, options.defaultDriver, plugin);
+};
+
+/**
+ * Expand selected cases into suite items, one per harness permutation. Cases
+ * without permutations keep today's single-driver item.
+ */
+export const planHarnessItems = async (
+  cases: readonly { path: string; id: string }[],
+  options: HarnessPlanOptions,
+): Promise<HarnessPlan> => {
+  const items: SuiteItem[] = [];
+  const excluded: string[] = [];
+  const unavailable: HarnessPlan["unavailable"] = [];
+  for (const selected of cases) {
+    const evalCase = await loadEvalCase(selected.path);
+    const pinned = evalCase.driver ? driverNameOf(evalCase.driver) : undefined;
+    if (
+      options.focus.driver !== undefined &&
+      pinned !== undefined &&
+      pinned !== options.focus.driver
+    ) {
+      excluded.push(selected.id);
+      continue;
+    }
+    let permutations = casePermutations(evalCase, options.matrix);
+    const fromMatrix = permutations.length > 0;
+    if (permutations.length === 0 && hasHarnessFocus(options.focus)) {
+      const base =
+        pinned ?? (options.defaultDriver ? driverNameOf(options.defaultDriver) : undefined);
+      const selecting = options.focus.model !== undefined || options.focus.effort !== undefined;
+      if (
+        base !== undefined &&
+        selecting &&
+        options.focus.driver === undefined &&
+        !options.registry.getDriver(base)?.selectHarness
+      ) {
+        excluded.push(selected.id);
+        continue;
+      }
+      permutations = base === undefined ? [] : [{ driver: base }];
+    }
+    permutations = focusHarnessPermutations(permutations, options.focus);
+    if (permutations.length === 0) {
+      items.push({ path: selected.path, id: selected.id });
+      continue;
+    }
+    for (const permutation of permutations) {
+      const id = permutationId(permutation);
+      const driver = bindPermutation(permutation, options);
+      const label = permutationLabel(selected.id, id);
+      if (fromMatrix) {
+        const effective = resolveDriverReference(evalCase, options.defaultDriver, driver);
+        const runtime = referencedDriverRuntime(effective, options.registry);
+        const config = (typeof effective === "string" ? undefined : effective.config) as
+          | { cwd?: string; env?: NodeJS.ProcessEnv }
+          | undefined;
+        const environment = mergeProcessEnvironment(process.env, config?.env);
+        const commands = runtime.commands.filter(
+          (command) => !resolveExecutable(command, environment, undefined, config?.cwd),
+        );
+        if (commands.length > 0) {
+          unavailable.push({ id: label, commands });
+          continue;
+        }
+      }
+      items.push({
+        path: selected.path,
+        id: label,
+        caseId: selected.id,
+        permutationId: id,
+        driver,
+      });
+    }
+  }
+  return { items, excluded, unavailable };
+};