@prismatic-io/lux 0.0.1

package/src/index.ts

@@ -0,0 +1,594 @@
+// Public surface of @prismatic-io/lux. The package root exposes authoring
+// contracts, extension points, built-ins, and supported programmatic
+// lifecycles. Explicit exports keep implementation and test helpers private.
+
+export { z } from "zod";
+
+// Bundled answerers and drivers. Test injection and protocol-normalization
+// helpers stay in their implementation modules.
+export {
+  type ClaudeCodeAnswererConfig,
+  ClaudeCodeAnswererConfigSchema,
+  claudeCodeAnswerer,
+} from "./answerers/claude-code/index.js";
+export {
+  type PersonaAnswererConfig,
+  PersonaAnswererConfigSchema,
+  personaAnswerer,
+} from "./answerers/persona/index.js";
+export {
+  type ScriptedAnswererConfig,
+  ScriptedAnswererConfigSchema,
+  scriptedAnswerer,
+} from "./answerers/scripted/index.js";
+export {
+  type TerminalAnswererConfig,
+  TerminalAnswererConfigSchema,
+  terminalAnswerer,
+} from "./answerers/terminal/index.js";
+// Bundled deterministic and judge assertions.
+export {
+  type AgentScope,
+  type ArtifactBytesUnderConfig,
+  ArtifactBytesUnderConfigSchema,
+  allCoreAssertions,
+  artifactBytesUnderAssertion,
+  type CommandExitsZeroConfig,
+  CommandExitsZeroConfigSchema,
+  type ContainsConfig,
+  ContainsConfigSchema,
+  type CostUnderConfig,
+  CostUnderConfigSchema,
+  commandExitsZeroAssertion,
+  containsAssertion,
+  costUnderAssertion,
+  type FileContainsConfig,
+  FileContainsConfigSchema,
+  type FileExistsConfig,
+  FileExistsConfigSchema,
+  type FileMatchesConfig,
+  FileMatchesConfigSchema,
+  type FileRegexCountConfig,
+  FileRegexCountConfigSchema,
+  type FileRegexSetConfig,
+  FileRegexSetConfigSchema,
+  fileContainsAssertion,
+  fileExistsAssertion,
+  fileMatchesAssertion,
+  fileRegexCountAssertion,
+  fileRegexSetAssertion,
+  type GlobCountConfig,
+  GlobCountConfigSchema,
+  globCountAssertion,
+  type InterruptCountConfig,
+  InterruptCountConfigSchema,
+  interruptCountAssertion,
+  type JsonPointerEqualsConfig,
+  JsonPointerEqualsConfigSchema,
+  jsonPointerEqualsAssertion,
+  type LatencyUnderConfig,
+  LatencyUnderConfigSchema,
+  latencyUnderAssertion,
+  matchesAgentScope,
+  type NoErrorResultsConfig,
+  NoErrorResultsConfigSchema,
+  type NotContainsConfig,
+  NotContainsConfigSchema,
+  noErrorResultsAssertion,
+  notContainsAssertion,
+  type PredicateConfig,
+  PredicateConfigSchema,
+  type PredicateFn,
+  predicateAssertion,
+  type RegexConfig,
+  RegexConfigSchema,
+  type RunSucceededConfig,
+  RunSucceededConfigSchema,
+  regexAssertion,
+  registerCoreAssertions,
+  runSucceededAssertion,
+  subagentToolCalls,
+  type TokensUnderConfig,
+  TokensUnderConfigSchema,
+  type ToolCall,
+  type ToolCallCountConfig,
+  ToolCallCountConfigSchema,
+  type ToolCalledConfig,
+  ToolCalledConfigSchema,
+  type ToolCalledWithConfig,
+  ToolCalledWithConfigSchema,
+  type ToolInvocationConfig,
+  ToolInvocationConfigSchema,
+  type ToolResultContainsConfig,
+  ToolResultContainsConfigSchema,
+  type ToolResultIsErrorConfig,
+  ToolResultIsErrorConfigSchema,
+  type ToolResultJsonPointerEqualsConfig,
+  ToolResultJsonPointerEqualsConfigSchema,
+  tokensUnderAssertion,
+  toolCallCountAssertion,
+  toolCalledAssertion,
+  toolCalledWithAssertion,
+  toolCallInputs,
+  toolCalls,
+  toolInvocationAssertion,
+  toolResultContainsAssertion,
+  toolResultIsErrorAssertion,
+  toolResultJsonPointerEqualsAssertion,
+} from "./assertions/core/index.js";
+export {
+  allMcpAssertions,
+  type CapabilityAbsentConfig,
+  CapabilityAbsentConfigSchema,
+  capabilityAbsentAssertion,
+  registerMcpAssertions,
+  type ServerInfoEqualsConfig,
+  ServerInfoEqualsConfigSchema,
+  serverInfoEqualsAssertion,
+  type ToolListedConfig,
+  ToolListedConfigSchema,
+  type ToolsListedCountConfig,
+  ToolsListedCountConfigSchema,
+  toolListedAssertion,
+  toolsListedCountAssertion,
+} from "./assertions/mcp/index.js";
+export {
+  createRubricAssertion,
+  type RubricConfig,
+  RubricConfigSchema,
+  type RubricEvidence,
+  RubricEvidenceSchema,
+  RubricModelAccountingDetailSchema,
+  rubricAssertion,
+} from "./assertions/rubric/index.js";
+// Authoring and extension contracts.
+export {
+  defineGraderAnnotations,
+  type GraderAnnotation,
+  type GraderAnnotationDataset,
+  type GraderAnnotationDatasetInput,
+  GraderAnnotationDatasetSchema,
+  GraderAnnotationSchema,
+} from "./core/annotation.js";
+export {
+  type Answerer,
+  type AnswererContext,
+  type AnswererModelAccounting,
+  AnswererModelAccountingSchema,
+  type AnswererModelCallAccounting,
+  AnswererModelCallAccountingSchema,
+  type AnswererPlugin,
+  type AnswerHistoryEntry,
+} from "./core/answerer.js";
+export {
+  inspectRunArtifact,
+  type ReadRunArtifactTextOptions,
+  type RunArtifactInspection,
+  readRunArtifactText,
+} from "./core/artifact-evidence.js";
+export {
+  ARTIFACT_SNAPSHOT_MANIFEST,
+  type ArtifactSnapshotContent,
+  type ArtifactSnapshotFile,
+  ArtifactSnapshotFileSchema,
+  type ArtifactSnapshotFiles,
+  type ArtifactSnapshotManifest,
+  ArtifactSnapshotManifestSchema,
+  readArtifactSnapshotManifest,
+  type WriteArtifactSnapshotOptions,
+  writeArtifactSnapshot,
+} from "./core/artifact-snapshot.js";
+export {
+  type AssertionImpl,
+  type AssertionPlugin,
+  type AssertionPluginDefinition,
+  type AssertionResult,
+  AssertionResultSchema,
+  defineAssertion,
+  type EvaluateContext,
+  type GradingAssertionEntry,
+  GradingAssertionEntrySchema,
+  type GradingReport,
+  GradingReportSchema,
+} from "./core/assertion.js";
+export {
+  type AnswererRef,
+  AnswererRefSchema,
+  type Assertion,
+  AssertionSchema,
+  type DriverRef,
+  DriverRefSchema,
+  defineEvalCase,
+  type EvalCase,
+  EvalCaseSchema,
+  type FixturesRef,
+  FixturesRefSchema,
+  type PersistedCase,
+  PersistedCaseSchema,
+} from "./core/case.js";
+export {
+  CONVERSATION_TURN_REQUEST,
+  type ConversationTurnRequest,
+  ConversationTurnRequestSchema,
+  type ConversationTurnResponse,
+  ConversationTurnResponseSchema,
+  conversationTurnInterrupt,
+  conversationTurnResponse,
+  isConversationTurnInterrupt,
+} from "./core/conversation-turn.js";
+export {
+  type AgentDriver,
+  type Artifact,
+  ArtifactSchema,
+  type DriverEventMap,
+  type DriverPlugin,
+  type ResolvedFixtures,
+  ResolvedFixturesSchema,
+  type StartContext,
+} from "./core/driver.js";
+export { type EventHandler, TypedEmitter } from "./core/emitter.js";
+export {
+  type Answer,
+  AnswerSchema,
+  ApproveAnswerSchema,
+  ApproveInterruptSchema,
+  AskAnswerSchema,
+  AskInterruptSchema,
+  CustomAnswerSchema,
+  CustomInterruptSchema,
+  type Done,
+  DoneSchema,
+  type ErrorEvent,
+  type ErrorEventReason,
+  ErrorEventReasonSchema,
+  ErrorEventSchema,
+  type Interrupt,
+  type InterruptKind,
+  InterruptKindSchema,
+  InterruptSchema,
+  type LifecycleStage,
+  LifecycleStageSchema,
+  type Progress,
+  ProgressSchema,
+  type ReadyState,
+  ReadyStateSchema,
+  type RunEvent,
+  RunEventSchema,
+} from "./core/events.js";
+export {
+  type ExecOptions,
+  type ExecResult,
+  exec,
+  type ManagedProcess,
+  type ManagedProcessExit,
+  type ManagedProcessOptions,
+  spawnManagedProcess,
+} from "./core/exec.js";
+export {
+  type Candidate,
+  type CandidateEvaluation,
+  CandidateEvaluationSchema,
+  CandidateIdSchema,
+  type CandidateProposal,
+  CandidateProposalSchema,
+  CandidateSchema,
+  defineExperiment,
+  type ExperimentBudget,
+  ExperimentBudgetSchema,
+  type ExperimentCampaign,
+  type ExperimentCampaignInput,
+  ExperimentCampaignSchema,
+  type ExperimentConstraint,
+  ExperimentConstraintSchema,
+  type ExperimentObjective,
+  ExperimentObjectiveSchema,
+  type ExperimentSelector,
+  ExperimentSelectorSchema,
+  type ExperimentVariant,
+  ExperimentVariantSchema,
+  type FileSetSubject,
+  FileSetSubjectSchema,
+  type Optimizer,
+  type OptimizerInput,
+} from "./core/experiment.js";
+export {
+  defineFixtures,
+  type LifecycleFixture,
+  type LifecycleFixturesDefinition,
+  type LifecycleFixtureUse,
+} from "./core/lifecycle-fixtures.js";
+export {
+  type CaseLifecycleHookContext,
+  defineLifecycleHooks,
+  type LifecycleHookCleanup,
+  type LifecycleHookSetupResult,
+  type LifecycleHooksPlugin,
+  lifecycleHooksPluginIssue,
+  type SuiteLifecycleHookContext,
+  type SuiteLifecycleHookItem,
+} from "./core/lifecycle-hooks.js";
+export {
+  hasMcpInit,
+  hasMcpToolsList,
+  isMcpInitEvent,
+  isMcpToolsListEvent,
+  type ListedTool,
+  listedTools,
+  MCP_INIT_KIND,
+  MCP_TOOLS_LIST_KIND,
+  type McpInitPayload,
+  type McpToolsListPayload,
+  mcpInit,
+} from "./core/mcp-events.js";
+export {
+  DEFAULT_CLAUDE_MODEL,
+  DEFAULT_CODEX_MODEL,
+  DEFAULT_CODEX_REASONING_EFFORT,
+  defaultHarnessModel,
+  type HarnessModelConfig,
+  HarnessModelConfigSchema,
+  type ModelCliProvider,
+  ModelCliProviderSchema,
+  type ReasoningEffort,
+  ReasoningEffortSchema,
+} from "./core/model-cli.js";
+export {
+  type AssertionProgress,
+  AssertionProgressSchema,
+  type RunObserver,
+} from "./core/observer.js";
+export { Registry } from "./core/registry.js";
+export {
+  defineRubric,
+  type RubricDefinition,
+  type RubricDefinitionInput,
+  RubricDefinitionSchema,
+} from "./core/rubric.js";
+export {
+  type OperationalFailure,
+  OperationalFailureSchema,
+  type PhaseDurations,
+  PhaseDurationsSchema,
+  RUN_SCHEMA_VERSION,
+  type Run,
+  type RunMetadata,
+  RunMetadataSchema,
+  type RunProvenance,
+  RunProvenanceSchema,
+  RunSchema,
+} from "./core/run.js";
+export {
+  bootstrapMeanConfidenceInterval,
+  type ConfidenceConfig,
+  ConfidenceConfigSchema,
+  type ConfidenceInterval,
+  ConfidenceIntervalSchema,
+} from "./core/statistics.js";
+export { SUBJECT_ROOT_TOKEN, subjectPath } from "./core/subject.js";
+export {
+  correlateToolEvents,
+  createToolOutcomeAccumulator,
+  isToolCallEvent,
+  isToolResultEvent,
+  malformedToolEventCount,
+  summarizeToolOutcomes,
+  TOOL_CALL_KIND,
+  TOOL_RESULT_KIND,
+  type ToolCallPayload,
+  ToolCallPayloadSchema,
+  type ToolEventCorrelation,
+  type ToolInvocation,
+  type ToolInvocationOutcome,
+  type ToolOutcomeAccumulator,
+  type ToolOutcomeSummary,
+  ToolOutcomeSummarySchema,
+  type ToolResultPayload,
+  ToolResultPayloadSchema,
+  toolCallProgress,
+  toolEventCalls,
+  toolInvocationOutcome,
+  toolInvocations,
+  toolResultProgress,
+  toolResults,
+  toolResultValue,
+} from "./core/tool-events.js";
+export {
+  addUsage,
+  emptyRunUsage,
+  emptyUsage,
+  harnessUsage,
+  isUsageEmpty,
+  type RunUsage,
+  RunUsageSchema,
+  sumUsage,
+  type TokenUsage,
+  TokenUsageSchema,
+  type TurnUsage,
+  TurnUsageSchema,
+  totalTokens,
+} from "./core/usage.js";
+export {
+  type FormatAnswerFn,
+  formatWireAnswer,
+  type ParsedDriverEvent,
+  type ParseEventFn,
+  parseWireEvent,
+  type WireEvent,
+  WireEventSchema,
+} from "./core/wire.js";
+export {
+  type ClaudeCodeDriverConfig,
+  ClaudeCodeDriverConfigSchema,
+  claudeCodeDriver,
+} from "./drivers/claude-code/index.js";
+export {
+  type CodexDriverConfig,
+  CodexDriverConfigSchema,
+  codexDriver,
+} from "./drivers/codex/index.js";
+export {
+  type McpProbeDriverConfig,
+  McpProbeDriverConfigSchema,
+  mcpProbeDriver,
+  type ProbeStep,
+} from "./drivers/mcp/index.js";
+export {
+  type SubprocessDriverConfig,
+  SubprocessDriverConfigSchema,
+  subprocessDriver,
+} from "./drivers/subprocess/index.js";
+
+export {
+  createReflectiveOptimizer,
+  type ReflectiveOptimizerConfig,
+  ReflectiveOptimizerConfigSchema,
+} from "./optimization/reflective.js";
+
+// Supported programmatic orchestration.
+export {
+  applyCandidate,
+  assertDisjointAnnotationSplits,
+  buildRegistryFromConfig,
+  buildSuiteSummary,
+  type CampaignExecutionOptions,
+  type CampaignExecutionResult,
+  CampaignExecutionResultSchema,
+  type CaseFilter,
+  type CaseObserver,
+  type CaseSummary,
+  type ComparisonInput,
+  captureSeedCandidate,
+  compareGraderAnnotation,
+  compareRuns,
+  compareSuiteRuns,
+  compareSuiteSummaries,
+  createCandidate,
+  createExperimentDecisionReport,
+  type DiscoveredCase,
+  type DiscoveryProblem,
+  type DiscoveryResult,
+  type DoctorCampaignReport,
+  DoctorCampaignReportSchema,
+  type DoctorDiagnostic,
+  DoctorDiagnosticSchema,
+  type DoctorOptions,
+  DoctorOptionsSchema,
+  type DoctorReport,
+  DoctorReportSchema,
+  deriveExperimentSelection,
+  discoverCases,
+  doctorProject,
+  type ExperimentBudgetUsage,
+  type ExperimentCampaignPlan,
+  ExperimentCampaignPlanSchema,
+  type ExperimentCommandMode,
+  ExperimentCommandModeSchema,
+  type ExperimentDecisionCandidate,
+  ExperimentDecisionCandidateSchema,
+  type ExperimentDecisionReport,
+  ExperimentDecisionReportSchema,
+  type ExperimentEvaluation,
+  type ExperimentProfile,
+  ExperimentProfileSchema,
+  type ExperimentRun,
+  ExperimentRunSchema,
+  type ExperimentSelection,
+  ExperimentSelectionSchema,
+  evaluateGraderCandidate,
+  experimentCampaignMode,
+  findConfigFile,
+  formatComparisonText,
+  formatDoctorReport,
+  formatExperimentDecisionMarkdown,
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  formatRunDetail,
+  formatRunsTable,
+  formatSuiteComparisonText,
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  GRADING_SNAPSHOT_SCHEMA_VERSION,
+  type GraderEvaluationEvidence,
+  GraderEvaluationEvidenceSchema,
+  type GraderJudgment,
+  type GradingDelta,
+  type GradingSnapshot,
+  GradingSnapshotSchema,
+  gradeRun,
+  hashSubjectTree,
+  hasRegression,
+  hasSuiteRegression,
+  isExperimentRun,
+  type ListedRun,
+  type LoadedExperimentCampaign,
+  LoadedExperimentCampaignSchema,
+  type LoadedGraderAnnotations,
+  LoadedGraderAnnotationsSchema,
+  type LuxConfig,
+  listExperimentRuns,
+  listRuns,
+  listSuiteRuns,
+  loadEvalCase,
+  loadExperimentCampaign,
+  loadExperimentCandidate,
+  loadExperimentCandidates,
+  loadExperimentDecisionReport,
+  loadExperimentRun,
+  loadFrozenRegradableRun,
+  loadGraderAnnotations,
+  loadLuxConfig,
+  loadRegradableRun,
+  loadRubricDefinition,
+  loadRunBundle,
+  loadRunDetail,
+  loadSuiteRun,
+  loadSuiteSummary,
+  loadVerifiedExperimentDecisionSnapshot,
+  type MetricSummary,
+  MetricSummarySchema,
+  materializeCandidate,
+  migrateExperimentCandidateIntegrity,
+  Orchestrator,
+  type OrchestratorOptions,
+  type OutcomeSummary,
+  optimizeExperimentCampaign,
+  paretoFrontier,
+  planExperimentCampaign,
+  type ResolvedLuxConfig,
+  type RunBundle,
+  type RunComparison,
+  type RunDetail,
+  type RunRequest,
+  type RunResult,
+  rankCandidates,
+  regradeRun,
+  releaseFrozenRegradableRun,
+  resolveExperimentCampaignPath,
+  runExperimentCampaign,
+  type SuiteCaseComparison,
+  SuiteCaseComparisonSchema,
+  type SuiteCaseResult,
+  type SuiteComparison,
+  SuiteComparisonSchema,
+  type SuiteItem,
+  type SuiteObserver,
+  type SuiteRun,
+  type SuiteRunItem,
+  SuiteRunSchema,
+  type SuiteSample,
+  SuiteSampleSchema,
+  type SuiteSummary,
+  SuiteSummarySchema,
+  selectCases,
+  selectGraderAnnotations,
+  summarizeGraderEvidence,
+  summarizeNumbers,
+  upsertGraderAnnotation,
+  type VerifiedExperimentDecisionSnapshot,
+  VerifiedExperimentDecisionSnapshotSchema,
+  validateAssertionPlan,
+  validateCandidate,
+  verifyExperimentDecisionIntegrity,
+  verifyExperimentRunIntegrity,
+  writeSuiteSummary,
+} from "./orchestrator/index.js";