@elevasis/core 0.2.0 → 0.3.0

package/src/execution/calibration/schemas.ts

@@ -1,121 +0,0 @@
-import { z } from 'zod'
-
-// ============================================================================
-// LLM Model Schema
-// ============================================================================
-
-/**
- * LLM model validation schema
- * Matches LLMModel type from model-info.ts
- */
-const LLMModelSchema = z.enum([
-  'gpt-5',
-  'gpt-5.4-mini',
-  'gpt-5.4-nano',
-  'openrouter/z-ai/glm-5',
-  'gemini-3-flash-preview',
-  'gemini-3.1-flash-lite-preview',
-  'mock'
-])
-
-// ============================================================================
-// Grading Schemas
-// ============================================================================
-
-/**
- * Grading criterion schema
- * Defines a single criterion for evaluating AI outputs
- */
-const GradingCriterionSchema = z.object({
-  name: z.string().min(1),
-  weight: z.number().min(0).max(1),
-  description: z.string().min(1),
-  scoringGuide: z.string().min(1)
-})
-
-/**
- * Grading rubric schema
- * Collection of criteria with passing threshold
- */
-const GradingRubricSchema = z.object({
-  passingThreshold: z.number().min(0).max(1),
-  criteria: z.array(GradingCriterionSchema).min(1)
-})
-
-// ============================================================================
-// Config Variant Schema
-// ============================================================================
-
-/**
- * Configuration variant schema
- * Defines a test variant with optional overrides
- */
-const ConfigVariantSchema = z.object({
-  variantName: z.string().min(1),
-  definitionOverrides: z.record(z.string(), z.unknown()).optional() // Flexible for agent/workflow overrides
-})
-
-// ============================================================================
-// Calibration Project Schemas
-// ============================================================================
-
-/**
- * Create calibration project schema
- * Validates request body for creating a new calibration project
- */
-export const CreateCalibrationProjectSchema = z.object({
-  resourceId: z.string().min(1),
-  resourceType: z.enum(['agent', 'workflow']),
-  name: z.string().min(1),
-  description: z.string().optional()
-})
-
-/**
- * Update calibration project schema
- * Validates request body for updating an existing calibration project
- */
-export const UpdateCalibrationProjectSchema = z.object({
-  name: z.string().min(1).optional(),
-  description: z.string().optional()
-})
-
-// ============================================================================
-// Calibration Run Schemas
-// ============================================================================
-
-/**
- * Create calibration run schema
- * Validates request body for creating a new calibration run
- * Includes refinement for graderModel requirement when gradingRubric is provided
- */
-export const CreateCalibrationRunSchema = z
-  .object({
-    projectId: z.string().uuid(),
-    name: z.string().min(1),
-    description: z.string().optional(),
-    executionMode: z.enum(['single', 'session']).default('single'),
-    testInputs: z.array(z.unknown()).min(1).max(50), // Min 1, max 50 inputs
-    configVariants: z.array(ConfigVariantSchema).min(1).max(10), // Min 1, max 10 variants
-    gradingRubric: GradingRubricSchema.optional(),
-    graderModel: LLMModelSchema.optional()
-  })
-  .refine((data) => !data.gradingRubric || data.graderModel, {
-    message: 'graderModel is required when gradingRubric is provided'
-  })
-
-// ============================================================================
-// Exported Schemas
-// ============================================================================
-
-export { GradingRubricSchema, GradingCriterionSchema, ConfigVariantSchema }
-
-// ============================================================================
-// Inferred Types
-// ============================================================================
-
-export type CreateCalibrationProjectInput = z.infer<typeof CreateCalibrationProjectSchema>
-export type UpdateCalibrationProjectInput = z.infer<typeof UpdateCalibrationProjectSchema>
-export type CreateCalibrationRunInput = z.infer<typeof CreateCalibrationRunSchema>
-export type GradingRubricInput = z.infer<typeof GradingRubricSchema>
-export type GradingCriterionInput = z.infer<typeof GradingCriterionSchema>
-export type ConfigVariantInput = z.infer<typeof ConfigVariantSchema>

package/src/execution/calibration/sse-events.ts

@@ -1,125 +0,0 @@
-/**
- * Calibration SSE Event Types
- *
- * Shared event type definitions for calibration real-time streaming.
- * Used by both the API broadcaster and the command-center UI.
- */
-
-// Single-turn events
-
-export interface CalibrationExecutionStartedEvent {
-  type: 'execution-started'
-  variantName: string
-  inputIndex: number
-  timestamp: number
-}
-
-export interface CalibrationExecutionCompletedEvent {
-  type: 'execution-completed'
-  variantName: string
-  inputIndex: number
-  executionId: string
-  timestamp: number
-}
-
-export interface CalibrationExecutionFailedEvent {
-  type: 'execution-failed'
-  variantName: string
-  inputIndex?: number
-  error: string
-  timestamp: number
-}
-
-// Session events
-
-export interface CalibrationSessionStartedEvent {
-  type: 'session-started'
-  variantName: string
-  sessionId: string
-  timestamp: number
-}
-
-export interface CalibrationTurnStartedEvent {
-  type: 'turn-started'
-  variantName: string
-  turnNumber: number
-  timestamp: number
-}
-
-export interface CalibrationTurnCompletedEvent {
-  type: 'turn-completed'
-  variantName: string
-  turnNumber: number
-  executionId: string
-  timestamp: number
-}
-
-export interface CalibrationSessionCompletedEvent {
-  type: 'session-completed'
-  variantName: string
-  sessionId: string
-  turnCount: number
-  timestamp: number
-}
-
-// Grading events
-
-export interface CalibrationGradingStartedEvent {
-  type: 'grading-started'
-  variantName: string
-  timestamp: number
-}
-
-export interface CalibrationGradingCompletedEvent {
-  type: 'grading-completed'
-  variantName: string
-  score: number
-  timestamp: number
-}
-
-export interface CalibrationGradingFailedEvent {
-  type: 'grading-failed'
-  variantName: string
-  error: string
-  timestamp: number
-}
-
-// Completion events
-
-export interface CalibrationCompletedEvent {
-  type: 'calibration-completed'
-  summary: { total: number; completed: number; failed: number }
-  timestamp: number
-}
-
-export interface CalibrationFailedEvent {
-  type: 'calibration-failed'
-  error: string
-  timestamp: number
-}
-
-// Connection event
-
-export interface CalibrationConnectedEvent {
-  type: 'connected'
-  timestamp: number
-  data?: undefined
-}
-
-/**
- * Union of all calibration SSE events
- */
-export type CalibrationSSEEvent =
-  | CalibrationExecutionStartedEvent
-  | CalibrationExecutionCompletedEvent
-  | CalibrationExecutionFailedEvent
-  | CalibrationSessionStartedEvent
-  | CalibrationTurnStartedEvent
-  | CalibrationTurnCompletedEvent
-  | CalibrationSessionCompletedEvent
-  | CalibrationGradingStartedEvent
-  | CalibrationGradingCompletedEvent
-  | CalibrationGradingFailedEvent
-  | CalibrationCompletedEvent
-  | CalibrationFailedEvent
-  | CalibrationConnectedEvent

package/src/execution/calibration/types.ts

@@ -1,190 +0,0 @@
-/**
- * Calibration Lab Type Definitions
- * Core types for AI model configuration optimization with cost/performance comparison
- */
-
-import type { AgentDefinition } from '../engine/agent/core/types'
-import type { WorkflowDefinition } from '../engine/workflow/types'
-import type { ExecutionMetricsDetail } from '../../operations/observability/types'
-
-// ============================================================================
-// Calibration Project Types
-// ============================================================================
-
-/**
- * Calibration project - groups related optimization runs
- * Provides organizational structure for iterative testing
- */
-export interface CalibrationProject {
-  id: string
-  organizationId: string
-  resourceId: string
-  resourceType: 'agent' | 'workflow'
-  name: string
-  description?: string | null
-  createdAt: Date
-  updatedAt: Date
-}
-
-// ============================================================================
-// Configuration Variant Types
-// ============================================================================
-
-/**
- * Configuration variant for testing
- * Defines what to override in the base agent/workflow definition
- */
-export interface ConfigVariant {
-  variantName: string
-  definitionOverrides?: AgentCalibrationOverrides | WorkflowCalibrationOverrides
-}
-
-/**
- * Agent definition overrides for calibration
- * Excludes identity/metadata fields that should not be modified
- */
-export type AgentCalibrationOverrides = Omit<
-  Partial<AgentDefinition>,
-  'config' | 'contract' | 'tools' | 'metricsConfig' | 'interface'
->
-
-/**
- * Workflow definition overrides for calibration
- * Excludes identity/metadata fields that should not be modified
- */
-export type WorkflowCalibrationOverrides = Omit<
-  Partial<WorkflowDefinition>,
-  'config' | 'contract' | 'steps' | 'entryPoint' | 'metricsConfig' | 'interface'
->
-
-// ============================================================================
-// Grading System Types
-// ============================================================================
-
-/**
- * Grading rubric for LLM-as-judge evaluation
- * Defines criteria and passing threshold
- */
-export interface GradingRubric {
-  passingThreshold: number // 0-1, e.g., 0.7 means 70% to pass
-  criteria: GradingCriterion[]
-}
-
-/**
- * Individual grading criterion
- * Weight should be 0-1, and all weights should sum to 1
- */
-export interface GradingCriterion {
-  name: string
-  weight: number // 0-1, weights should sum to 1
-  description: string
-  scoringGuide: string
-}
-
-/**
- * Grading result for a single execution or session
- * Contains overall score and per-criterion breakdown
- */
-export interface GradeResult {
-  score: number // 0-1
-  passed: boolean
-  details: Record<string, { score: number; justification: string }>
-}
-
-// ============================================================================
-// Result Types
-// ============================================================================
-
-/**
- * Single-turn calibration result
- * One result per (variant × input) combination
- */
-export interface SingleCalibrationResult {
-  executionId: string // Reference to execution_logs
-  variantName: string
-  inputIndex: number // Which input from testInputs array (0-based)
-  appliedOverrides?: AgentCalibrationOverrides | WorkflowCalibrationOverrides
-  status: 'pending' | 'running' | 'completed' | 'failed'
-  errorMessage?: string
-  grade?: GradeResult
-  gradeError?: string
-}
-
-/**
- * Multi-turn session calibration result
- * Leverages existing sessions infrastructure
- */
-export interface SessionCalibrationResult {
-  sessionId: string // Reference to sessions table
-  variantName: string
-  appliedOverrides?: AgentCalibrationOverrides | WorkflowCalibrationOverrides
-  status: 'pending' | 'running' | 'completed' | 'failed'
-  errorMessage?: string
-  turnCount: number
-  grade?: GradeResult
-  gradeError?: string
-}
-
-// ============================================================================
-// Calibration Run Types
-// ============================================================================
-
-/**
- * Calibration run - individual test execution within a project
- * Contains configuration, results, and grading information
- */
-export interface CalibrationRun {
-  id: string
-  organizationId: string
-  projectId: string
-  name: string
-  description?: string | null
-  executionMode: 'single' | 'session'
-  testInputs: unknown[]
-  configVariants: ConfigVariant[]
-  gradingRubric?: GradingRubric | null
-  graderModel?: string | null
-  results: (SingleCalibrationResult | SessionCalibrationResult)[]
-  status: 'pending' | 'running' | 'completed' | 'partial' | 'failed'
-  createdAt: Date
-  completedAt?: Date | null
-}
-
-// ============================================================================
-// Combined Response Types
-// ============================================================================
-
-/**
- * Execution log subset for comparison view
- * Contains essential data without full execution context
- */
-export interface ExecutionLog {
-  executionId: string
-  status: 'running' | 'completed' | 'failed'
-  input: unknown
-  output: unknown | null
-  error: CalibrationExecutionError | null
-  startedAt: string
-  completedAt: string | null
-}
-
-/**
- * Execution error information for calibration display
- * Subset of full execution error for UI display purposes
- * Named differently from ExecutionError class to avoid type conflicts
- */
-export interface CalibrationExecutionError {
-  message: string
-  category?: string
-  type?: string
-}
-
-/**
- * Combined calibration run data for comparison view
- * Single API call returns run + execution logs + metrics
- */
-export interface CalibrationRunWithFullData {
-  run: CalibrationRun
-  logs: Record<string, ExecutionLog> // keyed by executionId
-  metrics: Record<string, ExecutionMetricsDetail> // keyed by executionId
-}