@elevasis/core 0.2.1 → 0.4.0

package/src/execution/calibration/__tests__/schemas.test.ts

@@ -1,320 +0,0 @@
-import { describe, it, expect } from 'vitest'
-import {
-  CreateCalibrationProjectSchema,
-  UpdateCalibrationProjectSchema,
-  CreateCalibrationRunSchema,
-  GradingRubricSchema,
-  GradingCriterionSchema,
-  ConfigVariantSchema
-} from '../schemas'
-
-describe('Calibration Schemas', () => {
-  describe('GradingCriterionSchema', () => {
-    it('validates a valid grading criterion', () => {
-      const valid = {
-        name: 'Accuracy',
-        weight: 0.5,
-        description: 'Measures factual correctness',
-        scoringGuide: 'Check all facts are correct'
-      }
-
-      const result = GradingCriterionSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('rejects invalid weight (> 1)', () => {
-      const invalid = {
-        name: 'Accuracy',
-        weight: 1.5,
-        description: 'Measures factual correctness',
-        scoringGuide: 'Check all facts are correct'
-      }
-
-      const result = GradingCriterionSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('rejects invalid weight (< 0)', () => {
-      const invalid = {
-        name: 'Accuracy',
-        weight: -0.1,
-        description: 'Measures factual correctness',
-        scoringGuide: 'Check all facts are correct'
-      }
-
-      const result = GradingCriterionSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-  })
-
-  describe('GradingRubricSchema', () => {
-    it('validates a valid grading rubric', () => {
-      const valid = {
-        passingThreshold: 0.7,
-        criteria: [
-          {
-            name: 'Accuracy',
-            weight: 0.4,
-            description: 'Factual correctness',
-            scoringGuide: 'Verify claims'
-          },
-          {
-            name: 'Completeness',
-            weight: 0.6,
-            description: 'Covers all aspects',
-            scoringGuide: 'Check coverage'
-          }
-        ]
-      }
-
-      const result = GradingRubricSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('rejects empty criteria array', () => {
-      const invalid = {
-        passingThreshold: 0.7,
-        criteria: []
-      }
-
-      const result = GradingRubricSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-  })
-
-  describe('ConfigVariantSchema', () => {
-    it('validates a variant with overrides', () => {
-      const valid = {
-        variantName: 'GPT-5 High Reasoning',
-        definitionOverrides: {
-          modelConfig: { model: 'gpt-5', temperature: 1 }
-        }
-      }
-
-      const result = ConfigVariantSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('validates a variant without overrides', () => {
-      const valid = {
-        variantName: 'Baseline'
-      }
-
-      const result = ConfigVariantSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-  })
-
-  describe('CreateCalibrationProjectSchema', () => {
-    it('validates a valid project creation', () => {
-      const valid = {
-        resourceId: 'research-agent',
-        resourceType: 'agent' as const,
-        name: 'Q1 2025 Model Selection',
-        description: 'Finding optimal model'
-      }
-
-      const result = CreateCalibrationProjectSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('validates project without description', () => {
-      const valid = {
-        resourceId: 'research-agent',
-        resourceType: 'workflow' as const,
-        name: 'Q1 2025 Model Selection'
-      }
-
-      const result = CreateCalibrationProjectSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('rejects invalid resourceType', () => {
-      const invalid = {
-        resourceId: 'research-agent',
-        resourceType: 'invalid',
-        name: 'Q1 2025 Model Selection'
-      }
-
-      const result = CreateCalibrationProjectSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-  })
-
-  describe('UpdateCalibrationProjectSchema', () => {
-    it('validates name update', () => {
-      const valid = {
-        name: 'Updated Name'
-      }
-
-      const result = UpdateCalibrationProjectSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('validates description update', () => {
-      const valid = {
-        description: 'Updated description'
-      }
-
-      const result = UpdateCalibrationProjectSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('validates both fields', () => {
-      const valid = {
-        name: 'Updated Name',
-        description: 'Updated description'
-      }
-
-      const result = UpdateCalibrationProjectSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-  })
-
-  describe('CreateCalibrationRunSchema', () => {
-    it('validates a valid calibration run', () => {
-      const valid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        executionMode: 'single' as const,
-        testInputs: [{ query: 'Test query' }],
-        configVariants: [
-          {
-            variantName: 'GPT-5',
-            definitionOverrides: { modelConfig: { model: 'gpt-5' } }
-          }
-        ]
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('validates run with grading rubric and model', () => {
-      const valid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [{ query: 'Test query' }],
-        configVariants: [
-          {
-            variantName: 'GPT-5'
-          }
-        ],
-        gradingRubric: {
-          passingThreshold: 0.7,
-          criteria: [
-            {
-              name: 'Accuracy',
-              weight: 1,
-              description: 'Factual correctness',
-              scoringGuide: 'Verify claims'
-            }
-          ]
-        },
-        graderModel: 'gpt-5' as const
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(valid)
-      expect(result.success).toBe(true)
-    })
-
-    it('rejects grading rubric without grader model', () => {
-      const invalid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [{ query: 'Test query' }],
-        configVariants: [
-          {
-            variantName: 'GPT-5'
-          }
-        ],
-        gradingRubric: {
-          passingThreshold: 0.7,
-          criteria: [
-            {
-              name: 'Accuracy',
-              weight: 1,
-              description: 'Factual correctness',
-              scoringGuide: 'Verify claims'
-            }
-          ]
-        }
-        // Missing graderModel
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('rejects empty testInputs array', () => {
-      const invalid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [],
-        configVariants: [
-          {
-            variantName: 'GPT-5'
-          }
-        ]
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('rejects more than 50 testInputs', () => {
-      const invalid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: Array(51).fill({ query: 'Test' }),
-        configVariants: [
-          {
-            variantName: 'GPT-5'
-          }
-        ]
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('rejects empty configVariants array', () => {
-      const invalid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [{ query: 'Test' }],
-        configVariants: []
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('rejects more than 10 configVariants', () => {
-      const invalid = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [{ query: 'Test' }],
-        configVariants: Array(11).fill({ variantName: 'Test' })
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(invalid)
-      expect(result.success).toBe(false)
-    })
-
-    it('applies default executionMode', () => {
-      const input = {
-        projectId: '123e4567-e89b-12d3-a456-426614174000',
-        name: 'Model comparison',
-        testInputs: [{ query: 'Test' }],
-        configVariants: [{ variantName: 'GPT-5' }]
-      }
-
-      const result = CreateCalibrationRunSchema.safeParse(input)
-      expect(result.success).toBe(true)
-      if (result.success) {
-        expect(result.data.executionMode).toBe('single')
-      }
-    })
-  })
-})

package/src/execution/calibration/index.ts

@@ -1,3 +0,0 @@
-export * from './types'
-export * from './schemas'
-export * from './sse-events'

package/src/execution/calibration/schemas.ts

@@ -1,121 +0,0 @@
-import { z } from 'zod'
-
-// ============================================================================
-// LLM Model Schema
-// ============================================================================
-
-/**
- * LLM model validation schema
- * Matches LLMModel type from model-info.ts
- */
-const LLMModelSchema = z.enum([
-  'gpt-5',
-  'gpt-5.4-mini',
-  'gpt-5.4-nano',
-  'openrouter/z-ai/glm-5',
-  'gemini-3-flash-preview',
-  'gemini-3.1-flash-lite-preview',
-  'mock'
-])
-
-// ============================================================================
-// Grading Schemas
-// ============================================================================
-
-/**
- * Grading criterion schema
- * Defines a single criterion for evaluating AI outputs
- */
-const GradingCriterionSchema = z.object({
-  name: z.string().min(1),
-  weight: z.number().min(0).max(1),
-  description: z.string().min(1),
-  scoringGuide: z.string().min(1)
-})
-
-/**
- * Grading rubric schema
- * Collection of criteria with passing threshold
- */
-const GradingRubricSchema = z.object({
-  passingThreshold: z.number().min(0).max(1),
-  criteria: z.array(GradingCriterionSchema).min(1)
-})
-
-// ============================================================================
-// Config Variant Schema
-// ============================================================================
-
-/**
- * Configuration variant schema
- * Defines a test variant with optional overrides
- */
-const ConfigVariantSchema = z.object({
-  variantName: z.string().min(1),
-  definitionOverrides: z.record(z.string(), z.unknown()).optional() // Flexible for agent/workflow overrides
-})
-
-// ============================================================================
-// Calibration Project Schemas
-// ============================================================================
-
-/**
- * Create calibration project schema
- * Validates request body for creating a new calibration project
- */
-export const CreateCalibrationProjectSchema = z.object({
-  resourceId: z.string().min(1),
-  resourceType: z.enum(['agent', 'workflow']),
-  name: z.string().min(1),
-  description: z.string().optional()
-})
-
-/**
- * Update calibration project schema
- * Validates request body for updating an existing calibration project
- */
-export const UpdateCalibrationProjectSchema = z.object({
-  name: z.string().min(1).optional(),
-  description: z.string().optional()
-})
-
-// ============================================================================
-// Calibration Run Schemas
-// ============================================================================
-
-/**
- * Create calibration run schema
- * Validates request body for creating a new calibration run
- * Includes refinement for graderModel requirement when gradingRubric is provided
- */
-export const CreateCalibrationRunSchema = z
-  .object({
-    projectId: z.string().uuid(),
-    name: z.string().min(1),
-    description: z.string().optional(),
-    executionMode: z.enum(['single', 'session']).default('single'),
-    testInputs: z.array(z.unknown()).min(1).max(50), // Min 1, max 50 inputs
-    configVariants: z.array(ConfigVariantSchema).min(1).max(10), // Min 1, max 10 variants
-    gradingRubric: GradingRubricSchema.optional(),
-    graderModel: LLMModelSchema.optional()
-  })
-  .refine((data) => !data.gradingRubric || data.graderModel, {
-    message: 'graderModel is required when gradingRubric is provided'
-  })
-
-// ============================================================================
-// Exported Schemas
-// ============================================================================
-
-export { GradingRubricSchema, GradingCriterionSchema, ConfigVariantSchema }
-
-// ============================================================================
-// Inferred Types
-// ============================================================================
-
-export type CreateCalibrationProjectInput = z.infer<typeof CreateCalibrationProjectSchema>
-export type UpdateCalibrationProjectInput = z.infer<typeof UpdateCalibrationProjectSchema>
-export type CreateCalibrationRunInput = z.infer<typeof CreateCalibrationRunSchema>
-export type GradingRubricInput = z.infer<typeof GradingRubricSchema>
-export type GradingCriterionInput = z.infer<typeof GradingCriterionSchema>
-export type ConfigVariantInput = z.infer<typeof ConfigVariantSchema>

package/src/execution/calibration/sse-events.ts

@@ -1,125 +0,0 @@
-/**
- * Calibration SSE Event Types
- *
- * Shared event type definitions for calibration real-time streaming.
- * Used by both the API broadcaster and the command-center UI.
- */
-
-// Single-turn events
-
-export interface CalibrationExecutionStartedEvent {
-  type: 'execution-started'
-  variantName: string
-  inputIndex: number
-  timestamp: number
-}
-
-export interface CalibrationExecutionCompletedEvent {
-  type: 'execution-completed'
-  variantName: string
-  inputIndex: number
-  executionId: string
-  timestamp: number
-}
-
-export interface CalibrationExecutionFailedEvent {
-  type: 'execution-failed'
-  variantName: string
-  inputIndex?: number
-  error: string
-  timestamp: number
-}
-
-// Session events
-
-export interface CalibrationSessionStartedEvent {
-  type: 'session-started'
-  variantName: string
-  sessionId: string
-  timestamp: number
-}
-
-export interface CalibrationTurnStartedEvent {
-  type: 'turn-started'
-  variantName: string
-  turnNumber: number
-  timestamp: number
-}
-
-export interface CalibrationTurnCompletedEvent {
-  type: 'turn-completed'
-  variantName: string
-  turnNumber: number
-  executionId: string
-  timestamp: number
-}
-
-export interface CalibrationSessionCompletedEvent {
-  type: 'session-completed'
-  variantName: string
-  sessionId: string
-  turnCount: number
-  timestamp: number
-}
-
-// Grading events
-
-export interface CalibrationGradingStartedEvent {
-  type: 'grading-started'
-  variantName: string
-  timestamp: number
-}
-
-export interface CalibrationGradingCompletedEvent {
-  type: 'grading-completed'
-  variantName: string
-  score: number
-  timestamp: number
-}
-
-export interface CalibrationGradingFailedEvent {
-  type: 'grading-failed'
-  variantName: string
-  error: string
-  timestamp: number
-}
-
-// Completion events
-
-export interface CalibrationCompletedEvent {
-  type: 'calibration-completed'
-  summary: { total: number; completed: number; failed: number }
-  timestamp: number
-}
-
-export interface CalibrationFailedEvent {
-  type: 'calibration-failed'
-  error: string
-  timestamp: number
-}
-
-// Connection event
-
-export interface CalibrationConnectedEvent {
-  type: 'connected'
-  timestamp: number
-  data?: undefined
-}
-
-/**
- * Union of all calibration SSE events
- */
-export type CalibrationSSEEvent =
-  | CalibrationExecutionStartedEvent
-  | CalibrationExecutionCompletedEvent
-  | CalibrationExecutionFailedEvent
-  | CalibrationSessionStartedEvent
-  | CalibrationTurnStartedEvent
-  | CalibrationTurnCompletedEvent
-  | CalibrationSessionCompletedEvent
-  | CalibrationGradingStartedEvent
-  | CalibrationGradingCompletedEvent
-  | CalibrationGradingFailedEvent
-  | CalibrationCompletedEvent
-  | CalibrationFailedEvent
-  | CalibrationConnectedEvent