@elevasis/core 0.1.0

package/src/execution/calibration/__tests__/schemas.test.ts

@@ -0,0 +1,320 @@
+import { describe, it, expect } from 'vitest'
+import {
+  CreateCalibrationProjectSchema,
+  UpdateCalibrationProjectSchema,
+  CreateCalibrationRunSchema,
+  GradingRubricSchema,
+  GradingCriterionSchema,
+  ConfigVariantSchema
+} from '../schemas'
+
+describe('Calibration Schemas', () => {
+  describe('GradingCriterionSchema', () => {
+    it('validates a valid grading criterion', () => {
+      const valid = {
+        name: 'Accuracy',
+        weight: 0.5,
+        description: 'Measures factual correctness',
+        scoringGuide: 'Check all facts are correct'
+      }
+
+      const result = GradingCriterionSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects invalid weight (> 1)', () => {
+      const invalid = {
+        name: 'Accuracy',
+        weight: 1.5,
+        description: 'Measures factual correctness',
+        scoringGuide: 'Check all facts are correct'
+      }
+
+      const result = GradingCriterionSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects invalid weight (< 0)', () => {
+      const invalid = {
+        name: 'Accuracy',
+        weight: -0.1,
+        description: 'Measures factual correctness',
+        scoringGuide: 'Check all facts are correct'
+      }
+
+      const result = GradingCriterionSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+  })
+
+  describe('GradingRubricSchema', () => {
+    it('validates a valid grading rubric', () => {
+      const valid = {
+        passingThreshold: 0.7,
+        criteria: [
+          {
+            name: 'Accuracy',
+            weight: 0.4,
+            description: 'Factual correctness',
+            scoringGuide: 'Verify claims'
+          },
+          {
+            name: 'Completeness',
+            weight: 0.6,
+            description: 'Covers all aspects',
+            scoringGuide: 'Check coverage'
+          }
+        ]
+      }
+
+      const result = GradingRubricSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects empty criteria array', () => {
+      const invalid = {
+        passingThreshold: 0.7,
+        criteria: []
+      }
+
+      const result = GradingRubricSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+  })
+
+  describe('ConfigVariantSchema', () => {
+    it('validates a variant with overrides', () => {
+      const valid = {
+        variantName: 'GPT-5 High Reasoning',
+        definitionOverrides: {
+          modelConfig: { model: 'gpt-5', temperature: 1 }
+        }
+      }
+
+      const result = ConfigVariantSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates a variant without overrides', () => {
+      const valid = {
+        variantName: 'Baseline'
+      }
+
+      const result = ConfigVariantSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+  })
+
+  describe('CreateCalibrationProjectSchema', () => {
+    it('validates a valid project creation', () => {
+      const valid = {
+        resourceId: 'research-agent',
+        resourceType: 'agent' as const,
+        name: 'Q1 2025 Model Selection',
+        description: 'Finding optimal model'
+      }
+
+      const result = CreateCalibrationProjectSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates project without description', () => {
+      const valid = {
+        resourceId: 'research-agent',
+        resourceType: 'workflow' as const,
+        name: 'Q1 2025 Model Selection'
+      }
+
+      const result = CreateCalibrationProjectSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects invalid resourceType', () => {
+      const invalid = {
+        resourceId: 'research-agent',
+        resourceType: 'invalid',
+        name: 'Q1 2025 Model Selection'
+      }
+
+      const result = CreateCalibrationProjectSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+  })
+
+  describe('UpdateCalibrationProjectSchema', () => {
+    it('validates name update', () => {
+      const valid = {
+        name: 'Updated Name'
+      }
+
+      const result = UpdateCalibrationProjectSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates description update', () => {
+      const valid = {
+        description: 'Updated description'
+      }
+
+      const result = UpdateCalibrationProjectSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates both fields', () => {
+      const valid = {
+        name: 'Updated Name',
+        description: 'Updated description'
+      }
+
+      const result = UpdateCalibrationProjectSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+  })
+
+  describe('CreateCalibrationRunSchema', () => {
+    it('validates a valid calibration run', () => {
+      const valid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        executionMode: 'single' as const,
+        testInputs: [{ query: 'Test query' }],
+        configVariants: [
+          {
+            variantName: 'GPT-5',
+            definitionOverrides: { modelConfig: { model: 'gpt-5' } }
+          }
+        ]
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('validates run with grading rubric and model', () => {
+      const valid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [{ query: 'Test query' }],
+        configVariants: [
+          {
+            variantName: 'GPT-5'
+          }
+        ],
+        gradingRubric: {
+          passingThreshold: 0.7,
+          criteria: [
+            {
+              name: 'Accuracy',
+              weight: 1,
+              description: 'Factual correctness',
+              scoringGuide: 'Verify claims'
+            }
+          ]
+        },
+        graderModel: 'gpt-5' as const
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(valid)
+      expect(result.success).toBe(true)
+    })
+
+    it('rejects grading rubric without grader model', () => {
+      const invalid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [{ query: 'Test query' }],
+        configVariants: [
+          {
+            variantName: 'GPT-5'
+          }
+        ],
+        gradingRubric: {
+          passingThreshold: 0.7,
+          criteria: [
+            {
+              name: 'Accuracy',
+              weight: 1,
+              description: 'Factual correctness',
+              scoringGuide: 'Verify claims'
+            }
+          ]
+        }
+        // Missing graderModel
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects empty testInputs array', () => {
+      const invalid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [],
+        configVariants: [
+          {
+            variantName: 'GPT-5'
+          }
+        ]
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects more than 50 testInputs', () => {
+      const invalid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: Array(51).fill({ query: 'Test' }),
+        configVariants: [
+          {
+            variantName: 'GPT-5'
+          }
+        ]
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects empty configVariants array', () => {
+      const invalid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [{ query: 'Test' }],
+        configVariants: []
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('rejects more than 10 configVariants', () => {
+      const invalid = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [{ query: 'Test' }],
+        configVariants: Array(11).fill({ variantName: 'Test' })
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(invalid)
+      expect(result.success).toBe(false)
+    })
+
+    it('applies default executionMode', () => {
+      const input = {
+        projectId: '123e4567-e89b-12d3-a456-426614174000',
+        name: 'Model comparison',
+        testInputs: [{ query: 'Test' }],
+        configVariants: [{ variantName: 'GPT-5' }]
+      }
+
+      const result = CreateCalibrationRunSchema.safeParse(input)
+      expect(result.success).toBe(true)
+      if (result.success) {
+        expect(result.data.executionMode).toBe('single')
+      }
+    })
+  })
+})

package/src/execution/calibration/index.ts

@@ -0,0 +1,3 @@
+export * from './types'
+export * from './schemas'
+export * from './sse-events'

package/src/execution/calibration/schemas.ts

@@ -0,0 +1,121 @@
+import { z } from 'zod'
+
+// ============================================================================
+// LLM Model Schema
+// ============================================================================
+
+/**
+ * LLM model validation schema
+ * Matches LLMModel type from model-info.ts
+ */
+const LLMModelSchema = z.enum([
+  'gpt-5',
+  'gpt-5.4-mini',
+  'gpt-5.4-nano',
+  'openrouter/z-ai/glm-5',
+  'gemini-3-flash-preview',
+  'gemini-3.1-flash-lite-preview',
+  'mock'
+])
+
+// ============================================================================
+// Grading Schemas
+// ============================================================================
+
+/**
+ * Grading criterion schema
+ * Defines a single criterion for evaluating AI outputs
+ */
+const GradingCriterionSchema = z.object({
+  name: z.string().min(1),
+  weight: z.number().min(0).max(1),
+  description: z.string().min(1),
+  scoringGuide: z.string().min(1)
+})
+
+/**
+ * Grading rubric schema
+ * Collection of criteria with passing threshold
+ */
+const GradingRubricSchema = z.object({
+  passingThreshold: z.number().min(0).max(1),
+  criteria: z.array(GradingCriterionSchema).min(1)
+})
+
+// ============================================================================
+// Config Variant Schema
+// ============================================================================
+
+/**
+ * Configuration variant schema
+ * Defines a test variant with optional overrides
+ */
+const ConfigVariantSchema = z.object({
+  variantName: z.string().min(1),
+  definitionOverrides: z.record(z.string(), z.unknown()).optional() // Flexible for agent/workflow overrides
+})
+
+// ============================================================================
+// Calibration Project Schemas
+// ============================================================================
+
+/**
+ * Create calibration project schema
+ * Validates request body for creating a new calibration project
+ */
+export const CreateCalibrationProjectSchema = z.object({
+  resourceId: z.string().min(1),
+  resourceType: z.enum(['agent', 'workflow']),
+  name: z.string().min(1),
+  description: z.string().optional()
+})
+
+/**
+ * Update calibration project schema
+ * Validates request body for updating an existing calibration project
+ */
+export const UpdateCalibrationProjectSchema = z.object({
+  name: z.string().min(1).optional(),
+  description: z.string().optional()
+})
+
+// ============================================================================
+// Calibration Run Schemas
+// ============================================================================
+
+/**
+ * Create calibration run schema
+ * Validates request body for creating a new calibration run
+ * Includes refinement for graderModel requirement when gradingRubric is provided
+ */
+export const CreateCalibrationRunSchema = z
+  .object({
+    projectId: z.string().uuid(),
+    name: z.string().min(1),
+    description: z.string().optional(),
+    executionMode: z.enum(['single', 'session']).default('single'),
+    testInputs: z.array(z.unknown()).min(1).max(50), // Min 1, max 50 inputs
+    configVariants: z.array(ConfigVariantSchema).min(1).max(10), // Min 1, max 10 variants
+    gradingRubric: GradingRubricSchema.optional(),
+    graderModel: LLMModelSchema.optional()
+  })
+  .refine((data) => !data.gradingRubric || data.graderModel, {
+    message: 'graderModel is required when gradingRubric is provided'
+  })
+
+// ============================================================================
+// Exported Schemas
+// ============================================================================
+
+export { GradingRubricSchema, GradingCriterionSchema, ConfigVariantSchema }
+
+// ============================================================================
+// Inferred Types
+// ============================================================================
+
+export type CreateCalibrationProjectInput = z.infer<typeof CreateCalibrationProjectSchema>
+export type UpdateCalibrationProjectInput = z.infer<typeof UpdateCalibrationProjectSchema>
+export type CreateCalibrationRunInput = z.infer<typeof CreateCalibrationRunSchema>
+export type GradingRubricInput = z.infer<typeof GradingRubricSchema>
+export type GradingCriterionInput = z.infer<typeof GradingCriterionSchema>
+export type ConfigVariantInput = z.infer<typeof ConfigVariantSchema>

package/src/execution/calibration/sse-events.ts

@@ -0,0 +1,125 @@
+/**
+ * Calibration SSE Event Types
+ *
+ * Shared event type definitions for calibration real-time streaming.
+ * Used by both the API broadcaster and the command-center UI.
+ */
+
+// Single-turn events
+
+export interface CalibrationExecutionStartedEvent {
+  type: 'execution-started'
+  variantName: string
+  inputIndex: number
+  timestamp: number
+}
+
+export interface CalibrationExecutionCompletedEvent {
+  type: 'execution-completed'
+  variantName: string
+  inputIndex: number
+  executionId: string
+  timestamp: number
+}
+
+export interface CalibrationExecutionFailedEvent {
+  type: 'execution-failed'
+  variantName: string
+  inputIndex?: number
+  error: string
+  timestamp: number
+}
+
+// Session events
+
+export interface CalibrationSessionStartedEvent {
+  type: 'session-started'
+  variantName: string
+  sessionId: string
+  timestamp: number
+}
+
+export interface CalibrationTurnStartedEvent {
+  type: 'turn-started'
+  variantName: string
+  turnNumber: number
+  timestamp: number
+}
+
+export interface CalibrationTurnCompletedEvent {
+  type: 'turn-completed'
+  variantName: string
+  turnNumber: number
+  executionId: string
+  timestamp: number
+}
+
+export interface CalibrationSessionCompletedEvent {
+  type: 'session-completed'
+  variantName: string
+  sessionId: string
+  turnCount: number
+  timestamp: number
+}
+
+// Grading events
+
+export interface CalibrationGradingStartedEvent {
+  type: 'grading-started'
+  variantName: string
+  timestamp: number
+}
+
+export interface CalibrationGradingCompletedEvent {
+  type: 'grading-completed'
+  variantName: string
+  score: number
+  timestamp: number
+}
+
+export interface CalibrationGradingFailedEvent {
+  type: 'grading-failed'
+  variantName: string
+  error: string
+  timestamp: number
+}
+
+// Completion events
+
+export interface CalibrationCompletedEvent {
+  type: 'calibration-completed'
+  summary: { total: number; completed: number; failed: number }
+  timestamp: number
+}
+
+export interface CalibrationFailedEvent {
+  type: 'calibration-failed'
+  error: string
+  timestamp: number
+}
+
+// Connection event
+
+export interface CalibrationConnectedEvent {
+  type: 'connected'
+  timestamp: number
+  data?: undefined
+}
+
+/**
+ * Union of all calibration SSE events
+ */
+export type CalibrationSSEEvent =
+  | CalibrationExecutionStartedEvent
+  | CalibrationExecutionCompletedEvent
+  | CalibrationExecutionFailedEvent
+  | CalibrationSessionStartedEvent
+  | CalibrationTurnStartedEvent
+  | CalibrationTurnCompletedEvent
+  | CalibrationSessionCompletedEvent
+  | CalibrationGradingStartedEvent
+  | CalibrationGradingCompletedEvent
+  | CalibrationGradingFailedEvent
+  | CalibrationCompletedEvent
+  | CalibrationFailedEvent
+  | CalibrationConnectedEvent