@od-oneapp/ai-platform 0.1.0

package/src/output/validator.ts

@@ -0,0 +1,495 @@
+/**
+ * @fileoverview Output Validator Module
+ *
+ * Provides schema-based validation for AI model outputs with configurable
+ * error handling strategies including throw, partial recovery, and retry.
+ *
+ * @module @od-oneapp/ai-platform/output/validator
+ *
+ * @example
+ * ```typescript
+ * import { createOutputValidator } from '@od-oneapp/ai-platform/output';
+ * import { z } from 'zod';
+ *
+ * const PersonSchema = z.object({
+ *   name: z.string(),
+ *   age: z.number(),
+ *   email: z.string().email(),
+ * });
+ *
+ * // Simple validation with throw strategy
+ * const strictValidator = createOutputValidator({
+ *   schema: PersonSchema,
+ *   onValidationError: 'throw',
+ * });
+ *
+ * try {
+ *   const person = strictValidator.validate(rawOutput);
+ * } catch (error) {
+ *   console.error('Validation failed:', error);
+ * }
+ *
+ * // Partial recovery strategy
+ * const lenientValidator = createOutputValidator({
+ *   schema: PersonSchema,
+ *   onValidationError: 'partial',
+ * });
+ *
+ * const result = lenientValidator.validate(rawOutput);
+ * if (result.success) {
+ *   console.log('Valid:', result.data);
+ * } else {
+ *   console.log('Partial:', result.partial, 'Errors:', result.errors);
+ * }
+ *
+ * // Retry strategy with model
+ * const retryValidator = createOutputValidator({
+ *   schema: PersonSchema,
+ *   onValidationError: 'retry',
+ *   maxRetries: 2,
+ * });
+ *
+ * const validated = await retryValidator.validateAsync(rawOutput, {
+ *   model: openai('gpt-4'),
+ *   prompt: 'Extract person info from: John, 30 years old',
+ * });
+ * ```
+ */
+
+import { logWarn, logInfo } from '@repo/shared/logs';
+import { type ZodType, type ZodError, type ZodIssue } from 'zod';
+
+import { generateObject } from '../shared';
+
+import type { SDKLanguageModelV3 } from '../shared';
+
+// -----------------------------------------------------------------------------
+// Types
+// -----------------------------------------------------------------------------
+
+/**
+ * Error handling strategy for validation failures.
+ */
+export type ValidationErrorStrategy = 'throw' | 'partial' | 'retry';
+
+/**
+ * Configuration for the output validator.
+ */
+export interface OutputValidatorConfig<T> {
+  /** Schema for validation */
+  readonly schema: ZodType<T>;
+  /** Error handling strategy */
+  readonly onValidationError: ValidationErrorStrategy;
+  /** Max retries when strategy is 'retry' (default: 2) */
+  readonly maxRetries?: number;
+  /** Delay between retries in ms (default: 1000) */
+  readonly retryDelay?: number;
+  /** Custom error message prefix */
+  readonly errorMessage?: string;
+  /** Called on validation error (before retry or throw) */
+  readonly onError?: (error: ZodError, attempt: number) => void;
+}
+
+/**
+ * Context for async validation with retry support.
+ */
+export interface RetryContext {
+  /** Language model for regeneration */
+  readonly model: SDKLanguageModelV3;
+  /** Original prompt for regeneration */
+  readonly prompt: string;
+  /** System prompt (optional) */
+  readonly system?: string;
+  /** Temperature for regeneration */
+  readonly temperature?: number;
+}
+
+/**
+ * Result of validation with 'partial' strategy.
+ */
+export interface PartialValidationResult<T> {
+  /** Whether validation succeeded */
+  readonly success: boolean;
+  /** Validated data (if success) */
+  readonly data?: T;
+  /** Partial data extracted (if failure) */
+  readonly partial?: Partial<T>;
+  /** Validation errors (if failure) */
+  readonly errors?: ZodIssue[];
+  /** Raw error object */
+  readonly rawError?: ZodError;
+}
+
+/**
+ * Output validator instance.
+ */
+export interface OutputValidator<T> {
+  /**
+   * Validate output synchronously.
+   * - 'throw': throws ZodError on failure
+   * - 'partial': returns PartialValidationResult
+   * - 'retry': throws (use validateAsync for retry support)
+   */
+  validate(output: unknown): T | PartialValidationResult<T>;
+
+  /**
+   * Validate output asynchronously with retry support.
+   * - 'throw': throws ZodError on failure
+   * - 'partial': returns PartialValidationResult
+   * - 'retry': retries with model until success or max retries
+   */
+  validateAsync(
+    output: unknown,
+    retryContext?: RetryContext,
+  ): Promise<T | PartialValidationResult<T>>;
+
+  /** Get the validation schema */
+  getSchema(): ZodType<T>;
+
+  /** Get the error handling strategy */
+  getStrategy(): ValidationErrorStrategy;
+}
+
+// -----------------------------------------------------------------------------
+// Custom Error
+// -----------------------------------------------------------------------------
+
+/**
+ * Custom error for validation failures with context.
+ */
+export class OutputValidationError extends Error {
+  readonly zodError: ZodError;
+  readonly strategy: ValidationErrorStrategy;
+  readonly attempts: number;
+
+  constructor(
+    message: string,
+    zodError: ZodError,
+    strategy: ValidationErrorStrategy,
+    attempts: number = 1,
+  ) {
+    super(message);
+    this.name = 'OutputValidationError';
+    this.zodError = zodError;
+    this.strategy = strategy;
+    this.attempts = attempts;
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Helper Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Extract partial data from validation errors.
+ * Attempts to keep valid fields while noting invalid ones.
+ */
+function extractPartialData<T>(output: unknown, zodError: ZodError): Partial<T> | undefined {
+  if (typeof output !== 'object' || output === null) {
+    return undefined;
+  }
+
+  const result: Record<string, unknown> = { ...(output as Record<string, unknown>) };
+  const invalidPaths = new Set<string>();
+
+  // Collect paths with errors
+  for (const issue of zodError.issues) {
+    if (issue.path.length > 0) {
+      invalidPaths.add(String(issue.path[0]));
+    }
+  }
+
+  // Remove invalid top-level fields
+  for (const path of invalidPaths) {
+    delete result[path];
+  }
+
+  return Object.keys(result).length > 0 ? (result as Partial<T>) : undefined;
+}
+
+/**
+ * Build error message with custom prefix if provided.
+ */
+function buildErrorMessage(zodError: ZodError, customPrefix?: string): string {
+  const issueMessages = zodError.issues
+    .map(issue => {
+      const path = issue.path.length > 0 ? issue.path.join('.') : 'root';
+      return `${path}: ${issue.message}`;
+    })
+    .join('; ');
+
+  return customPrefix ? `${customPrefix}: ${issueMessages}` : `Validation failed: ${issueMessages}`;
+}
+
+/**
+ * Delay helper for retries.
+ */
+function delay(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+// -----------------------------------------------------------------------------
+// Factory
+// -----------------------------------------------------------------------------
+
+/**
+ * Create an output validator with configurable error handling.
+ *
+ * @param config - Validator configuration
+ * @returns Output validator instance
+ *
+ * @example
+ * ```typescript
+ * // Throw strategy (strict validation)
+ * const strictValidator = createOutputValidator({
+ *   schema: UserSchema,
+ *   onValidationError: 'throw',
+ * });
+ *
+ * // Partial strategy (lenient validation with error details)
+ * const lenientValidator = createOutputValidator({
+ *   schema: UserSchema,
+ *   onValidationError: 'partial',
+ * });
+ *
+ * // Retry strategy (regenerate with model on failure)
+ * const retryValidator = createOutputValidator({
+ *   schema: UserSchema,
+ *   onValidationError: 'retry',
+ *   maxRetries: 3,
+ *   retryDelay: 500,
+ * });
+ * ```
+ */
+export function createOutputValidator<T>(config: OutputValidatorConfig<T>): OutputValidator<T> {
+  const {
+    schema,
+    onValidationError,
+    maxRetries = 2,
+    retryDelay = 1000,
+    errorMessage,
+    onError,
+  } = config;
+
+  /**
+   * Synchronous validation.
+   */
+  function validate(output: unknown): T | PartialValidationResult<T> {
+    const result = schema.safeParse(output);
+
+    if (result.success) {
+      return result.data;
+    }
+
+    const zodError = result.error;
+    onError?.(zodError, 1);
+
+    switch (onValidationError) {
+      case 'throw':
+        throw new OutputValidationError(
+          buildErrorMessage(zodError, errorMessage),
+          zodError,
+          'throw',
+          1,
+        );
+
+      case 'partial':
+        return {
+          success: false,
+          partial: extractPartialData<T>(output, zodError),
+          errors: zodError.issues,
+          rawError: zodError,
+        };
+
+      case 'retry':
+        // For sync validation with retry strategy, we throw
+        // (use validateAsync for actual retry behavior)
+        throw new OutputValidationError(
+          `${buildErrorMessage(zodError, errorMessage)} (use validateAsync for retry support)`,
+          zodError,
+          'retry',
+          1,
+        );
+    }
+  }
+
+  /**
+   * Asynchronous validation with retry support.
+   */
+  async function validateAsync(
+    output: unknown,
+    retryContext?: RetryContext,
+  ): Promise<T | PartialValidationResult<T>> {
+    let currentOutput = output;
+    let lastError: ZodError | undefined;
+
+    for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+      const result = schema.safeParse(currentOutput);
+
+      if (result.success) {
+        if (attempt > 1) {
+          logInfo(`[OutputValidator] Validation succeeded on attempt ${attempt}`);
+        }
+        return result.data;
+      }
+
+      lastError = result.error;
+      onError?.(lastError, attempt);
+
+      // Handle based on strategy
+      if (onValidationError === 'partial') {
+        return {
+          success: false,
+          partial: extractPartialData<T>(currentOutput, lastError),
+          errors: lastError.issues,
+          rawError: lastError,
+        };
+      }
+
+      if (onValidationError === 'throw') {
+        throw new OutputValidationError(
+          buildErrorMessage(lastError, errorMessage),
+          lastError,
+          'throw',
+          attempt,
+        );
+      }
+
+      // Retry strategy
+      if (attempt <= maxRetries) {
+        if (!retryContext) {
+          throw new OutputValidationError(
+            'Retry strategy requires retryContext with model and prompt',
+            lastError,
+            'retry',
+            attempt,
+          );
+        }
+
+        logWarn(
+          `[OutputValidator] Validation failed on attempt ${attempt}/${maxRetries + 1}. ` +
+            `Errors: ${lastError.issues.map(i => i.message).join(', ')}. Retrying...`,
+        );
+
+        // Wait before retry
+        await delay(retryDelay * attempt);
+
+        // Regenerate with model
+        try {
+          const { object } = await generateObject({
+            model: retryContext.model,
+            schema,
+            prompt: retryContext.prompt,
+            system: retryContext.system,
+            temperature: retryContext.temperature,
+          });
+
+          currentOutput = object;
+        } catch (genError) {
+          logWarn(
+            `[OutputValidator] Regeneration failed on attempt ${attempt}: ${genError instanceof Error ? genError.message : String(genError)}`,
+          );
+          // Continue to next iteration, will fail validation again
+        }
+      }
+    }
+
+    // All retries exhausted
+    if (lastError) {
+      throw new OutputValidationError(
+        `${buildErrorMessage(lastError, errorMessage)} (exhausted ${maxRetries + 1} attempts)`,
+        lastError,
+        'retry',
+        maxRetries + 1,
+      );
+    }
+
+    // Should not reach here, but satisfy TypeScript
+    throw new Error('Validation failed with unknown error');
+  }
+
+  return {
+    validate,
+    validateAsync,
+    getSchema: () => schema,
+    getStrategy: () => onValidationError,
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Utility Functions
+// -----------------------------------------------------------------------------
+
+/**
+ * Check if an error is an OutputValidationError.
+ *
+ * @param error - Error to check
+ * @returns True if error is OutputValidationError
+ */
+export function isOutputValidationError(error: unknown): error is OutputValidationError {
+  return error instanceof OutputValidationError;
+}
+
+/**
+ * Create a simple validator that always throws on error.
+ * Shorthand for createOutputValidator with 'throw' strategy.
+ *
+ * @param schema - Zod schema for validation
+ * @returns Validator function
+ *
+ * @example
+ * ```typescript
+ * const validateUser = createStrictValidator(UserSchema);
+ * const user = validateUser(rawData); // throws if invalid
+ * ```
+ */
+export function createStrictValidator<T>(schema: ZodType<T>): (output: unknown) => T {
+  const validator = createOutputValidator({
+    schema,
+    onValidationError: 'throw',
+  });
+
+  return (output: unknown): T => {
+    return validator.validate(output) as T;
+  };
+}
+
+/**
+ * Create a validator that returns partial data on error.
+ * Shorthand for createOutputValidator with 'partial' strategy.
+ *
+ * @param schema - Zod schema for validation
+ * @returns Validator function returning PartialValidationResult
+ *
+ * @example
+ * ```typescript
+ * const validateUser = createPartialValidator(UserSchema);
+ * const result = validateUser(rawData);
+ * if (!result.success) {
+ *   console.log('Partial data:', result.partial);
+ * }
+ * ```
+ */
+export function createPartialValidator<T>(
+  schema: ZodType<T>,
+): (output: unknown) => PartialValidationResult<T> {
+  const validator = createOutputValidator({
+    schema,
+    onValidationError: 'partial',
+  });
+
+  return (output: unknown): PartialValidationResult<T> => {
+    const result = validator.validate(output);
+    // If validation succeeded, wrap in PartialValidationResult
+    if (!result || typeof result !== 'object' || !('success' in result)) {
+      return { success: true, data: result };
+    }
+    return result;
+  };
+}
+
+// -----------------------------------------------------------------------------
+// Re-export types for convenience
+// -----------------------------------------------------------------------------
+
+export type { ZodType, ZodError, ZodIssue };