illuma-agents 1.0.38 → 1.0.39

package/src/schemas/validate.test.ts

@@ -0,0 +1,358 @@
+// src/schemas/validate.test.ts
+import {
+  validateStructuredOutput,
+  createValidationErrorMessage,
+  isValidJsonSchema,
+  normalizeJsonSchema,
+} from './validate';
+import type { StructuredOutputConfig } from '@/types';
+
+describe('validateStructuredOutput', () => {
+  const simpleObjectSchema = {
+    type: 'object',
+    properties: {
+      name: { type: 'string' },
+      age: { type: 'number' },
+    },
+    required: ['name'],
+  };
+
+  describe('basic validation', () => {
+    it('should validate a correct object', () => {
+      const result = validateStructuredOutput(
+        { name: 'John', age: 30 },
+        simpleObjectSchema
+      );
+      expect(result.success).toBe(true);
+      expect(result.data).toEqual({ name: 'John', age: 30 });
+      expect(result.error).toBeUndefined();
+    });
+
+    it('should validate an object with only required fields', () => {
+      const result = validateStructuredOutput(
+        { name: 'Jane' },
+        simpleObjectSchema
+      );
+      expect(result.success).toBe(true);
+      expect(result.data).toEqual({ name: 'Jane' });
+    });
+
+    it('should fail when required field is missing', () => {
+      const result = validateStructuredOutput({ age: 25 }, simpleObjectSchema);
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('missing required property \'name\'');
+      expect(result.raw).toEqual({ age: 25 });
+    });
+
+    it('should fail when type is incorrect', () => {
+      const result = validateStructuredOutput(
+        { name: 123 },
+        simpleObjectSchema
+      );
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('expected string');
+    });
+  });
+
+  describe('string input parsing', () => {
+    it('should parse valid JSON string input', () => {
+      const jsonString = '{"name": "Alice", "age": 28}';
+      const result = validateStructuredOutput(jsonString, simpleObjectSchema);
+      expect(result.success).toBe(true);
+      expect(result.data).toEqual({ name: 'Alice', age: 28 });
+    });
+
+    it('should fail on invalid JSON string', () => {
+      const invalidJson = '{ name: "Bob" }'; // Missing quotes around key
+      const result = validateStructuredOutput(invalidJson, simpleObjectSchema);
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('Failed to parse output');
+    });
+  });
+
+  describe('type validation', () => {
+    it('should validate string type when passed as object property', () => {
+      // String values at root level get parsed as JSON first, so test nested
+      const schema = {
+        type: 'object',
+        properties: {
+          value: { type: 'string' },
+        },
+      };
+      expect(validateStructuredOutput({ value: 'hello' }, schema).success).toBe(
+        true
+      );
+      expect(validateStructuredOutput({ value: 123 }, schema).success).toBe(
+        false
+      );
+    });
+
+    it('should validate number type when passed as object property', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          value: { type: 'number' },
+        },
+      };
+      expect(validateStructuredOutput({ value: 42 }, schema).success).toBe(
+        true
+      );
+      expect(validateStructuredOutput({ value: '42' }, schema).success).toBe(
+        false
+      );
+    });
+
+    it('should validate boolean type when passed as object property', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          value: { type: 'boolean' },
+        },
+      };
+      expect(validateStructuredOutput({ value: true }, schema).success).toBe(
+        true
+      );
+      expect(validateStructuredOutput({ value: false }, schema).success).toBe(
+        true
+      );
+      expect(validateStructuredOutput({ value: 'true' }, schema).success).toBe(
+        false
+      );
+    });
+
+    it('should validate array type', () => {
+      const schema = { type: 'array', items: { type: 'string' } };
+      expect(validateStructuredOutput(['a', 'b'], schema).success).toBe(true);
+      expect(validateStructuredOutput('not an array', schema).success).toBe(
+        false
+      );
+    });
+
+    it('should validate object type', () => {
+      const schema = { type: 'object' };
+      expect(validateStructuredOutput({}, schema).success).toBe(true);
+      expect(validateStructuredOutput('not an object', schema).success).toBe(
+        false
+      );
+    });
+  });
+
+  describe('nested object validation', () => {
+    const nestedSchema = {
+      type: 'object',
+      properties: {
+        user: {
+          type: 'object',
+          properties: {
+            name: { type: 'string' },
+            email: { type: 'string' },
+          },
+          required: ['name'],
+        },
+        active: { type: 'boolean' },
+      },
+      required: ['user'],
+    };
+
+    it('should validate correct nested object', () => {
+      const result = validateStructuredOutput(
+        { user: { name: 'John', email: 'john@example.com' }, active: true },
+        nestedSchema
+      );
+      expect(result.success).toBe(true);
+    });
+
+    it('should fail on missing nested required field', () => {
+      const result = validateStructuredOutput(
+        { user: { email: 'john@example.com' } },
+        nestedSchema
+      );
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('missing required property \'name\'');
+    });
+
+    it('should fail on incorrect nested type', () => {
+      const result = validateStructuredOutput(
+        { user: { name: 123 } },
+        nestedSchema
+      );
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('expected string');
+    });
+  });
+
+  describe('array item validation', () => {
+    const arraySchema = {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          id: { type: 'number' },
+          value: { type: 'string' },
+        },
+        required: ['id'],
+      },
+    };
+
+    it('should validate correct array items', () => {
+      const result = validateStructuredOutput(
+        [
+          { id: 1, value: 'one' },
+          { id: 2, value: 'two' },
+        ],
+        arraySchema
+      );
+      expect(result.success).toBe(true);
+    });
+
+    it('should fail on incorrect array item type', () => {
+      const result = validateStructuredOutput(
+        [{ id: 'not a number', value: 'one' }],
+        arraySchema
+      );
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('expected number');
+    });
+
+    it('should fail on missing required field in array item', () => {
+      const result = validateStructuredOutput(
+        [{ value: 'missing id' }],
+        arraySchema
+      );
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('missing required property \'id\'');
+    });
+  });
+
+  describe('enum validation', () => {
+    it('should validate valid enum value in object property', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          color: { type: 'string', enum: ['red', 'green', 'blue'] },
+        },
+      };
+      const result = validateStructuredOutput({ color: 'red' }, schema);
+      expect(result.success).toBe(true);
+    });
+
+    it('should fail on invalid enum value in object property', () => {
+      const schema = {
+        type: 'object',
+        properties: {
+          color: { type: 'string', enum: ['red', 'green', 'blue'] },
+        },
+      };
+      const result = validateStructuredOutput({ color: 'yellow' }, schema);
+      expect(result.success).toBe(false);
+      expect(result.error).toContain('not in enum');
+    });
+  });
+});
+
+describe('createValidationErrorMessage', () => {
+  it('should return custom message when provided', () => {
+    const result = { success: false, error: 'Some error' };
+    const message = createValidationErrorMessage(
+      result,
+      'Custom error message'
+    );
+    expect(message).toBe('Custom error message');
+  });
+
+  it('should create message from error when no custom message', () => {
+    const result = { success: false, error: 'Missing required field' };
+    const message = createValidationErrorMessage(result);
+    expect(message).toContain('Missing required field');
+    expect(message).toContain('did not match the expected schema');
+  });
+});
+
+describe('isValidJsonSchema', () => {
+  it('should return true for schema with type', () => {
+    expect(isValidJsonSchema({ type: 'object' })).toBe(true);
+    expect(isValidJsonSchema({ type: 'string' })).toBe(true);
+  });
+
+  it('should return true for schema with properties', () => {
+    expect(
+      isValidJsonSchema({ properties: { name: { type: 'string' } } })
+    ).toBe(true);
+  });
+
+  it('should return true for schema with $schema', () => {
+    expect(
+      isValidJsonSchema({ $schema: 'http://json-schema.org/draft-07/schema#' })
+    ).toBe(true);
+  });
+
+  it('should return false for non-object values', () => {
+    expect(isValidJsonSchema(null)).toBe(false);
+    expect(isValidJsonSchema(undefined)).toBe(false);
+    expect(isValidJsonSchema('string')).toBe(false);
+    expect(isValidJsonSchema(123)).toBe(false);
+  });
+
+  it('should return false for empty object', () => {
+    expect(isValidJsonSchema({})).toBe(false);
+  });
+});
+
+describe('normalizeJsonSchema', () => {
+  it('should add type: object when properties exist', () => {
+    const schema = { properties: { name: { type: 'string' } } };
+    const normalized = normalizeJsonSchema(schema);
+    expect(normalized.type).toBe('object');
+  });
+
+  it('should not override existing type', () => {
+    const schema = { type: 'array', items: { type: 'string' } };
+    const normalized = normalizeJsonSchema(schema);
+    expect(normalized.type).toBe('array');
+  });
+
+  it('should add title from config name', () => {
+    const schema = { type: 'object' };
+    const config: StructuredOutputConfig = { schema: {}, name: 'MySchema' };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.title).toBe('MySchema');
+  });
+
+  it('should not override existing title', () => {
+    const schema = { type: 'object', title: 'ExistingTitle' };
+    const config: StructuredOutputConfig = { schema: {}, name: 'NewTitle' };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.title).toBe('ExistingTitle');
+  });
+
+  it('should add description from config', () => {
+    const schema = { type: 'object' };
+    const config: StructuredOutputConfig = {
+      schema: {},
+      description: 'My description',
+    };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.description).toBe('My description');
+  });
+
+  it('should set additionalProperties: false in strict mode', () => {
+    const schema = { type: 'object' };
+    const config: StructuredOutputConfig = { schema: {}, strict: true };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.additionalProperties).toBe(false);
+  });
+
+  it('should preserve existing additionalProperties value', () => {
+    const schema = { type: 'object', additionalProperties: true };
+    const config: StructuredOutputConfig = { schema: {}, strict: true };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.additionalProperties).toBe(true);
+  });
+
+  it('should not modify additionalProperties when strict is false', () => {
+    const schema = { type: 'object' };
+    const config: StructuredOutputConfig = { schema: {}, strict: false };
+    const normalized = normalizeJsonSchema(schema, config);
+    expect(normalized.additionalProperties).toBeUndefined();
+  });
+});

package/src/schemas/validate.ts

@@ -0,0 +1,238 @@
+// src/schemas/validate.ts
+import { z } from 'zod';
+import type * as t from '@/types';
+
+/**
+ * Validation result from structured output
+ */
+export interface ValidationResult<T = Record<string, unknown>> {
+  success: boolean;
+  data?: T;
+  error?: string;
+  raw?: unknown;
+}
+
+/**
+ * Validates structured output against a JSON Schema.
+ *
+ * @param output - The output to validate
+ * @param schema - JSON Schema to validate against
+ * @returns Validation result with success flag and data or error
+ */
+export function validateStructuredOutput(
+  output: unknown,
+  schema: Record<string, unknown>
+): ValidationResult {
+  try {
+    // Parse output if it's a string
+    const data = typeof output === 'string' ? JSON.parse(output) : output;
+
+    // Basic JSON Schema validation
+    const errors = validateAgainstJsonSchema(data, schema);
+
+    if (errors.length > 0) {
+      return {
+        success: false,
+        error: `Validation failed: ${errors.join('; ')}`,
+        raw: data,
+      };
+    }
+
+    return {
+      success: true,
+      data: data as Record<string, unknown>,
+    };
+  } catch (e) {
+    return {
+      success: false,
+      error: `Failed to parse output: ${(e as Error).message}`,
+      raw: output,
+    };
+  }
+}
+
+/**
+ * Validates data against a JSON Schema (simplified implementation).
+ * For complex schemas, consider using ajv or similar library.
+ */
+function validateAgainstJsonSchema(
+  data: unknown,
+  schema: Record<string, unknown>,
+  path = ''
+): string[] {
+  const errors: string[] = [];
+
+  if (schema.type === 'object' && typeof data !== 'object') {
+    errors.push(`${path || 'root'}: expected object, got ${typeof data}`);
+    return errors;
+  }
+
+  if (schema.type === 'array' && !Array.isArray(data)) {
+    errors.push(`${path || 'root'}: expected array, got ${typeof data}`);
+    return errors;
+  }
+
+  if (schema.type === 'string' && typeof data !== 'string') {
+    errors.push(`${path || 'root'}: expected string, got ${typeof data}`);
+    return errors;
+  }
+
+  if (schema.type === 'number' && typeof data !== 'number') {
+    errors.push(`${path || 'root'}: expected number, got ${typeof data}`);
+    return errors;
+  }
+
+  if (schema.type === 'boolean' && typeof data !== 'boolean') {
+    errors.push(`${path || 'root'}: expected boolean, got ${typeof data}`);
+    return errors;
+  }
+
+  // Validate required properties
+  if (
+    schema.type === 'object' &&
+    Array.isArray(schema.required) &&
+    typeof data === 'object' &&
+    data !== null
+  ) {
+    for (const requiredProp of schema.required as string[]) {
+      if (!(requiredProp in data)) {
+        errors.push(
+          `${path || 'root'}: missing required property '${requiredProp}'`
+        );
+      }
+    }
+  }
+
+  // Validate nested properties
+  if (
+    schema.type === 'object' &&
+    schema.properties &&
+    typeof data === 'object' &&
+    data !== null
+  ) {
+    const properties = schema.properties as Record<
+      string,
+      Record<string, unknown>
+    >;
+    for (const [key, propSchema] of Object.entries(properties)) {
+      if (key in data) {
+        const nestedErrors = validateAgainstJsonSchema(
+          (data as Record<string, unknown>)[key],
+          propSchema,
+          path ? `${path}.${key}` : key
+        );
+        errors.push(...nestedErrors);
+      }
+    }
+  }
+
+  // Validate array items
+  if (schema.type === 'array' && schema.items && Array.isArray(data)) {
+    const itemSchema = schema.items as Record<string, unknown>;
+    for (let i = 0; i < data.length; i++) {
+      const nestedErrors = validateAgainstJsonSchema(
+        data[i],
+        itemSchema,
+        `${path || 'root'}[${i}]`
+      );
+      errors.push(...nestedErrors);
+    }
+  }
+
+  // Validate enum values
+  if (schema.enum && Array.isArray(schema.enum)) {
+    if (!schema.enum.includes(data)) {
+      errors.push(
+        `${path || 'root'}: value '${data}' not in enum [${(schema.enum as unknown[]).join(', ')}]`
+      );
+    }
+  }
+
+  return errors;
+}
+
+/**
+ * Converts a Zod schema to JSON Schema format.
+ * This is a simplified converter for common types.
+ */
+export function zodToJsonSchema(zodSchema: z.ZodType): Record<string, unknown> {
+  // Use the zod-to-json-schema library if available
+  // For now, provide a basic implementation
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-require-imports
+    const { zodToJsonSchema: convert } = require('zod-to-json-schema');
+    return convert(zodSchema) as Record<string, unknown>;
+  } catch {
+    // Fallback: return a generic object schema
+    return {
+      type: 'object',
+      additionalProperties: true,
+    };
+  }
+}
+
+/**
+ * Creates a structured output error message for retry.
+ */
+export function createValidationErrorMessage(
+  result: ValidationResult,
+  customMessage?: string
+): string {
+  if (customMessage) {
+    return customMessage;
+  }
+
+  return `The response did not match the expected schema. Error: ${result.error}. Please try again and ensure your response exactly matches the required format.`;
+}
+
+/**
+ * Checks if a value is a valid JSON Schema object.
+ */
+export function isValidJsonSchema(
+  value: unknown
+): value is Record<string, unknown> {
+  if (typeof value !== 'object' || value === null) {
+    return false;
+  }
+
+  const schema = value as Record<string, unknown>;
+
+  // Basic check: must have a type or properties
+  return (
+    typeof schema.type === 'string' ||
+    typeof schema.properties === 'object' ||
+    typeof schema.$schema === 'string'
+  );
+}
+
+/**
+ * Normalizes a JSON Schema by adding defaults and cleaning up.
+ */
+export function normalizeJsonSchema(
+  schema: Record<string, unknown>,
+  config?: t.StructuredOutputConfig
+): Record<string, unknown> {
+  const normalized = { ...schema };
+
+  // Ensure type is set
+  if (!normalized.type && normalized.properties) {
+    normalized.type = 'object';
+  }
+
+  // Add title from config name
+  if (config?.name && !normalized.title) {
+    normalized.title = config.name;
+  }
+
+  // Add description from config
+  if (config?.description && !normalized.description) {
+    normalized.description = config.description;
+  }
+
+  // Enable additionalProperties: false for strict mode
+  if (config?.strict !== false && normalized.type === 'object') {
+    normalized.additionalProperties = normalized.additionalProperties ?? false;
+  }
+
+  return normalized;
+}