eval-harness 0.2.18

package/src/eval/schema.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8-sig"))
+
+
+def schema_type_matches(value: Any, expected: str) -> bool:
+    if expected == "object":
+        return isinstance(value, dict)
+    if expected == "array":
+        return isinstance(value, list)
+    if expected == "string":
+        return isinstance(value, str)
+    if expected == "integer":
+        return isinstance(value, int) and not isinstance(value, bool)
+    if expected == "number":
+        return isinstance(value, (int, float)) and not isinstance(value, bool)
+    if expected == "boolean":
+        return isinstance(value, bool)
+    if expected == "null":
+        return value is None
+    return True
+
+
+def schema_value_type(value: Any) -> str:
+    if value is None:
+        return "null"
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, int):
+        return "integer"
+    if isinstance(value, float):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    return type(value).__name__
+
+
+def validate_schema_document(data: Any, schema: dict[str, Any], path: str = "$") -> list[str]:
+    errors: list[str] = []
+    expected_type = schema.get("type")
+    expected_types = expected_type if isinstance(expected_type, list) else ([expected_type] if expected_type else [])
+    if expected_types and not any(schema_type_matches(data, str(item)) for item in expected_types):
+        errors.append(f"{path}: expected {'/'.join(str(item) for item in expected_types)}, got {schema_value_type(data)}")
+        return errors
+
+    if "const" in schema and data != schema["const"]:
+        errors.append(f"{path}: expected const {schema['const']!r}, got {data!r}")
+    if "enum" in schema and data not in schema.get("enum", []):
+        errors.append(f"{path}: expected one of {schema.get('enum')!r}, got {data!r}")
+
+    if isinstance(data, dict):
+        errors.extend(validate_object(data, schema, path))
+    if isinstance(data, list):
+        errors.extend(validate_array(data, schema, path))
+    if isinstance(data, str):
+        errors.extend(validate_string(data, schema, path))
+    if isinstance(data, (int, float)) and not isinstance(data, bool):
+        errors.extend(validate_number(data, schema, path))
+    return errors
+
+
+def validate_object(data: dict[str, Any], schema: dict[str, Any], path: str) -> list[str]:
+    errors: list[str] = []
+    for key in schema.get("required", []) or []:
+        if key not in data:
+            errors.append(f"{path}.{key}: missing required property")
+
+    properties = schema.get("properties") or {}
+    for key, child_schema in properties.items():
+        if key in data and isinstance(child_schema, dict):
+            errors.extend(validate_schema_document(data[key], child_schema, f"{path}.{key}"))
+
+    additional = schema.get("additionalProperties", True)
+    if additional is False:
+        extra = sorted(key for key in data if key not in properties)
+        errors.extend(f"{path}.{key}: additional property not allowed" for key in extra)
+    elif isinstance(additional, dict):
+        for key, value in data.items():
+            if key not in properties:
+                errors.extend(validate_schema_document(value, additional, f"{path}.{key}"))
+    return errors
+
+
+def validate_array(data: list[Any], schema: dict[str, Any], path: str) -> list[str]:
+    errors: list[str] = []
+    min_items = schema.get("minItems")
+    max_items = schema.get("maxItems")
+    if isinstance(min_items, int) and len(data) < min_items:
+        errors.append(f"{path}: expected at least {min_items} item(s), got {len(data)}")
+    if isinstance(max_items, int) and len(data) > max_items:
+        errors.append(f"{path}: expected at most {max_items} item(s), got {len(data)}")
+    if isinstance(schema.get("items"), dict):
+        for index, item in enumerate(data):
+            errors.extend(validate_schema_document(item, schema["items"], f"{path}[{index}]"))
+    return errors
+
+
+def validate_string(data: str, schema: dict[str, Any], path: str) -> list[str]:
+    errors: list[str] = []
+    min_length = schema.get("minLength")
+    max_length = schema.get("maxLength")
+    if isinstance(min_length, int) and len(data) < min_length:
+        errors.append(f"{path}: expected length >= {min_length}, got {len(data)}")
+    if isinstance(max_length, int) and len(data) > max_length:
+        errors.append(f"{path}: expected length <= {max_length}, got {len(data)}")
+    pattern = schema.get("pattern")
+    if isinstance(pattern, str) and re.search(pattern, data) is None:
+        errors.append(f"{path}: expected string matching pattern {pattern!r}, got {data!r}")
+    return errors
+
+
+def validate_number(data: int | float, schema: dict[str, Any], path: str) -> list[str]:
+    errors: list[str] = []
+    minimum = schema.get("minimum")
+    maximum = schema.get("maximum")
+    if isinstance(minimum, (int, float)) and data < minimum:
+        errors.append(f"{path}: expected >= {minimum}, got {data}")
+    if isinstance(maximum, (int, float)) and data > maximum:
+        errors.append(f"{path}: expected <= {maximum}, got {data}")
+    return errors
+
+
+def load_schema(schema_dir: Path, schema_name: str) -> dict[str, Any]:
+    schema_path = schema_dir / schema_name
+    if not schema_path.exists() and not schema_dir.is_absolute():
+        source_tree_schema = Path(__file__).resolve().parents[2] / schema_dir / schema_name
+        if source_tree_schema.exists():
+            schema_path = source_tree_schema
+    if not schema_path.exists():
+        raise RuntimeError(f"schema not found: {schema_path}")
+    return read_json(schema_path)
+
+
+def validate_json_artifact(path: Path, schema_dir: Path, schema_name: str) -> list[str]:
+    try:
+        data = read_json(path)
+    except Exception as exc:  # noqa: BLE001 - validation command reports exact broken artifact.
+        return [f"{path}: invalid json: {exc}"]
+    schema = load_schema(schema_dir, schema_name)
+    return [f"{path}: {error}" for error in validate_schema_document(data, schema)]