@microsoft/m365-copilot-eval 1.1.0-preview.1 → 1.2.0-preview.1

package/src/clients/cli/schema_handler.py

@@ -0,0 +1,349 @@
+"""Schema validation, version management, backup, and document upgrade for eval documents.
+
+This module provides the core infrastructure for JSON Schema contract versioning:
+- SchemaValidator: Validates eval documents against the JSON Schema
+- SchemaVersionManager: Reads and compares schema versions
+- FileBackupManager: Creates timestamped backups with atomic writes
+- DocumentUpgrader: Orchestrates the full upgrade flow
+"""
+
+import json
+import os
+import shutil
+import logging
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import Any, Optional
+
+
+class VersionRelation(Enum):
+    """Result of comparing a document's schema version against the current version."""
+    CURRENT = "current"
+    OLDER = "older"
+    NEWER_MAJOR = "newer_major"
+    UNSUPPORTED = "unsupported"
+
+from jsonschema import Draft202012Validator, ValidationError
+from jsonschema.exceptions import best_match
+
+logger = logging.getLogger(__name__)
+
+# Resolve repo root and schema paths relative to this file
+_REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+_DEFAULT_VERSION_PATH = _REPO_ROOT / "schema" / "version.json"
+
+
+def _resolve_schema_path(version_path: Path = _DEFAULT_VERSION_PATH) -> Path:
+    """Derive the schema file path from the major version in version.json."""
+    try:
+        with open(version_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        major = data["version"].split(".")[0]
+    except Exception:
+        major = "1"
+    return _REPO_ROOT / "schema" / f"v{major}" / "eval-document.schema.json"
+
+
+class SchemaValidator:
+    """Validates eval documents against the JSON Schema using Draft 2020-12."""
+
+    def __init__(self, schema_path: Optional[Path] = None):
+        schema_path = schema_path or _resolve_schema_path()
+        with open(schema_path, "r", encoding="utf-8") as f:
+            self.schema = json.load(f)
+
+        Draft202012Validator.check_schema(self.schema)
+
+        self.validator = Draft202012Validator(
+            schema=self.schema,
+            format_checker=Draft202012Validator.FORMAT_CHECKER,
+        )
+
+    def validate(self, document: dict[str, Any]) -> tuple[bool, list[ValidationError] | None]:
+        """Validate a document against the schema.
+
+        Returns:
+            (True, None) if valid, or (False, list_of_errors) if invalid.
+        """
+        errors = list(self.validator.iter_errors(document))
+        if not errors:
+            return True, None
+        return False, errors
+
+    @staticmethod
+    def format_errors(errors: list[ValidationError]) -> str:
+        """Format validation errors into a human-readable string.
+
+        Uses best_match to surface the most relevant error first, then lists
+        remaining errors with JSON paths.
+        """
+        if not errors:
+            return ""
+
+        best = best_match(errors)
+        lines = []
+
+        if best is not None:
+            path_str = " -> ".join(str(p) for p in best.absolute_path) or "(root)"
+            lines.append(f"Most relevant error at '{path_str}': {best.message}")
+
+        for err in errors:
+            if err is best:
+                continue
+            path_str = " -> ".join(str(p) for p in err.absolute_path) or "(root)"
+            lines.append(f"  - At '{path_str}': {err.message}")
+
+        lines.append("")
+        lines.append("Review the schema changelog for details: schema/CHANGELOG.md")
+        return "\n".join(lines)
+
+
+class SchemaVersionManager:
+    """Reads and compares schema versions from version.json."""
+
+    def __init__(self, version_path: Optional[Path] = None):
+        self._version_path = version_path or _DEFAULT_VERSION_PATH
+
+    def get_current_version(self) -> str:
+        """Read the current schema version from version.json."""
+        with open(self._version_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return data["version"]
+
+    def get_schema_path(self) -> Path:
+        """Return the resolved path to the current schema file."""
+        return _resolve_schema_path()
+
+    @staticmethod
+    def compare_versions(doc_version: str, current_version: str) -> VersionRelation:
+        """Compare document schema version against the current version."""
+        try:
+            doc_parts = [int(x) for x in doc_version.split(".")]
+            cur_parts = [int(x) for x in current_version.split(".")]
+        except (ValueError, AttributeError):
+            return VersionRelation.UNSUPPORTED
+
+        if len(doc_parts) != 3 or len(cur_parts) != 3:
+            return VersionRelation.UNSUPPORTED
+
+        doc_major, doc_minor, doc_patch = doc_parts
+        cur_major, cur_minor, cur_patch = cur_parts
+
+        if doc_major > cur_major:
+            return VersionRelation.NEWER_MAJOR
+
+        if doc_parts == cur_parts:
+            return VersionRelation.CURRENT
+
+        if doc_major == cur_major and (doc_minor, doc_patch) < (cur_minor, cur_patch):
+            return VersionRelation.OLDER
+
+        # Same major, doc is newer minor/patch than current — shouldn't happen normally,
+        # but treat as current (forward-compatible within major).
+        if doc_major == cur_major:
+            return VersionRelation.CURRENT
+
+        # doc_major < cur_major — older major version
+        return VersionRelation.OLDER
+
+
+class FileBackupManager:
+    """Creates timestamped backups and performs atomic file writes."""
+
+    @staticmethod
+    def create_backup_path(original_path: Path) -> Path:
+        """Generate a timestamped backup path for the given file."""
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        backup_name = f"{original_path.name}.bak.{timestamp}"
+        return original_path.parent / backup_name
+
+    @staticmethod
+    def create_backup(file_path: Path) -> Path:
+        """Create a backup copy of the file with a timestamped name.
+
+        Returns the path to the backup file.
+        """
+        backup_path = FileBackupManager.create_backup_path(file_path)
+        shutil.copy2(file_path, backup_path)
+        return backup_path
+
+    @staticmethod
+    def atomic_write(file_path: Path, content: str) -> None:
+        """Write content to file atomically using temp file + os.replace()."""
+        temp_path = file_path.with_suffix(file_path.suffix + ".tmp")
+        try:
+            with open(temp_path, "w", encoding="utf-8") as f:
+                f.write(content)
+            os.replace(temp_path, file_path)
+        except Exception:
+            # Clean up temp file on failure
+            if temp_path.exists():
+                temp_path.unlink()
+            raise
+
+
+@dataclass
+class UpgradeResult:
+    """Result of a document upgrade operation."""
+    upgraded: bool
+    old_version: str
+    new_version: str
+    document: Optional[dict] = None
+    backup_path: Optional[Path] = None
+    message: str = ""
+    error: Optional[str] = None
+
+
+class DocumentUpgrader:
+    """Orchestrates document validation, backup, and schema version upgrade.
+
+    Upgrade flow:
+    1. Load JSON document
+    2. Detect schemaVersion (assume "1.0.0" if missing per FR-030)
+    3. Reject unknown future versions
+    4. Validate against current schema
+    5. If valid + legacy (no schemaVersion): backup + upgrade
+    6. If valid + older (same major): upgrade without backup (ADR-007)
+    7. If invalid: return error without modifying file (FR-034)
+    8. Write updated document atomically
+    """
+
+    def __init__(
+        self,
+        validator: Optional[SchemaValidator] = None,
+        version_manager: Optional[SchemaVersionManager] = None,
+    ):
+        self._validator = validator or SchemaValidator()
+        self._version_manager = version_manager or SchemaVersionManager()
+
+    def upgrade(self, file_path: Path) -> UpgradeResult:
+        """Validate and upgrade an eval document file.
+
+        Args:
+            file_path: Path to the eval document JSON file.
+
+        Returns:
+            UpgradeResult with details of what happened.
+        """
+        current_version = self._version_manager.get_current_version()
+
+        # 1. Load document
+        with open(file_path, "r", encoding="utf-8") as f:
+            document = json.load(f)
+
+        # 2. Detect document type and schemaVersion
+        is_legacy = False
+        if isinstance(document, list):
+            # Bare array format — wrap into eval document format
+            document = {"items": document}
+            is_legacy = True
+        elif isinstance(document, dict) and "items" not in document:
+            if "prompts" in document:
+                # Dict format with prompts/expected_responses — convert
+                prompts = document.get("prompts", [])
+                expected_responses = document.get("expected_responses", [])
+                items = []
+                for i, prompt in enumerate(prompts):
+                    item: dict[str, Any] = {"prompt": prompt}
+                    if i < len(expected_responses) and expected_responses[i]:
+                        item["expected_response"] = expected_responses[i]
+                    items.append(item)
+                document = {"items": items}
+                is_legacy = True
+
+        doc_version = document.get("schemaVersion")
+        if doc_version is None:
+            doc_version = "1.0.0"
+            is_legacy = True
+
+        # 3. Reject unsupported or newer major versions
+        comparison = SchemaVersionManager.compare_versions(doc_version, current_version)
+        if comparison == VersionRelation.NEWER_MAJOR:
+            return UpgradeResult(
+                upgraded=False,
+                old_version=doc_version,
+                new_version=current_version,
+                error=(
+                    f"Document uses schema v{doc_version}, but this CLI only supports "
+                    f"v{current_version}. Please update the CLI to a version that "
+                    f"supports schema v{doc_version.split('.')[0]}.x."
+                ),
+            )
+        if comparison == VersionRelation.UNSUPPORTED:
+            return UpgradeResult(
+                upgraded=False,
+                old_version=doc_version,
+                new_version=current_version,
+                error=(
+                    f"Document has an invalid schema version '{doc_version}'. "
+                    "Expected a valid SemVer string (e.g., '1.0.0'). "
+                    "Please check the document source."
+                ),
+            )
+
+        # 4. Validate against current schema — inject schemaVersion temporarily for validation
+        validation_doc = dict(document)
+        if "schemaVersion" not in validation_doc:
+            validation_doc["schemaVersion"] = current_version
+
+        is_valid, errors = self._validator.validate(validation_doc)
+
+        if not is_valid:
+            error_msg = SchemaValidator.format_errors(errors)
+            return UpgradeResult(
+                upgraded=False,
+                old_version=doc_version,
+                new_version=current_version,
+                error=f"Document validation failed:\n{error_msg}",
+            )
+
+        # 5/6. Document is valid — decide upgrade strategy
+        if comparison == VersionRelation.CURRENT and not is_legacy:
+            # Already at current version and not legacy — no-op
+            return UpgradeResult(
+                upgraded=False,
+                old_version=doc_version,
+                new_version=current_version,
+                document=document,
+                message="Document is already at current schema version.",
+            )
+
+        # Upgrade needed — three scenarios reach here:
+        #   1. is_legacy=True, comparison=CURRENT  → legacy format (array/dict/no schemaVersion)
+        #      at current version; rewrite as eval document with backup
+        #   2. is_legacy=True, comparison=OLDER    → legacy format at an older version;
+        #      rewrite as eval document with backup
+        #   3. is_legacy=False, comparison=OLDER   → proper eval document at an older version;
+        #      bump schemaVersion in-place, no backup needed (ADR-007)
+        backup_path = None
+        if is_legacy:
+            # Legacy document — create backup before structural conversion (FR-032)
+            backup_path = FileBackupManager.create_backup(file_path)
+
+        # Update schemaVersion
+        document["schemaVersion"] = current_version
+
+        # Write atomically
+        updated_content = json.dumps(document, indent=2, ensure_ascii=False) + "\n"
+        FileBackupManager.atomic_write(file_path, updated_content)
+
+        if backup_path:
+            message = (
+                f"Document upgraded from {doc_version} to {current_version}. "
+                f"Original backed up to {backup_path}"
+            )
+        else:
+            message = f"Document upgraded from {doc_version} to {current_version}."
+
+        logger.info(message)
+
+        return UpgradeResult(
+            upgraded=True,
+            old_version=doc_version,
+            new_version=current_version,
+            document=document,
+            backup_path=backup_path,
+            message=message,
+        )

package/src/clients/cli/version_check.py

@@ -0,0 +1,139 @@
+"""
+Minimum version check for the M365 Copilot Agent Evaluation CLI.
+
+Strategy:
+  The published package.json contains a custom "minCliVersion" field.
+  At runtime the CLI fetches the latest published package metadata via
+  an aka.ms redirect that points to the npm registry API:
+
+      https://aka.ms/m365-evals-min-version
+        -> https://registry.npmjs.org/@microsoft/m365-copilot-eval/latest
+
+  The response is the full published package.json as JSON. We simply
+  read the "minCliVersion" field and compare it against the local
+  version.
+
+  Using aka.ms as an intermediary lets us change the target endpoint
+  without shipping a new CLI version.
+"""
+
+import json
+import os
+import urllib.request
+import urllib.error
+from typing import Optional
+
+from packaging.version import Version, InvalidVersion
+
+# The public npm package name
+NPM_PACKAGE_NAME = "@microsoft/m365-copilot-eval"
+
+# npm registry API URL via aka.ms redirect
+MIN_VERSION_URL = "https://aka.ms/m365-evals-min-version"
+
+# Timeout for fetching from the registry (seconds) — kept short to fail fast
+FETCH_TIMEOUT = 3
+
+
+def get_cli_version(quiet: bool = False) -> Optional[Version]:
+    """
+    Read the current CLI version from package.json at the repo root.
+
+    release-please keeps package.json's "version" field up to date,
+    so there is no need for a hardcoded version constant.
+
+    Args:
+        quiet: If True, suppress warning output on failure.
+
+    Returns:
+        A parsed Version object (e.g. Version('1.1.1-preview.1')),
+        or None if package.json cannot be read or the version is
+        unparseable.
+    """
+    try:
+        # Walk up from this file (src/clients/cli/) to the repo root
+        here = os.path.dirname(os.path.abspath(__file__))
+        package_json_path = os.path.join(here, "..", "..", "..", "package.json")
+        package_json_path = os.path.normpath(package_json_path)
+        with open(package_json_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return Version(data.get("version"))
+    except Exception as e:
+        if not quiet:
+            print(
+                f"\033[33mWarning: Could not determine current CLI version: {e}\033[0m"
+            )
+        return None
+
+
+def fetch_min_version(quiet: bool = False) -> Optional[Version]:
+    """
+    Fetch the minimum required CLI version from the npm registry.
+
+    How it works:
+      1. GET https://registry.npmjs.org/@microsoft/m365-copilot-eval/latest
+      2. The response is the published package.json as plain JSON.
+      3. Read the "minCliVersion" field.
+
+    Args:
+        quiet: If True, suppress warning output on failure.
+
+    Returns:
+        A parsed Version object (e.g. Version('1.3.0')),
+        or None on any failure.
+    """
+    try:
+        req = urllib.request.Request(
+            MIN_VERSION_URL,
+            headers={
+                "Accept": "application/json",
+            },
+        )
+        with urllib.request.urlopen(req, timeout=FETCH_TIMEOUT) as resp:
+            data = json.loads(resp.read().decode("utf-8"))
+
+        return Version(data.get("minCliVersion"))
+
+    except Exception as e:
+        if not quiet:
+            print(
+                f"\033[33mWarning: Could not fetch minimum version from registry: {e}\033[0m"
+            )
+        return None
+
+
+def check_min_version(current_version: Optional[Version], quiet: bool = False) -> bool:
+    """
+    Check whether the current CLI version meets the minimum required version
+    published in the npm registry.
+
+    Args:
+        current_version: The parsed Version this CLI is running
+                         (from get_cli_version()), or None if version
+                         cannot be determined.
+        quiet:           If True, suppress informational output.
+
+    Returns:
+        True if the version is acceptable (>= minVersion or check failed
+        gracefully), False if the current version is below the minimum.
+    """
+    # Fail open if current version unknown (package.json missing/unreadable)
+    if current_version is None:
+        return True
+
+    min_version = fetch_min_version(quiet=quiet)
+
+    if min_version is None:
+        return True
+
+    if current_version < min_version:
+        print(
+            f"\033[91m"
+            f"This version, {current_version}, of the M365 Evals CLI is no longer functional, "
+            f"you must update your tool to continue using it.\n"
+            f"Update by running: npm install -g {NPM_PACKAGE_NAME}@latest"
+            f"\033[0m"
+        )
+        return False
+
+    return True