failproofai 1.0.3 → 1.0.4-beta.2

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/authoring.py

@@ -0,0 +1,404 @@
+"""Evaluator definition registry and typed author results."""
+
+from __future__ import annotations
+
+import hashlib
+import inspect
+import json
+import math
+import re
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from failproofai_sdk.evaluator.protocol import (
+    MAX_CATALOG_DEFINITIONS,
+    MAX_DESCRIPTION_BYTES,
+    MAX_DISPLAY_NAME_BYTES,
+    MAX_DISPLAY_VALUE_BYTES,
+    MAX_EVAL_KEY_BYTES,
+    MAX_LABEL_BYTES,
+    MAX_LABELS_PER_RESULT,
+    MAX_REASONING_BYTES,
+    MAX_RESULTS_PER_RUN,
+    MAX_SUMMARY_BYTES,
+    MAX_UNIT_BYTES,
+    MAX_VERSION_BYTES,
+    CatalogDefinition,
+    ResultItem,
+    ResultKind,
+    SessionTranscript,
+)
+
+_KEY = re.compile(r"^[a-z][a-z0-9_]*$")
+EvalFunction = Callable[[SessionTranscript], "EvalResult | Awaitable[EvalResult]"]
+ConditionFunction = Callable[
+    [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]"
+]
+CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"]
+
+
+def _bounded(value: str, *, field_name: str, maximum: int) -> str:
+    if not isinstance(value, str):
+        raise TypeError(f"{field_name} must be a string")
+    if not value:
+        raise ValueError(f"{field_name} must not be empty")
+    size = len(value.encode("utf-8"))
+    if size > maximum:
+        raise ValueError(f"{field_name} is {size} bytes; maximum is {maximum}")
+    # Reject C0 control characters and DEL, matching the server's `check_bounded`
+    # (server/src/evaluator/protocol.rs). Without this the SDK accepts a string —
+    # e.g. reasoning/summary quoting transcript text that contains an ANSI escape
+    # or NUL — that the server then rejects with a NON-RETRYABLE 422, so a
+    # successful evaluation is silently lost and its assignment dead-letters.
+    # TAB, LF and CR are kept because real multi-line reasoning uses them.
+    bad = next(
+        (
+            ch
+            for ch in value
+            if (ord(ch) < 0x20 and ch not in "\t\n\r") or ord(ch) == 0x7F
+        ),
+        None,
+    )
+    if bad is not None:
+        raise ValueError(
+            f"{field_name} must not contain control characters "
+            f"(found U+{ord(bad):04X})"
+        )
+    return value
+
+
+def _finite(value: float, field_name: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise TypeError(f"{field_name} must be a number")
+    result = float(value)
+    if not math.isfinite(result):
+        raise ValueError(f"{field_name} must be finite")
+    return result
+
+
+def _labels(values: tuple[str, ...] | list[str]) -> tuple[str, ...]:
+    if len(values) > MAX_LABELS_PER_RESULT:
+        raise ValueError(f"at most {MAX_LABELS_PER_RESULT} labels are allowed")
+    normalized = []
+    for label in values:
+        normalized.append(_bounded(label, field_name="label", maximum=MAX_LABEL_BYTES))
+    if len(set(normalized)) != len(normalized):
+        raise ValueError("labels must be unique")
+    return tuple(sorted(normalized))
+
+
+@dataclass(frozen=True)
+class Score:
+    value: float
+    passed: bool | None = None
+    unit: str = "ratio"
+    display_value: str | None = None
+    description: str | None = None
+
+    def __post_init__(self) -> None:
+        value = _finite(self.value, "score value")
+        if not 0 <= value <= 1:
+            raise ValueError("score value must be between 0 and 1")
+        object.__setattr__(self, "value", value)
+        if self.passed is not None and not isinstance(self.passed, bool):
+            raise TypeError("score passed must be a boolean or None")
+        _validate_result_text(self.unit, self.display_value, self.description)
+
+
+@dataclass(frozen=True)
+class Metric:
+    value: float
+    unit: str = ""
+    display_value: str | None = None
+    description: str | None = None
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "value", _finite(self.value, "metric value"))
+        _validate_result_text(self.unit, self.display_value, self.description)
+
+
+@dataclass(frozen=True)
+class Assertion:
+    passed: bool
+    description: str | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.passed, bool):
+            raise TypeError("assertion passed must be a boolean")
+        if self.description is not None:
+            _bounded(
+                self.description,
+                field_name="description",
+                maximum=MAX_DESCRIPTION_BYTES,
+            )
+
+
+@dataclass(frozen=True)
+class ConditionResult:
+    applicable: bool
+    reason_code: str = "condition_false"
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.applicable, bool):
+            raise TypeError("condition applicable must be a boolean")
+        _validate_key(self.reason_code, "condition reason code")
+
+
+def _validate_result_text(
+    unit: str, display_value: str | None, description: str | None
+) -> None:
+    if unit:
+        _bounded(unit, field_name="unit", maximum=MAX_UNIT_BYTES)
+    if display_value is not None:
+        _bounded(
+            display_value,
+            field_name="display value",
+            maximum=MAX_DISPLAY_VALUE_BYTES,
+        )
+    if description is not None:
+        _bounded(
+            description,
+            field_name="description",
+            maximum=MAX_DESCRIPTION_BYTES,
+        )
+
+
+@dataclass(frozen=True)
+class EvalResult:
+    score: Score | None = None
+    metrics: Mapping[str, Metric | float] = field(default_factory=dict)
+    assertions: Mapping[str, Assertion | bool] = field(default_factory=dict)
+    reasoning: str | None = None
+    summary: str | None = None
+    labels: tuple[str, ...] = ()
+
+    def __post_init__(self) -> None:
+        if self.reasoning is not None:
+            _bounded(
+                self.reasoning,
+                field_name="reasoning",
+                maximum=MAX_REASONING_BYTES,
+            )
+        if self.summary is not None:
+            _bounded(self.summary, field_name="summary", maximum=MAX_SUMMARY_BYTES)
+        object.__setattr__(self, "labels", _labels(list(self.labels)))
+
+    def result_items(self, eval_key: str) -> tuple[ResultItem, ...]:
+        items: list[ResultItem] = []
+        if self.score is not None:
+            items.append(
+                ResultItem(
+                    result_key=eval_key,
+                    result_kind=ResultKind.SCORE,
+                    numeric_value=self.score.value,
+                    bool_value=self.score.passed,
+                    unit=self.score.unit,
+                    display_value=self.score.display_value,
+                    description=self.score.description,
+                    reasoning=self.reasoning,
+                    labels=self.labels,
+                )
+            )
+        for key, raw_metric in sorted(self.metrics.items()):
+            _validate_key(key, "metric key")
+            metric = (
+                raw_metric if isinstance(raw_metric, Metric) else Metric(raw_metric)
+            )
+            items.append(
+                ResultItem(
+                    result_key=key,
+                    result_kind=ResultKind.METRIC,
+                    numeric_value=metric.value,
+                    unit=metric.unit,
+                    display_value=metric.display_value,
+                    description=metric.description,
+                    # A metric-kind eval's primary result IS the metric whose
+                    # key equals eval_key; attach the eval's reasoning there so
+                    # it is not silently dropped for non-score evals.
+                    reasoning=self.reasoning if key == eval_key else None,
+                    labels=self.labels,
+                )
+            )
+        for key, raw_assertion in sorted(self.assertions.items()):
+            _validate_key(key, "assertion key")
+            assertion = (
+                raw_assertion
+                if isinstance(raw_assertion, Assertion)
+                else Assertion(raw_assertion)
+            )
+            items.append(
+                ResultItem(
+                    result_key=key,
+                    result_kind=ResultKind.ASSERTION,
+                    bool_value=assertion.passed,
+                    description=assertion.description,
+                    # An assertion-kind eval's primary result is the assertion
+                    # whose key equals eval_key; carry the eval's reasoning there
+                    # so a non-score eval does not lose it.
+                    reasoning=self.reasoning if key == eval_key else None,
+                    labels=self.labels,
+                )
+            )
+        if not items:
+            raise ValueError("an EvalResult must contain a score, metric, or assertion")
+        if len(items) > MAX_RESULTS_PER_RUN:
+            raise ValueError(
+                f"an EvalResult may contain at most {MAX_RESULTS_PER_RUN} results"
+            )
+        keys = [item.result_key for item in items]
+        if len(keys) != len(set(keys)):
+            raise ValueError("result keys must be unique within one evaluation run")
+        return tuple(items)
+
+
+def _validate_key(value: str, field_name: str = "eval_key") -> str:
+    _bounded(value, field_name=field_name, maximum=MAX_EVAL_KEY_BYTES)
+    if not _KEY.fullmatch(value):
+        raise ValueError(f"{field_name} must match {_KEY.pattern}")
+    return value
+
+
+@dataclass(frozen=True)
+class EvalDefinition:
+    eval_key: str
+    display_name: str
+    eval_version: str
+    result_kind: ResultKind
+    labels: tuple[str, ...]
+    function: EvalFunction
+    condition: ConditionFunction | None
+    on_cancel: CancellationFunction | None
+    timeout_seconds: float | None
+
+    def catalog_definition(self) -> CatalogDefinition:
+        return CatalogDefinition(
+            eval_key=self.eval_key,
+            display_name=self.display_name,
+            eval_version=self.eval_version,
+            result_kind=self.result_kind,
+            labels=self.labels,
+        )
+
+
+class Evaluator:
+    """A process-local collection of explicitly versioned evaluations."""
+
+    def __init__(self, *, name: str, version: str) -> None:
+        self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES)
+        self.version = _bounded(
+            version, field_name="version", maximum=MAX_VERSION_BYTES
+        )
+        self._definitions: dict[str, EvalDefinition] = {}
+
+    def eval(
+        self,
+        eval_key: str,
+        *,
+        version: str,
+        display_name: str | None = None,
+        result_kind: ResultKind | str = ResultKind.SCORE,
+        labels: tuple[str, ...] | list[str] = (),
+        when: ConditionFunction | None = None,
+        on_cancel: CancellationFunction | None = None,
+        timeout_seconds: float | None = None,
+    ) -> Callable[[EvalFunction], EvalFunction]:
+        key = _validate_key(eval_key)
+        eval_version = _bounded(
+            version, field_name="eval version", maximum=MAX_VERSION_BYTES
+        )
+        display = _bounded(
+            display_name or eval_key.replace("_", " ").capitalize(),
+            field_name="display name",
+            maximum=MAX_DISPLAY_NAME_BYTES,
+        )
+        kind = ResultKind(result_kind)
+        normalized_labels = _labels(list(labels))
+        if timeout_seconds is not None:
+            timeout_seconds = _finite(timeout_seconds, "timeout_seconds")
+            if timeout_seconds <= 0:
+                raise ValueError("timeout_seconds must be greater than zero")
+
+        def register(function: EvalFunction) -> EvalFunction:
+            if key in self._definitions:
+                raise ValueError(f"duplicate eval key: {key}")
+            if len(self._definitions) >= MAX_CATALOG_DEFINITIONS:
+                raise ValueError(
+                    f"an evaluator may define at most {MAX_CATALOG_DEFINITIONS} evaluations"
+                )
+            if not callable(function):
+                raise TypeError("evaluation must be callable")
+            if when is not None and not callable(when):
+                raise TypeError("when must be callable")
+            if on_cancel is not None and not callable(on_cancel):
+                raise TypeError("on_cancel must be callable")
+            self._definitions[key] = EvalDefinition(
+                eval_key=key,
+                display_name=display,
+                eval_version=eval_version,
+                result_kind=kind,
+                labels=normalized_labels,
+                function=function,
+                condition=when,
+                on_cancel=on_cancel,
+                timeout_seconds=timeout_seconds,
+            )
+            return function
+
+        return register
+
+    @property
+    def definitions(self) -> tuple[EvalDefinition, ...]:
+        return tuple(self._definitions[key] for key in sorted(self._definitions))
+
+    def catalog(self) -> tuple[CatalogDefinition, ...]:
+        return tuple(definition.catalog_definition() for definition in self.definitions)
+
+    @property
+    def catalog_revision(self) -> str:
+        payload = [item.to_wire() for item in self.catalog()]
+        canonical = json.dumps(
+            payload,
+            ensure_ascii=False,
+            allow_nan=False,
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode("utf-8")
+        return "sha256:" + hashlib.sha256(canonical).hexdigest()
+
+    def definition(self, eval_key: str) -> EvalDefinition:
+        try:
+            return self._definitions[eval_key]
+        except KeyError as error:
+            raise KeyError(f"unknown eval key: {eval_key}") from error
+
+    def run_from_env(self) -> None:
+        """Run this evaluator until the process receives a stop request."""
+        import asyncio
+        import signal
+
+        from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime
+
+        async def run() -> None:
+            runtime = WorkerRuntime(self, WorkerConfig.from_env())
+            loop = asyncio.get_running_loop()
+            for name in ("SIGINT", "SIGTERM"):
+                process_signal = getattr(signal, name, None)
+                if process_signal is None:
+                    continue
+                try:
+                    loop.add_signal_handler(process_signal, runtime.stop)
+                except (NotImplementedError, RuntimeError):
+                    pass
+            await runtime.run_forever()
+
+        asyncio.run(run())
+
+    @staticmethod
+    async def call(
+        function: EvalFunction | ConditionFunction, session: SessionTranscript
+    ) -> Any:
+        result = function(session)
+        if inspect.isawaitable(result):
+            return await result
+        return result

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/client.py

@@ -0,0 +1,299 @@
+"""Standard-library HTTP client for the Evaluator v2 worker protocol."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import random
+import time
+from collections.abc import Callable, Mapping
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+from failproofai_sdk.evaluator.protocol import (
+    CLAIM_PATH,
+    DEFINITIONS_PATH,
+    HEARTBEAT_PATH,
+    LEASE_GENERATION_HEADER,
+    MAX_TRANSCRIPT_BYTES,
+    PLAN_PATH,
+    REGISTER_PATH,
+    RESULT_PATH,
+    WORKER_ID_HEADER,
+    Assignment,
+    DefinitionsResponse,
+    ClaimRequest,
+    ClaimResponse,
+    ErrorResponse,
+    HeartbeatRequest,
+    HeartbeatResponse,
+    PlanRequest,
+    PlanResponse,
+    RegisterRequest,
+    RegisterResponse,
+    ResultRequest,
+    ResultResponse,
+    SessionTranscript,
+    WireModel,
+)
+
+_DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024
+_RETRYABLE_HTTP_STATUSES = frozenset({429, 502, 503, 504})
+
+
+class _RejectRedirects(HTTPRedirectHandler):
+    def redirect_request(self, request, file_pointer, code, message, headers, new_url):
+        return None
+
+
+def _open_without_redirects(request: Request, *, timeout: float):
+    return build_opener(_RejectRedirects()).open(request, timeout=timeout)
+
+
+class EvaluatorAPIError(RuntimeError):
+    def __init__(
+        self,
+        *,
+        status: int | None,
+        code: str,
+        message: str,
+        retryable: bool,
+        request_id: str | None = None,
+    ) -> None:
+        super().__init__(f"{code}: {message}")
+        self.status = status
+        self.code = code
+        self.retryable = retryable
+        self.request_id = request_id
+
+
+class EvaluatorClient:
+    """Client for the public Evaluator v2 machine API.
+
+    Hosted workers normally use the FailproofAI dashboard origin. Its ``/v1``
+    passthrough forwards this worker's bearer credential to the private server.
+    """
+
+    def __init__(
+        self,
+        *,
+        base_url: str,
+        credential: str,
+        timeout_seconds: float = 30,
+        max_retries: int = 3,
+        allow_insecure_http: bool = False,
+        opener: Callable[..., Any] | None = None,
+        sleeper: Callable[[float], None] = time.sleep,
+    ) -> None:
+        parsed = urlsplit(base_url)
+        if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+            raise ValueError("base_url must be an absolute http(s) URL")
+        hostname = parsed.hostname
+        loopback = hostname == "localhost"
+        if hostname is not None and not loopback:
+            try:
+                loopback = ipaddress.ip_address(hostname).is_loopback
+            except ValueError:
+                loopback = False
+        if parsed.scheme != "https" and not loopback and not allow_insecure_http:
+            raise ValueError("base_url must use https unless it targets loopback")
+        if not credential or not credential.strip():
+            raise ValueError("credential must not be empty")
+        if any(
+            ord(character) < 32 or ord(character) == 127 for character in credential
+        ):
+            raise ValueError("credential must not contain control characters")
+        if timeout_seconds <= 0:
+            raise ValueError("timeout_seconds must be greater than zero")
+        if max_retries < 0:
+            raise ValueError("max_retries must not be negative")
+        self._base_url = base_url.rstrip("/") + "/"
+        self._origin = (parsed.scheme, parsed.netloc)
+        self._credential = credential
+        self._timeout_seconds = timeout_seconds
+        self._max_retries = max_retries
+        self._opener = opener or _open_without_redirects
+        self._sleeper = sleeper
+
+    def register(self, request: RegisterRequest) -> RegisterResponse:
+        return RegisterResponse.from_wire(
+            self._json("POST", REGISTER_PATH, request, retry=True)
+        )
+
+    def claim(self, request: ClaimRequest) -> ClaimResponse:
+        # A lost claim response may already have leased work. Do not hide a
+        # second claim behind transport retry; the runtime recalculates capacity.
+        return ClaimResponse.from_wire(
+            self._json("POST", CLAIM_PATH, request, retry=False)
+        )
+
+    def transcript(
+        self, assignment: Assignment, *, worker_id: str
+    ) -> SessionTranscript:
+        headers = {
+            WORKER_ID_HEADER: worker_id,
+            LEASE_GENERATION_HEADER: str(assignment.lease_generation),
+        }
+        return SessionTranscript.from_wire(
+            self._json(
+                "GET",
+                assignment.transcript_url,
+                None,
+                retry=True,
+                headers=headers,
+                response_limit=MAX_TRANSCRIPT_BYTES,
+            )
+        )
+
+    def definitions(
+        self, assignment: Assignment, *, worker_id: str
+    ) -> DefinitionsResponse:
+        headers = {
+            WORKER_ID_HEADER: worker_id,
+            LEASE_GENERATION_HEADER: str(assignment.lease_generation),
+        }
+        path = assignment.definitions_url or DEFINITIONS_PATH.format(
+            assignment_id=assignment.assignment_id
+        )
+        return DefinitionsResponse.from_wire(
+            self._json("GET", path, None, retry=True, headers=headers)
+        )
+
+    def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse:
+        return PlanResponse.from_wire(
+            self._json(
+                "POST",
+                PLAN_PATH.format(assignment_id=assignment_id),
+                request,
+                retry=True,
+            )
+        )
+
+    def heartbeat(self, request: HeartbeatRequest) -> HeartbeatResponse:
+        return HeartbeatResponse.from_wire(
+            self._json("POST", HEARTBEAT_PATH, request, retry=True)
+        )
+
+    def submit_result(self, run_id: str, request: ResultRequest) -> ResultResponse:
+        return ResultResponse.from_wire(
+            self._json(
+                "POST",
+                RESULT_PATH.format(evaluation_run_id=run_id),
+                request,
+                retry=True,
+            )
+        )
+
+    def _url(self, path: str) -> str:
+        url = urljoin(self._base_url, path)
+        parsed = urlsplit(url)
+        if (parsed.scheme, parsed.netloc) != self._origin:
+            raise EvaluatorAPIError(
+                status=None,
+                code="invalid_transcript_url",
+                message="server supplied a URL outside the configured API origin",
+                retryable=False,
+            )
+        return url
+
+    def _json(
+        self,
+        method: str,
+        path: str,
+        body: WireModel | None,
+        *,
+        retry: bool,
+        headers: Mapping[str, str] | None = None,
+        response_limit: int = _DEFAULT_RESPONSE_LIMIT,
+    ) -> dict[str, Any]:
+        encoded = None
+        request_headers = {
+            "Accept": "application/json",
+            "Authorization": f"Bearer {self._credential}",
+            "User-Agent": "failproofai-sdk-evaluator/2",
+        }
+        if body is not None:
+            encoded = json.dumps(
+                body.to_wire(),
+                allow_nan=False,
+                ensure_ascii=False,
+                separators=(",", ":"),
+            ).encode("utf-8")
+            request_headers["Content-Type"] = "application/json"
+        if headers:
+            request_headers.update(headers)
+
+        attempts = self._max_retries + 1 if retry else 1
+        for attempt in range(attempts):
+            request = Request(
+                self._url(path), data=encoded, headers=request_headers, method=method
+            )
+            try:
+                with self._opener(request, timeout=self._timeout_seconds) as response:
+                    return self._decode(
+                        response.read(response_limit + 1), response_limit
+                    )
+            except HTTPError as error:
+                api_error = self._http_error(error, response_limit)
+                if attempt + 1 == attempts or not api_error.retryable:
+                    raise api_error from error
+            except (URLError, TimeoutError, OSError) as error:
+                if attempt + 1 == attempts:
+                    raise EvaluatorAPIError(
+                        status=None,
+                        code="transport_error",
+                        message=str(error),
+                        retryable=True,
+                    ) from error
+            # Jitter is scheduling noise, not a security decision.
+            self._sleeper(random.uniform(0, min(0.25 * (2**attempt), 2.0)))  # nosec B311
+        raise AssertionError("retry loop exhausted without returning or raising")
+
+    @staticmethod
+    def _decode(raw: bytes, limit: int) -> dict[str, Any]:
+        if len(raw) > limit:
+            raise EvaluatorAPIError(
+                status=None,
+                code="response_too_large",
+                message=f"server response exceeds {limit} bytes",
+                retryable=False,
+            )
+        try:
+            value = json.loads(raw)
+        except (UnicodeDecodeError, json.JSONDecodeError) as error:
+            raise EvaluatorAPIError(
+                status=None,
+                code="invalid_response",
+                message="server response was not valid JSON",
+                retryable=False,
+            ) from error
+        if not isinstance(value, dict):
+            raise EvaluatorAPIError(
+                status=None,
+                code="invalid_response",
+                message="server response must be a JSON object",
+                retryable=False,
+            )
+        return value
+
+    @classmethod
+    def _http_error(cls, error: HTTPError, limit: int) -> EvaluatorAPIError:
+        raw = error.read(limit + 1)
+        try:
+            response = ErrorResponse.from_wire(cls._decode(raw, limit))
+        except (ValueError, EvaluatorAPIError):
+            return EvaluatorAPIError(
+                status=error.code,
+                code="http_error",
+                message=f"server returned HTTP {error.code}",
+                retryable=error.code in _RETRYABLE_HTTP_STATUSES,
+            )
+        return EvaluatorAPIError(
+            status=error.code,
+            code=response.error.code,
+            message=response.error.message,
+            retryable=response.error.retryable,
+            request_id=response.error.request_id,
+        )