failproofai 1.0.3 → 1.0.4-beta.2

package/.next/standalone/sdk/python/examples/evaluator_worker.py

@@ -0,0 +1,121 @@
+"""Customer evaluator with deterministic and optional async judge checks."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import json
+import os
+from urllib.parse import urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+from failproofai_sdk.evaluator import (
+    ConditionResult,
+    EvalResult,
+    Evaluator,
+    Metric,
+    Score,
+)
+
+app = Evaluator(name="customer-production", version="2026.08.1")
+
+
+class _RejectRedirects(HTTPRedirectHandler):
+    def redirect_request(self, request, file_pointer, code, message, headers, new_url):
+        return None
+
+
+@app.eval(
+    "tool_efficiency",
+    version="1.0.0",
+    labels=["tools", "deterministic"],
+    when=lambda session: ConditionResult(
+        session.count("tool_use") > 0, "no_tool_calls"
+    ),
+)
+def tool_efficiency(session):
+    calls = session.events_of_type("tool_use")
+    distinct = {
+        event.payload.get("tool_name")
+        for event in calls
+        if event.payload.get("tool_name")
+    }
+    value = len(distinct) / len(calls)
+    return EvalResult(
+        score=Score(value, passed=value >= 0.7),
+        metrics={
+            "tool_call_count": Metric(len(calls), unit="events"),
+            "distinct_tool_count": Metric(len(distinct), unit="tools"),
+        },
+        reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls",
+    )
+
+
+def _judge_configured(session):
+    configured = bool(os.environ.get("EXAMPLE_JUDGE_URL"))
+    return ConditionResult(configured, "judge_not_configured")
+
+
+def _last_content(session, event_type):
+    events = session.events_of_type(event_type)
+    if not events:
+        return None
+    payload = events[-1].payload
+    fields = {
+        "human_input": ("response",),
+        "model_response": ("content",),
+        "agent_end": ("summary",),
+    }.get(event_type, ("content", "summary", "response"))
+    return next((payload.get(field) for field in fields if payload.get(field)), None)
+
+
+def _call_judge(question, answer):
+    url = os.environ["EXAMPLE_JUDGE_URL"]
+    parsed = urlsplit(url)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL")
+    hostname = parsed.hostname
+    loopback = hostname == "localhost"
+    if hostname is not None and not loopback:
+        try:
+            loopback = ipaddress.ip_address(hostname).is_loopback
+        except ValueError:
+            loopback = False
+    if parsed.scheme != "https" and not loopback:
+        raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback")
+    token = os.environ.get("EXAMPLE_JUDGE_TOKEN")
+    body = json.dumps({"question": question, "answer": answer}).encode("utf-8")
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    request = Request(url, data=body, headers=headers, method="POST")
+    with build_opener(_RejectRedirects()).open(request, timeout=25) as response:  # nosec B310
+        result = json.loads(response.read(64 * 1024))
+    return float(result["score"]), str(
+        result.get("reasoning") or "Judge returned no reasoning"
+    )
+
+
+@app.eval(
+    "answer_relevance",
+    version="judge-api-v1",
+    labels=["llm_judge", "relevance"],
+    when=_judge_configured,
+    timeout_seconds=30,
+)
+async def answer_relevance(session):
+    question = _last_content(session, "human_input")
+    answer = _last_content(session, "model_response")
+    if question is None or answer is None:
+        raise ValueError("answer relevance requires human input and model output")
+    value, reasoning = await asyncio.to_thread(_call_judge, question, answer)
+    value = min(max(value, 0.0), 1.0)
+    return EvalResult(
+        score=Score(value, passed=value >= 0.7),
+        reasoning=reasoning,
+        labels=("llm_judge", "relevance"),
+    )
+
+
+if __name__ == "__main__":
+    app.run_from_env()

package/.next/standalone/sdk/python/failproofai_sdk/_version.py

@@ -1 +1 @@
-__version__ = "0.0.1b2"
+__version__ = "0.0.1b3"

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/__init__.py

@@ -0,0 +1,101 @@
+"""Authoring and worker primitives for FailproofAI Evaluator v2.
+
+This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users
+who only emit telemetry do not import evaluator networking or runtime code.
+"""
+
+from failproofai_sdk.evaluator.authoring import (
+    Assertion,
+    ConditionResult,
+    EvalDefinition,
+    EvalResult,
+    Evaluator,
+    Metric,
+    Score,
+)
+from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient
+from failproofai_sdk.evaluator.protocol import (
+    Assignment,
+    AssignmentDefinition,
+    CatalogDefinition,
+    ClaimRequest,
+    ClaimResponse,
+    ErrorResponse,
+    EvalSelection,
+    ExecutionMode,
+    EvaluatorKind,
+    HeartbeatRequest,
+    HeartbeatResponse,
+    HeartbeatRun,
+    PlannedRun,
+    PlanRequest,
+    PlanResponse,
+    DefinitionsResponse,
+    ProtocolError,
+    RegisterRequest,
+    RegisterResponse,
+    RemoteError,
+    ResultItem,
+    ResultKind,
+    ResultRequest,
+    ResultResponse,
+    SessionTranscript,
+    SkippedEval,
+    TerminalRunStatus,
+    TranscriptEvent,
+    UnsupportedProtocolVersion,
+)
+from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime
+from failproofai_sdk.evaluator.source import (
+    UnsafeEvaluatorSource,
+    compile_condition,
+    compile_evaluator,
+    source_checksum,
+)
+
+__all__ = [
+    "Assertion",
+    "Assignment",
+    "AssignmentDefinition",
+    "CatalogDefinition",
+    "ClaimRequest",
+    "ClaimResponse",
+    "ConditionResult",
+    "ErrorResponse",
+    "EvalDefinition",
+    "EvalResult",
+    "EvalSelection",
+    "ExecutionMode",
+    "Evaluator",
+    "EvaluatorAPIError",
+    "EvaluatorClient",
+    "EvaluatorKind",
+    "HeartbeatRequest",
+    "HeartbeatResponse",
+    "HeartbeatRun",
+    "Metric",
+    "PlanRequest",
+    "PlanResponse",
+    "DefinitionsResponse",
+    "PlannedRun",
+    "ProtocolError",
+    "RegisterRequest",
+    "RegisterResponse",
+    "RemoteError",
+    "ResultItem",
+    "ResultKind",
+    "ResultRequest",
+    "ResultResponse",
+    "Score",
+    "SessionTranscript",
+    "SkippedEval",
+    "TerminalRunStatus",
+    "TranscriptEvent",
+    "UnsupportedProtocolVersion",
+    "WorkerConfig",
+    "WorkerRuntime",
+    "UnsafeEvaluatorSource",
+    "compile_condition",
+    "compile_evaluator",
+    "source_checksum",
+]

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/__main__.py

@@ -0,0 +1,49 @@
+"""Run an evaluator declared as ``module:attribute``."""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import os
+from collections.abc import Sequence
+
+from failproofai_sdk.evaluator.authoring import Evaluator
+
+
+def load_evaluator(spec: str) -> Evaluator:
+    module_name, separator, attribute = spec.partition(":")
+    if not module_name:
+        raise ValueError("evaluator module must not be empty")
+    if not separator:
+        attribute = "app"
+    if not attribute:
+        raise ValueError("evaluator attribute must not be empty")
+    module = importlib.import_module(module_name)
+    try:
+        evaluator = getattr(module, attribute)
+    except AttributeError as error:
+        raise ValueError(f"{spec!r} does not define {attribute!r}") from error
+    if not isinstance(evaluator, Evaluator):
+        raise TypeError(
+            f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator"
+        )
+    return evaluator
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator")
+    parser.add_argument(
+        "module",
+        nargs="?",
+        default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"),
+        help="Python module and optional attribute (for example my_evals:app)",
+    )
+    args = parser.parse_args(argv)
+    if not args.module:
+        parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)")
+    load_evaluator(args.module).run_from_env()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py

@@ -0,0 +1,63 @@
+"""Subprocess entry point for the managed-source sandbox.
+
+Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner <input-file>`` by
+``source._run_sandboxed``. The input file holds a pickled
+``(kind, source, session_wire, cpu_seconds, mem_bytes, eval_key)`` tuple. This
+process installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates
+the re-validated server-authored source against the reconstructed transcript,
+validates + bounds the result, and writes a pickled ``("ok", result)`` /
+``("err", type_name, message)`` outcome to stdout.
+
+This is a FRESH exec'd process — never a fork of the multi-threaded worker — so
+there is no inherited-lock deadlock (forking a process that has an asyncio loop,
+an executor pool and a writer daemon hangs the child). The parent enforces the
+wall-clock bound and the output-size bound by reading only so far and killing
+this process on timeout or overflow.
+"""
+
+from __future__ import annotations
+
+import pickle
+import sys
+
+
+def _main() -> int:
+    with open(sys.argv[1], "rb") as handle:
+        kind, source, session_wire, cpu_seconds, mem_bytes, eval_key = pickle.loads(
+            handle.read()
+        )
+    # Imported here, in the child, so the import cost is never on the worker's path.
+    from failproofai_sdk.evaluator.protocol import SessionTranscript
+    from failproofai_sdk.evaluator.source import (
+        SANDBOX_MAX_RESULT_BYTES,
+        _install_limits,
+        _raw_eval,
+    )
+
+    try:
+        session = SessionTranscript.from_wire(session_wire)
+        # Compile + re-validate BEFORE the limits so validation cost is not charged
+        # against the eval's CPU budget; the limits bind the eval itself.
+        run = _raw_eval(source, kind)
+        _install_limits(cpu_seconds, mem_bytes)
+        result = run(session)
+        # Bound the result INSIDE the sandbox before it crosses back: result_items
+        # enforces the 25-result limit + field validation, so a huge result
+        # (`metrics={str(x):1 for x in range(100000)}`) raises here instead of
+        # being serialized and shipped to the parent.
+        if kind == "evaluator":
+            result.result_items(eval_key or "result")
+        payload = pickle.dumps(("ok", result))
+        if len(payload) > SANDBOX_MAX_RESULT_BYTES:
+            payload = pickle.dumps(
+                ("err", "ResultTooLarge", "evaluation result exceeds the size limit")
+            )
+    except BaseException as error:  # noqa: BLE001 - relay type+msg, this process is the boundary
+        payload = pickle.dumps(("err", type(error).__name__, str(error)[:500]))
+    sys.stdout.buffer.write(payload)
+    sys.stdout.buffer.flush()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(_main())