failproofai 1.0.3 → 1.0.4-beta.2

package/.next/standalone/sdk/python/tests/fixtures/evaluator_v2/README.md

@@ -0,0 +1,28 @@
+# Evaluator v2 contract fixtures
+
+`contract.json` is the Checkpoint 0 wire contract shared by the Rust server and
+the zero-dependency Python SDK. The matching copy lives at
+`server/tests/fixtures/evaluator_v2/contract.json` in the `agenteye` repository.
+Change both copies together.
+
+Contract rules:
+
+- The only accepted protocol major is the exact string `"2"`. Unsupported
+  majors return `426 unsupported_protocol_version`.
+- Unknown JSON fields are ignored so either side may add optional fields within
+  major version 2. Removing, renaming, or changing the meaning of a field needs
+  a new major version.
+- Worker payloads never carry authoritative tenant or evaluator-instance
+  identity. The server derives those from the credential and leased record.
+- `lease_generation` is the fencing token. `409 lease_lost` is terminal for the
+  affected local execution; the SDK must stop heartbeating or submitting it.
+- `submission_id` is an idempotency key. Replaying identical content succeeds;
+  reusing it for different content returns `409 submission_conflict`.
+- Transcript overflow is terminal in v2 (`413 transcript_too_large`); the server
+  never silently truncates the evaluated input.
+- Only errors marked `retryable` may be retried automatically. HTTP method alone
+  is not enough to decide whether a protocol operation is safe to replay.
+
+The timing and payload limits in the fixture are normative defaults. A register
+response may lower the worker's effective concurrency, heartbeat interval, or
+lease duration, but may not raise a client-side payload bound.

package/.next/standalone/sdk/python/tests/fixtures/evaluator_v2/contract.json

@@ -0,0 +1,252 @@
+{
+  "fixture_revision": "evaluator-v2-2026-08-28.3",
+  "protocol": {
+    "supported_major_versions": ["2"],
+    "transcript_schema_version": "2",
+    "result_schema_version": "2"
+  },
+  "http": {
+    "register": "/v1/evaluator/workers/register",
+    "claim": "/v1/evaluator/assignments/claim",
+    "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript",
+    "definitions": "/v1/evaluator/assignments/{assignment_id}/definitions",
+    "plan": "/v1/evaluator/assignments/{assignment_id}/plan",
+    "heartbeat": "/v1/evaluator/runs/heartbeat",
+    "result": "/v1/evaluator/runs/{evaluation_run_id}/result",
+    "worker_id_header": "X-FailproofAI-Worker-Id",
+    "lease_generation_header": "X-FailproofAI-Lease-Generation"
+  },
+  "timing": {
+    "heartbeat_interval_seconds": 30,
+    "lease_duration_seconds": 120,
+    "poll_interval_seconds": 10,
+    "max_attempts": 5
+  },
+  "limits": {
+    "max_catalog_definitions": 100,
+    "max_claim_capacity": 32,
+    "max_transcript_bytes": 26214400,
+    "max_results_per_run": 25,
+    "max_eval_key_bytes": 128,
+    "max_display_name_bytes": 128,
+    "max_version_bytes": 128,
+    "max_worker_id_bytes": 128,
+    "max_label_bytes": 64,
+    "max_labels_per_result": 20,
+    "max_summary_bytes": 4096,
+    "max_reasoning_bytes": 16384,
+    "max_unit_bytes": 64,
+    "max_display_value_bytes": 256,
+    "max_description_bytes": 1000,
+    "max_error_code_bytes": 64,
+    "max_error_message_bytes": 4096
+  },
+  "errors": {
+    "invalid_credentials": {"http_status": 401, "retryable": false},
+    "instance_disabled": {"http_status": 403, "retryable": false},
+    "insufficient_permissions": {"http_status": 403, "retryable": false},
+    "assignment_not_found": {"http_status": 404, "retryable": false},
+    "run_not_found": {"http_status": 404, "retryable": false},
+    "catalog_mismatch": {"http_status": 409, "retryable": false},
+    "lease_lost": {"http_status": 409, "retryable": false},
+    "plan_conflict": {"http_status": 409, "retryable": false},
+    "submission_conflict": {"http_status": 409, "retryable": false},
+    "retry_budget_exhausted": {"http_status": 409, "retryable": false},
+    "transcript_too_large": {"http_status": 413, "retryable": false},
+    "invalid_request": {"http_status": 422, "retryable": false},
+    "invalid_catalog": {"http_status": 422, "retryable": false},
+    "incomplete_plan": {"http_status": 422, "retryable": false},
+    "unsupported_protocol_version": {"http_status": 426, "retryable": false},
+    "internal_error": {"http_status": 500, "retryable": true}
+  },
+  "samples": {
+    "register_request": {
+      "protocol_version": "2",
+      "worker_id": "pod-7f8c9",
+      "sdk_version": "0.0.1b2",
+      "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb",
+      "max_concurrency": 4,
+      "definitions": [
+        {
+          "eval_key": "tool_efficiency",
+          "display_name": "Tool efficiency",
+          "eval_version": "1.2.0",
+          "result_kind": "score",
+          "labels": ["tools", "deterministic"]
+        }
+      ]
+    },
+    "register_response": {
+      "protocol_version": "2",
+      "evaluator_instance_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23101",
+      "evaluator_kind": "customer",
+      "heartbeat_interval_seconds": 30,
+      "lease_duration_seconds": 120,
+      "poll_interval_seconds": 10,
+      "claim_limit": 4,
+      "disabled_definitions": []
+    },
+    "claim_request": {
+      "protocol_version": "2",
+      "worker_id": "pod-7f8c9",
+      "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb",
+      "capacity": 2
+    },
+    "claim_response": {
+      "protocol_version": "2",
+      "assignments": [
+        {
+          "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102",
+          "lease_generation": 3,
+          "lease_expires_at": "2026-08-28T12:02:00.000000Z",
+          "session_id": "session-42",
+          "session_revision_id": "evt-agent-end-42",
+          "agent_id": "support-agent",
+          "environment": "production",
+          "trigger_reason": "agent_end",
+          "event_count": 42,
+          "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript",
+          "definitions_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/definitions"
+        }
+      ]
+    },
+    "definitions_response": {
+      "protocol_version": "2",
+      "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102",
+      "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb",
+      "definitions": [
+        {
+          "eval_key": "tool_efficiency",
+          "display_name": "Tool efficiency",
+          "eval_version": "1.2.0",
+          "result_kind": "score",
+          "labels": ["tools", "deterministic"],
+          "execution_mode": "python",
+          "condition_source": null,
+          "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea",
+          "timeout_seconds": 30.0
+        }
+      ]
+    },
+    "transcript_response": {
+      "schema_version": "2",
+      "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102",
+      "session_id": "session-42",
+      "session_revision_id": "evt-agent-end-42",
+      "agent_id": "support-agent",
+      "environment": "production",
+      "started_at": "2026-08-28T11:58:00.000000Z",
+      "ended_at": "2026-08-28T12:00:00.000000Z",
+      "event_count": 2,
+      "events": [
+        {
+          "id": "evt-tool-1",
+          "ts": "2026-08-28T11:59:00.000000Z",
+          "event_type": "tool_use",
+          "payload": {"tool_name": "search"}
+        },
+        {
+          "id": "evt-end-1",
+          "ts": "2026-08-28T12:00:00.000000Z",
+          "event_type": "agent_end",
+          "payload": {"summary": "Done"}
+        }
+      ]
+    },
+    "plan_request": {
+      "protocol_version": "2",
+      "worker_id": "pod-7f8c9",
+      "lease_generation": 3,
+      "selected": [
+        {"eval_key": "tool_efficiency", "eval_version": "1.2.0"}
+      ],
+      "skipped": [
+        {
+          "eval_key": "answer_groundedness",
+          "eval_version": "2.1.0",
+          "reason_code": "no_retrieval_events"
+        }
+      ]
+    },
+    "plan_response": {
+      "protocol_version": "2",
+      "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102",
+      "assignment_status": "planned",
+      "idempotent_replay": false,
+      "runs": [
+        {
+          "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103",
+          "eval_key": "tool_efficiency",
+          "eval_version": "1.2.0",
+          "execution_mode": "python",
+          "evaluator_source": "EvalResult(score=Score(1.0))",
+          "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea",
+          "timeout_seconds": 30.0
+        }
+      ]
+    },
+    "heartbeat_request": {
+      "protocol_version": "2",
+      "worker_id": "pod-7f8c9",
+      "lease_generation": 3,
+      "runs": [
+        {
+          "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103",
+          "state": "running",
+          "progress": 0.5
+        }
+      ]
+    },
+    "heartbeat_response": {
+      "protocol_version": "2",
+      "lease_expires_at": "2026-08-28T12:02:30.000000Z",
+      "accepted_run_ids": ["018f47a8-7c1d-7e21-a22a-79f7a4d23103"]
+    },
+    "result_request": {
+      "protocol_version": "2",
+      "result_schema_version": "2",
+      "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104",
+      "worker_id": "pod-7f8c9",
+      "lease_generation": 3,
+      "status": "succeeded",
+      "started_at": "2026-08-28T12:00:10.000000Z",
+      "finished_at": "2026-08-28T12:00:10.812000Z",
+      "duration_ms": 812,
+      "summary": "Used a compact tool set without retries.",
+      "results": [
+        {
+          "result_key": "tool_efficiency",
+          "result_kind": "score",
+          "numeric_value": 0.92,
+          "bool_value": true,
+          "text_value": null,
+          "unit": "ratio",
+          "display_value": "92%",
+          "description": "Distinct tools divided by total tool calls",
+          "reasoning": "3 distinct tools across 3 calls",
+          "labels": ["tools", "deterministic"]
+        }
+      ],
+      "error_code": null,
+      "error_message": null
+    },
+    "result_response": {
+      "protocol_version": "2",
+      "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103",
+      "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104",
+      "status": "committed",
+      "idempotent_replay": false,
+      "result_count": 1,
+      "result_checksum": "sha256:8cbd34f2d95d"
+    },
+    "error_response": {
+      "protocol_version": "2",
+      "error": {
+        "code": "lease_lost",
+        "message": "The assignment lease is no longer owned by this worker.",
+        "retryable": false,
+        "request_id": "req-018f47a8"
+      }
+    }
+  }
+}

package/.next/standalone/sdk/python/tests/test_evaluator_authoring.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import asyncio
+import math
+
+import pytest
+
+from failproofai_sdk.evaluator import (
+    Assertion,
+    EvalResult,
+    Evaluator,
+    Metric,
+    ResultKind,
+    Score,
+)
+
+
+def test_catalog_is_stable_across_registration_order():
+    first = Evaluator(name="acme", version="2026.08.1")
+    second = Evaluator(name="acme", version="2026.08.1")
+
+    @first.eval("zeta_check", version="1", labels=["z", "a"])
+    def first_zeta(session):
+        return EvalResult(score=Score(1))
+
+    @first.eval("alpha_check", version="1")
+    def first_alpha(session):
+        return EvalResult(score=Score(1))
+
+    @second.eval("alpha_check", version="1")
+    def second_alpha(session):
+        return EvalResult(score=Score(1))
+
+    @second.eval("zeta_check", version="1", labels=["a", "z"])
+    def second_zeta(session):
+        return EvalResult(score=Score(1))
+
+    assert first.catalog_revision == second.catalog_revision
+    assert [item.eval_key for item in first.catalog()] == ["alpha_check", "zeta_check"]
+
+
+def test_duplicate_eval_keys_are_rejected_even_when_versions_differ():
+    evaluator = Evaluator(name="acme", version="1")
+
+    @evaluator.eval("quality", version="1")
+    def quality_v1(session):
+        return EvalResult(score=Score(1))
+
+    with pytest.raises(ValueError, match="duplicate eval key"):
+
+        @evaluator.eval("quality", version="2")
+        def quality_v2(session):
+            return EvalResult(score=Score(1))
+
+
+@pytest.mark.parametrize("value", [-0.01, 1.01, math.nan, math.inf])
+def test_scores_are_finite_ratios(value):
+    with pytest.raises(ValueError):
+        Score(value)
+
+
+def test_result_presentation_fields_are_bounded_before_networking():
+    with pytest.raises(ValueError, match="unit is 65 bytes"):
+        Metric(1, unit="u" * 65)
+    with pytest.raises(ValueError, match="display value is 257 bytes"):
+        Score(1, display_value="x" * 257)
+    with pytest.raises(ValueError, match="description is 1001 bytes"):
+        Assertion(True, description="x" * 1001)
+
+
+def test_eval_result_expands_to_typed_long_form_rows():
+    result = EvalResult(
+        score=Score(0.75, passed=True, unit="ratio"),
+        metrics={"call_count": Metric(4, unit="calls")},
+        assertions={"had_output": Assertion(True)},
+        reasoning="Three useful calls out of four.",
+        labels=("tools",),
+    )
+
+    items = result.result_items("tool_efficiency")
+    assert [item.result_kind for item in items] == [
+        ResultKind.SCORE,
+        ResultKind.METRIC,
+        ResultKind.ASSERTION,
+    ]
+    assert items[0].reasoning == "Three useful calls out of four."
+    assert items[1].numeric_value == 4
+    assert items[2].bool_value is True
+
+
+def test_empty_eval_result_is_rejected_when_serialized():
+    with pytest.raises(ValueError, match="must contain"):
+        EvalResult().result_items("quality")
+
+
+def test_result_keys_must_be_unique_across_kinds():
+    result = EvalResult(score=Score(1), metrics={"quality": 1})
+    with pytest.raises(ValueError, match="result keys must be unique"):
+        result.result_items("quality")
+
+
+def test_sync_and_async_functions_share_one_call_path():
+    async def async_eval(session):
+        return EvalResult(score=Score(1))
+
+    def sync_eval(session):
+        return EvalResult(score=Score(0.5))
+
+    async def exercise():
+        sync_result = await Evaluator.call(sync_eval, None)
+        async_result = await Evaluator.call(async_eval, None)
+        return sync_result, async_result
+
+    sync_result, async_result = asyncio.run(exercise())
+    assert sync_result.score.value == 0.5
+    assert async_result.score.value == 1
+
+
+def test_keys_are_machine_safe_and_versions_are_explicit():
+    evaluator = Evaluator(name="acme", version="1")
+    with pytest.raises(ValueError, match="must match"):
+        evaluator.eval("Not Safe", version="1")
+    with pytest.raises(ValueError, match="must not be empty"):
+        evaluator.eval("safe", version="")

package/.next/standalone/sdk/python/tests/test_evaluator_client.py

@@ -0,0 +1,253 @@
+from __future__ import annotations
+
+import io
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+
+import pytest
+
+from failproofai_sdk.evaluator import (
+    Assignment,
+    ClaimRequest,
+    EvaluatorAPIError,
+    EvaluatorClient,
+    ResultRequest,
+)
+
+FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json"
+
+
+def _samples():
+    return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"]
+
+
+class Response:
+    def __init__(self, body):
+        self.body = json.dumps(body).encode()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        return None
+
+    def read(self, amount):
+        return self.body[:amount]
+
+
+class RawResponse(Response):
+    def __init__(self, body: bytes):
+        self.body = body
+
+
+def test_claim_sends_bearer_auth_and_does_not_retry():
+    calls = []
+
+    def opener(request, timeout):
+        calls.append((request, timeout))
+        raise URLError("offline")
+
+    client = EvaluatorClient(
+        base_url="https://cloud.example/api/",
+        credential="secret",
+        opener=opener,
+        sleeper=lambda _: None,
+    )
+    with pytest.raises(EvaluatorAPIError, match="transport_error"):
+        client.claim(ClaimRequest("worker", "sha256:x", 1, 20))
+
+    assert len(calls) == 1
+    request, timeout = calls[0]
+    assert request.full_url == "https://cloud.example/v1/evaluator/assignments/claim"
+    assert request.get_header("Authorization") == "Bearer secret"
+    assert timeout == 30
+
+
+def test_idempotent_result_submission_retries_transport_failure():
+    samples = _samples()
+    calls = 0
+
+    def opener(request, timeout):
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise URLError("reset")
+        return Response(samples["result_response"])
+
+    client = EvaluatorClient(
+        base_url="https://cloud.example/",
+        credential="secret",
+        opener=opener,
+        sleeper=lambda _: None,
+    )
+    response = client.submit_result(
+        samples["result_response"]["evaluation_run_id"],
+        ResultRequest.from_wire(samples["result_request"]),
+    )
+    assert response.status == "committed"
+    assert calls == 2
+
+
+def test_transcript_url_cannot_exfiltrate_the_worker_credential():
+    sample = _samples()["claim_response"]["assignments"][0]
+    assignment = Assignment.from_wire(
+        {**sample, "transcript_url": "https://evil.test/read"}
+    )
+    client = EvaluatorClient(
+        base_url="https://cloud.example/",
+        credential="secret",
+        opener=lambda *_args, **_kwargs: pytest.fail("network must not be reached"),
+    )
+    with pytest.raises(EvaluatorAPIError, match="outside the configured API origin"):
+        client.transcript(assignment, worker_id="worker")
+
+
+def test_machine_error_envelope_controls_retryability():
+    body = json.dumps(_samples()["error_response"]).encode()
+
+    def opener(request, timeout):
+        raise HTTPError(request.full_url, 409, "Conflict", {}, io.BytesIO(body))
+
+    client = EvaluatorClient(
+        base_url="https://cloud.example/", credential="secret", opener=opener
+    )
+    with pytest.raises(EvaluatorAPIError) as caught:
+        client.claim(ClaimRequest("worker", "sha256:x", 1, 20))
+    assert caught.value.code == "lease_lost"
+    assert caught.value.status == 409
+    assert caught.value.retryable is False
+    assert caught.value.request_id == "req-018f47a8"
+
+
+@pytest.mark.parametrize(
+    ("body", "code"),
+    [
+        (b"not-json", "invalid_response"),
+        (b"[]", "invalid_response"),
+        (
+            b"{" + b'"padding":"' + b"x" * (2 * 1024 * 1024) + b'"}',
+            "response_too_large",
+        ),
+    ],
+)
+def test_malformed_or_oversized_server_responses_fail_closed(body, code):
+    client = EvaluatorClient(
+        base_url="https://cloud.example/",
+        credential="secret",
+        opener=lambda request, timeout: RawResponse(body),
+    )
+    with pytest.raises(EvaluatorAPIError) as caught:
+        client.claim(ClaimRequest("worker", "sha256:x", 1, 20))
+    assert caught.value.code == code
+    assert caught.value.retryable is False
+
+
+def test_transcript_identity_is_sent_as_fencing_headers():
+    samples = _samples()
+    assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0])
+    captured = None
+
+    def opener(request, timeout):
+        nonlocal captured
+        captured = request
+        return Response(samples["transcript_response"])
+
+    client = EvaluatorClient(
+        base_url="https://cloud.example/", credential="secret", opener=opener
+    )
+    client.transcript(assignment, worker_id="worker-7")
+    assert captured.get_header("X-failproofai-worker-id") == "worker-7"
+    assert captured.get_header("X-failproofai-lease-generation") == "3"
+
+
+def test_definitions_use_the_server_supplied_path_and_fencing_headers():
+    samples = _samples()
+    assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0])
+    captured = None
+
+    def opener(request, timeout):
+        nonlocal captured
+        captured = request
+        return Response(samples["definitions_response"])
+
+    client = EvaluatorClient(
+        base_url="https://cloud.example/", credential="secret", opener=opener
+    )
+    response = client.definitions(assignment, worker_id="worker-7")
+
+    assert response.assignment_id == assignment.assignment_id
+    assert response.definitions[0].execution_mode.value == "python"
+    assert captured.full_url.endswith(assignment.definitions_url)
+    assert captured.get_header("X-failproofai-worker-id") == "worker-7"
+    assert captured.get_header("X-failproofai-lease-generation") == "3"
+
+
+def test_constructor_rejects_unsafe_or_incomplete_configuration():
+    with pytest.raises(ValueError, match="absolute"):
+        EvaluatorClient(base_url="localhost:8080", credential="secret")
+    with pytest.raises(ValueError, match="credential"):
+        EvaluatorClient(base_url="https://cloud.example", credential="")
+    with pytest.raises(ValueError, match="control characters"):
+        EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak")
+
+
+def test_private_cluster_http_requires_an_explicit_opt_in():
+    with pytest.raises(ValueError, match="must use https"):
+        EvaluatorClient(base_url="http://server:8080", credential="secret")
+    EvaluatorClient(
+        base_url="http://server:8080",
+        credential="secret",
+        allow_insecure_http=True,
+    )
+
+
+def test_protocol_redirect_does_not_forward_the_bearer_credential():
+    exfiltration_attempts = []
+
+    class Sink(BaseHTTPRequestHandler):
+        def do_POST(self):
+            exfiltration_attempts.append(self.headers.get("Authorization"))
+            self.send_response(200)
+            self.end_headers()
+
+        def log_message(self, format, *args):
+            return
+
+    sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink)
+    sink_thread = threading.Thread(target=sink.serve_forever, daemon=True)
+    sink_thread.start()
+
+    location = f"http://127.0.0.1:{sink.server_address[1]}/steal"
+
+    class Redirector(BaseHTTPRequestHandler):
+        def do_POST(self):
+            self.send_response(307)
+            self.send_header("Location", location)
+            self.end_headers()
+
+        def log_message(self, format, *args):
+            return
+
+    redirector = ThreadingHTTPServer(("127.0.0.1", 0), Redirector)
+    redirector_thread = threading.Thread(target=redirector.serve_forever, daemon=True)
+    redirector_thread.start()
+    try:
+        client = EvaluatorClient(
+            base_url=f"http://127.0.0.1:{redirector.server_address[1]}",
+            credential="must-not-leak",
+            max_retries=0,
+        )
+        with pytest.raises(EvaluatorAPIError) as caught:
+            client.claim(ClaimRequest("worker", "sha256:x", 1, 0))
+        assert caught.value.status == 307
+        assert exfiltration_attempts == []
+    finally:
+        redirector.shutdown()
+        redirector.server_close()
+        redirector_thread.join(timeout=5)
+        sink.shutdown()
+        sink.server_close()
+        sink_thread.join(timeout=5)

package/.next/standalone/sdk/python/tests/test_evaluator_example.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+from failproofai_sdk.evaluator import ConditionResult
+
+
+def _example_module():
+    path = Path(__file__).parents[1] / "examples" / "evaluator_worker.py"
+    spec = importlib.util.spec_from_file_location("evaluator_worker_example", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_example_registers_deterministic_and_async_evals(monkeypatch):
+    monkeypatch.delenv("EXAMPLE_JUDGE_URL", raising=False)
+    module = _example_module()
+    definitions = {item.eval_key: item for item in module.app.definitions}
+    assert set(definitions) == {"answer_relevance", "tool_efficiency"}
+    assert definitions["answer_relevance"].eval_version == "judge-api-v1"
+
+    skipped = definitions["answer_relevance"].condition(None)
+    assert skipped == ConditionResult(False, "judge_not_configured")
+
+
+def test_example_rejects_non_http_judge_urls(monkeypatch):
+    module = _example_module()
+    monkeypatch.setenv("EXAMPLE_JUDGE_URL", "file:///etc/passwd")
+    with pytest.raises(ValueError, match="absolute http"):
+        module._call_judge("question", "answer")