failproofai 1.0.3 → 1.0.4-beta.2

package/.next/standalone/sdk/python/tests/test_evaluator_protocol.py

@@ -0,0 +1,246 @@
+from __future__ import annotations
+
+import copy
+import json
+from pathlib import Path
+
+import pytest
+
+from failproofai_sdk.evaluator import (
+    ClaimRequest,
+    ClaimResponse,
+    DefinitionsResponse,
+    ErrorResponse,
+    HeartbeatRequest,
+    HeartbeatResponse,
+    PlanRequest,
+    PlanResponse,
+    ProtocolError,
+    RegisterRequest,
+    RegisterResponse,
+    ResultRequest,
+    ResultResponse,
+    SessionTranscript,
+    UnsupportedProtocolVersion,
+    protocol,
+)
+
+FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json"
+
+
+def _contract():
+    return json.loads(FIXTURE.read_text(encoding="utf-8"))
+
+
+@pytest.mark.parametrize(
+    ("sample", "model"),
+    [
+        ("register_request", RegisterRequest),
+        ("register_response", RegisterResponse),
+        ("claim_request", ClaimRequest),
+        ("claim_response", ClaimResponse),
+        ("definitions_response", DefinitionsResponse),
+        ("transcript_response", SessionTranscript),
+        ("plan_request", PlanRequest),
+        ("plan_response", PlanResponse),
+        ("heartbeat_request", HeartbeatRequest),
+        ("heartbeat_response", HeartbeatResponse),
+        ("result_request", ResultRequest),
+        ("result_response", ResultResponse),
+        ("error_response", ErrorResponse),
+    ],
+)
+def test_golden_messages_round_trip(sample, model):
+    wire = _contract()["samples"][sample]
+    assert model.from_wire(wire).to_wire() == wire
+
+
+def test_unknown_additive_fields_are_tolerated():
+    wire = dict(_contract()["samples"]["claim_request"])
+    wire["future_optional_field"] = True
+    assert ClaimRequest.from_wire(wire).capacity == 2
+
+
+def test_unsupported_major_version_fails_loudly():
+    wire = dict(_contract()["samples"]["claim_request"])
+    wire["protocol_version"] = "3"
+    with pytest.raises(
+        UnsupportedProtocolVersion, match="supported major version is 2"
+    ):
+        ClaimRequest.from_wire(wire)
+
+
+def test_transcript_event_count_is_an_integrity_check():
+    wire = dict(_contract()["samples"]["transcript_response"])
+    wire["event_count"] = 99
+    with pytest.raises(ProtocolError, match="transcript contains 2 events"):
+        SessionTranscript.from_wire(wire)
+
+
+@pytest.mark.parametrize(
+    ("sample", "model", "path", "value", "message"),
+    [
+        (
+            "claim_response",
+            ClaimResponse,
+            ("assignments", 0),
+            42,
+            r"assignments\[0\] must be an object",
+        ),
+        (
+            "register_response",
+            RegisterResponse,
+            ("disabled_definitions",),
+            [42],
+            r"disabled_definitions\[0\] must be a string",
+        ),
+        (
+            "heartbeat_response",
+            HeartbeatResponse,
+            ("accepted_run_ids", 0),
+            None,
+            r"accepted_run_ids\[0\] must be a string",
+        ),
+        (
+            "plan_response",
+            PlanResponse,
+            ("runs", 0),
+            "not-an-object",
+            r"runs\[0\] must be an object",
+        ),
+        (
+            "result_request",
+            ResultRequest,
+            ("results", 0, "labels", 0),
+            7,
+            r"labels\[0\] must be a string",
+        ),
+        (
+            "register_request",
+            RegisterRequest,
+            ("definitions", 0, "result_kind"),
+            "unknown",
+            "result_kind must be one of",
+        ),
+    ],
+)
+def test_nested_wire_values_fail_with_protocol_errors(
+    sample, model, path, value, message
+):
+    wire = copy.deepcopy(_contract()["samples"][sample])
+    target = wire
+    for part in path[:-1]:
+        target = target[part]
+    target[path[-1]] = value
+    with pytest.raises(ProtocolError, match=message):
+        model.from_wire(wire)
+
+
+@pytest.mark.parametrize(
+    ("sample", "model", "field", "value", "message"),
+    [
+        (
+            "claim_response",
+            ClaimResponse,
+            "lease_generation",
+            0,
+            "lease_generation must be greater than zero",
+        ),
+        (
+            "claim_response",
+            ClaimResponse,
+            "event_count",
+            -1,
+            "event_count must not be negative",
+        ),
+        (
+            "result_response",
+            ResultResponse,
+            "result_count",
+            -1,
+            "result_count must not be negative",
+        ),
+    ],
+)
+def test_server_response_counters_and_generations_are_bounded(
+    sample, model, field, value, message
+):
+    wire = copy.deepcopy(_contract()["samples"][sample])
+    if sample == "claim_response":
+        wire["assignments"][0][field] = value
+    else:
+        wire[field] = value
+    with pytest.raises(ProtocolError, match=message):
+        model.from_wire(wire)
+
+
+def test_fixture_constants_match_the_sdk_contract():
+    contract = _contract()
+    assert contract["protocol"] == {
+        "supported_major_versions": [protocol.PROTOCOL_VERSION],
+        "transcript_schema_version": protocol.TRANSCRIPT_SCHEMA_VERSION,
+        "result_schema_version": protocol.RESULT_SCHEMA_VERSION,
+    }
+    assert contract["http"] == {
+        "register": protocol.REGISTER_PATH,
+            "claim": protocol.CLAIM_PATH,
+            "transcript": protocol.TRANSCRIPT_PATH,
+            "definitions": protocol.DEFINITIONS_PATH,
+        "plan": protocol.PLAN_PATH,
+        "heartbeat": protocol.HEARTBEAT_PATH,
+        "result": protocol.RESULT_PATH,
+        "worker_id_header": protocol.WORKER_ID_HEADER,
+        "lease_generation_header": protocol.LEASE_GENERATION_HEADER,
+    }
+    assert contract["timing"] == {
+        "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS,
+        "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS,
+        "poll_interval_seconds": protocol.DEFAULT_POLL_INTERVAL_SECONDS,
+        "max_attempts": protocol.MAX_ATTEMPTS,
+    }
+    assert contract["limits"] == {
+        "max_catalog_definitions": protocol.MAX_CATALOG_DEFINITIONS,
+        "max_claim_capacity": protocol.MAX_CLAIM_CAPACITY,
+        "max_transcript_bytes": protocol.MAX_TRANSCRIPT_BYTES,
+        "max_results_per_run": protocol.MAX_RESULTS_PER_RUN,
+        "max_eval_key_bytes": protocol.MAX_EVAL_KEY_BYTES,
+        "max_display_name_bytes": protocol.MAX_DISPLAY_NAME_BYTES,
+        "max_version_bytes": protocol.MAX_VERSION_BYTES,
+        "max_worker_id_bytes": protocol.MAX_WORKER_ID_BYTES,
+        "max_label_bytes": protocol.MAX_LABEL_BYTES,
+        "max_labels_per_result": protocol.MAX_LABELS_PER_RESULT,
+        "max_summary_bytes": protocol.MAX_SUMMARY_BYTES,
+        "max_reasoning_bytes": protocol.MAX_REASONING_BYTES,
+        "max_unit_bytes": protocol.MAX_UNIT_BYTES,
+        "max_display_value_bytes": protocol.MAX_DISPLAY_VALUE_BYTES,
+        "max_description_bytes": protocol.MAX_DESCRIPTION_BYTES,
+        "max_error_code_bytes": protocol.MAX_ERROR_CODE_BYTES,
+        "max_error_message_bytes": protocol.MAX_ERROR_MESSAGE_BYTES,
+    }
+    assert contract["errors"] == protocol.ERROR_SPECS
+
+
+def test_session_helpers_use_the_protocol_event_vocabulary():
+    session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"])
+    assert session.count("tool_use") == 1
+    assert session.events_of_type("agent_end")[0].payload["summary"] == "Done"
+
+
+def test_falsy_or_missing_execution_mode_is_rejected_not_defaulted():
+    # F2: a falsy/absent execution_mode was silently coerced to 'local', which
+    # could run a server-authored ('python') definition down the customer path.
+    # It must now be a hard protocol error, not a default.
+    samples = _contract()["samples"]
+    for bad in ("", None):
+        defs = json.loads(json.dumps(samples["definitions_response"]))
+        defs["definitions"][0]["execution_mode"] = bad
+        with pytest.raises(ProtocolError, match="execution_mode"):
+            DefinitionsResponse.from_wire(defs)
+        plan = json.loads(json.dumps(samples["plan_response"]))
+        plan["runs"][0]["execution_mode"] = bad
+        with pytest.raises(ProtocolError, match="execution_mode"):
+            PlanResponse.from_wire(plan)
+    absent = json.loads(json.dumps(samples["definitions_response"]))
+    del absent["definitions"][0]["execution_mode"]
+    with pytest.raises(ProtocolError, match="execution_mode"):
+        DefinitionsResponse.from_wire(absent)

package/.next/standalone/sdk/python/tests/test_evaluator_review_fixes.py

@@ -0,0 +1,164 @@
+"""Regression tests for Evaluator v2 PR-review fixes.
+
+Covers: control-character rejection in bounded result text, reasoning carried on
+non-score primary results, and the object-repr guard no longer false-rejecting
+ordinary hex literals.
+"""
+from __future__ import annotations
+
+import pytest
+
+from failproofai_sdk.evaluator.authoring import EvalResult, Metric, Score
+from failproofai_sdk.evaluator.source import _forbid_object_reprs, UnsafeEvaluatorSource
+
+
+def test_bounded_rejects_c0_control_characters():
+    # NUL / ESC in reasoning would be accepted by the SDK but rejected by the
+    # server with a non-retryable 422, silently losing a successful eval.
+    with pytest.raises(ValueError):
+        EvalResult(score=Score(1.0, passed=True), reasoning="bad\x00value")
+    with pytest.raises(ValueError):
+        EvalResult(score=Score(1.0, passed=True), summary="esc\x1b[31m")
+
+
+def test_bounded_keeps_tab_newline_cr():
+    r = EvalResult(score=Score(1.0, passed=True), reasoning="line one\nline\ttwo\r")
+    items = r.result_items("q")
+    assert items[0].reasoning == "line one\nline\ttwo\r"
+
+
+def test_reasoning_carried_on_metric_primary():
+    # A metric-kind eval's primary result is the metric whose key == eval_key.
+    r = EvalResult(metrics={"latency": Metric(12.0)}, reasoning="slow tail")
+    items = {i.result_key: i for i in r.result_items("latency")}
+    assert items["latency"].reasoning == "slow tail"
+
+
+def test_reasoning_not_smeared_onto_secondary_metrics():
+    r = EvalResult(
+        score=Score(1.0, passed=True),
+        metrics={"aux": Metric(3.0)},
+        reasoning="about the score",
+    )
+    items = {i.result_key: i for i in r.result_items("q")}
+    assert items["q"].reasoning == "about the score"
+    assert items["aux"].reasoning is None
+
+
+def test_object_repr_guard_allows_plain_hex_literals():
+    # A hex colour / digest in result text must not be mistaken for a pointer repr.
+    assert _forbid_object_reprs("summary", "background 0xFFFFFF, sha 0xdeadbeef1234") == (
+        "background 0xFFFFFF, sha 0xdeadbeef1234"
+    )
+
+
+def test_object_repr_guard_still_rejects_pointer_reprs():
+    with pytest.raises(UnsafeEvaluatorSource):
+        _forbid_object_reprs("summary", "<foo.Bar object at 0x7f9c1a2b3c4d>")
+    # ...even with the leading '<' stripped (the reshape bypass).
+    with pytest.raises(UnsafeEvaluatorSource):
+        _forbid_object_reprs("summary", "foo.Bar object at 0x7f9c1a2b3c4d>")
+
+
+# ---- runtime fixes (Section 1/2 of the follow-up review) --------------------
+import asyncio
+from dataclasses import replace
+
+from failproofai_sdk.evaluator import (
+    ClaimResponse, EvaluatorAPIError, PlannedRun, PlanResponse, SessionTranscript,
+    WorkerConfig, WorkerRuntime,
+)
+from failproofai_sdk.evaluator import RegisterResponse, HeartbeatResponse
+import json as _json
+from pathlib import Path as _Path
+
+
+def _samples_fx():
+    return _json.loads(
+        (_Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json").read_text()
+    )["samples"]
+
+
+class _FakeClient:
+    def __init__(self):
+        s = _samples_fx()
+        self.assignment = replace(
+            ClaimResponse.from_wire(s["claim_response"]).assignments[0],
+            definitions_url="",
+        )
+        self.session = SessionTranscript.from_wire(s["transcript_response"])
+        self.plans, self.submissions, self.heartbeats = [], [], []
+
+    def transcript(self, assignment, *, worker_id):
+        return self.session
+
+    def plan(self, assignment_id, request):
+        self.plans.append(request)
+        return PlanResponse(
+            assignment_id=assignment_id,
+            assignment_status="planned" if request.selected else "skipped",
+            runs=tuple(
+                PlannedRun(f"run-{i.eval_key}", i.eval_key, i.eval_version)
+                for i in request.selected
+            ),
+        )
+
+    def submit_result(self, run_id, request):
+        self.submissions.append((run_id, request))
+
+    def heartbeat(self, request):
+        self.heartbeats.append(request)
+        return HeartbeatResponse(
+            lease_expires_at="2026-08-28T12:02:30.000000Z",
+            accepted_run_ids=tuple(r.evaluation_run_id for r in request.runs),
+        )
+
+
+def _rt(evaluator, client):
+    return WorkerRuntime(
+        evaluator,
+        WorkerConfig(server_url="https://cloud.example", credential="x",
+                     worker_id="worker-test", max_concurrency=2),
+        client=client,
+    )
+
+
+def test_transcript_too_large_skips_assignment_without_crashing():
+    from failproofai_sdk.evaluator import Evaluator, EvalResult, Score
+    ev = Evaluator(name="t", version="1")
+    ev.eval("quality", version="1")(lambda s: EvalResult(score=Score(1.0, passed=True)))
+
+    class TooLarge(_FakeClient):
+        def transcript(self, assignment, *, worker_id):
+            raise EvaluatorAPIError(
+                status=413, code="transcript_too_large",
+                message="transcript exceeds 26214400 bytes", retryable=False,
+            )
+
+    c = TooLarge()
+    # returns cleanly — no plan, no submission, no exception
+    asyncio.run(_rt(ev, c).process_assignment(c.assignment))
+    assert c.plans == [] and c.submissions == []
+
+
+def test_idempotent_replay_runs_a_definition_this_attempt_skipped():
+    from failproofai_sdk.evaluator import Evaluator, EvalResult, Score, ConditionResult
+    ev = Evaluator(name="t", version="1")
+
+    @ev.eval("quality", version="1", when=lambda s: ConditionResult(False, "nope"))
+    def quality(session):
+        return EvalResult(score=Score(1.0, passed=True))
+
+    class ReplaySkipped(_FakeClient):
+        def plan(self, assignment_id, request):
+            # This attempt selected nothing, but the server replays the first
+            # attempt's run for the now-skipped eval.
+            return PlanResponse(
+                assignment_id=assignment_id, assignment_status="planned",
+                runs=(PlannedRun("run-quality", "quality", "1"),),
+                idempotent_replay=True,
+            )
+
+    c = ReplaySkipped()
+    asyncio.run(_rt(ev, c).process_assignment(c.assignment))
+    assert [rid for rid, _ in c.submissions] == ["run-quality"]