claude-dev-env 1.94.0 → 2.0.0

package/skills/codex-review/scripts/test_codex_down_classifier.py

@@ -0,0 +1,143 @@
+"""Behavioral tests for the Codex down-classifier."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+SCRIPTS_DIRECTORY = Path(__file__).resolve().parent
+if str(SCRIPTS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIRECTORY))
+
+import codex_down_classifier as classifier  # noqa: E402
+from codex_review_scripts_constants.classifier_constants import (  # noqa: E402
+    FAILURE_CLASS_AUTH_FAILURE,
+    FAILURE_CLASS_CONFIG_ERROR,
+    FAILURE_CLASS_MODEL_ERROR,
+    FAILURE_CLASS_UNKNOWN,
+    FAILURE_CLASS_USAGE_LIMIT,
+)
+from codex_review_scripts_constants.run_constants import (  # noqa: E402
+    OUTCOME_CLASS_CODEX_DOWN,
+    OUTCOME_CLASS_COMPLETED,
+)
+
+FIXTURES_DIRECTORY = SCRIPTS_DIRECTORY / "fixtures"
+
+
+def _read_fixture(fixture_name: str) -> str:
+    return (FIXTURES_DIRECTORY / fixture_name).read_text(encoding="utf-8")
+
+
+def test_config_load_failure_fixture_maps_to_config_error() -> None:
+    stream_text = _read_fixture("config_load_failure_v0.125.0.txt")
+
+    classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_CONFIG_ERROR
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_model_rejection_fixture_maps_to_model_error() -> None:
+    stream_text = _read_fixture("model_rejection_v0.125.0.jsonl")
+
+    classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_MODEL_ERROR
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_success_stream_fixture_maps_to_completed() -> None:
+    stream_text = _read_fixture("success_stream_v0.144.3.jsonl")
+
+    classification = classifier.classify_codex_run(
+        exit_code=0,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == OUTCOME_CLASS_COMPLETED
+    assert classification.outcome_class == OUTCOME_CLASS_COMPLETED
+
+
+def test_usage_limit_maps_to_codex_down() -> None:
+    stream_text = _read_fixture("usage_limit_synthetic.txt")
+
+    classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_USAGE_LIMIT
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_auth_failure_maps_to_codex_down() -> None:
+    stream_text = _read_fixture("auth_failure_synthetic.txt")
+
+    classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_AUTH_FAILURE
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_unknown_nonzero_exit_maps_to_codex_down() -> None:
+    stream_text = _read_fixture("unknown_failure_synthetic.txt")
+
+    classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text=stream_text,
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_UNKNOWN
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_unparseable_nonzero_exit_maps_to_codex_down() -> None:
+    classification = classifier.classify_codex_run(
+        exit_code=2,
+        stream_text="",
+    )
+
+    assert classification.detail_class == FAILURE_CLASS_UNKNOWN
+    assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_bare_token_streams_map_to_unknown_not_usage_or_auth() -> None:
+    all_false_positive_streams = (
+        "failed opening ticket/429/summary",
+        "disk quota exceeded on volume",
+        "failed to parse credits field in CSV",
+        "could not render login form widget",
+        "request id ab401cd failed",
+    )
+
+    for each_stream_text in all_false_positive_streams:
+        classification = classifier.classify_codex_run(
+            exit_code=1,
+            stream_text=each_stream_text,
+        )
+        assert classification.detail_class == FAILURE_CLASS_UNKNOWN, each_stream_text
+        assert classification.outcome_class == OUTCOME_CLASS_CODEX_DOWN
+
+
+def test_contextual_usage_and_auth_phrases_still_classify() -> None:
+    usage_classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text="HTTP 429: rate limit hit; api quota exhausted for this key",
+    )
+    auth_classification = classifier.classify_codex_run(
+        exit_code=1,
+        stream_text="authentication failed; login required",
+    )
+
+    assert usage_classification.detail_class == FAILURE_CLASS_USAGE_LIMIT
+    assert auth_classification.detail_class == FAILURE_CLASS_AUTH_FAILURE