@plumbus/ai-decision-laya 0.2.0

package/service/test_audit.py

@@ -0,0 +1,170 @@
+"""Adversarial HTTP and inference-boundary regressions; fake models only."""
+
+import http.client
+import json
+import socket
+import threading
+import unittest
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import test_server as fixtures
+from server import LayaBackend, RequestError, check_token_budget, create_server, validate_body
+
+
+class HttpAudit(unittest.TestCase):
+    def setUp(self):
+        self.backend = fixtures.FakeBackend()
+        self.server = create_server(("127.0.0.1", 0), self.backend, "test-key")
+        self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join()
+
+    def send(self, data=None, extra=(), content_type="application/json", length=None, incomplete=False):
+        data = json.dumps(fixtures.request_body()).encode() if data is None else data
+        conn = http.client.HTTPConnection(*self.server.server_address, timeout=2)
+        try:
+            conn.putrequest("POST", "/v1/systemone")
+            conn.putheader("Authorization", "Bearer test-key")
+            conn.putheader("Content-Type", content_type)
+            conn.putheader("Content-Length", str(len(data) if length is None else length))
+            for k, v in extra:
+                conn.putheader(k, v)
+            conn.endheaders(data)
+            if incomplete:
+                conn.sock.shutdown(socket.SHUT_WR)
+            response = conn.getresponse()
+            return response.status, json.loads(response.read())
+        finally:
+            conn.close()
+
+    def test_L01_numeric_overflow_cannot_enter_inference(self):
+        data = b'{"state":{"value":1e999},"questions":{"p":{"type":"noul","instructions":"?"}}}'
+        self.assertEqual(self.send(data)[0], 422)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L02_duplicate_json_keys_are_rejected(self):
+        data = b'{"state":"first","state":"second","questions":{"p":{"type":"noul","instructions":"?"}}}'
+        self.assertEqual(self.send(data)[0], 422)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L03_duplicate_content_lengths_are_rejected(self):
+        data = json.dumps(fixtures.request_body()).encode()
+        self.assertEqual(self.send(data, extra=[("Content-Length", str(len(data)))])[0], 400)
+
+    def test_L04_duplicate_auth_headers_are_rejected(self):
+        self.assertEqual(self.send(extra=[("Authorization", "Bearer other")])[0], 401)
+
+    def test_L05_incomplete_body_is_not_run_as_a_valid_shorter_request(self):
+        self.assertEqual(self.send(length=999, incomplete=True)[0], 400)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L06_unsupported_content_encoding_is_explicit(self):
+        self.assertEqual(self.send(extra=[("Content-Encoding", "gzip")])[0], 415)
+
+    def test_L07_wrong_content_type_is_rejected(self):
+        self.assertEqual(self.send(content_type="text/plain")[0], 415)
+
+    def test_L08_utf16_is_not_accepted_as_utf8_json(self):
+        data = json.dumps(fixtures.request_body()).encode("utf-16")
+        self.assertEqual(self.send(data)[0], 422)
+
+    def test_L09_inference_value_errors_are_server_errors(self):
+        self.backend.failure = ValueError("private checkpoint failure")
+        self.assertEqual(self.send(), (500, {"error": "Laya inference failed"}))
+
+    def test_L10_invalid_model_output_is_server_error(self):
+        self.backend.predict = lambda _body: {"answers": {"p": float("nan")}}
+        self.assertEqual(self.send()[0], 500)
+
+    def test_L11_output_size_is_bounded(self):
+        self.backend.predict = lambda _body: {"data": "x" * (2 * 1024 * 1024)}
+        self.assertEqual(self.send()[0], 500)
+
+    def test_L12_unpaired_unicode_surrogates_are_rejected_before_inference(self):
+        data = b'{"state":"\\ud800","questions":{"p":{"type":"noul","instructions":"?"}}}'
+        self.assertEqual(self.send(data)[0], 422)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L13_non_ascii_service_keys_fail_configuration(self):
+        with self.assertRaises(RequestError):
+            server = create_server(("127.0.0.1", 0), self.backend, "secret-ש")
+            server.server_close()
+
+    def test_L14_non_ascii_text_roundtrips_with_byte_content_length(self):
+        body = fixtures.request_body()
+        body["state"] = {"text": "נא להחזיר כסף 😀"}
+        self.assertEqual(self.send(json.dumps(body, ensure_ascii=False).encode())[0], 200)
+        self.assertEqual(self.backend.calls[0]["state"], body["state"])
+
+
+class BackendAudit(unittest.TestCase):
+    def make_backend(self, model="english"):
+        backend = LayaBackend.__new__(LayaBackend)
+        backend.names = ("english",)
+        backend.lock = threading.Lock()
+        class Route(dict):
+            @property
+            def model(self):
+                return self["model"]
+        agent = fixtures.BudgetTests().agent()
+        agent.predict = Mock(return_value={"answers": {}})
+        backend.router = SimpleNamespace(route=Mock(return_value=Route(model=model, repo="fake", reason="test")), load=Mock(return_value=agent))
+        backend.render_options = lambda _q: ["false", "true"]
+        backend.serialize_state = lambda value: value
+        return backend, agent
+
+    def test_L15_unpreloaded_checkpoint_does_not_trigger_loading(self):
+        backend, _ = self.make_backend("multilingual")
+        with self.assertRaises(RequestError):
+            backend.predict(fixtures.request_body())
+        backend.router.load.assert_not_called()
+
+    def test_L16_failed_inference_releases_device_lock(self):
+        backend, agent = self.make_backend()
+        agent.predict.side_effect = [RuntimeError("failure"), {"answers": {}}]
+        with self.assertRaises(RuntimeError):
+            backend.predict(fixtures.request_body())
+        self.assertIsNotNone(backend.predict(fixtures.request_body()))
+
+    def test_L17_concurrent_inference_does_not_load_twice(self):
+        backend, agent = self.make_backend()
+        entered, finish = threading.Event(), threading.Event()
+        def slow(_state, _questions):
+            entered.set()
+            finish.wait(2)
+            return {"answers": {}}
+        agent.predict.side_effect = slow
+        thread = threading.Thread(target=lambda: backend.predict(fixtures.request_body()))
+        thread.start()
+        try:
+            self.assertTrue(entered.wait(1))
+            self.assertIsNone(backend.predict(fixtures.request_body()))
+            backend.router.load.assert_called_once()
+        finally:
+            finish.set()
+            thread.join()
+
+    def test_L18_exact_state_token_budget_then_one_token_over(self):
+        agent = fixtures.BudgetTests().agent()
+        question = {"q": {"type": "noul", "instructions": "?"}}
+        # head 3 + two option markers/labels 4 + special tokens 4 = 11.
+        check_token_budget(agent, "word " * 21, question, lambda _q: ["false", "true"], lambda s: s)
+        with self.assertRaises(RequestError):
+            check_token_budget(agent, "word " * 22, question, lambda _q: ["false", "true"], lambda s: s)
+
+    def test_L19_blank_model_and_language_fail_request_validation(self):
+        for field in ("model", "lang"):
+            with self.subTest(field=field), self.assertRaises(RequestError):
+                validate_body({**fixtures.request_body(), field: "   "})
+
+    def test_L20_invalid_route_is_a_client_error_without_loading(self):
+        backend, _ = self.make_backend()
+        backend.router.route.side_effect = ValueError("Unknown checkpoint")
+        with self.assertRaises(RequestError):
+            backend.predict(fixtures.request_body())
+        backend.router.load.assert_not_called()

package/service/test_audit_round_two.py

@@ -0,0 +1,165 @@
+"""Second service audit: lifecycle, HTTP metadata and batch boundaries."""
+import http.client
+import json
+import socket
+import struct
+import sys
+import threading
+import unittest
+from types import ModuleType
+from unittest.mock import Mock, patch
+
+import test_audit as previous
+import test_server as fixtures
+from server import LayaBackend, RequestError, check_token_budget, create_server
+
+
+class ServiceRoundTwo(unittest.TestCase):
+    def setUp(self):
+        self.backend = fixtures.FakeBackend()
+        self.server = create_server(("127.0.0.1", 0), self.backend, "test-key")
+        self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join()
+
+    def send(self, extra=(), length="auto", auth="Bearer test-key", address=None):
+        data = json.dumps(fixtures.request_body()).encode()
+        conn = http.client.HTTPConnection(*(address or self.server.server_address), timeout=2)
+        try:
+            conn.putrequest("POST", "/v1/systemone")
+            conn.putheader("Authorization", auth)
+            conn.putheader("Content-Type", "application/json")
+            if length is not None:
+                conn.putheader("Content-Length", str(len(data)) if length == "auto" else length)
+            for key, value in extra:
+                conn.putheader(key, value)
+            conn.endheaders(data)
+            response = conn.getresponse()
+            return response.status, json.loads(response.read())
+        finally:
+            conn.close()
+
+    def test_L23_missing_length_cannot_start_inference(self):
+        self.assertEqual(self.send(length=None)[0], 400)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L24_signed_length_cannot_start_inference(self):
+        self.assertEqual(self.send(length="+100")[0], 400)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_L25_conflicting_content_types_are_rejected(self):
+        self.assertEqual(self.send(extra=[("Content-Type", "text/plain")])[0], 415)
+
+    def test_L26_conflicting_content_encodings_are_rejected(self):
+        self.assertEqual(self.send(extra=[("Content-Encoding", "identity"), ("Content-Encoding", "gzip")])[0], 415)
+
+    def test_L27_authentication_scheme_is_case_insensitive(self):
+        self.assertEqual(self.send(auth="bEaReR test-key")[0], 200)
+
+    def test_L28_client_disconnect_during_inference_does_not_escape_handler(self):
+        entered, finish = threading.Event(), threading.Event()
+        original = self.backend.predict
+        def wait(body):
+            entered.set()
+            finish.wait(2)
+            return original(body)
+        self.backend.predict = wait
+        self.server.handle_error = Mock()
+        connection = http.client.HTTPConnection(*self.server.server_address, timeout=2)
+        body = json.dumps(fixtures.request_body())
+        try:
+            connection.request("POST", "/v1/systemone", body=body, headers={"Authorization": "Bearer test-key", "Content-Type": "application/json"})
+            self.assertTrue(entered.wait(1))
+            connection.sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
+            connection.close()
+        finally:
+            connection.close()
+            finish.set()
+        self.server.shutdown()
+        self.server.server_close()  # Join handlers before inspecting their error outcome.
+        self.server.handle_error.assert_not_called()
+
+    def test_L29_connections_are_bounded_and_capacity_is_reusable(self):
+        backend = fixtures.FakeBackend()
+        entered, finish = threading.Event(), threading.Event()
+        original = backend.predict
+        def wait(body):
+            entered.set()
+            finish.wait(2)
+            return original(body)
+        backend.predict = wait
+        server = create_server(("127.0.0.1", 0), backend, "test-key", max_connections=1)
+        thread = threading.Thread(target=lambda: server.serve_forever(poll_interval=0.01), daemon=True)
+        results = []
+        first = threading.Thread(target=lambda: results.append(self.send(address=server.server_address)[0]))
+        thread.start()
+        first.start()
+        try:
+            self.assertTrue(entered.wait(1))
+            self.assertEqual(self.send(address=server.server_address)[0], 503)
+            finish.set()
+            first.join(2)
+            self.assertEqual(results, [200])
+            self.assertEqual(self.send(address=server.server_address)[0], 200)
+        finally:
+            finish.set()
+            first.join(2)
+            server.shutdown()
+            server.server_close()
+            thread.join()
+
+    def test_L30_invalid_unicode_output_is_a_safe_server_error(self):
+        self.backend.predict = lambda _body: {"result": "\ud800"}
+        self.assertEqual(self.send(), (500, {"error": "Laya inference failed"}))
+
+    def fake_modules(self):
+        laya, common = ModuleType("laya"), ModuleType("laya.common")
+        laya.Router = Mock()
+        common.render_options = Mock()
+        common.serialize_state = Mock()
+        return laya, {"laya": laya, "laya.common": common}
+
+    def test_L31_invalid_preload_list_does_not_construct_router(self):
+        module, modules = self.fake_modules()
+        with patch.dict(sys.modules, modules), patch.dict("os.environ", {"LAYA_MODELS": "invalid"}):
+            with self.assertRaises(RequestError):
+                LayaBackend()
+            module.Router.assert_not_called()
+
+    def test_L32_duplicate_preload_names_do_not_increase_model_count(self):
+        module, modules = self.fake_modules()
+        with patch.dict(sys.modules, modules), patch.dict("os.environ", {"LAYA_MODELS": "english,english,multilingual", "LAYA_DEVICE": "cpu"}):
+            LayaBackend()
+            module.Router.assert_called_once_with(device="cpu", max_loaded=2)
+            module.Router.return_value.preload.assert_called_once_with(["english", "multilingual"])
+
+    def test_L33_exact_option_budget_keeps_all_48_description_tokens(self):
+        agent = fixtures.BudgetTests().agent()
+        agent.cfg = {"max_len": 256, "head_max_len": 192}
+        check_token_budget(agent, "short", {"q": {"type": "choice", "instructions": "?"}}, lambda _q: ["word " * 48, "other"], lambda s: s)
+        with self.assertRaises(RequestError):
+            check_token_budget(agent, "short", {"q": {"type": "choice", "instructions": "?"}}, lambda _q: ["word " * 49, "other"], lambda s: s)
+
+    def test_L34_structured_state_reaches_model_without_replacement_by_serialized_text(self):
+        backend, agent = previous.BackendAudit().make_backend()
+        backend.serialize_state = lambda value: json.dumps(value, ensure_ascii=False)
+        body = {**fixtures.request_body(), "state": {"text": "שלום"}}
+        backend.predict(body)
+        agent.predict.assert_called_once_with(body["state"], body["questions"])
+
+    def test_L35_one_oversized_question_rejects_the_whole_batch_before_inference(self):
+        backend, agent = previous.BackendAudit().make_backend()
+        body = fixtures.request_body()
+        body["questions"]["too_long"] = {"type": "noul", "instructions": "word " * 100}
+        with self.assertRaises(RequestError):
+            backend.predict(body)
+        agent.predict.assert_not_called()
+
+    def test_L36_callers_cannot_override_the_server_device(self):
+        from server import validate_body
+        with self.assertRaises(RequestError):
+            validate_body({**fixtures.request_body(), "device": "cuda:9"})

package/service/test_server.py

@@ -0,0 +1,145 @@
+"""Offline service tests. No Laya/torch import, model downloads, or GPU required."""
+
+import http.client
+import json
+import threading
+import unittest
+import urllib.error
+import urllib.request
+from types import SimpleNamespace
+
+from server import LayaBackend, RequestError, check_token_budget, create_server, validate_body
+
+
+class FakeBackend:
+    names = ("english",)
+
+    def __init__(self):
+        self.calls = []
+        self.busy = False
+        self.failure = None
+
+    def predict(self, body):
+        self.calls.append(body)
+        if self.failure:
+            raise self.failure
+        if self.busy:
+            return None
+        return {"model": "fake-checkpoint", "usage": {"input_tokens": 10, "output_tokens": 0}, "answers": {"refund": {"type": "noul", "noul": 0.9}}}
+
+
+def request_body():
+    return {"state": "Refund please", "questions": {"refund": {"type": "noul", "instructions": "Refund requested?"}}}
+
+
+class ServiceTests(unittest.TestCase):
+    def setUp(self):
+        self.backend = FakeBackend()
+        self.server = create_server(("127.0.0.1", 0), self.backend, "test-key")
+        self.thread = threading.Thread(target=lambda: self.server.serve_forever(poll_interval=0.01), daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join()
+
+    def call(self, body=None, key="test-key", path="/v1/systemone", method="POST"):
+        data = json.dumps(body if body is not None else request_body()).encode() if method == "POST" else None
+        req = urllib.request.Request(f"http://127.0.0.1:{self.server.server_address[1]}{path}", data=data, method=method, headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"})
+        try:
+            response = urllib.request.urlopen(req, timeout=2)
+        except urllib.error.HTTPError as error:
+            response = error
+        with response:
+            return response.status, json.loads(response.read())
+
+    def test_predict(self):
+        status, result = self.call()
+        self.assertEqual(status, 200)
+        self.assertEqual(result["answers"]["refund"]["noul"], 0.9)
+        self.assertEqual(len(self.backend.calls), 1)
+
+    def test_auth_before_inference(self):
+        self.assertEqual(self.call(key="wrong")[0], 401)
+        self.assertEqual(self.backend.calls, [])
+
+    def test_health_and_not_found(self):
+        self.assertEqual(self.call(path="/healthz", method="GET"), (200, {"ready": True, "models": ["english"]}))
+        self.assertEqual(self.call(path="/bad")[0], 404)
+
+    def test_invalid_request_and_oversize(self):
+        self.assertEqual(self.call(body={"state": None, "questions": {}})[0], 422)
+        # Send only the oversized Content-Length. Rejection must precede reading
+        # the body; sending megabytes races the server closing the connection.
+        connection = http.client.HTTPConnection(*self.server.server_address, timeout=2)
+        try:
+            connection.putrequest("POST", "/v1/systemone")
+            connection.putheader("Authorization", "Bearer test-key")
+            connection.putheader("Content-Length", str(2 * 1024 * 1024 + 1))
+            connection.endheaders()
+            response = connection.getresponse()
+            self.assertEqual(response.status, 413)
+            response.read()
+        finally:
+            connection.close()
+        self.assertEqual(self.backend.calls, [])
+
+    def test_busy_and_safe_failure(self):
+        self.backend.busy = True
+        self.assertEqual(self.call()[0], 503)
+        self.backend.failure = RuntimeError("private-user-input")
+        self.assertEqual(self.call(), (500, {"error": "Laya inference failed"}))
+
+    def test_truncation_failure(self):
+        self.backend.failure = RequestError("Too long")
+        self.assertEqual(self.call()[0], 422)
+
+    def test_no_unauthenticated_configuration(self):
+        with self.assertRaises(RequestError):
+            create_server(("127.0.0.1", 0), self.backend, "")
+
+
+class BudgetTests(unittest.TestCase):
+    def agent(self):
+        class Tokenizer:
+            mask_token = "[MASK]"
+
+            def __call__(self, text, **_kwargs):
+                return {"input_ids": text.split()}
+
+        return SimpleNamespace(tok=Tokenizer(), cfg={"max_len": 32, "head_max_len": 24}, _to_internal=lambda q: {"t": q["type"], "ins": q["instructions"]})
+
+    def check(self, state, instructions="Refund?", options=None):
+        check_token_budget(self.agent(), state, {"q": {"type": "noul", "instructions": instructions}}, lambda _q: options or ["false", "true"], lambda value: value)
+
+    def test_short_request_and_truncation_boundaries(self):
+        self.check("Refund please")
+        for state, instructions, options in [
+            ("word " * 40, "Refund?", None),
+            ("short", "instruction " * 40, None),
+            ("short", "Refund?", ["option " * 49, "other"]),
+        ]:
+            with self.assertRaises(RequestError):
+                self.check(state, instructions, options)
+
+    def test_busy_inference_returns_without_loading_a_checkpoint(self):
+        backend = LayaBackend.__new__(LayaBackend)
+        backend.lock = threading.Lock()
+        backend.lock.acquire()
+        self.assertIsNone(backend.predict(request_body()))
+        backend.lock.release()
+
+    def test_wire_question_validation(self):
+        for question in [
+            {"type": "choice", "instructions": "?", "criteria": {"only": None}},
+            {"type": "score", "instructions": "?", "criteria": ["one"]},
+            {"type": "noul", "instructions": "?", "criteria": {"invalid": "yes"}},
+            {"type": "unknown", "instructions": "?"},
+        ]:
+            with self.assertRaises(RequestError):
+                validate_body({"state": "text", "questions": {"q": question}})
+
+
+if __name__ == "__main__":
+    unittest.main()