@1aboveio/skills 1.0.15 → 1.0.17

package/skills/app-dev/android-development/scripts/test_failure_capture.py

@@ -0,0 +1,378 @@
+"""Focused tests for failure_capture.py: no device, no adb, no emulator.
+
+Run directly (`python3 test_failure_capture.py`) or through
+`scripts/failure-capture.test.mjs`, which is what CI discovers.
+"""
+
+import contextlib
+import io
+import json
+import os
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from failure_capture import (  # noqa: E402
+    LOGCAT_NAME,
+    NATIVE_FAILURE_PREFIX,
+    RECORD_KEYS,
+    RECORD_NAME,
+    SCREENSHOT_NAME,
+    UI_DUMP_NAME,
+    AdbClient,
+    CaptureError,
+    FailureRecorder,
+    read_record,
+)
+
+
+class FakeDevice:
+    """Stands in for AdbClient: records calls and replays configured outcomes."""
+
+    def __init__(self, fail=()):
+        self.fail = set(fail)
+        self.calls = []
+
+    def _capture(self, kind, destination, payload):
+        self.calls.append((kind, destination))
+        if kind in self.fail:
+            raise CaptureError("%s failed" % kind)
+        if payload is None:
+            return
+        with open(destination, "wb") as stream:
+            stream.write(payload)
+
+    def screencap(self, destination, timeout):
+        self.timeout = timeout
+        self._capture("screenshot", destination, b"\x89PNG\r\n")
+
+    def ui_dump(self, destination, timeout):
+        self.timeout = timeout
+        self._capture("uiDump", destination, b'<?xml version="1.0"?><hierarchy rotation="0"/>')
+
+    def logcat_tail(self, destination, lines, timeout):
+        self.timeout = timeout
+        self.lines = lines
+        self._capture("logcatTail", destination, b"E ActivityManager: crash\n")
+
+
+class RecorderTest(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.after = self.directory.name
+        self.evidence = os.path.join(self.directory.name, "logs")
+        self.device = FakeDevice()
+        self.output = io.StringIO()
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def recorder(self, **options):
+        options.setdefault("client", self.device)
+        return FailureRecorder(
+            "checkout smoke", self.evidence, stream=self.output, **options
+        )
+
+    def record(self):
+        with open(os.path.join(self.evidence, RECORD_NAME), encoding="utf-8") as stream:
+            return json.load(stream)
+
+    def native_lines(self):
+        return [line for line in self.output.getvalue().splitlines() if line]
+
+    @staticmethod
+    def bytes(path):
+        with open(path, "rb") as stream:
+            return stream.read()
+
+    def test_a_passing_step_writes_nothing(self):
+        recorder = self.recorder()
+        with recorder.step("tap Pay"):
+            pass
+        self.assertFalse(os.path.exists(os.path.join(self.evidence, RECORD_NAME)))
+        self.assertEqual(self.device.calls, [])
+        self.assertEqual(self.output.getvalue(), "")
+        self.assertIsNone(recorder.failure)
+
+    def test_a_failing_step_records_evidence_and_re_raises(self):
+        recorder = self.recorder()
+        with self.assertRaisesRegex(ValueError, "terminal 500"):
+            with recorder.step("wait for receipt"):
+                raise ValueError("terminal 500")
+
+        record = self.record()
+        self.assertEqual(list(record), list(RECORD_KEYS))
+        self.assertEqual(record["case"], "checkout smoke")
+        self.assertEqual(record["step"], "wait for receipt")
+        self.assertEqual(record["error"], "ValueError: terminal 500")
+        self.assertEqual(record["screenshot"], SCREENSHOT_NAME)
+        self.assertEqual(record["uiDump"], UI_DUMP_NAME)
+        self.assertEqual(record["logcatTail"], LOGCAT_NAME)
+        self.assertEqual(record["evidenceDir"], self.evidence)
+        self.assertEqual(record["cleanupErrors"], [])
+        for name in (SCREENSHOT_NAME, UI_DUMP_NAME, LOGCAT_NAME):
+            self.assertTrue(os.path.isfile(os.path.join(self.evidence, name)), name)
+
+        line = self.native_lines()
+        self.assertEqual(len(line), 1)
+        self.assertTrue(line[0].startswith(NATIVE_FAILURE_PREFIX))
+        self.assertNotIn("\n", line[0])
+        compact = json.loads(line[0][len(NATIVE_FAILURE_PREFIX):])
+        self.assertEqual(list(compact), ["case", "step", "error", "evidenceDir"])
+        self.assertEqual(compact["error"], "ValueError: terminal 500")
+        self.assertEqual(compact["evidenceDir"], self.evidence)
+
+    def test_a_capture_failure_keeps_the_original_error(self):
+        recorder = self.recorder(client=FakeDevice(fail={"screenshot", "uiDump", "logcatTail"}))
+        with self.assertRaisesRegex(RuntimeError, "app crashed"):
+            with recorder.step("launch"):
+                raise RuntimeError("app crashed")
+
+        record = self.record()
+        self.assertEqual(record["error"], "RuntimeError: app crashed")
+        self.assertIsNone(record["screenshot"])
+        self.assertIsNone(record["uiDump"])
+        self.assertIsNone(record["logcatTail"])
+        self.assertEqual(len(record["cleanupErrors"]), 3)
+        self.assertTrue(record["cleanupErrors"][0].startswith("screenshot: "))
+        self.assertEqual(len(self.native_lines()), 1)
+
+    def test_only_the_innermost_step_is_recorded(self):
+        recorder = self.recorder()
+        with self.assertRaises(AssertionError):
+            with recorder.step("outer"):
+                with recorder.step("inner"):
+                    raise AssertionError("selector mismatch")
+        self.assertEqual(self.record()["step"], "inner")
+        self.assertEqual(len(self.native_lines()), 1)
+        self.assertEqual(len(self.device.calls), 3)
+
+    def test_an_existing_record_is_never_overwritten(self):
+        os.makedirs(self.evidence)
+        kept = {
+            "case": "earlier case",
+            "step": "earlier step",
+            "error": "KeyError: 'Terminal'",
+            "screenshot": None,
+            "uiDump": None,
+            "logcatTail": None,
+            "evidenceDir": self.evidence,
+            "cleanupErrors": ["screenshot: adb vanished"],
+        }
+        path = os.path.join(self.evidence, RECORD_NAME)
+        with open(path, "w", encoding="utf-8") as stream:
+            json.dump(kept, stream)
+        before = self.bytes(path)
+
+        recorder = self.recorder()
+        with self.assertRaises(ValueError):
+            with recorder.step("later step"):
+                raise ValueError("later failure")
+
+        self.assertEqual(self.bytes(path), before)
+        self.assertEqual(self.device.calls, [])
+        parsed = json.loads(self.native_lines()[0][len(NATIVE_FAILURE_PREFIX):])
+        self.assertEqual(parsed["step"], "earlier step")
+        self.assertEqual(parsed["error"], "KeyError: 'Terminal'")
+
+    def test_the_second_failure_keeps_the_first_record(self):
+        recorder = self.recorder()
+        with self.assertRaises(ValueError):
+            with recorder.step("first"):
+                raise ValueError("first")
+        with self.assertRaises(ValueError):
+            with recorder.step("second"):
+                raise ValueError("second")
+        self.assertEqual(self.record()["step"], "first")
+        self.assertEqual(len(self.native_lines()), 1)
+        self.assertEqual(len(self.device.calls), 3)
+
+    def test_a_decorated_step_records_the_failure(self):
+        recorder = self.recorder()
+
+        @recorder.step("restart app")
+        def restart():
+            raise TimeoutError("package did not start")
+
+        with self.assertRaises(TimeoutError):
+            restart()
+        self.assertEqual(self.record()["step"], "restart app")
+        self.assertEqual(self.record()["error"], "TimeoutError: package did not start")
+
+    def test_a_cancelled_run_is_not_a_failed_case(self):
+        recorder = self.recorder()
+        with self.assertRaises(KeyboardInterrupt):
+            with recorder.step("long wait"):
+                raise KeyboardInterrupt()
+        self.assertIsNone(recorder.failure)
+        self.assertFalse(os.path.exists(os.path.join(self.evidence, RECORD_NAME)))
+        self.assertEqual(self.output.getvalue(), "")
+
+    def test_capture_settings_are_passed_to_the_device(self):
+        recorder = self.recorder(capture_timeout=2.5, logcat_lines=25)
+        with self.assertRaises(ValueError):
+            with recorder.step("tap"):
+                raise ValueError("tap")
+        self.assertEqual(self.device.timeout, 2.5)
+        self.assertEqual(self.device.lines, 25)
+
+    def test_an_unwritable_evidence_directory_is_reported_without_masking(self):
+        blocker = os.path.join(self.after, "blocked")
+        with open(blocker, "w", encoding="utf-8") as stream:
+            stream.write("not a directory")
+        recorder = FailureRecorder(
+            "case", os.path.join(blocker, "logs"), client=self.device, stream=self.output
+        )
+        with contextlib.redirect_stderr(io.StringIO()) as reported:
+            with self.assertRaisesRegex(ValueError, "boom"):
+                with recorder.step("step"):
+                    raise ValueError("boom")
+        self.assertIn("failure record not written", reported.getvalue())
+        self.assertEqual(recorder.failure.error, "ValueError: boom")
+        self.assertTrue(recorder.failure.cleanupErrors)
+        self.assertEqual(len(self.native_lines()), 1)
+
+    def test_the_default_evidence_directory_follows_log_dir(self):
+        previous = os.environ.get("LOG_DIR")
+        os.environ["LOG_DIR"] = self.evidence
+        try:
+            recorder = FailureRecorder("case", client=self.device, stream=self.output)
+            self.assertEqual(recorder.evidence_dir, self.evidence)
+        finally:
+            if previous is None:
+                os.environ.pop("LOG_DIR")
+            else:
+                os.environ["LOG_DIR"] = previous
+
+    def test_case_defaults_to_the_device_case_variable(self):
+        previous = os.environ.get("DEVICE_CASE")
+        os.environ["DEVICE_CASE"] = "from-env"
+        try:
+            recorder = FailureRecorder(
+                None, self.evidence, client=self.device, stream=self.output
+            )
+            with self.assertRaises(ValueError):
+                with recorder.step("step"):
+                    raise ValueError("boom")
+            self.assertEqual(self.record()["case"], "from-env")
+        finally:
+            if previous is None:
+                os.environ.pop("DEVICE_CASE")
+            else:
+                os.environ["DEVICE_CASE"] = previous
+
+
+class Result:
+    def __init__(self, returncode=0, stdout=b"", stderr=b""):
+        self.returncode = returncode
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+class AdbClientTest(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.after = self.directory.name
+        self.commands = []
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def client(self, responses, serial="emulator-5580", **options):
+        def adb(command, timeout):
+            self.commands.append((command, timeout))
+            outcome = responses.pop(0)
+            if isinstance(outcome, BaseException):
+                raise outcome
+            return outcome
+
+        return AdbClient(serial=serial, adb=adb, **options)
+
+    def path(self, name):
+        return os.path.join(self.after, name)
+
+    @staticmethod
+    def bytes(path):
+        with open(path, "rb") as stream:
+            return stream.read()
+
+    def test_every_capture_names_the_serial_and_stays_bounded(self):
+        client = self.client(
+            [
+                Result(stdout=b"\x89PNG"),
+                Result(stdout=b"UI hiercharchy dumped"),
+                Result(stdout=b"<hierarchy/>"),
+                Result(stdout=b"line\n"),
+            ]
+        )
+        client.screencap(self.path("s.png"), 3.0)
+        client.ui_dump(self.path("ui.xml"), 4.0)
+        client.logcat_tail(self.path("logcat.txt"), 120, 5.0)
+
+        self.assertEqual(
+            self.commands,
+            [
+                (["adb", "-s", "emulator-5580", "exec-out", "screencap", "-p"], 3.0),
+                (["adb", "-s", "emulator-5580", "shell", "uiautomator", "dump", "/sdcard/failure-capture.xml"], 4.0),
+                (["adb", "-s", "emulator-5580", "exec-out", "cat", "/sdcard/failure-capture.xml"], 4.0),
+                (["adb", "-s", "emulator-5580", "logcat", "-d", "-t", "120"], 5.0),
+            ],
+        )
+        self.assertEqual(self.bytes(self.path("s.png")), b"\x89PNG")
+        self.assertEqual(self.bytes(self.path("logcat.txt")), b"line\n")
+
+    def test_without_a_serial_adb_uses_its_default_device(self):
+        client = self.client([Result(stdout=b"png")], serial=None)
+        client.screencap(self.path("s.png"), 1.0)
+        self.assertEqual(self.commands[0][0], ["adb", "exec-out", "screencap", "-p"])
+
+    def test_a_timeout_is_a_capture_error(self):
+        client = self.client([__import__("subprocess").TimeoutExpired("adb", 1.0)])
+        with self.assertRaisesRegex(CaptureError, "timed out after 1.0s"):
+            client.screencap(self.path("s.png"), 1.0)
+        self.assertFalse(os.path.exists(self.path("s.png")))
+
+    def test_a_non_zero_exit_is_a_capture_error_with_its_stderr(self):
+        client = self.client([Result(returncode=1, stderr=b"device offline")])
+        with self.assertRaisesRegex(CaptureError, "device offline"):
+            client.logcat_tail(self.path("logcat.txt"), 10, 1.0)
+
+    def test_empty_or_wrong_output_is_a_capture_error(self):
+        client = self.client([Result(stdout=b""), Result(stdout=b"UI hiercharchy dumped"), Result(stdout=b"junk")])
+        with self.assertRaisesRegex(CaptureError, "no image"):
+            client.screencap(self.path("s.png"), 1.0)
+        with self.assertRaisesRegex(CaptureError, "no hierarchy"):
+            client.ui_dump(self.path("ui.xml"), 1.0)
+        self.assertFalse(os.path.exists(self.path("s.png")))
+        self.assertFalse(os.path.exists(self.path("ui.xml")))
+
+
+class RecordTest(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.evidence = self.directory.name
+
+    def tearDown(self):
+        self.directory.cleanup()
+
+    def test_an_unreadable_record_is_replaced(self):
+        path = os.path.join(self.evidence, RECORD_NAME)
+        with open(path, "w", encoding="utf-8") as stream:
+            stream.write("{not json")
+
+        recorder = FailureRecorder("case", self.evidence, client=FakeDevice(), stream=io.StringIO())
+        with self.assertRaises(ValueError):
+            with recorder.step("step"):
+                raise ValueError("boom")
+
+        record = read_record(self.evidence)
+        self.assertEqual(record.step, "step")
+        self.assertTrue(
+            any("replaced an unreadable record" in error for error in record.cleanupErrors)
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()