@descryy/runtime-test-runner 0.4.12 → 0.4.14

package/reporters/phpunit/DescryPhpUnitExtension.php

@@ -0,0 +1,209 @@
+<?php declare(strict_types=1);
+
+/*
+ * §17's PHPUnit reporter — a runner extension writing the same NDJSON
+ * envelope every other reporter in this package writes (RT-109).
+ *
+ * ## Why an extension and not the JUnit XML or TestDox output
+ *
+ * PHPUnit emits JUnit XML, and RT-088 measured what that format costs:
+ * `@Disabled` and an aborted assumption collapse into an identical bare
+ * `<skipped>`. PHPUnit has the same problem with more to lose —
+ * `markTestSkipped()` and `markTestIncomplete()` are different facts about
+ * coverage and the XML writes both as `<skipped>`.
+ *
+ * PHPUnit's **event system** is its own structured stream. It also removes
+ * the shape RT-088 warned about generally: a document written at the end of
+ * the run produces nothing at all if the run is killed.
+ *
+ * `TestListener` is not an option here and should not be looked for: it was
+ * removed in PHPUnit 10 and replaced by this.
+ *
+ * ## The four outcomes, and the one that needs a snapshot
+ *
+ * `MarkedIncomplete` -> `not-implemented`. PHPUnit's own word for it is
+ * "incomplete", meaning the test is a placeholder for behaviour that is not
+ * written. That is `test.todo` exactly.
+ *
+ * `Errored` and `Failed` are both `test:fail`, named apart in the payload:
+ * an assertion that did not hold and an exception that escaped are different
+ * things to go and look at, and PHPUnit is the only layer that knows which
+ * happened.
+ *
+ * `Skipped` is the one that needs care, because PHPUnit reaches it two ways
+ * — a `#[Requires*]` attribute the runner evaluated *before* the test, and
+ * `markTestSkipped()` *inside* the body. Measured: `Test\Prepared` fires only
+ * for the second. So a `Skipped` with a preceding `Prepared` ran real code
+ * and bailed on an assumption (`precondition-unmet`, JUnit's `ABORTED`), and
+ * one without never started (`disabled`).
+ *
+ * That is the same shape RSpec needed (its metadata is mutated by execution)
+ * and pytest needed (its markers are only readable at collection). Three
+ * runners, three different mechanisms, one rule: **the distinction is read
+ * from the runner's own event order, never parsed out of a message string.**
+ *
+ * ## The envelope version
+ *
+ * Declared before the first event, as a number: `{"version":"1"}` is read as
+ * a broken reporter rather than a matching one. Keep in step with
+ * `ENVELOPE_VERSION` in `test-runner-collector.ts` — that file is the
+ * authority, not this comment.
+ */
+
+use PHPUnit\Event\Test\Errored;
+use PHPUnit\Event\Test\ErroredSubscriber;
+use PHPUnit\Event\Test\Failed;
+use PHPUnit\Event\Test\FailedSubscriber;
+use PHPUnit\Event\Test\MarkedIncomplete;
+use PHPUnit\Event\Test\MarkedIncompleteSubscriber;
+use PHPUnit\Event\Test\Passed;
+use PHPUnit\Event\Test\PassedSubscriber;
+use PHPUnit\Event\Test\Prepared;
+use PHPUnit\Event\Test\PreparedSubscriber;
+use PHPUnit\Event\Test\Skipped;
+use PHPUnit\Event\Test\SkippedSubscriber;
+use PHPUnit\Event\TestRunner\Started as RunnerStarted;
+use PHPUnit\Event\TestRunner\StartedSubscriber as RunnerStartedSubscriber;
+use PHPUnit\Runner\Extension\Extension;
+use PHPUnit\Runner\Extension\Facade;
+use PHPUnit\Runner\Extension\ParameterCollection;
+use PHPUnit\TextUI\Configuration\Configuration;
+
+final class DescryEmitter
+{
+    /** Must equal ENVELOPE_VERSION in test-runner-collector.ts. A mismatch is refused there, loudly. */
+    public const ENVELOPE_VERSION = 1;
+
+    /** Test ids that reached `Prepared`, i.e. whose body actually started. */
+    private static array $prepared = [];
+
+    public static function envelope(): void
+    {
+        self::write(['type' => 'descry:envelope', 'data' => ['version' => self::ENVELOPE_VERSION]]);
+    }
+
+    public static function started(Prepared $event): void
+    {
+        self::$prepared[$event->test()->id()] = true;
+        self::write(['type' => 'test:start', 'data' => self::common($event)]);
+    }
+
+    public static function passed(Passed $event): void
+    {
+        self::write(['type' => 'test:pass', 'data' => self::common($event)]);
+    }
+
+    public static function failed(Failed $event): void
+    {
+        $data = self::common($event);
+        $data['details'] = ['error' => ['message' => $event->throwable()->asString()]];
+        self::write(['type' => 'test:fail', 'data' => $data]);
+    }
+
+    public static function errored(Errored $event): void
+    {
+        $data = self::common($event);
+        // Named apart from a failed assertion. PHPUnit is the only layer that
+        // knows which of the two happened, and they send a developer to
+        // different places.
+        $data['details'] = ['error' => ['message' => $event->throwable()->asString(), 'failureType' => 'error']];
+        self::write(['type' => 'test:fail', 'data' => $data]);
+    }
+
+    public static function skipped(Skipped $event): void
+    {
+        $data = self::common($event);
+        $data['skip'] = true;
+        // Read from the event order, never from the message. A `#[Requires*]`
+        // attribute is evaluated before the test and produces no `Prepared`;
+        // `markTestSkipped()` runs inside the body, which necessarily did.
+        $data['skipReason'] = isset(self::$prepared[$event->test()->id()]) ? 'precondition-unmet' : 'disabled';
+        $data['message'] = $event->message();
+        self::write(['type' => 'test:pass', 'data' => $data]);
+    }
+
+    public static function incomplete(MarkedIncomplete $event): void
+    {
+        $data = self::common($event);
+        $data['skip'] = true;
+        // PHPUnit's own word is "incomplete": a placeholder for behaviour
+        // that is not written. node:test's `todo`, not its `skip`.
+        $data['skipReason'] = 'not-implemented';
+        $data['message'] = $event->throwable()->message();
+        self::write(['type' => 'test:pass', 'data' => $data]);
+    }
+
+    /**
+     * `name` is the method (with a data-set suffix for a data provider);
+     * `qualifiedName` is `Class::method`, which is what a graph resolves
+     * against. Both come off `TestMethod`, so nothing is stripped or
+     * reconstructed downstream.
+     */
+    private static function common($event): array
+    {
+        $test = $event->test();
+        $data = ['name' => $test->name()];
+
+        if ($test instanceof \PHPUnit\Event\Code\TestMethod) {
+            // `id()` is `Class::method` plus a data-provider suffix when
+            // there is one; `name()` is the bare method and drops it. The
+            // display name keeps the suffix so two data sets are two
+            // results, and `qualifiedName` is the declaring method so both
+            // resolve to one graph node. Neither is reconstructed by string
+            // surgery on the other beyond removing the class prefix `id()`
+            // itself put there.
+            $prefix                = $test->className() . '::';
+            $data['name']          = str_starts_with($test->id(), $prefix) ? substr($test->id(), strlen($prefix)) : $test->name();
+            $data['qualifiedName'] = $prefix . $test->methodName();
+            $data['file']          = $test->file();
+            $data['line']          = $test->line();
+        }
+
+        return $data;
+    }
+
+    /**
+     * **The leading newline is not cosmetic**, and it was measured on pytest
+     * rather than guessed at here: a runner that writes progress characters
+     * with no trailing newline puts a report on the same line as one, and
+     * the line stops being JSON. A collector then reads every result as
+     * noise while the starts arrive intact — a reporter that looks healthy
+     * and reports nothing. PHPUnit's default progress output does exactly
+     * that (`.`, `F`, `S`, `I`, in columns of 63).
+     */
+    private static function write(array $object): void
+    {
+        fwrite(STDOUT, "\n" . json_encode($object, JSON_UNESCAPED_SLASHES) . "\n");
+        fflush(STDOUT);
+    }
+}
+
+final class DescryPhpUnitExtension implements Extension
+{
+    public function bootstrap(Configuration $configuration, Facade $facade, ParameterCollection $parameters): void
+    {
+        $facade->registerSubscribers(
+            new class implements RunnerStartedSubscriber {
+                public function notify(RunnerStarted $event): void { DescryEmitter::envelope(); }
+            },
+            new class implements PreparedSubscriber {
+                public function notify(Prepared $event): void { DescryEmitter::started($event); }
+            },
+            new class implements PassedSubscriber {
+                public function notify(Passed $event): void { DescryEmitter::passed($event); }
+            },
+            new class implements FailedSubscriber {
+                public function notify(Failed $event): void { DescryEmitter::failed($event); }
+            },
+            new class implements ErroredSubscriber {
+                public function notify(Errored $event): void { DescryEmitter::errored($event); }
+            },
+            new class implements SkippedSubscriber {
+                public function notify(Skipped $event): void { DescryEmitter::skipped($event); }
+            },
+            new class implements MarkedIncompleteSubscriber {
+                public function notify(MarkedIncomplete $event): void { DescryEmitter::incomplete($event); }
+            },
+        );
+    }
+}

package/reporters/pytest/descry_pytest_plugin.py

@@ -0,0 +1,397 @@
+"""§17's pytest reporter — a plugin writing the same NDJSON envelope
+``reporter.ts``, ``jest-reporter.ts``, ``vitest-reporter.ts`` and
+``DescryJUnitListener.java`` write (RT-107).
+
+## Why a plugin and not ``--junit-xml``
+
+pytest can emit JUnit XML, and it is the wrong source for the same reason
+RT-088 gave for JUnit itself: the XML collapses distinct outcomes into one
+bare ``<skipped>``. pytest has **more** to lose than JUnit does — ``skip``,
+``skipif``, ``xfail`` and a runtime ``pytest.skip()`` are four different
+facts about coverage, and the report format keeps two of them apart at best.
+
+``pytest_runtest_logreport`` is pytest's own structured reporting hook. It
+receives the outcome pytest itself computed, so nothing here re-derives a
+verdict from printed text.
+
+## The four outcomes, and where each one is *read* rather than inferred
+
+``skipReason`` is set here, never downstream, for the same reason
+``isSuite`` is: only the reporter knows what its runner can tell apart.
+
+===========================  ==========================  ====================
+what the developer wrote     what it means for coverage  ``skipReason``
+===========================  ==========================  ====================
+``@pytest.mark.skip``        switched off deliberately   ``disabled``
+``@pytest.mark.skipif(c)``   the environment said no     ``precondition-unmet``
+``@pytest.mark.xfail``       known-broken, not absent    ``not-implemented``
+``pytest.skip()`` at runtime ran real code, then bailed  ``precondition-unmet``
+===========================  ==========================  ====================
+
+``xfail`` -> ``not-implemented`` is the one worth arguing. The vocabulary
+defines it as "declared as absent rather than switched off", and an ``xfail``
+test is written, does run, and is declared not to work yet — which is what
+``test.todo`` means on node:test, and is not what ``test.skip`` means. The
+alternative reading (``disabled``) would put ``xfail`` in the same bucket as
+``skip``, and the whole reason this reporter exists is that those two answer
+"is this code covered" differently.
+
+A runtime ``pytest.skip()`` gets ``precondition-unmet`` and not
+``unspecified``: it is JUnit's ``ABORTED`` exactly — the test body executed
+and bailed on an assumption.
+
+## Which marker applied is read at collection, not parsed from the report
+
+``report.longrepr`` for a skip is a ``(path, lineno, reason)`` tuple whose
+reason is a *string* like ``"Skipped: no database"``. Distinguishing
+``skip`` from ``skipif`` by parsing it would be a heuristic over prose. So
+markers are read off the item in ``pytest_collection_modifyitems``, where
+pytest hands over the real ``Mark`` objects, and looked up by ``nodeid``.
+
+The same pass reads ``module.__name__``, the class and the original function
+name, so ``qualifiedName`` is what pytest knows rather than a nodeid this
+plugin reformatted into something that looks like a dotted path.
+
+## Setup and teardown failures are real failures
+
+A fixture that raises produces ``failed`` on the ``setup`` phase and the test
+body never runs. Reported as ``test:fail``: it is the single signal that the
+test did not execute, and dropping it to make the phase handling uniform
+would trade a real failure for a silence. The ``call`` phase is what
+produces passes, so a passed ``setup``/``teardown`` is not forwarded — the
+same asymmetry the translator applies to node:test containers and the JUnit
+listener applies to its own.
+
+## The envelope version
+
+Declared before the first event, as a **number**: ``{"version": "1"}`` is
+read as a broken reporter rather than a matching one. Keep in step with
+``ENVELOPE_VERSION`` in ``test-runner-collector.ts`` — that file is the
+authority, not this docstring.
+"""
+
+import json
+import os
+import sys
+import threading
+
+import pytest
+
+#: Must equal ENVELOPE_VERSION in test-runner-collector.ts. A mismatch is refused there, loudly.
+ENVELOPE_VERSION = 1
+
+#: nodeid -> what collection could see about this item. Populated once, read per report.
+_COLLECTED: dict = {}
+
+#: Set from the config so writes can step outside pytest's own output capture.
+_CAPTURE_MANAGER = None
+
+_COVERAGE_DIRECTORY_VARIABLE = "DESCRY_COVERAGE_DIR"
+_COVERAGE_ROOT_VARIABLE = "DESCRY_COVERAGE_ROOT"
+_EXCLUDED_SEGMENTS = frozenset(
+    {"site-packages", "dist-packages", "node_modules", ".venv", "venv", "vendor", "__pycache__", ".tox", ".git"}
+)
+
+_RECORDER = None
+_TEST_FILES = set()
+
+
+class _LineRecorder:
+    def __init__(self, root):
+        self.root = root
+        self.executed = {}
+        self.mechanism = None
+        self.tool_id = None
+        self.notes = []
+        self.relative_cache = {}
+        self.own_file = os.path.realpath(__file__)
+
+    def relative(self, filename):
+        cached = self.relative_cache.get(filename)
+        if cached is not None:
+            return cached
+        result = ""
+        try:
+            if filename.endswith(".py") and os.path.isabs(filename):
+                real = os.path.realpath(filename)
+                prefix = self.root + os.sep
+                if real.startswith(prefix) and real != self.own_file:
+                    parts = real[len(prefix):].split(os.sep)
+                    if not _EXCLUDED_SEGMENTS.intersection(parts):
+                        result = "/".join(parts)
+        except (OSError, ValueError):
+            result = ""
+        self.relative_cache[filename] = result
+        return result
+
+    def lines_of(self, relative):
+        lines = self.executed.get(relative)
+        if lines is None:
+            lines = self.executed[relative] = set()
+        return lines
+
+    def on_line(self, code, line_number):
+        relative = self.relative(code.co_filename)
+        if relative and line_number > 0:
+            self.lines_of(relative).add(line_number)
+        return sys.monitoring.DISABLE
+
+    def global_trace(self, frame, event, argument):
+        relative = self.relative(frame.f_code.co_filename)
+        if not relative:
+            return None
+        lines = self.lines_of(relative)
+
+        def local_trace(local_frame, local_event, local_argument):
+            if local_event == "line":
+                lines.add(local_frame.f_lineno)
+            return local_trace
+
+        return local_trace
+
+    def start_monitoring(self):
+        monitoring = getattr(sys, "monitoring", None)
+        if monitoring is None:
+            return False
+        for tool_id in dict.fromkeys((monitoring.COVERAGE_ID, 3, 4, 2, 5)):
+            try:
+                monitoring.use_tool_id(tool_id, "descry-coverage")
+            except ValueError:
+                continue
+            monitoring.register_callback(tool_id, monitoring.events.LINE, self.on_line)
+            monitoring.set_events(tool_id, monitoring.events.LINE)
+            self.mechanism = "monitoring"
+            self.tool_id = tool_id
+            if tool_id != monitoring.COVERAGE_ID:
+                self.notes.append(
+                    "sys.monitoring COVERAGE_ID was taken by another tool, so tool id %d was used" % tool_id
+                )
+            return True
+        self.notes.append("every sys.monitoring tool id in use, fell back to sys.settrace")
+        return False
+
+    def start_settrace(self):
+        if sys.gettrace() is not None:
+            raise RuntimeError("another sys.settrace tracer (a debugger or coverage tool) is already active")
+        sys.settrace(self.global_trace)
+        threading.settrace(self.global_trace)
+        self.mechanism = "settrace"
+        self.notes.append("recorded with sys.settrace, which slows the test run noticeably")
+
+    def start(self):
+        if not self.start_monitoring():
+            self.start_settrace()
+
+    def stop(self):
+        if self.mechanism == "monitoring":
+            monitoring = sys.monitoring
+            monitoring.set_events(self.tool_id, 0)
+            monitoring.register_callback(self.tool_id, monitoring.events.LINE, None)
+            monitoring.free_tool_id(self.tool_id)
+        elif self.mechanism == "settrace":
+            sys.settrace(None)
+            threading.settrace(None)
+
+
+def _coverage_directory():
+    return os.environ.get(_COVERAGE_DIRECTORY_VARIABLE) or None
+
+
+def _write_coverage_file(payload):
+    directory = _coverage_directory()
+    if directory is None:
+        return
+    pid = os.getpid()
+    temporary = os.path.join(directory, ".coverage-%d.tmp" % pid)
+    with open(temporary, "w", encoding="utf-8") as handle:
+        json.dump(payload, handle)
+    os.replace(temporary, os.path.join(directory, "coverage-%d.json" % pid))
+
+
+def _write_unavailable(reason):
+    try:
+        _write_coverage_file({"unavailable": reason})
+    except Exception:
+        return
+
+
+def _start_recording():
+    global _RECORDER
+    root = os.environ.get(_COVERAGE_ROOT_VARIABLE)
+    if _RECORDER is not None or _coverage_directory() is None or not root:
+        return
+    recorder = _LineRecorder(os.path.realpath(root))
+    try:
+        recorder.start()
+    except Exception as error:
+        _write_unavailable("%s: %s" % (type(error).__name__, error))
+        return
+    _RECORDER = recorder
+
+
+def _finish_recording():
+    global _RECORDER
+    recorder, _RECORDER = _RECORDER, None
+    if recorder is None:
+        return
+    try:
+        recorder.stop()
+        files = {
+            relative: sorted(lines)
+            for relative, lines in recorder.executed.items()
+            if relative not in _TEST_FILES and relative.rsplit("/", 1)[-1] != "conftest.py"
+        }
+        _write_coverage_file({"tool": recorder.mechanism, "notes": recorder.notes, "files": files})
+    except Exception as error:
+        _write_unavailable("%s: %s" % (type(error).__name__, error))
+
+
+def _note_test_files(items):
+    recorder = _RECORDER
+    if recorder is None:
+        return
+    for item in items:
+        path = getattr(item, "path", None)
+        filename = str(path) if path is not None else str(item.fspath)
+        relative = recorder.relative(filename)
+        if relative:
+            _TEST_FILES.add(relative)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_load_initial_conftests(early_config, parser, args):
+    _start_recording()
+
+
+def _emit(line: str) -> None:
+    """Write one NDJSON line to the real stdout.
+
+    Two hazards, both measured rather than anticipated.
+
+    **Capture.** pytest captures stdout at the *file-descriptor* level by
+    default (``--capture=fd``), which swallows a plain ``print`` from a hook
+    as readily as one from a test body. The capture manager's own context is
+    the supported way out, and is what pytest's terminal reporter uses for
+    the same purpose.
+
+    **The leading newline is not cosmetic.** pytest's progress characters
+    are written with no trailing newline, so a report emitted from
+    ``pytest_runtest_logreport`` lands on the same line::
+
+        .{"type": "test:pass", ...}
+        F{"type": "test:fail", ...}
+
+    That line is not JSON. The collector would read every result as noise
+    and keep only ``test:start`` — measured exactly that way on the first
+    run of this plugin: eight starts, zero outcomes, and no error anywhere,
+    because unparseable stdout is *supposed* to be ignorable. A reporter
+    that silently reports only starts is the shape §17 exists to ban, so the
+    fix is to always begin on a fresh line rather than to depend on a
+    verbosity setting where it happens not to bite.
+    """
+    text = "\n" + line + "\n"
+    if _CAPTURE_MANAGER is None:
+        sys.stdout.write(text)
+        sys.stdout.flush()
+        return
+    with _CAPTURE_MANAGER.global_and_fixture_disabled():
+        sys.stdout.write(text)
+        sys.stdout.flush()
+
+
+def pytest_configure(config):
+    global _CAPTURE_MANAGER
+    _CAPTURE_MANAGER = config.pluginmanager.getplugin("capturemanager")
+    _start_recording()
+    _emit(json.dumps({"type": "descry:envelope", "data": {"version": ENVELOPE_VERSION}}))
+
+
+def pytest_collection_modifyitems(items):
+    """Read the markers and the real names off each item, once."""
+    _note_test_files(items)
+    for item in items:
+        markers = {mark.name for mark in item.iter_markers()}
+        module = getattr(item, "module", None)
+        cls = getattr(item, "cls", None)
+        parts = [getattr(module, "__name__", None), getattr(cls, "__name__", None), getattr(item, "originalname", None) or item.name]
+        location = getattr(item, "location", (None, None, None))
+        _COLLECTED[item.nodeid] = {
+            "markers": markers,
+            # `originalname` is the declaring function for a parameterised
+            # case, whose `name` carries its arguments. Both are emitted so
+            # the translator is never left to strip anything.
+            "qualifiedName": ".".join(p for p in parts if p),
+            "file": _absolute_file(item, location[0]),
+            # pytest's `location` line is 0-based; every other reporter in
+            # this envelope emits 1-based, which is what an editor opens.
+            "line": None if location[1] is None else location[1] + 1,
+        }
+
+
+def _absolute_file(item, relative):
+    path = getattr(item, "path", None) or getattr(item, "fspath", None)
+    return str(path) if path is not None else relative
+
+
+def _common(report) -> dict:
+    collected = _COLLECTED.get(report.nodeid, {})
+    data = {"name": report.nodeid.split("::")[-1]}
+    for key in ("qualifiedName", "file", "line"):
+        value = collected.get(key)
+        if value is not None:
+            data[key] = value
+    return data
+
+
+def _skip_reason(report) -> str:
+    markers = _COLLECTED.get(report.nodeid, {}).get("markers", set())
+    # `wasxfail` is set by pytest on any report an xfail marker governed,
+    # including one raised by `pytest.xfail()` at runtime, so it is checked
+    # before the markers rather than after.
+    if hasattr(report, "wasxfail") or "xfail" in markers:
+        return "not-implemented"
+    if "skipif" in markers:
+        return "precondition-unmet"
+    if "skip" in markers:
+        return "disabled"
+    # No marker: `pytest.skip()` or `pytest.importorskip()` inside the test
+    # body. It ran real code and bailed on an assumption — JUnit's ABORTED.
+    return "precondition-unmet"
+
+
+def pytest_runtest_logreport(report):
+    if report.when == "setup" and report.outcome == "passed":
+        _emit(json.dumps({"type": "test:start", "data": _common(report)}))
+        return
+
+    if report.outcome == "skipped":
+        data = _common(report)
+        data["skip"] = True
+        data["skipReason"] = _skip_reason(report)
+        if report.longreprtext:
+            data["message"] = report.longreprtext
+        _emit(json.dumps({"type": "test:pass", "data": data}))
+        return
+
+    if report.outcome == "failed":
+        data = _common(report)
+        data["details"] = {"error": {"message": report.longreprtext}}
+        if report.when != "call":
+            # A fixture failure. Named in the payload rather than collapsed,
+            # because "the test failed" and "the test never ran" are
+            # different facts and only this reporter can tell them apart.
+            data["details"]["error"]["failureType"] = f"{report.when}Error"
+        _emit(json.dumps({"type": "test:fail", "data": data}))
+        return
+
+    if report.when == "call" and report.outcome == "passed":
+        data = _common(report)
+        if report.duration is not None:
+            data["details"] = {"duration_ms": report.duration * 1000}
+        _emit(json.dumps({"type": "test:pass", "data": data}))
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_unconfigure(config):  # noqa: ARG001 - pytest's signature
+    _finish_recording()
+    _COLLECTED.clear()