@ajksunkang-aios/kgraph-linux-x64 0.1.2 → 0.1.3

package/lib/site-packages/trio/_core/_tests/test_thread_cache.py

@@ -0,0 +1,227 @@
+from __future__ import annotations
+
+import os
+import threading
+import time
+from contextlib import contextmanager
+from queue import Queue
+from typing import TYPE_CHECKING, NoReturn
+
+import pytest
+
+from .. import _thread_cache
+from .._thread_cache import ThreadCache, start_thread_soon
+from .tutil import gc_collect_harder, slow
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from outcome import Outcome
+
+
+def test_thread_cache_basics() -> None:
+    q: Queue[Outcome[object]] = Queue()
+
+    def fn() -> NoReturn:
+        raise RuntimeError("hi")
+
+    def deliver(outcome: Outcome[object]) -> None:
+        q.put(outcome)
+
+    start_thread_soon(fn, deliver)
+
+    outcome = q.get()
+    with pytest.raises(RuntimeError, match="hi"):
+        outcome.unwrap()
+
+
+def test_thread_cache_deref() -> None:
+    res = [False]
+
+    class del_me:
+        def __call__(self) -> int:
+            return 42
+
+        def __del__(self) -> None:
+            res[0] = True
+
+    q: Queue[Outcome[int]] = Queue()
+
+    def deliver(outcome: Outcome[int]) -> None:
+        q.put(outcome)
+
+    start_thread_soon(del_me(), deliver)
+    outcome = q.get()
+    assert outcome.unwrap() == 42
+
+    gc_collect_harder()
+    assert res[0]
+
+
+@slow
+def test_spawning_new_thread_from_deliver_reuses_starting_thread() -> None:
+    # We know that no-one else is using the thread cache, so if we keep
+    # submitting new jobs the instant the previous one is finished, we should
+    # keep getting the same thread over and over. This tests both that the
+    # thread cache is LIFO, and that threads can be assigned new work *before*
+    # deliver exits.
+
+    # Make sure there are a few threads running, so if we weren't LIFO then we
+    # could grab the wrong one.
+    q: Queue[Outcome[object]] = Queue()
+    COUNT = 5
+    for _ in range(COUNT):
+        start_thread_soon(lambda: time.sleep(1), lambda result: q.put(result))
+    for _ in range(COUNT):
+        q.get().unwrap()
+
+    seen_threads = set()
+    done = threading.Event()
+
+    def deliver(n: int, _: object) -> None:
+        print(n)
+        seen_threads.add(threading.current_thread())
+        if n == 0:
+            done.set()
+        else:
+            start_thread_soon(lambda: None, lambda _: deliver(n - 1, _))
+
+    start_thread_soon(lambda: None, lambda _: deliver(5, _))
+
+    done.wait()
+
+    assert len(seen_threads) == 1
+
+
+@slow
+def test_idle_threads_exit(monkeypatch: pytest.MonkeyPatch) -> None:
+    # Temporarily set the idle timeout to something tiny, to speed up the
+    # test. (But non-zero, so that the worker loop will at least yield the
+    # CPU.)
+    monkeypatch.setattr(_thread_cache, "IDLE_TIMEOUT", 0.0001)
+
+    q: Queue[threading.Thread] = Queue()
+    start_thread_soon(lambda: None, lambda _: q.put(threading.current_thread()))
+    seen_thread = q.get()
+    # Since the idle timeout is 0, after sleeping for 1 second, the thread
+    # should have exited
+    time.sleep(1)
+    assert not seen_thread.is_alive()
+
+
+@contextmanager
+def _join_started_threads() -> Iterator[None]:
+    before = frozenset(threading.enumerate())
+    try:
+        yield
+    finally:
+        for thread in threading.enumerate():
+            if thread not in before:
+                thread.join(timeout=1.0)
+                assert not thread.is_alive()
+
+
+def test_race_between_idle_exit_and_job_assignment(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # This is a lock where the first few times you try to acquire it with a
+    # timeout, it waits until the lock is available and then pretends to time
+    # out. Using this in our thread cache implementation causes the following
+    # sequence:
+    #
+    # 1. start_thread_soon grabs the worker thread, assigns it a job, and
+    #    releases its lock.
+    # 2. The worker thread wakes up (because the lock has been released), but
+    #    the JankyLock lies to it and tells it that the lock timed out. So the
+    #    worker thread tries to exit.
+    # 3. The worker thread checks for the race between exiting and being
+    #    assigned a job, and discovers that it *is* in the process of being
+    #    assigned a job, so it loops around and tries to acquire the lock
+    #    again.
+    # 4. Eventually the JankyLock admits that the lock is available, and
+    #    everything proceeds as normal.
+
+    class JankyLock:
+        def __init__(self) -> None:
+            self._lock = threading.Lock()
+            self._counter = 3
+
+        def acquire(self, timeout: int = -1) -> bool:
+            got_it = self._lock.acquire(timeout=timeout)
+            if timeout == -1:
+                return True
+            elif got_it:
+                if self._counter > 0:
+                    self._counter -= 1
+                    self._lock.release()
+                    return False
+                return True
+            else:
+                return False
+
+        def release(self) -> None:
+            self._lock.release()
+
+    monkeypatch.setattr(_thread_cache, "Lock", JankyLock)
+
+    with _join_started_threads():
+        tc = ThreadCache()
+        done = threading.Event()
+        tc.start_thread_soon(lambda: None, lambda _: done.set())
+        done.wait()
+        # Let's kill the thread we started, so it doesn't hang around until the
+        # test suite finishes. Doesn't really do any harm, but it can be confusing
+        # to see it in debug output.
+        monkeypatch.setattr(_thread_cache, "IDLE_TIMEOUT", 0.0001)
+        tc.start_thread_soon(lambda: None, lambda _: None)
+
+
+def test_raise_in_deliver(capfd: pytest.CaptureFixture[str]) -> None:
+    seen_threads = set()
+
+    def track_threads() -> None:
+        seen_threads.add(threading.current_thread())
+
+    def deliver(_: object) -> NoReturn:
+        done.set()
+        raise RuntimeError("don't do this")
+
+    done = threading.Event()
+    start_thread_soon(track_threads, deliver)
+    done.wait()
+    done = threading.Event()
+    start_thread_soon(track_threads, lambda _: done.set())
+    done.wait()
+    assert len(seen_threads) == 1
+    err = capfd.readouterr().err
+    assert "don't do this" in err
+    assert "delivering result" in err
+
+
+@pytest.mark.skipif(not hasattr(os, "fork"), reason="os.fork isn't supported")
+def test_clear_thread_cache_after_fork() -> None:
+    assert hasattr(os, "fork")
+
+    def foo() -> None:
+        pass
+
+    # ensure the thread cache exists
+    done = threading.Event()
+    start_thread_soon(foo, lambda _: done.set())
+    done.wait()
+
+    child_pid = os.fork()
+
+    # try using it
+    done = threading.Event()
+    start_thread_soon(foo, lambda _: done.set())
+    done.wait()
+
+    if child_pid != 0:
+        # if this test fails, this will hang, triggering a timeout.
+        os.waitpid(child_pid, 0)
+    else:  # pragma: no cover  # coverage is shut down by os._exit(0)
+        # we would *want* to allow coverage to take a snapshot here. check
+        # git blame for how to do that. however, that times out for some
+        # reason when having `tests/` in `source` for coverage.py.
+        os._exit(0)

package/lib/site-packages/trio/_core/_tests/test_tutil.py

@@ -0,0 +1,13 @@
+import pytest
+
+from .tutil import check_sequence_matches
+
+
+def test_check_sequence_matches() -> None:
+    check_sequence_matches([1, 2, 3], [1, 2, 3])
+    with pytest.raises(AssertionError):
+        check_sequence_matches([1, 3, 2], [1, 2, 3])
+    check_sequence_matches([1, 2, 3, 4], [1, {2, 3}, 4])
+    check_sequence_matches([1, 3, 2, 4], [1, {2, 3}, 4])
+    with pytest.raises(AssertionError):
+        check_sequence_matches([1, 2, 4, 3], [1, {2, 3}, 4])

package/lib/site-packages/trio/_core/_tests/test_unbounded_queue.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import itertools
+
+import pytest
+
+from ... import _core
+from ...testing import assert_checkpoints, wait_all_tasks_blocked
+
+pytestmark = pytest.mark.filterwarnings(
+    "ignore:.*UnboundedQueue:trio.TrioDeprecationWarning",
+)
+
+
+async def test_UnboundedQueue_basic() -> None:
+    q: _core.UnboundedQueue[str | int | None] = _core.UnboundedQueue()
+    q.put_nowait("hi")
+    assert await q.get_batch() == ["hi"]
+    with pytest.raises(_core.WouldBlock):
+        q.get_batch_nowait()
+    q.put_nowait(1)
+    q.put_nowait(2)
+    q.put_nowait(3)
+    assert q.get_batch_nowait() == [1, 2, 3]
+
+    assert q.empty()
+    assert q.qsize() == 0
+    q.put_nowait(None)
+    assert not q.empty()
+    assert q.qsize() == 1
+
+    stats = q.statistics()
+    assert stats.qsize == 1
+    assert stats.tasks_waiting == 0
+
+    # smoke test
+    repr(q)
+
+
+async def test_UnboundedQueue_blocking() -> None:
+    record = []
+    q = _core.UnboundedQueue[int]()
+
+    async def get_batch_consumer() -> None:
+        while True:
+            batch = await q.get_batch()
+            assert batch
+            record.append(batch)
+
+    async def aiter_consumer() -> None:
+        async for batch in q:
+            assert batch
+            record.append(batch)
+
+    for consumer in (get_batch_consumer, aiter_consumer):
+        record.clear()
+        async with _core.open_nursery() as nursery:
+            nursery.start_soon(consumer)
+            await _core.wait_all_tasks_blocked()
+            stats = q.statistics()
+            assert stats.qsize == 0
+            assert stats.tasks_waiting == 1
+            q.put_nowait(10)
+            q.put_nowait(11)
+            await _core.wait_all_tasks_blocked()
+            q.put_nowait(12)
+            await _core.wait_all_tasks_blocked()
+            assert record == [[10, 11], [12]]
+            nursery.cancel_scope.cancel()
+
+
+async def test_UnboundedQueue_fairness() -> None:
+    q = _core.UnboundedQueue[int]()
+
+    # If there's no-one else around, we can put stuff in and take it out
+    # again, no problem
+    q.put_nowait(1)
+    q.put_nowait(2)
+    assert q.get_batch_nowait() == [1, 2]
+
+    result = None
+
+    async def get_batch(q: _core.UnboundedQueue[int]) -> None:
+        nonlocal result
+        result = await q.get_batch()
+
+    # But if someone else is waiting to read, then they get dibs
+    async with _core.open_nursery() as nursery:
+        nursery.start_soon(get_batch, q)
+        await _core.wait_all_tasks_blocked()
+        q.put_nowait(3)
+        q.put_nowait(4)
+        with pytest.raises(_core.WouldBlock):
+            q.get_batch_nowait()
+    assert result == [3, 4]
+
+    # If two tasks are trying to read, they alternate
+    record = []
+
+    async def reader(name: str) -> None:
+        while True:
+            record.append((name, await q.get_batch()))
+
+    async with _core.open_nursery() as nursery:
+        nursery.start_soon(reader, "a")
+        await _core.wait_all_tasks_blocked()
+        nursery.start_soon(reader, "b")
+        await _core.wait_all_tasks_blocked()
+
+        for i in range(20):
+            q.put_nowait(i)
+            await _core.wait_all_tasks_blocked()
+
+        nursery.cancel_scope.cancel()
+
+    assert record == list(zip(itertools.cycle("ab"), [[i] for i in range(20)]))
+
+
+async def test_UnboundedQueue_trivial_yields() -> None:
+    q = _core.UnboundedQueue[None]()
+
+    q.put_nowait(None)
+    with assert_checkpoints():
+        await q.get_batch()
+
+    q.put_nowait(None)
+    with assert_checkpoints():
+        async for _ in q:  # pragma: no branch
+            break
+
+
+async def test_UnboundedQueue_no_spurious_wakeups() -> None:
+    # If we have two tasks waiting, and put two items into the queue... then
+    # only one task wakes up
+    record = []
+
+    async def getter(q: _core.UnboundedQueue[int], i: int) -> None:
+        got = await q.get_batch()
+        record.append((i, got))
+
+    async with _core.open_nursery() as nursery:
+        q = _core.UnboundedQueue[int]()
+        nursery.start_soon(getter, q, 1)
+        await wait_all_tasks_blocked()
+        nursery.start_soon(getter, q, 2)
+        await wait_all_tasks_blocked()
+
+        for i in range(10):
+            q.put_nowait(i)
+        await wait_all_tasks_blocked()
+
+        assert record == [(1, list(range(10)))]
+
+        nursery.cancel_scope.cancel()

package/lib/site-packages/trio/_core/_tests/test_windows.py

@@ -0,0 +1,305 @@
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+from unittest.mock import create_autospec
+
+import pytest
+
+on_windows = os.name == "nt"
+# Mark all the tests in this file as being windows-only
+pytestmark = pytest.mark.skipif(not on_windows, reason="windows only")
+
+assert (
+    sys.platform == "win32" or not TYPE_CHECKING
+)  # Skip type checking when not on Windows
+
+from ... import _core, sleep
+from ...testing import wait_all_tasks_blocked
+from .tutil import gc_collect_harder, restore_unraisablehook, slow
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+    from io import BufferedWriter
+
+if on_windows:
+    from .._windows_cffi import (
+        INVALID_HANDLE_VALUE,
+        FileFlags,
+        Handle,
+        ffi,
+        kernel32,
+        raise_winerror,
+    )
+
+
+def test_winerror(monkeypatch: pytest.MonkeyPatch) -> None:
+    # this is unfortunately needed as the generated ffi is read-only
+    class FFIWrapper:
+        def getwinerror(self) -> None:
+            raise NotImplementedError("this is a fake implementation")
+
+    mock = create_autospec(ffi.getwinerror)
+    monkeypatch.setattr("trio._core._windows_cffi.ffi", FFIWrapper)
+    monkeypatch.setattr("trio._core._windows_cffi.ffi.getwinerror", mock)
+
+    # Returning none = no error, should not happen.
+    mock.return_value = None
+    with pytest.raises(RuntimeError, match=r"^No error set\?$"):
+        raise_winerror()
+    mock.assert_called_once_with()
+    mock.reset_mock()
+
+    with pytest.raises(RuntimeError, match=r"^No error set\?$"):
+        raise_winerror(38)
+    mock.assert_called_once_with(38)
+    mock.reset_mock()
+
+    mock.return_value = (12, "test error")
+    with pytest.raises(
+        OSError,
+        match=r"^\[WinError 12\] test error: 'file_1' -> 'file_2'$",
+    ) as exc:
+        raise_winerror(filename="file_1", filename2="file_2")
+    mock.assert_called_once_with()
+    mock.reset_mock()
+    assert exc.value.winerror == 12
+    assert exc.value.strerror == "test error"
+    assert exc.value.filename == "file_1"
+    assert exc.value.filename2 == "file_2"
+
+    # With an explicit number passed in, it overrides what getwinerror() returns.
+    with pytest.raises(
+        OSError,
+        match=r"^\[WinError 18\] test error: 'a/file' -> 'b/file'$",
+    ) as exc:
+        raise_winerror(18, filename="a/file", filename2="b/file")
+    mock.assert_called_once_with(18)
+    mock.reset_mock()
+    assert exc.value.winerror == 18
+    assert exc.value.strerror == "test error"
+    assert exc.value.filename == "a/file"
+    assert exc.value.filename2 == "b/file"
+
+
+# The undocumented API that this is testing should be changed to stop using
+# UnboundedQueue (or just removed until we have time to redo it), but until
+# then we filter out the warning.
+@pytest.mark.filterwarnings("ignore:.*UnboundedQueue:trio.TrioDeprecationWarning")
+async def test_completion_key_listen() -> None:
+    from .. import _io_windows
+
+    async def post(key: int) -> None:
+        iocp = Handle(ffi.cast("HANDLE", _core.current_iocp()))
+        for i in range(10):
+            print("post", i)
+            if i % 3 == 0:
+                await _core.checkpoint()
+            success = kernel32.PostQueuedCompletionStatus(iocp, i, key, ffi.NULL)
+            assert success
+
+    with _core.monitor_completion_key() as (key, queue):
+        async with _core.open_nursery() as nursery:
+            nursery.start_soon(post, key)
+            i = 0
+            print("loop")
+            async for batch in queue:  # pragma: no branch
+                print("got some", batch)
+                for info in batch:
+                    assert isinstance(info, _io_windows.CompletionKeyEventInfo)
+                    assert info.lpOverlapped == 0
+                    assert info.dwNumberOfBytesTransferred == i
+                    i += 1
+                if i == 10:
+                    break
+            print("end loop")
+
+
+async def test_readinto_overlapped() -> None:
+    data = b"1" * 1024 + b"2" * 1024 + b"3" * 1024 + b"4" * 1024
+    buffer = bytearray(len(data))
+
+    with tempfile.TemporaryDirectory() as tdir:
+        tfile = os.path.join(tdir, "numbers.txt")
+        with open(  # noqa: ASYNC230  # This is a test, synchronous is ok
+            tfile,
+            "wb",
+        ) as fp:
+            fp.write(data)
+            fp.flush()
+
+        rawname = tfile.encode("utf-16le") + b"\0\0"
+        rawname_buf = ffi.from_buffer(rawname)
+        handle = kernel32.CreateFileW(
+            ffi.cast("LPCWSTR", rawname_buf),
+            FileFlags.GENERIC_READ,
+            FileFlags.FILE_SHARE_READ,
+            ffi.NULL,  # no security attributes
+            FileFlags.OPEN_EXISTING,
+            FileFlags.FILE_FLAG_OVERLAPPED,
+            ffi.NULL,  # no template file
+        )
+        if handle == INVALID_HANDLE_VALUE:  # pragma: no cover
+            raise_winerror()
+
+        try:
+            with memoryview(buffer) as buffer_view:
+
+                async def read_region(start: int, end: int) -> None:
+                    await _core.readinto_overlapped(
+                        handle,
+                        buffer_view[start:end],
+                        start,
+                    )
+
+                _core.register_with_iocp(handle)
+                async with _core.open_nursery() as nursery:
+                    for start in range(0, 4096, 512):
+                        nursery.start_soon(read_region, start, start + 512)
+
+                assert buffer == data
+
+                with pytest.raises((BufferError, TypeError)):
+                    await _core.readinto_overlapped(handle, b"immutable")
+        finally:
+            kernel32.CloseHandle(handle)
+
+
+@contextmanager
+def pipe_with_overlapped_read() -> Generator[tuple[BufferedWriter, int], None, None]:
+    import msvcrt
+    from asyncio.windows_utils import pipe
+
+    read_handle, write_handle = pipe(overlapped=(True, False))
+    try:
+        write_fd = msvcrt.open_osfhandle(write_handle, 0)
+        yield os.fdopen(write_fd, "wb", closefd=False), read_handle
+    finally:
+        kernel32.CloseHandle(Handle(ffi.cast("HANDLE", read_handle)))
+        kernel32.CloseHandle(Handle(ffi.cast("HANDLE", write_handle)))
+
+
+@restore_unraisablehook()
+def test_forgot_to_register_with_iocp() -> None:
+    with pipe_with_overlapped_read() as (write_fp, read_handle):
+        with write_fp:
+            write_fp.write(b"test\n")
+
+        left_run_yet = False
+
+        async def main() -> None:
+            target = bytearray(1)
+            try:
+                async with _core.open_nursery() as nursery:
+                    nursery.start_soon(
+                        _core.readinto_overlapped,
+                        read_handle,
+                        target,
+                        name="xyz",
+                    )
+                    await wait_all_tasks_blocked()
+                    nursery.cancel_scope.cancel()
+            finally:
+                # Run loop is exited without unwinding running tasks, so
+                # we don't get here until the main() coroutine is GC'ed
+                assert left_run_yet
+
+        with pytest.raises(_core.TrioInternalError) as exc_info:
+            _core.run(main)
+        left_run_yet = True
+        assert "Failed to cancel overlapped I/O in xyz " in str(exc_info.value)
+        assert "forget to call register_with_iocp()?" in str(exc_info.value)
+
+        # Make sure the Nursery.__del__ assertion about dangling children
+        # gets put with the correct test
+        del exc_info
+        gc_collect_harder()
+
+
+@slow
+async def test_too_late_to_cancel() -> None:
+    import time
+
+    with pipe_with_overlapped_read() as (write_fp, read_handle):
+        _core.register_with_iocp(read_handle)
+        target = bytearray(6)
+        async with _core.open_nursery() as nursery:
+            # Start an async read in the background
+            nursery.start_soon(_core.readinto_overlapped, read_handle, target)
+            await wait_all_tasks_blocked()
+
+            # Synchronous write to the other end of the pipe
+            with write_fp:
+                write_fp.write(b"test1\ntest2\n")
+
+            # Note: not trio.sleep! We're making sure the OS level
+            # ReadFile completes, before Trio has a chance to execute
+            # another checkpoint and notice it completed.
+            time.sleep(1)  # noqa: ASYNC251
+            nursery.cancel_scope.cancel()
+        assert target[:6] == b"test1\n"
+
+        # Do another I/O to make sure we've actually processed the
+        # fallback completion that was posted when CancelIoEx failed.
+        assert await _core.readinto_overlapped(read_handle, target) == 6
+        assert target[:6] == b"test2\n"
+
+
+def test_lsp_that_hooks_select_gives_good_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from .. import _io_windows
+    from .._windows_cffi import CData, WSAIoctls, _handle
+
+    def patched_get_underlying(
+        sock: int | CData,
+        *,
+        which: int = WSAIoctls.SIO_BASE_HANDLE,
+    ) -> CData:
+        if hasattr(sock, "fileno"):  # pragma: no branch
+            sock = sock.fileno()
+        if which == WSAIoctls.SIO_BSP_HANDLE_SELECT:
+            return _handle(sock + 1)
+        else:
+            return _handle(sock)
+
+    monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying)
+    with pytest.raises(
+        RuntimeError,
+        match="SIO_BASE_HANDLE and SIO_BSP_HANDLE_SELECT differ",
+    ):
+        _core.run(sleep, 0)
+
+
+def test_lsp_that_completely_hides_base_socket_gives_good_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # This tests behavior with an LSP that fails SIO_BASE_HANDLE and returns
+    # self for SIO_BSP_HANDLE_SELECT (like Komodia), but also returns
+    # self for SIO_BSP_HANDLE_POLL. No known LSP does this, but we want to
+    # make sure we get an error rather than an infinite loop.
+
+    from .. import _io_windows
+    from .._windows_cffi import CData, WSAIoctls, _handle
+
+    def patched_get_underlying(
+        sock: int | CData,
+        *,
+        which: int = WSAIoctls.SIO_BASE_HANDLE,
+    ) -> CData:
+        if hasattr(sock, "fileno"):  # pragma: no branch
+            sock = sock.fileno()
+        if which == WSAIoctls.SIO_BASE_HANDLE:
+            raise OSError("nope")
+        else:
+            return _handle(sock)
+
+    monkeypatch.setattr(_io_windows, "_get_underlying_socket", patched_get_underlying)
+    with pytest.raises(
+        RuntimeError,
+        match="SIO_BASE_HANDLE failed and SIO_BSP_HANDLE_POLL didn't return a diff",
+    ):
+        _core.run(sleep, 0)